Bump version: 0.16.8-rc.0 → 0.16.8

* repaint category value when color changes

* bug fix incorrect indexing of user colors

* add test for bug 2007

* lint

.bumpversion.cfg

@@ -1,5 +1,16 @@
 [bumpversion]
-current_version = 0.15.0
+current_version = 0.16.8
+commit = True
+parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(?:-(?P<prerel>rc)\.(?P<prerelversion>\d+))?
+serialize = 
+	{major}.{minor}.{patch}-{prerel}.{prerelversion}
+	{major}.{minor}.{patch}
+
+[bumpversion:part:prerel]
+optional_value = release
+values = 
+	rc
+	release
 
 [bumpversion:file:setup.py]
 search = version="{current_version}"
@@ -12,4 +23,3 @@ replace = "version": "{new_version}"
 [bumpversion:file:server/__init__.py]
 search = __version__ = "{current_version}"
 replace = __version__ = "{new_version}"
-

.github/ISSUE_TEMPLATE/---bug-report.md

@@ -0,0 +1,32 @@
+---
+name: "\U0001F41E Bug report"
+about: Create a report to identify a bug in cellxgene
+title: "[BUG]"
+labels: bug
+assignees: ''
+
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behavior:
+1. Go to '...'
+2. Click on '....'
+3. Scroll down to '....'
+4. See error
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Screenshots**
+If applicable, add screenshots to help explain your problem.
+
+**Version (please complete the following information):**
+ - Desktop or hosted?:
+ - Browser (if hosted) [e.g. chrome, safari]:
+ - Version [e.g. 0.13.0]:
+
+**Additional context**
+Add any other context about the problem here.

.github/ISSUE_TEMPLATE/---feature-request.md

@@ -0,0 +1,20 @@
+---
+name: "\U0001F4A1 Feature request"
+about: Suggest an idea for this project
+title: "[FEATURE REQUEST]"
+labels: user request
+assignees: ''
+
+---
+
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Describe alternatives you've considered**
+A clear and concise description of any alternative solutions or features you've considered.
+
+**Additional context**
+Add any other context or screenshots about the feature request here.

.github/ISSUE_TEMPLATE/---question-clarification.md

@@ -0,0 +1,10 @@
+---
+name: "\U0001F9D0 Question/Clarification"
+about: Ask a question or for a clarification
+title: "[QUESTION]"
+labels: question
+assignees: ''
+
+---
+
+

.github/workflows/codeql-analysis.yml

@@ -0,0 +1,67 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL Scan"
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [ main ]
+  schedule:
+    - cron: '0 8 * * *'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'javascript', 'python' ]
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ]
+        # Learn more:
+        # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v1
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+        # queries: ./path/to/local/query, your-org/your-repo/queries@main
+
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v1
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 https://git.io/JvXDl
+
+    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
+    #    and modify them (or add more) to build your code if your project
+    #    uses a compiled language
+
+    #- run: |
+    #   make bootstrap
+    #   make release
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v1

.github/workflows/compatibility_tests.yml

@@ -5,7 +5,7 @@ on:
     - cron: '0 8 7 * 2'
   push:
     branches:
-      - master
+      - main
 
 env:
   JEST_ENV: prod
@@ -22,13 +22,14 @@ jobs:
       - name: Build docker image
         run: docker build .
 
-  cellxgene-master-with-python-and-anndata-versions:
+  cellxgene-main-with-python-and-anndata-versions:
     name: python versions x anndata versions
     runs-on: ubuntu-latest
+    continue-on-error: true
     strategy:
       matrix:
-        python-version: [3.6, 3.7, 3.8]
-        anndata-version: [0.6.22.post1, 0.7.1]
+        python-version: [3.6, 3.7] # As of Oct 2020 Anndata is not compatible with 3.8
+        anndata-version: [0.7.0, 0.7.1, 0.7.2, 0.7.3, 0.7.4, 0.7.5]
         test-suite: [smoke-test, smoke-test-annotations]
     steps:
     - uses: actions/checkout@v2
@@ -67,11 +68,6 @@ jobs:
         uses: actions/checkout@v2
         with:
           path: cellxgene
-      - name: Checkout tools repo
-        uses: actions/checkout@v2
-        with:
-          repository: theislab/anndata
-          path: anndata
       - name: Install dependencies
         run: |
           cd cellxgene
@@ -82,12 +78,12 @@ jobs:
           # 2. install cellxgene
           pip install --upgrade cellxgene
           # 3. install anndata
-          cd ../anndata && pip install -e .
+          pip install git+https://github.com/theislab/anndata
       - name: Tests
         run: cd cellxgene && make unit-test ${{ matrix.test-suite }}
 
-  cellxgene-master-with-anndata-master:
-    name: cellxgene master with anndata master
+  cellxgene-main-with-anndata-master:
+    name: cellxgene main with anndata master
     runs-on: ubuntu-latest
     strategy:
       matrix:
@@ -102,17 +98,11 @@ jobs:
         uses: actions/checkout@v2
         with:
           path: cellxgene
-      - name: Checkout tools repo
-        uses: actions/checkout@v2
-        with:
-          repository: theislab/anndata
-          path: anndata
       - name: Install dependencies
         run: |
           cd cellxgene
           sed -i -E 's/^anndata[>=]=[0-9]+.[0-9]+.[0-9]+$/anndata/g' server/requirements.txt
           make pydist install-dist dev-env
-          cd ../anndata
-          pip install -e .
+          pip install git+https://github.com/theislab/anndata
       - name: Tests
         run: cd cellxgene && make unit-test ${{ matrix.test-suite }}

.github/workflows/deploy.yml

@@ -0,0 +1,13 @@
+name: Deploy via single cell infra repo
+
+on:
+  push:
+    branches: main
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - name: repository dispatch
+        run: |
+          curl -XPOST -u czi-sci-single-cell-eng:${{secrets.SCI_GITHUB_TOKEN}} -H "Accept: application/vnd.github.everest-preview+json"  -H "Content-Type: application/json" https://api.github.com/repos/chanzuckerberg/single-cell-infra/dispatches --data '{"event_type": "cellxgene-hook"}'

.github/workflows/push_tests.yml

@@ -1,6 +1,10 @@
 name: Push Tests
 
-on: [push]
+on:
+  push:
+    branches: main
+  pull_request:
+    branches: "*"
 
 env:
   JEST_ENV: prod
@@ -10,100 +14,120 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
-    - name: Set up Python 3.7
-      uses: actions/setup-python@v1
-      with:
-        python-version: 3.7
-    - name: Lint with flake8
-      run: |
-        pip install flake8
-        make lint
+      - uses: actions/checkout@v2
+      - run: |
+          git fetch --depth=1 origin +${{github.base_ref}}
+      - name: Set up Python 3.7
+        uses: actions/setup-python@v1
+        with:
+          python-version: 3.7
+      - name: Node cache
+        uses: actions/cache@v1
+        with:
+          path: ~/.npm
+          key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-node-
+      - name: Install dependencies
+        run: |
+          pip install flake8
+          pip install black
+          cd client
+          npm install
+      - name: Format with black and lint with flake8
+        run: |
+          make lint-server
+      - name: Lint src with eslint
+        working-directory: ./client
+        run: |
+          make lint
 
   unit-test:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
-    - name: Set up Python 3.7
-      uses: actions/setup-python@v1
-      with:
-        python-version: 3.7
-    - name: Python cache
-      uses: actions/cache@v1
-      with:
-        path: ~/.cache/pip
-        key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}
-        restore-keys: |
-          ${{ runner.os }}-pip-
-    - name: Node cache
-      uses: actions/cache@v1
-      with:
-        path: ~/.npm
-        key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
-        restore-keys: |
-          ${{ runner.os }}-node-
-    - name: Install dependencies
-      run: make pydist install-dist dev-env-server
-    - name: Unit tests
-      run: |
-        make unit-test
-        bash <(curl -s https://codecov.io/bash) -y .codecov.yml -k server -cF backend,python,unitTest
-        cd client && ./node_modules/codecov/bin/codecov --yml=../.codecov.yml --root=../ --gcov-root=../ -C -F frontend,javascript,unitTest
+      - uses: actions/checkout@v2
+      - name: Set up Python 3.7
+        uses: actions/setup-python@v1
+        with:
+          python-version: 3.7
+      - name: Python cache
+        uses: actions/cache@v1
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+      - name: Node cache
+        uses: actions/cache@v1
+        with:
+          path: ~/.npm
+          key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-node-
+      - name: Install dependencies
+        run: make pydist install-dist dev-env-server
+      - name: Unit tests
+        run: |
+          make unit-test
+          bash <(curl -s https://codecov.io/bash) -y .codecov.yml -k server -cF backend,python,unitTest
+          cd client && ./node_modules/codecov/bin/codecov --yml=../.codecov.yml --root=../ --gcov-root=../ -C -F frontend,javascript,unitTest
 
   smoke-tests:
-    runs-on: ubuntu-latest
+    runs-on: macos-latest
+    timeout-minutes: 20
     steps:
-    - uses: actions/checkout@v2
-    - name: Set up Python 3.7
-      uses: actions/setup-python@v1
-      with:
-        python-version: 3.7
-    - name: Python cache
-      uses: actions/cache@v1
-      with:
-        path: ~/.cache/pip
-        key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}
-        restore-keys: |
-          ${{ runner.os }}-pip-
-    - name: Node cache
-      uses: actions/cache@v1
-      with:
-        path: ~/.npm
-        key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
-        restore-keys: |
-          ${{ runner.os }}-node-
-    - name: Install dependencies
-      run: make pydist install-dist
-    - name: Smoke tests (without annotations feature)
-      run: |
-        cd client && make smoke-test
-        ./node_modules/codecov/bin/codecov --yml=../.codecov.yml --root=../ --gcov-root=../ -C -F frontend,javascript,smokeTest
+      - uses: actions/checkout@v2
+      - name: Set up Python 3.7
+        uses: actions/setup-python@v1
+        with:
+          python-version: 3.7
+      - name: Python cache
+        uses: actions/cache@v1
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+      - name: Node cache
+        uses: actions/cache@v1
+        with:
+          path: ~/.npm
+          key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-node-
+      - name: Install dependencies
+        run: make pydist install-dist
+      - name: Smoke tests (without annotations feature)
+        run: |
+          cd client && make smoke-test
+          ./node_modules/codecov/bin/codecov --yml=../.codecov.yml --root=../ --gcov-root=../ -C -F frontend,javascript,smokeTest
 
   smoke-tests-annotations:
     runs-on: ubuntu-latest
+    timeout-minutes: 20
     steps:
-    - uses: actions/checkout@v2
-    - name: Set up Python 3.7
-      uses: actions/setup-python@v1
-      with:
-        python-version: 3.7
-    - name: Python cache
-      uses: actions/cache@v1
-      with:
-        path: ~/.cache/pip
-        key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}
-        restore-keys: |
-          ${{ runner.os }}-pip-
-    - name: Node cache
-      uses: actions/cache@v1
-      with:
-        path: ~/.npm
-        key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
-        restore-keys: |
-          ${{ runner.os }}-node-
-    - name: Install dependencies
-      run: make pydist install-dist
-    - name: Smoke tests (with annotations feature)
-      run: |
-        cd client && make smoke-test-annotations
-        ./node_modules/codecov/bin/codecov --yml=../.codecov.yml --root=../ --gcov-root=../ -C -F frontend,javascript,smokeTestAnnotations
+      - uses: actions/checkout@v2
+      - name: Set up Python 3.7
+        uses: actions/setup-python@v1
+        with:
+          python-version: 3.7
+      - name: Python cache
+        uses: actions/cache@v1
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+      - name: Node cache
+        uses: actions/cache@v1
+        with:
+          path: ~/.npm
+          key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-node-
+      - name: Install dependencies
+        run: make pydist install-dist
+      - name: Smoke tests (with annotations feature)
+        run: |
+          cd client && make smoke-test-annotations
+          ./node_modules/codecov/bin/codecov --yml=../.codecov.yml --root=../ --gcov-root=../ -C -F frontend,javascript,smokeTestAnnotations

.github/workflows/scale-test.yml

@@ -0,0 +1,30 @@
+name: "Scale test cellxgene APIs for initial loading"
+
+on:
+  schedule:
+    - cron: "0 0 * * Sun"
+
+jobs:
+  locust-build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python 3.7
+        uses: actions/setup-python@v1
+        with:
+          python-version: 3.7
+      - name: Install dependencies
+        run: |
+          pip install -r server/test/locust/requirements-locust.txt
+      - name: Dev Scale Test
+        run: |
+          locust -f server/test/locust/locustfile.py --headless -u 30 -r 10 --host https://api.cellxgene.dev.single-cell.czi.technology/cellxgene/e/ --run-time 5m 2>&1 | tee locust_dev_stats.txt
+      - name: Slack success webhook
+        env:
+          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
+        run: |
+          DEV_STATS=$(tail -n 61 locust_dev_stats.txt)
+          DEV_MSG="\`\`\`CELLXGENE EXPLORER DEV SCALE TEST RESULTS: ${DEV_STATS}\`\`\`"
+          curl -X POST -H 'Content-type: application/json' --data "{'text':'${DEV_MSG}'}" $SLACK_WEBHOOK
+
+

.gitignore

@@ -19,12 +19,14 @@ venv/
 cellxgene/
 
 # client build
-server/common/web/static/css/
-server/common/web/static/img/
-server/common/web/static/media/
-server/common/web/static/fonts/
-server/common/web/static/js/
-server/common/web/templates/index\.html
+server/common/web/static/*
+server/common/web/templates/
+server/common/web/csp-hashes.json
+
+# eb build
+artifact.dir
+artifact.zip
+customize
 
 # Jupyter Notebook
 .ipynb_checkpoints
@@ -41,7 +43,14 @@ npm-debug.log
 __pycache__
 *.DS_Store*
 data
+tags
+myconfig.yaml
 
 # Jekyll
 docs/_site/
 docs/Gemfile.lock
+
+client/.eslintcache
+
+# E2E Testing
+ignoreE2E*

Dockerfile

@@ -4,7 +4,8 @@ ENV LC_ALL=C.UTF-8
 ENV LANG=C.UTF-8
 
 RUN apt-get update && \
-    apt-get install -y build-essential libxml2-dev python3-dev python3-pip zlib1g-dev python3-requests && \
+    apt-get install -y build-essential libxml2-dev python3-dev python3-pip zlib1g-dev python3-requests python3-aiohttp && \
+    python3 -m pip install --upgrade pip && \
     pip3 install cellxgene
 
 ENTRYPOINT ["cellxgene"]

LICENSE.txt

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2013
+Copyright (c) 2017-2020 Chan Zuckerberg Initiative
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of
 this software and associated documentation files (the "Software"), to deal in
@@ -17,4 +17,4 @@ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
 FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
 COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
 IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
-CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

MANIFEST.in

@@ -3,3 +3,5 @@ recursive-include server/common/web/static *
 
 include server/requirements.txt
 include server/requirements-prepare.txt
+include server/converters/schema/hgnc_complete_set.txt.gz
+include server/converters/schema/schema_definitions/*

Makefile

@@ -22,17 +22,13 @@ clean-%:
 
 # BUILDING PACKAGE
 
-.PHONY: build
-build: clean build-cli
-	@echo "done"
-
 .PHONY: build-client
 build-client:
 	cd client && $(MAKE) ci build
 
-.PHONY: build-cli
-build-cli: build-client
-	git ls-files server/ | cpio -pdm $(BUILDDIR)
+.PHONY: build
+build: clean build-client
+	git ls-files server/ | grep -v 'server/test/' | cpio -pdm $(BUILDDIR)
 	cp -r client/build/  $(CLIENTBUILD)
 	$(call copy_client_assets,$(CLIENTBUILD),$(SERVERBUILD))
 	cp MANIFEST.in README.md setup.cfg setup.py $(BUILDDIR)
@@ -42,6 +38,10 @@ build-cli: build-client
 build-for-server-dev: clean-server build-client
 	$(call copy_client_assets,client/build,server)
 
+.PHONY: copy-client-assets
+copy-client-assets:
+	$(call copy_client_assets,client/build,server)
+
 # TESTING
 .PHONY: test
 test: unit-test smoke-test
@@ -60,6 +60,11 @@ smoke-test:
 smoke-test-annotations:
 	cd client && $(MAKE) smoke-test-annotations
 
+.PHONY: test-db
+test-db:
+	cd server && $(MAKE) test-db
+
+
 # FORMATTING CODE
 
 .PHOHY: fmt
@@ -74,9 +79,16 @@ fmt-py:
 	black .
 
 .PHONY: lint
-lint:
-	flake8 server
+lint: lint-server lint-client
 
+.PHONY: lint-server
+lint-server: fmt-py
+	flake8 server --per-file-ignores='server/test/fixtures/dataset_config_outline.py:F821 server/test/fixtures/server_config_outline.py:F821 server/test/performance/scale_test_annotations.py:E501'
+
+
+.PHONY: lint-client
+lint-client:
+	cd client && $(MAKE) lint
 
 # CREATING DISTRIBUTION RELEASE
 
@@ -88,22 +100,32 @@ pydist: build
 
 # RELEASE HELPERS
 
-# create new version to commit to master
-.PHONY: release-stage-1
-release-stage-1: dev-env bump clean-lite gen-package-lock
+# Create new version to commit to main
+.PHONY: create-release-candidate
+create-release-candidate: dev-env bump-version clean-lite gen-package-lock
 	@echo "Version bumped part:$(PART) and client built. Ready to commit and push"
 
-# build dist and release to dev pypi
-.PHONY: release-stage-2
-release-stage-2: dev-env pydist twine
+# Bump the release candidate version if needed (i.e. the previous release candidate had errors).
+.PHONY: recreate-release-candidate
+recreate-release-candidate: dev-env bump-release-candidate clean-lite gen-package-lock
+	@echo "Version bumped part:$(PART) and client built. Ready to commit and push"
+
+# Build dist and release to Test PyPI
+.PHONY: release-candidate-to-test-pypi
+release-candidate-to-test-pypi: dev-env pydist twine
 	@echo "Dist built and uploaded to test.pypi.org"
 	@echo "Test the install:"
 	@echo "    make install-release-test"
-	@echo "Then upload to Pypi prod:"
-	@echo "    make twine-prod"
 
-.PHONY: release-stage-final
-release-stage-final: twine-prod
+# Build final dist (gets rid of the rc tag) and release final candidate to TestPyPI
+.PHONY: release-final-to-test-pypi
+release-final-to-test-pypi: dev-env bump-release clean-lite gen-package-lock pydist twine
+	@echo "Final release dist built and uploaded to test.pypi.org"
+	@echo "Test the install:"
+	@echo "    make install-release-test"
+
+.PHONY: release-final
+release-final: twine-prod
 	@echo "Release uploaded to pypi.org"
 
 # DANGER: releases directly to prod
@@ -125,11 +147,22 @@ dev-env-client:
 dev-env-server:
 	pip install -r server/requirements-dev.txt
 
-# give PART=[major, minor, part] as param to make bump
-.PHONY: bump
-bump:
+# Set PART=[major, minor, patch] as param to make bump.
+# This will create a release candidate. (i.e. 0.16.1 -> 0.16.2-rc.0 for a patch bump)
+.PHONY: bump-version
+bump-version:
 	bumpversion --config-file .bumpversion.cfg $(PART)
 
+# Increments the release candidate version (i.e. 0.16.2-rc.1 -> 0.16.2-rc.2)
+.PHONY: bump-release-candidate
+bump-release-candidate:
+	bumpversion --config-file .bumpversion.cfg prerelversion --allow-dirty
+
+# Finalizes the release candidate by removing the release candidate tag (i.e. 0.16.2-rc.2 -> 0.16.2).
+.PHONY: bump-release
+bump-release:
+	bumpversion --config-file .bumpversion.cfg prerel --allow-dirty
+
 .PHONY: twine
 twine:
 	twine upload --repository-url https://test.pypi.org/legacy/ dist/*
@@ -146,13 +179,6 @@ gen-package-lock:
 
 # INSTALL
 
-# setup.py sucks when you have your library in a separate folder, adding these in to help setup envs
-
-# install from build directory
-.PHONY: install
-install: uninstall
-	cd $(BUILDDIR); pip install -e .
-
 # install from source tree for development
 .PHONY: install-dev
 install-dev: uninstall

PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,11 @@
+#### Reviewers
+**Functional:** 
+
+**Readability:** 
+
+---
+
+## Changes
+- add
+- remove
+- modify

README.md

@@ -5,35 +5,39 @@ _an interactive explorer for single-cell transcriptomics data_
 [![DOI](https://zenodo.org/badge/105615409.svg)](https://zenodo.org/badge/latestdoi/105615409) [![PyPI](https://img.shields.io/pypi/v/cellxgene)](https://pypi.org/project/cellxgene/) [![PyPI - Downloads](https://img.shields.io/pypi/dm/cellxgene)](https://pypistats.org/packages/cellxgene) [![GitHub last commit](https://img.shields.io/github/last-commit/chanzuckerberg/cellxgene)](https://github.com/chanzuckerberg/cellxgene/pulse)
 [![Push Tests](https://github.com/chanzuckerberg/cellxgene/workflows/Push%20Tests/badge.svg)](https://github.com/chanzuckerberg/cellxgene/actions?query=workflow%3A%22Push+Tests%22)
 [![Compatibility Tests](https://github.com/chanzuckerberg/cellxgene/workflows/Compatibility%20Tests/badge.svg)](https://github.com/chanzuckerberg/cellxgene/actions?query=workflow%3A%22Compatibility+Tests%22)
-![Code Coverage](https://codecov.io/gh/chanzuckerberg/cellxgene/branch/master/graph/badge.svg)
+![Code Coverage](https://codecov.io/gh/chanzuckerberg/cellxgene/branch/main/graph/badge.svg)
 
 cellxgene (pronounced "cell-by-gene") is an interactive data explorer for single-cell transcriptomics datasets, such as those coming from the [Human Cell Atlas](https://humancellatlas.org). Leveraging modern web development techniques to enable fast visualizations of at least 1 million cells, we hope to enable biologists and computational researchers to explore their data.
 
 Whether you need to visualize one thousand cells or one million, cellxgene helps you gain insight into your single-cell data.
 
-<img src="https://github.com/chanzuckerberg/cellxgene/raw/master/docs/images/crossfilter.gif" width="350" height="200" hspace="30"><img src="https://github.com/chanzuckerberg/cellxgene/raw/master/docs/images/category-breakdown.gif" width="350" height="200" hspace="30">
+<img src="https://github.com/chanzuckerberg/cellxgene/raw/main/docs/images/crossfilter.gif" width="350" height="200" hspace="30"><img src="https://github.com/chanzuckerberg/cellxgene/raw/main/docs/images/category-breakdown.gif" width="350" height="200" hspace="30">
 
 # Getting started
+
 ### The comprehensive guide to cellxgene
+
 [The cellxgene documentation is your one-stop-shop for information about cellxgene](https://chanzuckerberg.github.io/cellxgene/)! You may be particularly interested in:
-* Seeing [what cellxgene can do](https://chanzuckerberg.github.io/cellxgene/posts/gallery)  
-* Learning more about cellxgene [installation](https://chanzuckerberg.github.io/cellxgene/posts/install) and [usage](https://chanzuckerberg.github.io/cellxgene/posts/launch)
-* [Preparing your own data](https://chanzuckerberg.github.io/cellxgene/posts/prepare) for use in cellxgene
-* Checking out [our roadmap](https://chanzuckerberg.github.io/cellxgene/posts/roadmap) for future development
-* [Contributing](https://chanzuckerberg.github.io/cellxgene/posts/contribute) to cellxgene
+
+- Seeing [what cellxgene can do](https://chanzuckerberg.github.io/cellxgene/posts/gallery)
+- Learning more about cellxgene [installation](https://chanzuckerberg.github.io/cellxgene/posts/install) and [usage](https://chanzuckerberg.github.io/cellxgene/posts/launch)
+- [Preparing your own data](https://chanzuckerberg.github.io/cellxgene/posts/prepare) for use in cellxgene
+- Checking out [our roadmap](https://chanzuckerberg.github.io/cellxgene/posts/roadmap) for future development
+- [Contributing](https://chanzuckerberg.github.io/cellxgene/posts/contribute) to cellxgene
 
 ### Quick start
 
 To install cellxgene you need Python 3.6+. We recommend [installing cellxgene into a conda or virtual environment.](https://chanzuckerberg.github.io/cellxgene/posts/install)
 
 Install the package.
-``` bash
+
+```bash
 pip install cellxgene
 ```
 
 Launch cellxgene with an example [anndata](https://anndata.readthedocs.io/en/latest/) file
 
-``` bash
+```bash
 cellxgene launch https://cellxgene-example-data.czi.technology/pbmc3k.h5ad
 ```
 
@@ -41,6 +45,17 @@ To explore more datasets already formatted for cellxgene, check out the [Demo da
 see [Preparing your data](https://chanzuckerberg.github.io/cellxgene/posts/prepare) to learn more about formatting your own
 data for cellxgene.
 
+### Supported browsers
+
+cellxgene currently supports the following browsers:
+
+- Google Chrome 61+
+- Edge 15+
+- Firefox 60+
+- Safari 10.1+
+
+Please [file an issue](https://github.com/chanzuckerberg/cellxgene/issues/new/choose) if you would like us to add support for an unsupported browser.
+
 ### Finding help
 
 We'd love to hear from you!
@@ -51,36 +66,31 @@ For any errors, [report bugs on Github](https://github.com/chanzuckerberg/cellxg
 # Developing with cellxgene
 
 ### Contributing
-We warmly welcome contributions from the community! Please see our [contributing guide](https://chanzuckerberg.github.io/cellxgene/posts/contribute) and don't hesitate to open an issue or send a pull request to improve cellxgene.
+
+We warmly welcome contributions from the community! Please see our [contributing guide](https://chanzuckerberg.github.io/cellxgene/posts/contribute) and don't hesitate to open an issue or send a pull request to improve cellxgene. Please see the [dev_docs](https://github.com/chanzuckerberg/cellxgene/tree/main/dev_docs) for pull request suggestions, unit test details, local documentation preview, and other development specifics. 
 
 This project adheres to the Contributor Covenant [code of conduct](https://github.com/chanzuckerberg/.github/blob/master/CODE_OF_CONDUCT.md). By participating, you are expected to uphold this code. Please report unacceptable behavior to opensource@chanzuckerberg.com.
 
 ### Reuse
 
-This project was started with the sole goal of empowering the scientific community to explore and understand their data. As such, we encourage other scientific tool builders in academia or industry to adopt the patterns, tools, and code from this project, and reach out to us with ideas or questions. All code is freely available for reuse under the [MIT license](https://opensource.org/licenses/MIT).
+This project was started with the sole goal of empowering the scientific community to explore and understand their data. 
+As such, we encourage other scientific tool builders in academia or industry to adopt the patterns, tools, and code from 
+this project. All code is freely available for reuse under the [MIT license](https://opensource.org/licenses/MIT).
+
+
+Before extending cellxgene, we encourage you to reach out to us with ideas or questions. It might be possible that an 
+extension could be directly contributed, which would make it available for a wider audience, or that it's on our 
+[roadmap](./docs/posts/roadmap.md) and under active development. 
+
+See the [cellxgene extensions](./docs/posts/extensions.md) section of our documentation for examples of community use and cellxgene extensions. 
 
 ### Security
 
 If you believe you have found a security issue, we would appreciate notification. Please send email to <security@chanzuckerberg.com>.
 
-# About
-### Core team
+# Inspiration
 
-The current core team:
-
-- Colin Megill, frontend & product design
-- Bruce Martin, software engineer
-- Sidney Bell, computational biologist
-- Lia Prins, designer
-- Severiano Badajoz, software engineer
-
-We would also like to gratefully acknowledge contributions from past core team members:
-
-- Charlotte Weaver, software engineer
-
-### Inspiration
-
-We've been heavily inspired by several other related single-cell visualization projects, including the [UCSC Cell Browswer](http://cells.ucsc.edu/), [Cytoscape](http://www.cytoscape.org/), [Xena](https://xena.ucsc.edu/), [ASAP](https://asap.epfl.ch/), [Gene Pattern](http://genepattern-notebook.org/), and many others. We hope to explore collaborations where useful as this community works together on improving interactive visualization for single-cell data.
+We've been heavily inspired by several other related single-cell visualization projects, including the [UCSC Cell Browser](http://cells.ucsc.edu/), [Cytoscape](http://www.cytoscape.org/), [Xena](https://xena.ucsc.edu/), [ASAP](https://asap.epfl.ch/), [GenePattern](http://genepattern-notebook.org/), and many others. We hope to explore collaborations where useful as this community works together on improving interactive visualization for single-cell data.
 
 We were inspired by Mike Bostock and the [crossfilter](https://github.com/crossfilter) team for the design of our filtering implementation.
 

app.json

@@ -1,21 +0,0 @@
-{
-  "name": "cellxgene",
-  "description": "An interactive explorer for single-cell transcriptomics data",
-  "repository": "https://github.com/chanzuckerberg/cellxgene",
-  "logo": "https://cellxgene-example-data.czi.technology/favicon.png",
-  "keywords": [
-    "scientific",
-    "visualization",
-    "scrna-seq",
-    "transcriptomics",
-    "dataviz"
-  ],
-  "stack": "container",
-  "env": {
-    "DATASET": {
-      "description": "Link to dataset",
-      "value": "https://cellxgene-example-data.czi.technology/pbmc3k.h5ad",
-      "required": "true"
-    }
-  }
-}

client/Makefile

@@ -1,80 +1,43 @@
 include ../common.mk
 
-DATASET := $(if $(DATASET),$(DATASET),../example-dataset/pbmc3k.h5ad)
-ANNOTATIONS := $(if $(ANNOTATIONS),$(ANNOTATIONS),../server/test/test_datasets/pbmc3k-annotations.csv)
+ANNOTATIONS := $(if $(ANNOTATIONS),$(ANNOTATIONS),../server/test/fixtures/pbmc3k-annotations.csv)
 ANNOTATIONS_FILENAME := $(shell basename $(ANNOTATIONS))
 
+CXG_CONFIG := $(if $(CXG_CONFIG), $(CXG_CONFIG), ./__tests__/e2e/test_config.yaml)
+
 # Packaging
 .PHONY: clean
 clean:
 	rm -rf node_modules
+	rm -f __tests__/screenshots/*.png
 
 .PHONY: ci
 ci:
-	npm ci client
+	npm ci
 
 .PHONY: install
 install:
-	npm install client
+	npm install
 
 .PHONY: build
+WEBPACK_CONFIG ?= configuration/webpack/webpack.config.prod.js
 build:
-	npm run build
-
+	npm run build $(WEBPACK_CONFIG)
 
 # Development convenience methods
-
-.PHONY: server-requirements
-server-requirements:
-	virtualenv -p python3 ../venv
-	source ../venv/bin/activate && \
-		pip install -r ../server/requirements-dev.txt && \
-		yes | pip uninstall cellxgene || true && \
-		pip install -e ..
-
 .PHONY: start-frontend
 start-frontend:
 	node server/development.js
 
-.PHONY: start-server
-start-server:
-	cellxgene launch -p $(CXG_SERVER_PORT) $(CXG_OPTIONS) $(DATASET)
-
-.PHONY: backend-dev
-backend-dev: server-requirements
-	source ../venv/bin/activate && $(MAKE) start-server
-
-.PHONY: backend-dev-anno-ontology
-backend-dev-anno-ontology: server-requirements
-	CXG_OPTIONS='--experimental-annotations-ontology' \
-		$(MAKE) backend-dev
-
-.PHONY: test
-test:
-	node node_modules/jest/bin/jest.js
-
-.PHONY: e2e
-e2e:
-	node node_modules/jest/bin/jest.js \
-		--verbose false \
-		--config __tests__/e2e/e2eJestConfig.json \
-		e2e/e2e.test.js
-
-.PHONY: e2e-annotations
-e2e-annotations:
-	node node_modules/jest/bin/jest.js \
-		--verbose false \
-		--config __tests__/e2e/e2eJestConfig.json \
-		e2e/e2eAnnotations.test.js
-
+# start an instance of cellxgene and run the end-to-end tests
 .PHONY: smoke-test
 smoke-test:
 	start_server_and_test \
-		'CXG_OPTIONS="--disable-annotations" $(MAKE) start-server' \
+		'CXG_OPTIONS="--config-file $(CXG_CONFIG)" $(MAKE) start-server' \
 		$(CXG_SERVER_PORT) \
-		'$(MAKE) e2e'
-
+		'CXG_URL_BASE="http://localhost:$(CXG_SERVER_PORT)" CXG_AUTH_TYPE="test" npm run e2e -- --verbose false'
 
+# start an instance of cellxgene and run the end-to-end annotations tests
 .PHONY: smoke-test-annotations
 smoke-test-annotations:
 	$(eval TMP_DIR := $(shell  mktemp -d /tmp/cellxgene_XXXXXX))
@@ -82,13 +45,12 @@ smoke-test-annotations:
 	start_server_and_test \
 		'CXG_OPTIONS="--annotations-file $(TMP_DIR)/$(ANNOTATIONS_FILENAME)" $(MAKE) start-server' \
 		$(CXG_SERVER_PORT) \
-		'$(MAKE) e2e-annotations'
+		'CXG_URL_BASE="http://localhost:$(CXG_SERVER_PORT)" npm run e2e-annotations -- --verbose false'
 	rm -rf $(TMP_DIR)
 
 .PHONY: unit-test
 unit-test:
-	node node_modules/jest/bin/jest.js \
-	    --testPathIgnorePatterns e2e
+	node node_modules/jest/bin/jest.js --testPathIgnorePatterns e2e
 
 # pass remaining commands through to npm run
 %:

client/__tests__/e2e/__snapshots__/e2e.test.js.snap

@@ -0,0 +1,5 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`did launch page launched 1`] = `"<span style=\\"max-width: 155px; display: flex; overflow: hidden; justify-content: flex-start; width: 100%; padding: 0px;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">pbm</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">c3k</span><span style=\\"position: absolute; right: 0px; color: inherit;\\">c3k</span></span></span>"`;
+
+exports[`metadata loads categories and values from dataset appear 1`] = `"<div style=\\"display: flex; justify-content: space-between; align-items: baseline;\\"><div style=\\"display: flex; justify-content: flex-start; align-items: flex-start;\\"><label class=\\"bp3-control bp3-checkbox\\" for=\\"category-select-louvain\\"><input id=\\"category-select-louvain\\" data-testclass=\\"category-select\\" data-testid=\\"louvain:category-select\\" type=\\"checkbox\\" checked=\\"\\"><span class=\\"bp3-control-indicator\\"></span></label><span role=\\"menuitem\\" tabindex=\\"0\\" data-testclass=\\"category-expand\\" data-testid=\\"louvain:category-expand\\" style=\\"cursor: pointer;\\"><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\" style=\\"max-width: 265px;\\"><span data-testid=\\"louvain:category-label\\" aria-label=\\"louvain\\" class=\\"\\" tabindex=\\"0\\" style=\\"max-width: 265px;\\"><span style=\\"max-width: 265px; display: flex; overflow: hidden; justify-content: flex-start; width: 100%; padding: 0px;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">lou</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">vain</span><span style=\\"position: absolute; right: 0px; color: inherit;\\">vain</span></span></span></span></span></span><svg stroke=\\"currentColor\\" fill=\\"currentColor\\" stroke-width=\\"0\\" viewBox=\\"0 0 320 512\\" data-testclass=\\"category-expand-is-not-expanded\\" height=\\"1em\\" width=\\"1em\\" xmlns=\\"http://www.w3.org/2000/svg\\" style=\\"font-size: 10px; margin-left: 5px;\\"><path d=\\"M285.476 272.971L91.132 467.314c-9.373 9.373-24.569 9.373-33.941 0l-22.667-22.667c-9.357-9.357-9.375-24.522-.04-33.901L188.505 256 34.484 101.255c-9.335-9.379-9.317-24.544.04-33.901l22.667-22.667c9.373-9.373 24.569-9.373 33.941 0L285.475 239.03c9.373 9.372 9.373 24.568.001 33.941z\\"></path></svg></span></div><div><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\"><a role=\\"button\\" data-testclass=\\"colorby\\" data-testid=\\"colorby-louvain\\" class=\\"bp3-button\\" tabindex=\\"0\\"><span icon=\\"tint\\" class=\\"bp3-icon bp3-icon-tint\\"><svg data-icon=\\"tint\\" width=\\"16\\" height=\\"16\\" viewBox=\\"0 0 16 16\\"><desc>tint</desc><path d=\\"M7.88 1s-4.9 6.28-4.9 8.9c.01 2.82 2.34 5.1 4.99 5.1 2.65-.01 5.03-2.3 5.03-5.13C12.99 7.17 7.88 1 7.88 1z\\" fill-rule=\\"evenodd\\"></path></svg></span></a></span></span></div></div><div style=\\"margin-left: 26px;\\"></div><div></div>"`;

client/__tests__/e2e/__snapshots__/e2eAnnotations.test.js.snap

@@ -0,0 +1,23 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`annotations stacked bar graph renders 1`] = `
+Array [
+  "<div class=\\"categorical__value___2RKaC\\" data-testclass=\\"categorical-row\\" style=\\"padding: 4px 0px 4px 7px; display: flex; align-items: baseline; justify-content: space-between; margin-bottom: 2px; border-radius: 2px;\\"><div style=\\"margin: 0px; padding: 0px; user-select: none; width: 220px; display: flex; justify-content: space-between;\\"><div style=\\"display: flex; align-items: baseline;\\"><label for=\\"value-toggle-checkbox-TEST-CATEGORY-TEST-LABEL\\" class=\\"bp3-control bp3-checkbox\\" style=\\"margin: 0px;\\"><input id=\\"value-toggle-checkbox-TEST-CATEGORY-TEST-LABEL\\" data-testclass=\\"categorical-value-select\\" data-testid=\\"categorical-value-select-TEST-CATEGORY-TEST-LABEL\\" type=\\"checkbox\\" checked=\\"\\"><span class=\\"bp3-control-indicator\\"></span></label><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\" style=\\"width: 63px; color: black; font-style: normal; display: inline-block; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px;\\"><span data-testid=\\"categorical-value-TEST-CATEGORY-TEST-LABEL\\" data-testclass=\\"categorical-value\\" aria-label=\\"TEST-LABEL\\" class=\\"\\" tabindex=\\"0\\" style=\\"width: 63px; color: black; font-style: normal; display: inline-block; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px;\\"><span style=\\"width: 100%; color: black; font-style: normal; display: flex; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px; justify-content: flex-start; padding: 0px;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">TEST-</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">LABEL</span><span style=\\"position: absolute; right: 0px; color: black;\\">LABEL</span></span></span></span></span></span></div><span style=\\"flex-shrink: 0;\\"></span></div><div><span><span data-testclass=\\"categorical-value-count\\" data-testid=\\"categorical-value-count-TEST-CATEGORY-TEST-LABEL\\" style=\\"color: black;\\">0</span><svg display=\\"none\\" style=\\"margin-left: 5px; width: 11px; height: 11px; background-color: inherit;\\"></svg><span><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\"><button type=\\"button\\" data-testclass=\\"seeActions\\" data-testid=\\"TEST-CATEGORY:TEST-LABEL:see-actions\\" class=\\"bp3-button bp3-minimal bp3-small\\" tabindex=\\"0\\" style=\\"margin-left: 2px; position: relative; top: -1px; min-height: 16px;\\"><span icon=\\"more\\" class=\\"bp3-icon bp3-icon-more\\"><svg data-icon=\\"more\\" width=\\"10\\" height=\\"10\\" viewBox=\\"0 0 16 16\\"><desc>more</desc><path d=\\"M2 6.03a2 2 0 100 4 2 2 0 100-4zM14 6.03a2 2 0 100 4 2 2 0 100-4zM8 6.03a2 2 0 100 4 2 2 0 100-4z\\" fill-rule=\\"evenodd\\"></path></svg></span></button></span></span></span></span></div></div>",
+  "<div class=\\"categorical__value___2RKaC\\" data-testclass=\\"categorical-row\\" style=\\"padding: 4px 0px 4px 7px; display: flex; align-items: baseline; justify-content: space-between; margin-bottom: 2px; border-radius: 2px;\\"><div style=\\"margin: 0px; padding: 0px; user-select: none; width: 220px; display: flex; justify-content: space-between;\\"><div style=\\"display: flex; align-items: baseline;\\"><label for=\\"value-toggle-checkbox-TEST-CATEGORY-unassigned\\" class=\\"bp3-control bp3-checkbox\\" style=\\"margin: 0px;\\"><input id=\\"value-toggle-checkbox-TEST-CATEGORY-unassigned\\" data-testclass=\\"categorical-value-select\\" data-testid=\\"categorical-value-select-TEST-CATEGORY-unassigned\\" type=\\"checkbox\\" checked=\\"\\"><span class=\\"bp3-control-indicator\\"></span></label><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\" style=\\"width: 63px; color: rgb(171, 171, 171); font-style: italic; display: inline-block; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px;\\"><span data-testid=\\"categorical-value-TEST-CATEGORY-unassigned\\" data-testclass=\\"categorical-value\\" aria-label=\\"unassigned\\" class=\\"\\" tabindex=\\"0\\" style=\\"width: 63px; color: rgb(171, 171, 171); font-style: italic; display: inline-block; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px;\\"><span style=\\"width: 100%; color: rgb(171, 171, 171); font-style: italic; display: flex; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px; justify-content: flex-start; padding: 0px;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">unass</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">igned</span><span style=\\"position: absolute; right: 0px; color: rgb(171, 171, 171);\\">igned</span></span></span></span></span></span></div><span style=\\"flex-shrink: 0;\\"><canvas width=\\"100\\" height=\\"11\\" style=\\"margin-right: 5px; width: 100px; height: 11px;\\"></canvas></span></div><div><span><span data-testclass=\\"categorical-value-count\\" data-testid=\\"categorical-value-count-TEST-CATEGORY-unassigned\\" style=\\"color: rgb(171, 171, 171); font-style: italic;\\">2133</span><svg display=\\"none\\" style=\\"margin-left: 5px; width: 11px; height: 11px; background-color: inherit;\\"></svg><span><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\"><button type=\\"button\\" data-testclass=\\"seeActions\\" data-testid=\\"TEST-CATEGORY:unassigned:see-actions\\" class=\\"bp3-button bp3-minimal bp3-small\\" tabindex=\\"0\\" style=\\"margin-left: 2px; position: relative; top: -1px; min-height: 16px;\\"><span icon=\\"more\\" class=\\"bp3-icon bp3-icon-more\\"><svg data-icon=\\"more\\" width=\\"10\\" height=\\"10\\" viewBox=\\"0 0 16 16\\"><desc>more</desc><path d=\\"M2 6.03a2 2 0 100 4 2 2 0 100-4zM14 6.03a2 2 0 100 4 2 2 0 100-4zM8 6.03a2 2 0 100 4 2 2 0 100-4z\\" fill-rule=\\"evenodd\\"></path></svg></span></button></span></span></span></span></div></div>",
+]
+`;
+
+exports[`annotations stacked bar graph renders 2`] = `
+Array [
+  "<div class=\\"categorical__value___2RKaC\\" data-testclass=\\"categorical-row\\" style=\\"padding: 4px 0px 4px 7px; display: flex; align-items: baseline; justify-content: space-between; margin-bottom: 2px; border-radius: 2px;\\"><div style=\\"margin: 0px; padding: 0px; user-select: none; width: 220px; display: flex; justify-content: space-between;\\"><div style=\\"display: flex; align-items: baseline;\\"><label for=\\"value-toggle-checkbox-TEST-CATEGORY-TEST-LABEL\\" class=\\"bp3-control bp3-checkbox\\" style=\\"margin: 0px;\\"><input id=\\"value-toggle-checkbox-TEST-CATEGORY-TEST-LABEL\\" data-testclass=\\"categorical-value-select\\" data-testid=\\"categorical-value-select-TEST-CATEGORY-TEST-LABEL\\" type=\\"checkbox\\" checked=\\"\\"><span class=\\"bp3-control-indicator\\"></span></label><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\" style=\\"width: 63px; color: black; font-style: normal; display: inline-block; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px;\\"><span data-testid=\\"categorical-value-TEST-CATEGORY-TEST-LABEL\\" data-testclass=\\"categorical-value\\" aria-label=\\"TEST-LABEL\\" class=\\"\\" tabindex=\\"0\\" style=\\"width: 63px; color: black; font-style: normal; display: inline-block; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px;\\"><span style=\\"width: 100%; color: black; font-style: normal; display: flex; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px; justify-content: flex-start; padding: 0px;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">TEST-</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">LABEL</span><span style=\\"position: absolute; right: 0px; color: black;\\">LABEL</span></span></span></span></span></span></div><span style=\\"flex-shrink: 0;\\"></span></div><div><span><span data-testclass=\\"categorical-value-count\\" data-testid=\\"categorical-value-count-TEST-CATEGORY-TEST-LABEL\\" style=\\"color: black;\\">0</span><svg display=\\"none\\" style=\\"margin-left: 5px; width: 11px; height: 11px; background-color: inherit;\\"></svg><span><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\"><button type=\\"button\\" data-testclass=\\"seeActions\\" data-testid=\\"TEST-CATEGORY:TEST-LABEL:see-actions\\" class=\\"bp3-button bp3-minimal bp3-small\\" tabindex=\\"0\\" style=\\"margin-left: 2px; position: relative; top: -1px; min-height: 16px;\\"><span icon=\\"more\\" class=\\"bp3-icon bp3-icon-more\\"><svg data-icon=\\"more\\" width=\\"10\\" height=\\"10\\" viewBox=\\"0 0 16 16\\"><desc>more</desc><path d=\\"M2 6.03a2 2 0 100 4 2 2 0 100-4zM14 6.03a2 2 0 100 4 2 2 0 100-4zM8 6.03a2 2 0 100 4 2 2 0 100-4z\\" fill-rule=\\"evenodd\\"></path></svg></span></button></span></span></span></span></div></div>",
+  "<div class=\\"categorical__value___2RKaC\\" data-testclass=\\"categorical-row\\" style=\\"padding: 4px 0px 4px 7px; display: flex; align-items: baseline; justify-content: space-between; margin-bottom: 2px; border-radius: 2px;\\"><div style=\\"margin: 0px; padding: 0px; user-select: none; width: 220px; display: flex; justify-content: space-between;\\"><div style=\\"display: flex; align-items: baseline;\\"><label for=\\"value-toggle-checkbox-TEST-CATEGORY-unassigned\\" class=\\"bp3-control bp3-checkbox\\" style=\\"margin: 0px;\\"><input id=\\"value-toggle-checkbox-TEST-CATEGORY-unassigned\\" data-testclass=\\"categorical-value-select\\" data-testid=\\"categorical-value-select-TEST-CATEGORY-unassigned\\" type=\\"checkbox\\" checked=\\"\\"><span class=\\"bp3-control-indicator\\"></span></label><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\" style=\\"width: 63px; color: rgb(171, 171, 171); font-style: italic; display: inline-block; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px;\\"><span data-testid=\\"categorical-value-TEST-CATEGORY-unassigned\\" data-testclass=\\"categorical-value\\" aria-label=\\"unassigned\\" class=\\"\\" tabindex=\\"0\\" style=\\"width: 63px; color: rgb(171, 171, 171); font-style: italic; display: inline-block; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px;\\"><span style=\\"width: 100%; color: rgb(171, 171, 171); font-style: italic; display: flex; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px; justify-content: flex-start; padding: 0px;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">unass</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">igned</span><span style=\\"position: absolute; right: 0px; color: rgb(171, 171, 171);\\">igned</span></span></span></span></span></span></div><span style=\\"flex-shrink: 0;\\"><canvas width=\\"100\\" height=\\"11\\" style=\\"margin-right: 5px; width: 100px; height: 11px;\\"></canvas></span></div><div><span><span data-testclass=\\"categorical-value-count\\" data-testid=\\"categorical-value-count-TEST-CATEGORY-unassigned\\" style=\\"color: rgb(171, 171, 171); font-style: italic;\\">2638</span><svg display=\\"none\\" style=\\"margin-left: 5px; width: 11px; height: 11px; background-color: inherit;\\"></svg><span><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\"><button type=\\"button\\" data-testclass=\\"seeActions\\" data-testid=\\"TEST-CATEGORY:unassigned:see-actions\\" class=\\"bp3-button bp3-minimal bp3-small\\" tabindex=\\"0\\" style=\\"margin-left: 2px; position: relative; top: -1px; min-height: 16px;\\"><span icon=\\"more\\" class=\\"bp3-icon bp3-icon-more\\"><svg data-icon=\\"more\\" width=\\"10\\" height=\\"10\\" viewBox=\\"0 0 16 16\\"><desc>more</desc><path d=\\"M2 6.03a2 2 0 100 4 2 2 0 100-4zM14 6.03a2 2 0 100 4 2 2 0 100-4zM8 6.03a2 2 0 100 4 2 2 0 100-4z\\" fill-rule=\\"evenodd\\"></path></svg></span></button></span></span></span></span></div></div>",
+]
+`;
+
+exports[`annotations truncate midpoint whitespace 1`] = `"<span data-testid=\\"categorical-value-TEST-CATEGORY-123 456\\" data-testclass=\\"categorical-value\\" aria-label=\\"123 456\\" class=\\"\\" tabindex=\\"0\\" style=\\"width: 187px; color: black; font-style: normal; display: inline-block; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px;\\"><span style=\\"width: 100%; color: black; font-style: normal; display: flex; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px; justify-content: flex-start; padding: 0px;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">123</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">&nbsp;456</span><span style=\\"position: absolute; right: 0px; color: black;\\">&nbsp;456</span></span></span></span>"`;
+
+exports[`annotations truncate midpoint whitespace 2`] = `"<span data-testid=\\"categorical-value-TEST-CATEGORY-123 456\\" data-testclass=\\"categorical-value\\" aria-label=\\"123 456\\" class=\\"\\" tabindex=\\"0\\" style=\\"width: 187px; color: black; font-style: normal; display: inline-block; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px;\\"><span style=\\"width: 100%; color: black; font-style: normal; display: flex; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px; justify-content: flex-start; padding: 0px;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">123</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">&nbsp;456</span><span style=\\"position: absolute; right: 0px; color: black;\\">&nbsp;456</span></span></span></span>"`;
+
+exports[`annotations truncate single character 1`] = `"<span data-testid=\\"categorical-value-TEST-CATEGORY-T\\" data-testclass=\\"categorical-value\\" aria-label=\\"T\\" class=\\"\\" tabindex=\\"0\\" style=\\"width: 187px; color: black; font-style: normal; display: inline-block; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px;\\"><span style=\\"width: 100%; color: black; font-style: normal; display: flex; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px; justify-content: flex-start; padding: 0px;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">T</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\"></span><span style=\\"position: absolute; right: 0px; color: black;\\"></span></span></span></span>"`;
+
+exports[`annotations truncate single character 2`] = `"<span data-testid=\\"categorical-value-TEST-CATEGORY-T\\" data-testclass=\\"categorical-value\\" aria-label=\\"T\\" class=\\"\\" tabindex=\\"0\\" style=\\"width: 187px; color: black; font-style: normal; display: inline-block; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px;\\"><span style=\\"width: 100%; color: black; font-style: normal; display: flex; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px; justify-content: flex-start; padding: 0px;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">T</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\"></span><span style=\\"position: absolute; right: 0px; color: black;\\"></span></span></span></span>"`;

client/__tests__/e2e/cellxgeneActions.js

@@ -1,196 +1,388 @@
+/* eslint-disable no-await-in-loop -- await in loop is needed to emulate sequential user actions  */
 import { strict as assert } from "assert";
+import {
+  clearInputAndTypeInto,
+  clickOn,
+  getAllByClass,
+  getOneElementInnerText,
+  typeInto,
+  waitByID,
+  waitByClass,
+  waitForAllByIds,
+  clickOnUntil,
+  getTestClass,
+  getTestId,
+  isElementPresent,
+  goToPage,
+} from "./puppeteerUtils";
 
-export const cellxgeneActions = (page, utils) => ({
+import { appUrlBase } from "./config";
 
-  async drag(testId, start, end, lasso = false) {
-    const layout = await utils.waitByID(testId);
-    const elBox = await layout.boxModel();
-    const x1 = elBox.content[0].x + start.x;
-    const x2 = elBox.content[0].x + end.x;
-    const y1 = elBox.content[0].y + start.y;
-    const y2 = elBox.content[0].y + end.y;
+export async function drag(testId, start, end, lasso = false) {
+  const layout = await waitByID(testId);
+  const elBox = await layout.boxModel();
+  const x1 = elBox.content[0].x + start.x;
+  const x2 = elBox.content[0].x + end.x;
+  const y1 = elBox.content[0].y + start.y;
+  const y2 = elBox.content[0].y + end.y;
+  await page.mouse.move(x1, y1);
+  await page.mouse.down();
+  if (lasso) {
+    await page.mouse.move(x2, y1);
+    await page.mouse.move(x2, y2);
+    await page.mouse.move(x1, y2);
     await page.mouse.move(x1, y1);
-    await page.mouse.down();
-    if (lasso) {
-      await page.mouse.move(x2, y1);
-      await page.mouse.move(x2, y2);
-      await page.mouse.move(x1, y2);
-      await page.mouse.move(x1, y1);
-    } else {
-      await page.mouse.move(x2, y2);
-    }
-    await page.mouse.up();
-  },
-
-  async clickOnCoordinate(testId, coord) {
-    const layout = await utils.waitByID(testId);
-    const elBox = await layout.boxModel();
-    const x = elBox.content[0].x + coord.x;
-    const y = elBox.content[0].y + coord.y;
-    await page.mouse.click(x, y);
-  },
-
-  async getAllHistograms(testclass, testIds) {
-    const histTestIds = testIds.map(tid => `histogram-${tid}`);
-    // these load asynchronously, so we need to wait for each histogram individually
-    await utils.waitForAllByIds(histTestIds);
-    const allHistograms = await utils.getAllByClass(testclass);
-    return allHistograms.map(hist => hist.replace(/^histogram-/, ""));
-  },
-
-  async getAllCategoriesAndCounts(category) {
-    await utils.waitByClass("categorical-row");
-    return page.$$eval(
-      `[data-testid="category-${category}"] [data-testclass='categorical-row']`,
-      rows => Object.fromEntries(rows.map(row => {
-        const cat = row.querySelector("[data-testclass='categorical-value']").innerText;
-        const count = row.querySelector("[data-testclass='categorical-value-count']").innerText;
-        return [cat, count];
-      }))
-    );
-  },
-
-  async cellSet(num) {
-    await utils.clickOn(`cellset-button-${num}`);
-    return utils.getOneElementInnerText(`[data-testid='cellset-count-${num}']`);
-  },
-
-  async resetCategory(category) {
-    const checkboxId = `${category}:category-select`;
-    await utils.waitByID(checkboxId);
-    const checkedPseudoclass = await page.$eval(
-      `[data-testid='${checkboxId}']`,
-      el => el.matches(":checked")
-    );
-    if (!checkedPseudoclass) await utils.clickOn(checkboxId);
-    try {
-      const categoryRow = await utils.waitByID(`${category}:category-expand`);
-      const isExpanded = await categoryRow.$("[data-testclass='category-expand-is-expanded']");
-      if (isExpanded) await utils.clickOn(`${category}:category-expand`);
-    } catch {}
-  },
-
-  async calcCoordinate(testId, xAsPercent, yAsPercent) {
-    const el = await utils.waitByID(testId);
-    const size = await el.boxModel();
-    return {
-      x: Math.floor(size.width * xAsPercent),
-      y: Math.floor(size.height * yAsPercent)
-    }
-  },
-
-  async calcDragCoordinates(testId, coordinateAsPercent) {
-    return {
-      start: await this.calcCoordinate(testId, coordinateAsPercent.x1, coordinateAsPercent.y1),
-      end: await this.calcCoordinate(testId, coordinateAsPercent.x2, coordinateAsPercent.y2)
-    };
-  },
-
-  async selectCategory(category, values, reset = true) {
-    if (reset) await this.resetCategory(category);
-    await utils.clickOn(`${category}:category-expand`);
-    await utils.clickOn(`${category}:category-select`);
-    for (const val of values) {
-      await utils.clickOn(`categorical-value-select-${category}-${val}`);
-    }
-  },
-
-  async expandCategory(category) {
-    const expand = await utils.waitByID(`${category}:category-expand`);
-    const notExpanded = await expand.$("[data-testclass='category-expand-is-not-expanded']");
-    if (notExpanded) await utils.clickOn(`${category}:category-expand`);
-  },
-
-  async clip(min = 0, max = 100) {
-    await utils.clickOn("visualization-settings");
-    await utils.clearInputAndTypeInto("clip-min-input", min);
-    await utils.clearInputAndTypeInto("clip-max-input", max);
-    await utils.clickOn("clip-commit");
-  },
-
-  async createCategory(categoryName) {
-    await utils.clickOn("open-annotation-dialog");
-    await utils.typeInto("new-category-name", categoryName);
-    await utils.clickOn("submit-category");
-  },
-
-  async renameCategory(oldCatgoryName, newCategoryName) {
-    await utils.clickOn(`${oldCatgoryName}:see-actions`);
-    await utils.clickOn(`${oldCatgoryName}:edit-category-mode`);
-    await utils.clearInputAndTypeInto(`${oldCatgoryName}:edit-category-name-text`, newCategoryName);
-    await utils.clickOn(`${oldCatgoryName}:submit-category-edit`);
-  },
-
-  async deleteCategory(categoryName) {
-    await utils.clickOn(`${categoryName}:see-actions`);
-    await utils.clickOn(`${categoryName}:delete-category`);
-  },
-
-  async createLabel(categoryName, labelName) {
-    await utils.clickOn(`${categoryName}:see-actions`);
-    await utils.clickOn(`${categoryName}:add-new-label-to-category`);
-    await utils.typeInto(`${categoryName}:new-label-name`, labelName);
-    await utils.clickOn(`${categoryName}:submit-label`);
-  },
-
-  async deleteLabel(categoryName, labelName) {
-    await this.expandCategory(categoryName);
-    await utils.clickOn(`${categoryName}:${labelName}:see-actions`);
-    await utils.clickOn( `${categoryName}:${labelName}:delete-label`);
-  },
-
-  async renameLabel(categoryName, oldLabelName, newLabelName) {
-    await this.expandCategory(categoryName);
-    await utils.clickOn(`${categoryName}:${oldLabelName}:see-actions`);
-    await utils.clickOn(`${categoryName}:${oldLabelName}:edit-label`);
-    await utils.clearInputAndTypeInto(
-      `${categoryName}:${oldLabelName}:edit-label-name`,
-      newLabelName
-    );
-    await utils.clickOn(`${categoryName}:${oldLabelName}:submit-label-edit`);
-  },
-
-  async addGeneToSearch(geneName) {
-    await utils.typeInto("gene-search", geneName);
-    await page.keyboard.press("Enter");
-    await page.waitForSelector(
-      `[data-testid='histogram-${geneName}']`
-    );
-  },
-
-  async subset(coordinatesAsPercent) {
-    // In order to deselect the selection after the subset, make sure we have some clear part
-    // of the scatterplot we can click on
-    assert(coordinatesAsPercent.x2 < 0.99 || coordinatesAsPercent.y2 < 0.99);
-    const lassoSelection = await this.calcDragCoordinates( "layout-graph", coordinatesAsPercent);
-    await this.drag("layout-graph", lassoSelection.start, lassoSelection.end, true );
-    await utils.clickOn("subset-button");
-    const clearCoordinate = await this.calcCoordinate(
-      "layout-graph",
-      0.5,
-      0.99
-    );
-    await this.clickOnCoordinate("layout-graph", clearCoordinate);
-  },
-
-  async setSellSet(cellSet, cellSetNum) {
-    for (const selection of cellSet.filter(sel => sel.kind === "categorical")) {
-      await this.selectCategory(selection.metadata, selection.values, true);
-    }
-    await this.cellSet(cellSetNum);
-  },
-
-  async runDiffExp(cellSet1, cellSet2) {
-    await this.setSellSet(cellSet1, 1);
-    await this.setSellSet(cellSet2, 2);
-    await utils.clickOn("diffexp-button");
-  },
-
-  async bulkAddGenes(geneNames) {
-    await utils.clickOn("section-bulk-add");
-    await utils.typeInto("input-bulk-add", geneNames.join(","));
-    await page.keyboard.press("Enter");
+  } else {
+    await page.mouse.move(x2, y2);
   }
-});
+  await page.mouse.up();
+}
 
+export async function clickOnCoordinate(testId, coord) {
+  const layout = await expect(page).toMatchElement(getTestId(testId));
+  const elBox = await layout.boxModel();
 
+  if (!elBox) {
+    throw Error("Layout's boxModel is not available!");
+  }
+
+  const x = elBox.content[0].x + coord.x;
+  const y = elBox.content[0].y + coord.y;
+  await page.mouse.click(x, y);
+}
+
+export async function getAllHistograms(testclass, testIds) {
+  const histTestIds = testIds.map((tid) => `histogram-${tid}`);
+
+  // these load asynchronously, so we need to wait for each histogram individually,
+  // and they may be quite slow in some cases.
+  await waitForAllByIds(histTestIds, { timeout: 4 * 60 * 1000 });
+
+  const allHistograms = await getAllByClass(testclass);
+
+  const testIDs = await Promise.all(
+    allHistograms.map((hist) => {
+      return page.evaluate((elem) => {
+        return elem.dataset.testid;
+      }, hist);
+    })
+  );
+
+  return testIDs.map((id) => id.replace(/^histogram-/, ""));
+}
+
+export async function getAllCategoriesAndCounts(category) {
+  // these load asynchronously, so we have to wait for the specific category.
+  await waitByID(`category-${category}`);
+
+  return page.$$eval(
+    `[data-testid="category-${category}"] [data-testclass='categorical-row']`,
+    (rows) =>
+      Object.fromEntries(
+        rows.map((row) => {
+          const cat = row
+            .querySelector("[data-testclass='categorical-value']")
+            .getAttribute("aria-label");
+
+          const count = row.querySelector(
+            "[data-testclass='categorical-value-count']"
+          ).innerText;
+
+          return [cat, count];
+        })
+      )
+  );
+}
+
+export async function getCellSetCount(num) {
+  await clickOn(`cellset-button-${num}`);
+  return getOneElementInnerText(`[data-testid='cellset-count-${num}']`);
+}
+
+export async function resetCategory(category) {
+  const checkboxId = `${category}:category-select`;
+  await waitByID(checkboxId);
+  const checkedPseudoclass = await page.$eval(
+    `[data-testid='${checkboxId}']`,
+    (el) => el.matches(":checked")
+  );
+  if (!checkedPseudoclass) await clickOn(checkboxId);
+
+  const categoryRow = await waitByID(`${category}:category-expand`);
+
+  const isExpanded = await categoryRow.$(
+    "[data-testclass='category-expand-is-expanded']"
+  );
+
+  if (isExpanded) await clickOn(`${category}:category-expand`);
+}
+
+export async function calcCoordinate(testId, xAsPercent, yAsPercent) {
+  const el = await waitByID(testId);
+  const size = await el.boxModel();
+  return {
+    x: Math.floor(size.width * xAsPercent),
+    y: Math.floor(size.height * yAsPercent),
+  };
+}
+
+export async function calcDragCoordinates(testId, coordinateAsPercent) {
+  return {
+    start: await calcCoordinate(
+      testId,
+      coordinateAsPercent.x1,
+      coordinateAsPercent.y1
+    ),
+    end: await calcCoordinate(
+      testId,
+      coordinateAsPercent.x2,
+      coordinateAsPercent.y2
+    ),
+  };
+}
+
+export async function selectCategory(category, values, reset = true) {
+  if (reset) await resetCategory(category);
+
+  await clickOn(`${category}:category-expand`);
+  await clickOn(`${category}:category-select`);
+
+  for (const value of values) {
+    await clickOn(`categorical-value-select-${category}-${value}`);
+  }
+}
+
+export async function expandCategory(category) {
+  const expand = await waitByID(`${category}:category-expand`);
+  const notExpanded = await expand.$(
+    "[data-testclass='category-expand-is-not-expanded']"
+  );
+  if (notExpanded) await clickOn(`${category}:category-expand`);
+}
+
+export async function clip(min = 0, max = 100) {
+  await clickOn("visualization-settings");
+  await clearInputAndTypeInto("clip-min-input", min);
+  await clearInputAndTypeInto("clip-max-input", max);
+  await clickOn("clip-commit");
+}
+
+export async function createCategory(categoryName) {
+  await clickOnUntil("open-annotation-dialog", async () => {
+    await expect(page).toMatchElement(getTestId("new-category-name"));
+  });
+
+  await typeInto("new-category-name", categoryName);
+  await clickOn("submit-category");
+}
+
+export async function duplicateCategory(categoryName) {
+  await clickOn("open-annotation-dialog");
+
+  await typeInto("new-category-name", categoryName);
+
+  const dropdownOptionClass = "duplicate-category-dropdown-option";
+
+  await clickOnUntil("duplicate-category-dropdown", async () => {
+    await expect(page).toMatchElement(getTestClass(dropdownOptionClass));
+  });
+
+  const option = await expect(page).toMatchElement(
+    getTestClass(dropdownOptionClass)
+  );
+
+  await option.click();
+
+  await clickOnUntil("submit-category", async () => {
+    await expect(page).toMatchElement(
+      getTestId(`${categoryName}:category-expand`)
+    );
+  });
+
+  await waitByClass("autosave-complete");
+}
+
+export async function renameCategory(oldCategoryName, newCategoryName) {
+  await clickOn(`${oldCategoryName}:see-actions`);
+  await clickOn(`${oldCategoryName}:edit-category-mode`);
+  await clearInputAndTypeInto(
+    `${oldCategoryName}:edit-category-name-text`,
+    newCategoryName
+  );
+  await clickOn(`${oldCategoryName}:submit-category-edit`);
+}
+
+export async function deleteCategory(categoryName) {
+  const targetId = `${categoryName}:delete-category`;
+
+  await clickOnUntil(`${categoryName}:see-actions`, async () => {
+    await expect(page).toMatchElement(getTestId(targetId));
+  });
+
+  await clickOn(targetId);
+
+  await assertCategoryDoesNotExist();
+}
+
+export async function createLabel(categoryName, labelName) {
+  /**
+   * (thuang): This explicit wait is needed, since currently showing
+   * the modal again quickly after the previous action dismissing the
+   * modal will persist the input value from the previous action.
+   *
+   * To reproduce:
+   * 1. Click on the plus sign to show the modal to add a new label to the category
+   * 2. Type `123` in the input box
+   * 3. Hover over your mouse over the plus sign and double click to quickly dismiss and
+   * invoke the modal again
+   * 4. You will see `123` is persisted in the input box
+   * 5. Expected behavior is to get an empty input box
+   */
+  await page.waitFor(500);
+
+  await clickOn(`${categoryName}:see-actions`);
+
+  await clickOn(`${categoryName}:add-new-label-to-category`);
+
+  await typeInto(`${categoryName}:new-label-name`, labelName);
+
+  await clickOn(`${categoryName}:submit-label`);
+}
+
+export async function deleteLabel(categoryName, labelName) {
+  await expandCategory(categoryName);
+  await clickOn(`${categoryName}:${labelName}:see-actions`);
+  await clickOn(`${categoryName}:${labelName}:delete-label`);
+}
+
+export async function renameLabel(categoryName, oldLabelName, newLabelName) {
+  await expandCategory(categoryName);
+  await clickOn(`${categoryName}:${oldLabelName}:see-actions`);
+  await clickOn(`${categoryName}:${oldLabelName}:edit-label`);
+  await clearInputAndTypeInto(
+    `${categoryName}:${oldLabelName}:edit-label-name`,
+    newLabelName
+  );
+  await clickOn(`${categoryName}:${oldLabelName}:submit-label-edit`);
+}
+
+export async function addGeneToSearch(geneName) {
+  await typeInto("gene-search", geneName);
+  await page.keyboard.press("Enter");
+  await page.waitForSelector(`[data-testid='histogram-${geneName}']`);
+}
+
+export async function subset(coordinatesAsPercent) {
+  // In order to deselect the selection after the subset, make sure we have some clear part
+  // of the scatterplot we can click on
+  assert(coordinatesAsPercent.x2 < 0.99 || coordinatesAsPercent.y2 < 0.99);
+  const lassoSelection = await calcDragCoordinates(
+    "layout-graph",
+    coordinatesAsPercent
+  );
+  await drag("layout-graph", lassoSelection.start, lassoSelection.end, true);
+  await clickOn("subset-button");
+  const clearCoordinate = await calcCoordinate("layout-graph", 0.5, 0.99);
+  await clickOnCoordinate("layout-graph", clearCoordinate);
+}
+
+export async function setSellSet(cellSet, cellSetNum) {
+  const selections = cellSet.filter((sel) => sel.kind === "categorical");
+
+  for (const selection of selections) {
+    await selectCategory(selection.metadata, selection.values, true);
+  }
+
+  await getCellSetCount(cellSetNum);
+}
+
+export async function runDiffExp(cellSet1, cellSet2) {
+  await setSellSet(cellSet1, 1);
+  await setSellSet(cellSet2, 2);
+  await clickOn("diffexp-button");
+}
+
+export async function bulkAddGenes(geneNames) {
+  await clickOn("section-bulk-add");
+  await typeInto("input-bulk-add", geneNames.join(","));
+  await page.keyboard.press("Enter");
+}
+
+export async function assertCategoryDoesNotExist(categoryName) {
+  const result = await isElementPresent(
+    getTestId(`${categoryName}:category-label`)
+  );
+
+  await expect(result).toBe(false);
+}
+
+export async function login() {
+  const email = `cellxgene-smoke-test+${process.env.DEPLOYMENT_STAGE}@chanzuckerberg.com`;
+  const password = "Test1111";
+
+  await goToPage(appUrlBase);
+
+  await clickOn("log-in");
+
+  // (thuang): Auth0 form is unstable and unsafe for input until verified
+  await waitUntilFormFieldStable('[name="email"]');
+
+  await expect(page).toFillForm("form", {
+    email,
+    password,
+  });
+
+  await Promise.all([
+    page.waitForNavigation({ waitUntil: "networkidle0" }),
+    expect(page).toClick('[name="submit"]'),
+  ]);
+
+  expect(page.url()).toContain(appUrlBase);
+}
+
+export async function logout() {
+  await clickOnUntil("user-info", async () => {
+    await waitByID("log-out");
+    await Promise.all([
+      page.waitForNavigation({ waitUntil: "networkidle0" }),
+      clickOn("log-out"),
+    ]);
+  });
+
+  await waitByID("log-in");
+}
+
+async function waitUntilFormFieldStable(selector) {
+  const MAX_RETRY = 10;
+  const WAIT_FOR_MS = 200;
+
+  const EXPECTED_VALUE = "aaa";
+
+  let retry = 0;
+
+  while (retry < MAX_RETRY) {
+    try {
+      await expect(page).toFill(selector, EXPECTED_VALUE);
+
+      const fieldHandle = await expect(page).toMatchElement(selector);
+
+      const fieldValue = await page.evaluate(
+        (input) => input.value,
+        fieldHandle
+      );
+
+      expect(fieldValue).toBe(EXPECTED_VALUE);
+
+      break;
+    } catch (error) {
+      retry += 1;
+
+      await page.waitFor(WAIT_FOR_MS);
+    }
+  }
+
+  if (retry === MAX_RETRY) {
+    throw Error("clickOnUntil() assertion failed!");
+  }
+}
+/* eslint-enable no-await-in-loop -- await in loop is needed to emulate sequential user actions */

client/__tests__/e2e/config.js

@@ -1,9 +1,8 @@
-export const jest_env = process.env.JEST_ENV;
-export const appPort = process.env.CXG_SERVER_PORT;
-export const appUrlBase = `http://localhost:${appPort}`;
-export const DEV = jest_env === "dev";
-export const DEBUG = jest_env === "debug";
-export const DATASET = "pbmc3k";
+import * as ENV_DEFAULT from "../../../environment.default.json";
 
-if (DEBUG) jest.setTimeout(100000);
-if (DEV) jest.setTimeout(10000);
+export const jestEnv = process.env.JEST_ENV || ENV_DEFAULT.JEST_ENV;
+export const appUrlBase =
+  process.env.CXG_URL_BASE || `http://localhost:${ENV_DEFAULT.CXG_CLIENT_PORT}`;
+export const DATASET = "pbmc3k";
+export const isDev = jestEnv === ENV_DEFAULT.DEV;
+export const isDebug = jestEnv === ENV_DEFAULT.DEBUG;

client/__tests__/e2e/data.js

@@ -4,7 +4,7 @@ export const datasets = {
     dataframe: {
       nObs: "2638",
       nVar: "1838",
-      type: "float32"
+      type: "float32",
     },
     categorical: {
       louvain: {
@@ -15,47 +15,47 @@ export const datasets = {
         "Dendritic cells": "37",
         "FCGR3A+ Monocytes": "150",
         Megakaryocytes: "15",
-        "NK cells": "154"
-      }
+        "NK cells": "154",
+      },
     },
     continuous: {
       n_genes: "int32",
       percent_mito: "float32",
-      n_counts: "float32"
+      n_counts: "float32",
     },
     cellsets: {
       lasso: [
         {
           "coordinates-as-percent": { x1: 0.1, y1: 0.25, x2: 0.7, y2: 0.75 },
-          count: "1181"
-        }
+          count: "1131",
+        },
       ],
       categorical: [
         {
           metadata: "louvain",
           values: ["B cells", "Megakaryocytes"],
-          count: "357"
-        }
+          count: "357",
+        },
       ],
       continuous: [
         {
           metadata: "n_genes",
           "coordinates-as-percent": { x1: 0.25, y1: 0.5, x2: 0.55, y2: 0.5 },
-          count: "1552"
-        }
-      ]
+          count: "1537",
+        },
+      ],
     },
 
     diffexp: {
       cellset1: [
-        { kind: "categorical", metadata: "louvain", values: ["B cells"] }
+        { kind: "categorical", metadata: "louvain", values: ["B cells"] },
       ],
       cellset2: [
         {
           kind: "categorical",
           metadata: "louvain",
-          values: ["CD4 T cells", "NK cells"]
-        }
+          values: ["CD4 T cells", "NK cells"],
+        },
       ],
       "gene-results": [
         "HLA-DRB1",
@@ -67,21 +67,21 @@ export const datasets = {
         "HLA-DQB1",
         "MS4A1",
         "IL32",
-        "CD37"
-      ]
+        "CD37",
+      ],
     },
 
     genes: {
       bulkadd: ["S100A8", "FCGR3A", "LGALS2", "GSTP1"],
-      search: "ACD"
+      search: "ACD",
     },
     subset: {
       cellset1: [
         {
           kind: "categorical",
           metadata: "louvain",
-          values: ["B cells", "Megakaryocytes"]
-        }
+          values: ["B cells", "Megakaryocytes"],
+        },
       ],
       count: "357",
       categorical: {
@@ -93,27 +93,27 @@ export const datasets = {
           "Dendritic cells": "0",
           "FCGR3A+ Monocytes": "0",
           Megakaryocytes: "15",
-          "NK cells": "0"
-        }
+          "NK cells": "0",
+        },
       },
       lasso: {
         "coordinates-as-percent": { x1: 0.25, y1: 0.05, x2: 0.75, y2: 0.55 },
-        count: "329"
-      }
+        count: "331",
+      },
     },
     scatter: {
-      genes: { x: "S100A8", y: "FCGR3A" }
+      genes: { x: "S100A8", y: "FCGR3A" },
     },
     pan: {
-      "coordinates-as-percent": { x1: 0.75, y1: 0.75, x2: 0.35, y2: 0.35 }
+      "coordinates-as-percent": { x1: 0.75, y1: 0.75, x2: 0.35, y2: 0.35 },
     },
     features: {
       panzoom: {
         lasso: {
           "coordinates-as-percent": { x1: 0.3, y1: 0.3, x2: 0.5, y2: 0.5 },
-          count: "24"
-        }
-      }
+          count: "38",
+        },
+      },
     },
     categoryLabel: {
       lasso: {
@@ -121,18 +121,18 @@ export const datasets = {
       },
       newCount: {
         bySubsetConfig: {
-          false: "600",
-          true: "591"
-        }
-      }
+          false: "668",
+          true: "659",
+        },
+      },
     },
     annotationsFromFile: {
       count: {
         bySubsetConfig: {
           false: "1161",
-          true: "856"
-        }
-      }
+          true: "852",
+        },
+      },
     },
     clip: {
       min: "30",
@@ -140,8 +140,8 @@ export const datasets = {
       metadata: "n_genes",
       gene: "S100A8",
       "coordinates-as-percent": { x1: 0.25, y1: 0.5, x2: 0.55, y2: 0.5 },
-      count: "392",
-      "gene-cell-count": "421"
-    }
-  }
+      count: "386",
+      "gene-cell-count": "416",
+    },
+  },
 };

client/__tests__/e2e/e2e.test.js

@@ -1,46 +1,72 @@
-/*
-Smoke test suite that will be run in Travis CI
-
-Tests included in this file are expected to be relatively stable and test core features
+/**
+ * Smoke test suite that will be run in Travis CI
+ * Tests included in this file are expected to be relatively stable and test core features
  */
+
+/* eslint-disable no-await-in-loop -- await in loop is needed to emulate sequential user actions  */
 import { appUrlBase, DATASET } from "./config";
-import { setupTestBrowser } from "./testBrowser";
+
 import { datasets } from "./data";
 
-let browser, page, utils, cxgActions;
+import {
+  clickOn,
+  getAllByClass,
+  getElementCoordinates,
+  getOneElementInnerHTML,
+  getTestId,
+  goToPage,
+  typeInto,
+  waitByID,
+  clickOnUntil,
+} from "./puppeteerUtils";
+
+import {
+  addGeneToSearch,
+  bulkAddGenes,
+  calcDragCoordinates,
+  clip,
+  drag,
+  getAllCategoriesAndCounts,
+  getAllHistograms,
+  getCellSetCount,
+  runDiffExp,
+  selectCategory,
+  subset,
+  login,
+  logout,
+} from "./cellxgeneActions";
+
 const data = datasets[DATASET];
 
-beforeAll(async () => {
-  const browserViewport = { width: 1280, height: 960 };
-  [browser, page, utils, cxgActions] = await setupTestBrowser(browserViewport);
-});
-
-beforeEach(async () => {
-  await page.goto(appUrlBase);
-});
-
-afterAll(() => {
-  if (browser !== undefined) browser.close();
-});
-
 describe("did launch", () => {
   test("page launched", async () => {
-    const element = await utils.getOneElementInnerHTML("[data-testid='header']");
-    expect(element).toBe(data.title);
+    await goToPage(appUrlBase);
+
+    const element = await getOneElementInnerHTML(getTestId("header"));
+
+    expect(element).toMatchSnapshot();
   });
 });
 
 describe("metadata loads", () => {
   test("categories and values from dataset appear", async () => {
-    for (const label in data.categorical) {
-      await utils.waitByID(`category-${label}`);
-      const categoryName = await utils.getOneElementInnerText(`[data-testid="category-${label}"]`);
-      expect(categoryName).toMatch(label);
-      await utils.clickOn(`${label}:category-expand`);
-      const categories = await cxgActions.getAllCategoriesAndCounts(label);
+    await goToPage(appUrlBase);
+
+    for (const label of Object.keys(data.categorical)) {
+      const element = await getOneElementInnerHTML(
+        getTestId(`category-${label}`)
+      );
+
+      expect(element).toMatchSnapshot();
+
+      await clickOn(`${label}:category-expand`);
+
+      const categories = await getAllCategoriesAndCounts(label);
+
       expect(Object.keys(categories)).toMatchObject(
         Object.keys(data.categorical[label])
       );
+
       expect(Object.values(categories)).toMatchObject(
         Object.values(data.categorical[label])
       );
@@ -48,100 +74,141 @@ describe("metadata loads", () => {
   });
 
   test("continuous data appears", async () => {
-    for (const label in data.continuous) {
-      await utils.waitByID(`histogram-${label}`);
+    await goToPage(appUrlBase);
+
+    for (const label of Object.keys(data.continuous)) {
+      await waitByID(`histogram-${label}`);
     }
   });
 });
 
 describe("cell selection", () => {
   test("selects all cells cellset 1", async () => {
-    const cellCount = await cxgActions.cellSet(1);
+    await goToPage(appUrlBase);
+
+    const cellCount = await getCellSetCount(1);
     expect(cellCount).toBe(data.dataframe.nObs);
   });
 
   test("selects all cells cellset 2", async () => {
-    const cellCount = await cxgActions.cellSet(2);
+    await goToPage(appUrlBase);
+
+    const cellCount = await getCellSetCount(2);
     expect(cellCount).toBe(data.dataframe.nObs);
   });
 
   test("selects cells via lasso", async () => {
+    await goToPage(appUrlBase);
+
     for (const cellset of data.cellsets.lasso) {
-      const cellset1 = await cxgActions.calcDragCoordinates(
+      const cellset1 = await calcDragCoordinates(
         "layout-graph",
         cellset["coordinates-as-percent"]
       );
-      await cxgActions.drag("layout-graph", cellset1.start, cellset1.end, true);
-      const cellCount = await cxgActions.cellSet(1);
+
+      await drag("layout-graph", cellset1.start, cellset1.end, true);
+      const cellCount = await getCellSetCount(1);
       expect(cellCount).toBe(cellset.count);
     }
   });
 
   test("selects cells via categorical", async () => {
+    await goToPage(appUrlBase);
+
     for (const cellset of data.cellsets.categorical) {
-      await utils.clickOn(`${cellset.metadata}:category-expand`);
-      await utils.clickOn(`${cellset.metadata}:category-select`);
-      for (const val of cellset.values) {
-        await utils.clickOn(`categorical-value-select-${cellset.metadata}-${val}`);
+      await clickOn(`${cellset.metadata}:category-expand`);
+      await clickOn(`${cellset.metadata}:category-select`);
+
+      for (const value of cellset.values) {
+        await clickOn(`categorical-value-select-${cellset.metadata}-${value}`);
       }
-      const cellCount = await cxgActions.cellSet(1);
+
+      const cellCount = await getCellSetCount(1);
+
       expect(cellCount).toBe(cellset.count);
     }
   });
 
   test("selects cells via continuous", async () => {
+    await goToPage(appUrlBase);
+
     for (const cellset of data.cellsets.continuous) {
       const histBrushableAreaId = `histogram-${cellset.metadata}-plot-brushable-area`;
-      const coords = await cxgActions.calcDragCoordinates(
+
+      const coords = await calcDragCoordinates(
         histBrushableAreaId,
         cellset["coordinates-as-percent"]
       );
-      await cxgActions.drag(histBrushableAreaId, coords.start, coords.end);
-      const cellCount = await cxgActions.cellSet(1);
+
+      await drag(histBrushableAreaId, coords.start, coords.end);
+
+      const cellCount = await getCellSetCount(1);
+
       expect(cellCount).toBe(cellset.count);
     }
   });
 });
 
 describe("gene entry", () => {
-  test("search for single gene", async () => cxgActions.addGeneToSearch(data.genes.search));
+  test("search for single gene", async () => {
+    await goToPage(appUrlBase);
+    await addGeneToSearch(data.genes.search);
+  });
 
   test("bulk add genes", async () => {
+    await goToPage(appUrlBase);
+
     const testGenes = data.genes.bulkadd;
-    await cxgActions.bulkAddGenes(testGenes);
-    const allHistograms = await cxgActions.getAllHistograms("histogram-user-gene", testGenes);
+
+    await bulkAddGenes(testGenes);
+    const allHistograms = await getAllHistograms(
+      "histogram-user-gene",
+      testGenes
+    );
+
     expect(allHistograms).toEqual(expect.arrayContaining(testGenes));
-    expect(allHistograms.length).toEqual(testGenes.length);
+    expect(allHistograms).toHaveLength(testGenes.length);
   });
 });
 
 describe("differential expression", () => {
   test("selects cells, saves them and performs diffexp", async () => {
-    await cxgActions.runDiffExp(data.diffexp.cellset1, data.diffexp.cellset2);
-    const allHistograms = await cxgActions.getAllHistograms(
+    await goToPage(appUrlBase);
+
+    await runDiffExp(data.diffexp.cellset1, data.diffexp.cellset2);
+
+    const allHistograms = await getAllHistograms(
       "histogram-diffexp",
       data.diffexp["gene-results"]
     );
+
     expect(allHistograms).toEqual(
       expect.arrayContaining(data.diffexp["gene-results"])
     );
-    expect(allHistograms.length).toEqual(data.diffexp["gene-results"].length);
+
+    expect(allHistograms).toHaveLength(data.diffexp["gene-results"].length);
   });
 });
 
 describe("subset", () => {
   test("subset - cell count matches", async () => {
+    await goToPage(appUrlBase);
+
     for (const select of data.subset.cellset1) {
       if (select.kind === "categorical") {
-        await cxgActions.selectCategory(select.metadata, select.values, true);
+        await selectCategory(select.metadata, select.values, true);
       }
     }
-    await utils.clickOn("subset-button");
-    for (const label in data.subset.categorical) {
-      const categories = await cxgActions.getAllCategoriesAndCounts(label);
+
+    await clickOn("subset-button");
+
+    for (const label of Object.keys(data.subset.categorical)) {
+      const categories = await getAllCategoriesAndCounts(label);
+
       expect(Object.keys(categories)).toMatchObject(
         Object.keys(data.subset.categorical[label])
       );
+
       expect(Object.values(categories)).toMatchObject(
         Object.values(data.subset.categorical[label])
       );
@@ -149,99 +216,152 @@ describe("subset", () => {
   });
 
   test("lasso after subset", async () => {
+    await goToPage(appUrlBase);
+
     for (const select of data.subset.cellset1) {
       if (select.kind === "categorical") {
-        await cxgActions.selectCategory(select.metadata, select.values, true);
+        await selectCategory(select.metadata, select.values, true);
       }
     }
-    await utils.clickOn("subset-button");
-    const lassoSelection = await cxgActions.calcDragCoordinates(
+
+    await clickOn("subset-button");
+
+    const lassoSelection = await calcDragCoordinates(
       "layout-graph",
       data.subset.lasso["coordinates-as-percent"]
     );
-    await cxgActions.drag(
-      "layout-graph",
-      lassoSelection.start,
-      lassoSelection.end,
-      true
-    );
-    const cellCount = await cxgActions.cellSet(1);
+
+    await drag("layout-graph", lassoSelection.start, lassoSelection.end, true);
+
+    const cellCount = await getCellSetCount(1);
     expect(cellCount).toBe(data.subset.lasso.count);
   });
 
   test("undo selection appends the top diff exp genes to user defined genes", async () => {
+    await goToPage(appUrlBase);
+
     const userDefinedGenes = data.genes.bulkadd;
     const diffExpGenes = data.diffexp["gene-results"];
-    await cxgActions.bulkAddGenes(userDefinedGenes);
-    const userDefinedHistograms = await cxgActions.getAllHistograms("histogram-user-gene", userDefinedGenes);
-    expect(userDefinedHistograms).toEqual(expect.arrayContaining(userDefinedGenes));
-    await cxgActions.subset({x1: 0.15, y1: 0.10, x2: 0.98, y2: 0.98});
-    await cxgActions.runDiffExp(data.diffexp.cellset1, data.diffexp.cellset2);
-    const diffExpHistograms = await cxgActions.getAllHistograms("histogram-diffexp", diffExpGenes);
+
+    await bulkAddGenes(userDefinedGenes);
+    const userDefinedHistograms = await getAllHistograms(
+      "histogram-user-gene",
+      userDefinedGenes
+    );
+
+    expect(userDefinedHistograms).toEqual(
+      expect.arrayContaining(userDefinedGenes)
+    );
+
+    await subset({ x1: 0.15, y1: 0.1, x2: 0.98, y2: 0.98 });
+    await runDiffExp(data.diffexp.cellset1, data.diffexp.cellset2);
+
+    const diffExpHistograms = await getAllHistograms(
+      "histogram-diffexp",
+      diffExpGenes
+    );
+
     expect(diffExpHistograms).toEqual(expect.arrayContaining(diffExpGenes));
-    await utils.clickOn("reset-subset-button");
+
+    await clickOn("reset-subset-button");
     const expected = [].concat(userDefinedGenes, diffExpGenes);
-    const userDefinedHistogramsAfterSubset = await cxgActions.getAllHistograms(
+    const userDefinedHistogramsAfterSubset = await getAllHistograms(
       "histogram-user-gene",
       expected
     );
-    expect(userDefinedHistogramsAfterSubset).toEqual(expect.arrayContaining(expected));
+
+    expect(userDefinedHistogramsAfterSubset).toEqual(
+      expect.arrayContaining(expected)
+    );
   });
 
   test("subset selection appends the top diff exp genes to user defined genes", async () => {
+    await goToPage(appUrlBase);
+
     const userDefinedGenes = data.genes.bulkadd;
     const diffExpGenes = data.diffexp["gene-results"];
-    await cxgActions.bulkAddGenes(userDefinedGenes);
-    const userDefinedHistograms = await cxgActions.getAllHistograms("histogram-user-gene", userDefinedGenes);
-    expect(userDefinedHistograms).toEqual(expect.arrayContaining(userDefinedGenes));
-    await cxgActions.subset({x1: 0.15, y1: 0.10, x2: 0.98, y2: 0.98});
-    await cxgActions.runDiffExp(data.diffexp.cellset1, data.diffexp.cellset2);
-    const diffExpHistograms = await cxgActions.getAllHistograms("histogram-diffexp", diffExpGenes);
+
+    await bulkAddGenes(userDefinedGenes);
+    const userDefinedHistograms = await getAllHistograms(
+      "histogram-user-gene",
+      userDefinedGenes
+    );
+
+    expect(userDefinedHistograms).toEqual(
+      expect.arrayContaining(userDefinedGenes)
+    );
+
+    await subset({ x1: 0.15, y1: 0.1, x2: 0.98, y2: 0.98 });
+    await runDiffExp(data.diffexp.cellset1, data.diffexp.cellset2);
+
+    const diffExpHistograms = await getAllHistograms(
+      "histogram-diffexp",
+      diffExpGenes
+    );
+
     expect(diffExpHistograms).toEqual(expect.arrayContaining(diffExpGenes));
-    await cxgActions.subset({x1: 0.16, y1: 0.11, x2: 0.97, y2: 0.97});
+
+    await subset({ x1: 0.16, y1: 0.11, x2: 0.97, y2: 0.97 });
+
     const expected = [].concat(userDefinedGenes, diffExpGenes);
-    const userDefinedHistogramsAfterSubset = await cxgActions.getAllHistograms(
+    const userDefinedHistogramsAfterSubset = await getAllHistograms(
       "histogram-user-gene",
       expected
     );
-    expect(userDefinedHistogramsAfterSubset).toEqual(expect.arrayContaining(expected));
+
+    expect(userDefinedHistogramsAfterSubset).toEqual(
+      expect.arrayContaining(expected)
+    );
   });
 });
 
 describe("scatter plot", () => {
   test("scatter plot appears", async () => {
-    await cxgActions.bulkAddGenes(Object.values(data.scatter.genes));
-    await utils.clickOn(`plot-x-${data.scatter.genes.x}`);
-    await utils.clickOn(`plot-y-${data.scatter.genes.y}`);
-    await utils.waitByID("scatterplot");
+    await goToPage(appUrlBase);
+
+    await bulkAddGenes(Object.values(data.scatter.genes));
+    await clickOn(`plot-x-${data.scatter.genes.x}`);
+    await clickOn(`plot-y-${data.scatter.genes.y}`);
+    await waitByID("scatterplot");
   });
 });
 
 describe("clipping", () => {
   test("clip continuous", async () => {
-    await cxgActions.clip(data.clip.min, data.clip.max);
+    await goToPage(appUrlBase);
+
+    await clip(data.clip.min, data.clip.max);
     const histBrushableAreaId = `histogram-${data.clip.metadata}-plot-brushable-area`;
-    const coords = await cxgActions.calcDragCoordinates(
+    const coords = await calcDragCoordinates(
       histBrushableAreaId,
       data.clip["coordinates-as-percent"]
     );
-    await cxgActions.drag(histBrushableAreaId, coords.start, coords.end);
-    const cellCount = await cxgActions.cellSet(1);
+    await drag(histBrushableAreaId, coords.start, coords.end);
+    const cellCount = await getCellSetCount(1);
     expect(cellCount).toBe(data.clip.count);
   });
 
   test("clip gene", async () => {
-    await utils.typeInto("gene-search", data.clip.gene);
+    await goToPage(appUrlBase);
+
+    await typeInto("gene-search", data.clip.gene);
     await page.keyboard.press("Enter");
+
     await page.waitForSelector(`[data-testid='histogram-${data.clip.gene}']`);
-    await cxgActions.clip(data.clip.min, data.clip.max);
+
+    await clip(data.clip.min, data.clip.max);
+
     const histBrushableAreaId = `histogram-${data.clip.gene}-plot-brushable-area`;
-    const coords = await cxgActions.calcDragCoordinates(
+
+    const coords = await calcDragCoordinates(
       histBrushableAreaId,
       data.clip["coordinates-as-percent"]
     );
-    await cxgActions.drag(histBrushableAreaId, coords.start, coords.end);
-    const cellCount = await cxgActions.cellSet(1);
+
+    await drag(histBrushableAreaId, coords.start, coords.end);
+
+    const cellCount = await getCellSetCount(1);
+
     expect(cellCount).toBe(data.clip["gene-cell-count"]);
   });
 });
@@ -249,82 +369,92 @@ describe("clipping", () => {
 // interact with UI elements just that they do not break
 describe("ui elements don't error", () => {
   test("color by", async () => {
-    for (const label in data.categorical) {
-      await utils.clickOn(`colorby-${label}`);
-    }
-    for (const label in data.continuous) {
-      await utils.clickOn(`colorby-${label}`);
+    await goToPage(appUrlBase);
+
+    const allLabels = [
+      ...Object.keys(data.categorical),
+      ...Object.keys(data.continuous),
+    ];
+
+    for (const label of allLabels) {
+      await clickOn(`colorby-${label}`);
     }
   });
 
   test("color by for gene", async () => {
-    await utils.typeInto("gene-search", data.genes.search);
+    await goToPage(appUrlBase);
+
+    await typeInto("gene-search", data.genes.search);
     await page.keyboard.press("Enter");
     await page.waitForSelector(
       `[data-testid='histogram-${data.genes.search}']`
     );
-    await utils.clickOn(`colorby-${data.genes.search}`);
+    await clickOn(`colorby-${data.genes.search}`);
   });
 
   test("pan and zoom", async () => {
-    await utils.clickOn("mode-pan-zoom");
-    const panCoords = await cxgActions.calcDragCoordinates(
+    await goToPage(appUrlBase);
+
+    await clickOn("mode-pan-zoom");
+    const panCoords = await calcDragCoordinates(
       "layout-graph",
       data.pan["coordinates-as-percent"]
     );
-    await cxgActions.drag(
-      "layout-graph",
-      panCoords.start,
-      panCoords.end,
-      false
-    );
+
+    await drag("layout-graph", panCoords.start, panCoords.end, false);
+
     await page.evaluate("window.scrollBy(0, 1000);");
   });
 });
 
 describe("centroid labels", () => {
   test("labels are created", async () => {
-    await utils.clickOn("centroid-label-toggle");
+    await goToPage(appUrlBase);
+
     const labels = Object.keys(data.categorical);
-    /* eslint-disable no-await-in-loop */
+
+    await clickOn(`colorby-${labels[0]}`);
+    await clickOn("centroid-label-toggle");
+
     // Toggle colorby for each category and check to see if labels are generated
     for (let i = 0, { length } = labels; i < length; i += 1) {
       const label = labels[i];
-      await utils.clickOn(`colorby-${label}`);
-      const generatedLabels = await utils.getAllByClass("centroid-label");
+      // first label is already enabled
+      if (i !== 0) await clickOn(`colorby-${label}`);
+      const generatedLabels = await getAllByClass("centroid-label");
       // Number of labels generated should be equal to size of the object
       expect(generatedLabels).toHaveLength(
         Object.keys(data.categorical[label]).length
       );
     }
-    /* eslint-enable no-await-in-loop */
   });
 });
 
 describe("graph overlay", () => {
   test("transform centroids correctly", async () => {
+    await goToPage(appUrlBase);
+
     const category = Object.keys(data.categorical)[0];
-    await utils.clickOn("centroid-label-toggle");
-    await utils.clickOn(`colorby-${category}`);
-    await utils.clickOn("mode-pan-zoom");
-    const panCoords = await cxgActions.calcDragCoordinates(
+
+    await clickOn(`colorby-${category}`);
+    await clickOn("centroid-label-toggle");
+    await clickOn("mode-pan-zoom");
+
+    const panCoords = await calcDragCoordinates(
       "layout-graph",
       data.pan["coordinates-as-percent"]
     );
 
     const categoryValue = Object.keys(data.categorical[category])[0];
-    const initialCoordinates = await utils.getElementCoordinates(
+    const initialCoordinates = await getElementCoordinates(
       `${categoryValue}-centroid-label`
     );
-    await cxgActions.drag(
-      "layout-graph",
-      panCoords.start,
-      panCoords.end,
-      false
-    );
-    const terminalCoordinates = await utils.getElementCoordinates(
+
+    await drag("layout-graph", panCoords.start, panCoords.end, false);
+    const terminalCoordinates = await getElementCoordinates(
       `${categoryValue}-centroid-label`
     );
+
     expect(terminalCoordinates[0] - initialCoordinates[0]).toBeCloseTo(
       panCoords.end.x - panCoords.start.x
     );
@@ -333,3 +463,92 @@ describe("graph overlay", () => {
     );
   });
 });
+
+test("pan zoom mode resets lasso selection", async () => {
+  await goToPage(appUrlBase);
+
+  const panzoomLasso = data.features.panzoom.lasso;
+
+  const lassoSelection = await calcDragCoordinates(
+    "layout-graph",
+    panzoomLasso["coordinates-as-percent"]
+  );
+
+  await drag("layout-graph", lassoSelection.start, lassoSelection.end, true);
+  await waitByID("lasso-element", { visible: true });
+
+  const initialCount = await getCellSetCount(1);
+
+  expect(initialCount).toBe(panzoomLasso.count);
+
+  await clickOn("mode-pan-zoom");
+  await clickOn("mode-lasso");
+
+  const modeSwitchCount = await getCellSetCount(1);
+
+  expect(modeSwitchCount).toBe(initialCount);
+});
+
+test("lasso moves after pan", async () => {
+  await goToPage(appUrlBase);
+
+  const panzoomLasso = data.features.panzoom.lasso;
+  const coordinatesAsPercent = panzoomLasso["coordinates-as-percent"];
+
+  const lassoSelection = await calcDragCoordinates(
+    "layout-graph",
+    coordinatesAsPercent
+  );
+
+  await drag("layout-graph", lassoSelection.start, lassoSelection.end, true);
+  await waitByID("lasso-element", { visible: true });
+
+  const initialCount = await getCellSetCount(1);
+
+  expect(initialCount).toBe(panzoomLasso.count);
+
+  await clickOn("mode-pan-zoom");
+
+  const panCoords = await calcDragCoordinates(
+    "layout-graph",
+    coordinatesAsPercent
+  );
+
+  await drag("layout-graph", panCoords.start, panCoords.end, false);
+  await clickOn("mode-lasso");
+
+  const panCount = await getCellSetCount(2);
+
+  expect(panCount).toBe(initialCount);
+});
+
+const describeIfCalledByMakeFileTarget =
+  process.env.CXG_AUTH_TYPE?.toLowerCase() === "test"
+    ? describe
+    : describe.skip;
+
+describeIfCalledByMakeFileTarget("auth buttons", () => {
+  test("login then logout", async () => {
+    await goToPage(appUrlBase);
+    await clickOnUntil("log-in", async () => {
+      await page.waitForNavigation({ waitUntil: "networkidle0" });
+      await waitByID("user-info");
+    });
+    await logout();
+  });
+});
+
+const conditionalDescribe =
+  process.env.TEST_AUTH_INTEGRATION === "true" ? describe : describe.skip;
+
+conditionalDescribe("AuthN Integration", () => {
+  it("logs in", async () => {
+    await login();
+  });
+
+  it("logs out", async () => {
+    await login();
+    await logout();
+  });
+});
+/* eslint-enable no-await-in-loop -- await in loop is needed to emulate sequential user actions */

client/__tests__/e2e/e2eAnnotations.test.js

@@ -2,221 +2,354 @@
 Tests included in this file are specific to annotation features
 */
 import { appUrlBase, DATASET } from "./config";
-import { setupTestBrowser } from "./testBrowser";
 import { datasets } from "./data";
 
-let browser, page, utils, actions;
+import {
+  clickOn,
+  goToPage,
+  waitByClass,
+  waitByID,
+  getTestId,
+  getTestClass,
+  getAllByClass,
+} from "./puppeteerUtils";
+
+import {
+  assertCategoryDoesNotExist,
+  calcDragCoordinates,
+  createCategory,
+  createLabel,
+  deleteCategory,
+  deleteLabel,
+  drag,
+  expandCategory,
+  renameCategory,
+  renameLabel,
+  subset,
+  duplicateCategory,
+} from "./cellxgeneActions";
+
 const data = datasets[DATASET];
 
-beforeAll(async () => {
-  const browserViewport = {width: 1280, height: 960};
-  [browser, page, utils, actions] = await setupTestBrowser(browserViewport);
-});
+const perTestCategoryName = "TEST-CATEGORY";
+const perTestLabelName = "TEST-LABEL";
 
-afterAll(() => {
-  if (browser !== undefined) browser.close()
-});
+async function setup(config) {
+  await goToPage(appUrlBase);
+
+  // setup the test fixtures
+  await createCategory(perTestCategoryName);
+  await createLabel(perTestCategoryName, perTestLabelName);
+
+  if (config.withSubset) {
+    await subset({ x1: 0.1, y1: 0.1, x2: 0.8, y2: 0.8 });
+  }
+
+  await waitByClass("autosave-complete");
+}
 
 describe.each([
-  {withSubset: true, tag: "subset"},
-  {withSubset: false, tag: "whole"}
+  { withSubset: true, tag: "subset" },
+  { withSubset: false, tag: "whole" },
 ])("annotations", (config) => {
-
-  const perTestCategoryName = "per-test-category";
-  const perTestLabelName = "per-test-label";
-
-  beforeEach(async () => {
-    await page.goto(appUrlBase);
-    // wait for the page to load
-    await utils.waitByClass("autosave-complete");
-    // setup the test fixtures
-    await actions.createCategory(perTestCategoryName);
-    await actions.createLabel(perTestCategoryName, perTestLabelName);
-    if (config.withSubset) await actions.subset({x1: 0.10, y1: 0.10, x2: 0.80, y2: 0.80});
-    await utils.waitByClass("autosave-complete");
-  });
-
-  afterEach(async () => {
-    await deleteCategoryIfExists(perTestCategoryName);
-    await utils.waitByClass("autosave-complete");
-  });
-
   test("create a category", async () => {
+    await setup(config);
+
     const categoryName = `category-created-${config.tag}`;
+
     await assertCategoryDoesNotExist(categoryName);
-    await actions.createCategory(categoryName);
+
+    await createCategory(categoryName);
+
     await assertCategoryExists(categoryName);
   });
 
   test("delete a category", async () => {
-    await actions.deleteCategory(perTestCategoryName);
+    await setup(config);
+
+    await deleteCategory(perTestCategoryName);
     await assertCategoryDoesNotExist(perTestCategoryName);
   });
 
   test("rename a category", async () => {
-    const newCategoryName = `cluster-for-real-${config.tag}`;
-    await actions.renameCategory(perTestCategoryName, newCategoryName);
+    await setup(config);
+
+    const newCategoryName = `NEW-${config.tag}`;
+
+    await renameCategory(perTestCategoryName, newCategoryName);
     await assertCategoryDoesNotExist(perTestCategoryName);
     await assertCategoryExists(newCategoryName);
   });
 
   test("create a label", async () => {
+    await setup(config);
+
     const labelName = `new-label-${config.tag}`;
+
     await assertLabelDoesNotExist(perTestCategoryName, labelName);
-    await actions.createLabel(perTestCategoryName, labelName);
+
+    await createLabel(perTestCategoryName, labelName);
+
     await assertLabelExists(perTestCategoryName, labelName);
   });
 
   test("delete a label", async () => {
-    await actions.deleteLabel(perTestCategoryName, perTestLabelName);
+    await setup(config);
+
+    await deleteLabel(perTestCategoryName, perTestLabelName);
     await assertLabelDoesNotExist(perTestCategoryName, perTestLabelName);
   });
 
   test("rename a label", async () => {
+    await setup(config);
+
     const newLabelName = "my-cool-new-label";
+
     await assertLabelDoesNotExist(perTestCategoryName, newLabelName);
-    await actions.renameLabel(perTestCategoryName, perTestLabelName, newLabelName);
+    await renameLabel(perTestCategoryName, perTestLabelName, newLabelName);
     await assertLabelDoesNotExist(perTestCategoryName, perTestLabelName);
     await assertLabelExists(perTestCategoryName, newLabelName);
   });
 
   test("check cell count for a label loaded from file", async () => {
-    const categoryName = "cluster-test";
-    const labelName = "four";
-    await actions.expandCategory(categoryName);
-    const result = await utils.waitByID(`categorical-value-count-${categoryName}-${labelName}`);
-    expect(await result.evaluate(node => node.innerText)).toBe(
-      data.annotationsFromFile.count.bySubsetConfig[config.withSubset]
+    await setup(config);
+
+    const duplicateCategoryName = "duplicate";
+    await duplicateCategory(duplicateCategoryName);
+
+    await page.reload({ waitUntil: ["networkidle0", "domcontentloaded"] });
+
+    const firstCategoryExpandIcon = await expect(page).toMatchElement(
+      getTestClass("category-expand")
     );
+
+    await firstCategoryExpandIcon.click();
+
+    const expectedCategoryRow = await expect(page).toMatchElement(
+      getTestClass("categorical-row")
+    );
+    const expectedLabelName = await getInnerText(
+      expectedCategoryRow,
+      "categorical-value"
+    );
+    const expectedLabelCount = await getInnerText(
+      expectedCategoryRow,
+      "categorical-value-count"
+    );
+
+    await expandCategory(duplicateCategoryName);
+
+    const expectedCategory = await expect(page).toMatchElement(
+      getTestClass("category")
+    );
+
+    const actualCategoryRow = await expect(expectedCategory).toMatchElement(
+      getTestClass("categorical-row")
+    );
+    const actualLabelName = await getInnerText(
+      actualCategoryRow,
+      "categorical-value"
+    );
+    const actualLabelCount = await getInnerText(
+      actualCategoryRow,
+      "categorical-value-count"
+    );
+
+    expect(actualLabelName).toBe(expectedLabelName);
+    expect(actualLabelCount).toBe(expectedLabelCount);
+
+    async function getInnerText(element, className) {
+      return element.$eval(getTestClass(className), (node) => node?.innerText);
+    }
   });
 
   test("assign cells to a label", async () => {
-    await actions.expandCategory(perTestCategoryName);
-    const lassoSelection = await actions.calcDragCoordinates(
+    await setup(config);
+
+    await expandCategory(perTestCategoryName);
+
+    const lassoSelection = await calcDragCoordinates(
       "layout-graph",
       data.categoryLabel.lasso["coordinates-as-percent"]
     );
-    await actions.drag(
-      "layout-graph",
-      lassoSelection.start,
-      lassoSelection.end,
-      true
+
+    await drag("layout-graph", lassoSelection.start, lassoSelection.end, true);
+    await waitByID("lasso-element", { visible: true });
+    await clickOn(`${perTestCategoryName}:${perTestLabelName}:see-actions`);
+    await clickOn(
+      `${perTestCategoryName}:${perTestLabelName}:add-current-selection-to-this-label`
     );
-    await utils.waitByID("lasso-element", {visible: true});
-    await utils.clickOn(`${perTestCategoryName}:${perTestLabelName}:see-actions`);
-    await utils.clickOn(`${perTestCategoryName}:${perTestLabelName}:add-current-selection-to-this-label`);
-    const result = await utils.waitByID(`categorical-value-count-${perTestCategoryName}-${perTestLabelName}`);
-    expect(await result.evaluate(node => node.innerText)).toBe(
+
+    const result = await waitByID(
+      `categorical-value-count-${perTestCategoryName}-${perTestLabelName}`
+    );
+
+    expect(await result.evaluate((node) => node.innerText)).toBe(
       data.categoryLabel.newCount.bySubsetConfig[config.withSubset]
     );
   });
 
   test("undo/redo category creation", async () => {
+    await setup(config);
+
     const categoryName = `category-created-undo-${config.tag}`;
+
     await assertCategoryDoesNotExist(categoryName);
-    await actions.createCategory(categoryName);
+    await createCategory(categoryName);
     await assertCategoryExists(categoryName);
-    await utils.clickOn("undo");
+    await clickOn("undo");
     await assertCategoryDoesNotExist(categoryName);
-    await utils.clickOn("redo");
+    await clickOn("redo");
     await assertCategoryExists(categoryName);
   });
 
   test("undo/redo category deletion", async () => {
+    await setup(config);
+
     const categoryName = `category-deleted-undo-${config.tag}`;
-    await actions.createCategory(categoryName);
+
+    await createCategory(categoryName);
     await assertCategoryExists(categoryName);
-    await actions.deleteCategory(categoryName);
+    await deleteCategory(categoryName);
     await assertCategoryDoesNotExist(categoryName);
-    await utils.clickOn("undo");
+    await clickOn("undo");
     await assertCategoryExists(categoryName);
-    await utils.clickOn("redo");
+    await clickOn("redo");
     await assertCategoryDoesNotExist(categoryName);
   });
 
   test("undo/redo category rename", async () => {
+    await setup(config);
+
     const newCategoryName = `category-renamed-undo-${config.tag}`;
+
     await assertCategoryDoesNotExist(newCategoryName);
-    await actions.renameCategory(perTestCategoryName, newCategoryName);
+    await renameCategory(perTestCategoryName, newCategoryName);
     await assertCategoryExists(newCategoryName);
     await assertCategoryDoesNotExist(perTestCategoryName);
-    await utils.clickOn("undo");
+    await clickOn("undo");
     await assertCategoryExists(perTestCategoryName);
     await assertCategoryDoesNotExist(newCategoryName);
-    await utils.clickOn("redo");
+    await clickOn("redo");
     await assertCategoryExists(newCategoryName);
     await assertCategoryDoesNotExist(perTestCategoryName);
   });
 
   test("undo/redo label creation", async () => {
+    await setup(config);
+
     const labelName = `label-created-undo-${config.tag}`;
+
     await assertLabelDoesNotExist(perTestCategoryName, labelName);
-    await actions.createLabel(perTestCategoryName, labelName);
+    await createLabel(perTestCategoryName, labelName);
     await assertLabelExists(perTestCategoryName, labelName);
-    await utils.clickOn("undo");
+    await clickOn("undo");
     await assertLabelDoesNotExist(perTestCategoryName);
-    await utils.clickOn("redo");
+    await clickOn("redo");
     await assertLabelExists(perTestCategoryName, labelName);
   });
 
   test("undo/redo label deletion", async () => {
-    await actions.deleteLabel(perTestCategoryName, perTestLabelName);
+    await setup(config);
+
+    await deleteLabel(perTestCategoryName, perTestLabelName);
     await assertLabelDoesNotExist(perTestCategoryName);
-    await utils.clickOn("undo");
+    await clickOn("undo");
     await assertLabelExists(perTestCategoryName, perTestLabelName);
-    await utils.clickOn("redo");
+    await clickOn("redo");
     await assertLabelDoesNotExist(perTestCategoryName);
   });
 
   test("undo/redo label rename", async () => {
+    await setup(config);
+
     const newLabelName = `label-renamed-undo-${config.tag}`;
+
     await assertLabelDoesNotExist(perTestCategoryName, newLabelName);
-    await actions.renameLabel(perTestCategoryName, perTestLabelName, newLabelName);
+    await renameLabel(perTestCategoryName, perTestLabelName, newLabelName);
     await assertLabelExists(perTestCategoryName, newLabelName);
     await assertLabelDoesNotExist(perTestCategoryName, perTestLabelName);
-    await utils.clickOn("undo");
+    await clickOn("undo");
     await assertLabelExists(perTestCategoryName, perTestLabelName);
     await assertLabelDoesNotExist(perTestCategoryName, newLabelName);
-    await utils.clickOn("redo");
+    await clickOn("redo");
     await assertLabelExists(perTestCategoryName, newLabelName);
     await assertLabelDoesNotExist(perTestCategoryName, perTestLabelName);
   });
 
-  async function assertCategoryExists(categoryName) {
-    const handle = await utils.waitByID(`${categoryName}:category-expand`);
-    const result = await handle.evaluate(node => node.innerText);
-    // slice beginning and end of category name result to account for truncation of long names
-    expect(result.slice(0, 10)).toBe(categoryName.slice(0, 10));
-    expect(result.slice(-10)).toBe(categoryName.slice(-10));
-  }
+  test("stacked bar graph renders", async () => {
+    await setup(config);
 
-  async function assertCategoryDoesNotExist(categoryName) {
-    const result = await page.$(`[data-testid='${categoryName}:category-expand']`);
-    expect(result).toBeNull();
+    await expandCategory(perTestCategoryName);
+
+    await clickOn(`colorby-louvain`);
+
+    const labels = await getAllByClass("categorical-row");
+
+    const result = await Promise.all(
+      labels.map((label) => {
+        return page.evaluate((element) => {
+          return element.outerHTML;
+        }, label);
+      })
+    );
+
+    expect(result).toMatchSnapshot();
+  });
+
+  test("truncate midpoint whitespace", async () => {
+    await setup(config);
+    const newLabelName = "123 456";
+    await renameLabel(perTestCategoryName, perTestLabelName, newLabelName);
+    const value = await waitByID(
+      `categorical-value-${perTestCategoryName}-${newLabelName}`
+    );
+    const result = await page.evaluate((elem) => elem.outerHTML, value);
+    expect(result).toMatchSnapshot();
+  });
+
+  test("truncate single character", async () => {
+    await setup(config);
+    const newLabelName = "T";
+    await renameLabel(perTestCategoryName, perTestLabelName, newLabelName);
+    const value = await waitByID(
+      `categorical-value-${perTestCategoryName}-${newLabelName}`
+    );
+    const result = await page.evaluate((elem) => elem.outerHTML, value);
+    expect(result).toMatchSnapshot();
+  });
+
+  async function assertCategoryExists(categoryName) {
+    const handle = await waitByID(`${categoryName}:category-label`);
+
+    const result = await handle.evaluate((node) =>
+      node.getAttribute("aria-label")
+    );
+
+    return expect(result).toBe(categoryName);
   }
 
   async function assertLabelExists(categoryName, labelName) {
-    const category = await utils.waitByID(`${categoryName}:category-expand`);
-    expect(category).not.toBeNull();
-    await actions.expandCategory(categoryName);
-    const previous = await utils.waitByID(`categorical-value-${categoryName}-${labelName}`);
-    expect(await previous.evaluate(node => node.innerText)).toBe(labelName);
+    await expect(page).toMatchElement(
+      getTestId(`${categoryName}:category-expand`)
+    );
+
+    await expandCategory(categoryName);
+
+    const previous = await waitByID(
+      `categorical-value-${categoryName}-${labelName}`
+    );
+
+    expect(
+      await previous.evaluate((node) => node.getAttribute("aria-label"))
+    ).toBe(labelName);
   }
 
   async function assertLabelDoesNotExist(categoryName, labelName) {
-    await actions.expandCategory(categoryName);
-    const result = await page.$(`[data-testid='categorical-value-${categoryName}-${labelName}']`);
+    await expandCategory(categoryName);
+    const result = await page.$(
+      `[data-testid='categorical-value-${categoryName}-${labelName}']`
+    );
     expect(result).toBeNull();
   }
-
-  async function deleteCategoryIfExists(categoryName) {
-    try {
-      const category = await page.waitForSelector(
-        `[data-testid='${categoryName}:category-expand']`,
-        {timeout: 200}
-      );
-      if (category !== null) return await actions.deleteCategory(categoryName);
-    } catch {}
-    return null
-  }
 });

client/__tests__/e2e/e2eJestConfig.json

@@ -1,10 +1,10 @@
 {
+  "testRunner": "jest-circus/runner",
   "preset": "jest-puppeteer",
-  "testMatch": [
-    "**/__tests__/**/?(*.)(spec|test).js?(x)"
-  ],
-  "testURL": "http://localhost/",
-  "setupFiles": [
-    "../setupMissingGlobals.js"
-  ]
+  "testMatch": ["**/__tests__/**/?(*.)(spec|test).js?(x)"],
+  "setupFiles": ["../setupMissingGlobals.js"],
+  "setupFilesAfterEnv": ["expect-puppeteer", "./puppeteer.setup.js"],
+  "globalSetup": "jest-environment-puppeteer/setup",
+  "globalTeardown": "jest-environment-puppeteer/teardown",
+  "testEnvironment": "./screenshot_env.js"
 }

client/__tests__/e2e/feature.test.js

@@ -1,118 +0,0 @@
-/*
-NOT run in Travis CI
-
-UX tests using puppeteer to be run locally.
-
-To run locally, ensure you are running the client is running on port 3000.
-Then run jest --verbose false --config __tests__/e2e/e2eJestConfig.json feature.
- */
-
-import puppeteer from "puppeteer";
-import { appUrlBase, DEBUG, DEV, DATASET } from "./config";
-import { puppeteerUtils, cellxgeneActions } from "./puppeteerUtils";
-import { datasets } from "./data";
-
-let browser, page, utils, cxgActions, spy;
-const browserViewport = { width: 1280, height: 960 };
-let data = datasets[DATASET].features;
-
-if (DEBUG) jest.setTimeout(100000);
-if (DEV) jest.setTimeout(10000);
-
-beforeAll(async () => {
-  const browserParams = DEV
-    ? { headless: false, slowMo: 5 }
-    : DEBUG
-    ? { headless: false, slowMo: 100, devtools: true }
-    : {};
-  browser = await puppeteer.launch(browserParams);
-  page = await browser.newPage();
-  await page.setViewport(browserViewport);
-  if (DEV || DEBUG) {
-    page.on("console", msg => console.log(`PAGE LOG: ${msg.text()}`));
-  }
-  page.on("pageerror", err => {
-    throw new Error(`Console error: ${err}`);
-  });
-  utils = puppeteerUtils(page);
-  cxgActions = cellxgeneActions(page);
-});
-
-beforeEach(async () => {
-  await page.goto(appUrlBase);
-});
-
-afterAll(() => {
-  if (!DEBUG) {
-    browser.close();
-  }
-});
-
-describe("zoom interaction", async () => {
-  // Skip this test since UI is to hide lasso path when switching modes
-  test.skip("lasso visible after switching modes to pan/zoom", async () => {
-    const lassoSelection = await cxgActions.calcDragCoordinates(
-      "layout-graph",
-      data.panzoom.lasso["coordinates-as-percent"]
-    );
-    await cxgActions.drag(
-      "layout-graph",
-      lassoSelection.start,
-      lassoSelection.end,
-      true
-    );
-    await utils.waitByID("lasso-element", { visible: true });
-    await utils.clickOn("mode-pan-zoom");
-    await utils.waitByID("lasso-element", { visible: true });
-  });
-
-  test("pan zoom mode resets lasso selection", async () => {
-    const lassoSelection = await cxgActions.calcDragCoordinates(
-      "layout-graph",
-      data.panzoom.lasso["coordinates-as-percent"]
-    );
-    await cxgActions.drag(
-      "layout-graph",
-      lassoSelection.start,
-      lassoSelection.end,
-      true
-    );
-    await utils.waitByID("lasso-element", { visible: true });
-    const initialCount = await cxgActions.cellSet(1);
-    expect(initialCount).toBe(data.panzoom.lasso.count);
-    await utils.clickOn("mode-pan-zoom");
-    await utils.clickOn("mode-lasso");
-    const modeSwitchCount = await cxgActions.cellSet(1);
-    expect(modeSwitchCount).toBe(initialCount);
-  });
-
-  test("lasso moves after pan", async () => {
-    const lassoSelection = await cxgActions.calcDragCoordinates(
-      "layout-graph",
-      data.panzoom.lasso["coordinates-as-percent"]
-    );
-    await cxgActions.drag(
-      "layout-graph",
-      lassoSelection.start,
-      lassoSelection.end,
-      true
-    );
-    await utils.waitByID("lasso-element", { visible: true });
-    const initialCount = await cxgActions.cellSet(1);
-    expect(initialCount).toBe(data.panzoom.lasso.count);
-    await utils.clickOn("mode-pan-zoom");
-    const panCoords = await cxgActions.calcDragCoordinates(
-      "layout-graph",
-      data.panzoom.lasso["coordinates-as-percent"]
-    );
-    await cxgActions.drag(
-      "layout-graph",
-      panCoords.start,
-      panCoords.end,
-      false
-    );
-    await utils.clickOn("mode-lasso");
-    const panCount = await cxgActions.cellSet(2);
-    expect(panCount).toBe(initialCount);
-  });
-});

client/__tests__/e2e/puppeteer.setup.js

@@ -0,0 +1,58 @@
+/**
+ * `client/jest-puppeteer.config.js` is for configuring Puppeteer's launch config options
+ * `client/__tests__/e2e/puppeteer.setup.js` is for configuring `jest`, `browser`,
+ * and `page` objects
+ */
+
+import { setDefaultOptions } from "expect-puppeteer";
+import { isDebug, isDev } from "./config";
+import * as ENV_DEFAULT from "../../../environment.default.json";
+
+// (thuang): This is the max time a test can take to run.
+// Since when debugging, we run slowMo and !headless, this means
+// a test can take more time to finish, so we don't want
+// jest to shut off the test too soon
+jest.setTimeout(2 * 60 * 1000);
+setDefaultOptions({ timeout: 20 * 1000 });
+
+jest.retryTimes(ENV_DEFAULT.RETRY_ATTEMPTS);
+
+beforeEach(async () => {
+  await jestPuppeteer.resetBrowser();
+
+  const userAgent = await browser.userAgent();
+  await page.setUserAgent(`${userAgent}bot`);
+
+  await page._client.send("Animation.setPlaybackRate", { playbackRate: 12 });
+
+  page.on("pageerror", (err) => {
+    throw new Error(`Console error: ${err}`);
+  });
+
+  page.on("error", (err) => {
+    throw new Error(`Console error: ${err}`);
+  });
+
+  page.on("console", async (msg) => {
+    if (isDev || isDebug) {
+      // If there is a console.error but an error is not thrown, this will ensure the test fails
+      console.log(`PAGE LOG: ${msg.text()}`);
+      if (msg.type() === "error") {
+        // TODO: chromium does not currently support the CSP directive on the
+        // line below, so we swallow this error. Remove this when the test
+        // suite uses a browser version that supports this directive.
+        if (
+          msg.text() ===
+          "Unrecognized Content-Security-Policy directive 'require-trusted-types-for'.\n"
+        ) {
+          return;
+        }
+        const errorMsgText = await Promise.all(
+          // TODO can we do this without internal properties?
+          msg.args().map((arg) => arg._remoteObject.description)
+        );
+        throw new Error(`Console error: ${errorMsgText}`);
+      }
+    }
+  });
+});

client/__tests__/e2e/puppeteerUtils.js

@@ -1,69 +1,131 @@
-export const puppeteerUtils = page => ({
-  async waitByID(testId, props = {}) {
-    return page.waitForSelector(`[data-testid='${testId}']`, props);
-  },
+/* eslint-disable no-await-in-loop -- await in loop is needed to emulate sequential user actions */
+export function getTestId(id) {
+  return `[data-testid='${id}']`;
+}
 
-  async waitByClass(testClass, props = {}) {
-    return page.waitForSelector(`[data-testclass='${testClass}']`, props);
-  },
+export function getTestClass(className) {
+  return `[data-testclass='${className}']`;
+}
 
-  async waitForAllByIds(testIds) {
-    await Promise.all(
-      testIds.map(testId => page.waitForSelector(`[data-testid='${testId}']`))
-    );
-  },
+export async function waitByID(testId, props = {}) {
+  return page.waitForSelector(getTestId(testId), props);
+}
 
-  async getAllByClass(testClass) {
-    return page.$$eval(`[data-testclass=${testClass}]`, eles =>
-      eles.map(ele => ele.dataset.testid)
-    );
-  },
+export async function waitByClass(testClass, props = {}) {
+  await page.waitForSelector(`[data-testclass='${testClass}']`, props);
+}
 
-  async typeInto(testId, text) {
-    // blueprint's  typeahead is treating typing weird, clicking & waiting first solves this
-    // only works for text without special characters
-    await this.waitByID(testId);
-    const selector = `[data-testid='${testId}']`;
-    // type ahead can be annoying if you don't pause before you type
-    await page.click(selector);
-    await page.waitFor(200);
-    await page.type(selector, text);
-  },
+export async function waitForAllByIds(testIds) {
+  await Promise.all(
+    testIds.map((testId) => page.waitForSelector(getTestId(testId)))
+  );
+}
 
-  async clearInputAndTypeInto(testId, text) {
-    await this.waitByID(testId);
-    const selector = `[data-testid='${testId}']`;
-    // only works for text without special characters
-    // type ahead can be annoying if you don't pause before you type
-    await page.click(selector);
-    await page.waitFor(200);
-    // select all
-    await page.click(selector, { clickCount: 3 });
-    await page.keyboard.press("Backspace");
-    await page.type(selector, text);
-  },
+export async function getAllByClass(testClass) {
+  return page.$$(`[data-testclass=${testClass}]`);
+}
 
-  async clickOn(testid, options = {}) {
-    await this.waitByID(testid);
-    const click = await page.click(`[data-testid='${testid}']`, options);
-    await page.waitFor(50);
-    return click;
-  },
+export async function typeInto(testId, text) {
+  // blueprint's  typeahead is treating typing weird, clicking & waiting first solves this
+  // only works for text without special characters
+  await waitByID(testId);
+  const selector = getTestId(testId);
+  // type ahead can be annoying if you don't pause before you type
+  await page.click(selector);
+  await page.waitFor(200);
+  await page.type(selector, text);
+}
 
-  async getOneElementInnerHTML(selector) {
-    await page.waitForSelector(selector);
-    return page.$eval(selector, el => el.innerHTML);
-  },
+export async function clearInputAndTypeInto(testId, text) {
+  await waitByID(testId);
+  const selector = getTestId(testId);
+  // only works for text without special characters
+  // type ahead can be annoying if you don't pause before you type
+  await page.click(selector);
+  await page.waitFor(200);
+  // select all
+  await page.click(selector, { clickCount: 3 });
+  await page.keyboard.press("Backspace");
+  await page.type(selector, text);
+}
 
-  async getOneElementInnerText(selector) {
-    await page.waitForSelector(selector);
-    return page.$eval(selector, el => el.innerText);
-  },
+export async function clickOn(testId, options = {}) {
+  await expect(page).toClick(getTestId(testId), options);
+}
 
-  async getElementCoordinates(testid) {
-    return page.$eval(`[data-testid='${testid}']`, elem => {
-      const { left, top } = elem.getBoundingClientRect();
-      return [left, top];
-    });
+/**
+ * (thuang): There are times when Puppeteer clicks on a button and the page doesn't respond.
+ * So I added clickOnUntil() to retry clicking until a given condition is met.
+ */
+export async function clickOnUntil(testId, assert) {
+  const MAX_RETRY = 10;
+  const WAIT_FOR_MS = 200;
+
+  let retry = 0;
+
+  while (retry < MAX_RETRY) {
+    try {
+      await clickOn(testId);
+      await assert();
+
+      break;
+    } catch (error) {
+      retry += 1;
+
+      await page.waitFor(WAIT_FOR_MS);
+    }
   }
-});
+
+  if (retry === MAX_RETRY) {
+    throw Error("clickOnUntil() assertion failed!");
+  }
+}
+
+export async function getOneElementInnerHTML(selector, options = {}) {
+  await page.waitForSelector(selector, options);
+
+  return page.$eval(selector, (el) => el.innerHTML);
+}
+
+export async function getOneElementInnerText(selector) {
+  expect(page).toMatchElement(selector);
+
+  return page.$eval(selector, (el) => el.innerText);
+}
+
+export async function getElementCoordinates(testId) {
+  return page.$eval(getTestId(testId), (elem) => {
+    const { left, top } = elem.getBoundingClientRect();
+    return [left, top];
+  });
+}
+
+async function clickTermsOfService() {
+  if (!(await isElementPresent(getTestId("tos-cookies-accept")))) return;
+
+  await clickOn("tos-cookies-accept");
+}
+
+async function nameNewAnnotation() {
+  if (await isElementPresent(getTestId("annotation-dialog"))) {
+    await typeInto("new-annotation-name", "ignoreE2E");
+    await clickOn("submit-annotation");
+
+    // wait for the page to load
+    await waitByClass("autosave-complete");
+  }
+}
+
+export async function goToPage(url) {
+  await page.goto(url, {
+    waitUntil: "networkidle0",
+  });
+
+  await nameNewAnnotation();
+  await clickTermsOfService();
+}
+
+export async function isElementPresent(selector, options) {
+  return Boolean(await page.$(selector, options));
+}
+/* eslint-enable no-await-in-loop -- await in loop is needed to emulate sequential user actions */

client/__tests__/e2e/screenshot_env.js

@@ -0,0 +1,36 @@
+const PuppeteerEnvironment = require("jest-environment-puppeteer");
+require("jest-circus");
+const ENV_DEFAULT = require("../../../environment.default.json");
+
+const takeScreenshot = require("./takeScreenshot");
+
+class ScreenshotEnvironment extends PuppeteerEnvironment {
+  async handleTestEvent(event, state) {
+    if (["test_start", "test_done"].includes(event.name)) {
+      console.log("------------------event name:\n", event.name);
+      console.log("~~~~ Current test errors\n", new Date(), event.test.errors);
+      console.log("~~~~ Current test\n", new Date(), event.test);
+    }
+
+    if (event.name === "error") {
+      console.log("error event:", JSON.stringify(event));
+    }
+
+    if (event.name === "test_fn_failure" || event.name === "hook_failure") {
+      console.log("------------------event name:\n", event.name);
+      console.log(">>>> Current state\n", new Date(), state);
+      console.log("===> Failure event\n", new Date(), event);
+
+      // (thuang): We only want to take screenshot on the last try
+      if (
+        state.currentlyRunningTest.invocations <= ENV_DEFAULT.RETRY_ATTEMPTS
+      ) {
+        return;
+      }
+
+      await takeScreenshot(state.currentlyRunningTest.name, this.global.page);
+    }
+  }
+}
+
+module.exports = ScreenshotEnvironment;

client/__tests__/e2e/takeScreenshot.js

@@ -0,0 +1,17 @@
+function toFilename(name) {
+  return name.replace(/[^a-z0-9.-]+/gi, "-");
+}
+
+async function takeScreenshot(currentTestName, page) {
+  const testName = toFilename(currentTestName);
+
+  // Take a screenshot at the point of failure
+  const date = new Date().toISOString();
+  const screenshotName = `${date}-${testName}.png`;
+
+  await page.screenshot({
+    path: `./__tests__/screenshots/ignoreE2E-screenshot-${screenshotName}`,
+  });
+}
+
+module.exports = takeScreenshot;

client/__tests__/e2e/testBrowser.js

@@ -1,35 +0,0 @@
-import puppeteer from "puppeteer";
-import { DEBUG, DEV } from "./config";
-import { puppeteerUtils } from "./puppeteerUtils";
-import { cellxgeneActions } from "./cellxgeneActions";
-
-export async function setupTestBrowser(browserViewport) {
-  const browserParams = DEV
-    ? { headless: false, slowMo: 5 }
-    : DEBUG
-      ? { headless: false, slowMo: 100, devtools: true }
-      : {};
-  const browser = await puppeteer.launch(browserParams);
-  const page = await browser.pages().then(pages => pages[0]);
-  await page.setViewport(browserViewport);
-  if (DEV || DEBUG) {
-    page.on("console", async msg => {
-      // If there is a console.error but an error is not thrown, this will ensure the test fails
-      if (msg.type() === "error") {
-        const errorMsgText = await Promise.all(
-          // TODO can we do this without internal properties?
-          msg.args().map(arg => arg._remoteObject.description)
-        );
-        throw new Error(`Console error: ${errorMsgText}`);
-      }
-      console.log(`PAGE LOG: ${msg.text()}`);
-    });
-  }
-  page.on("pageerror", err => {
-    throw new Error(`Console error: ${err}`);
-  });
-  const utils = puppeteerUtils(page);
-  const cxgActions = cellxgeneActions(page, utils);
-  return [browser, page, utils, cxgActions];
-}
-

client/__tests__/e2e/test_config.yaml

@@ -0,0 +1,47 @@
+server:
+  app:
+    force_https: true
+
+    # By default, cellxgene will serve api requests from the same base url as the webpage.
+    # In general api_base_url and web_base_url will not need to be set.
+    # There are two reasons to set these parameters:
+    #  1. Oauth authentication is used; the oauth server will redirect back to the api_base_url after login,
+    #     which then redirects back to the web_base_url.  If the web_base_url is not set, it will default to
+    #     the api_base_url.  If oauth authentication is used, the api_base_url must be set.
+    #     For a local test (where the server runs on "http://localhost:<port>"), then the api_base_url may be
+    #     set to the string "local".
+    #  2. The cellxgene deploymnent is in an environment where the webpage and api have
+    #     different base urls.  In this case both api_base_url and web_base_url must be set.
+    #     It is up to the server admin to ensure that the networking is setup correctly for this environment.
+    api_base_url: http://localhost:5005
+    web_base_url: http://localhost:3000
+
+  authentication:
+    # The authentication types may be "none", "session", "oauth"
+    # none:  No authentication support, features like user_annotations must not be enabled.
+    # session:  A session based userid is automatically generated. (no params needed)
+    # oauth: oauth2 is used for authentication;  parameters are defined in params_oauth.
+    type: test
+
+dataset:
+  app:
+    about_legal_tos: null
+    about_legal_privacy: null
+
+  presentation:
+    max_categories: 1000
+    custom_colors: true
+
+  user_annotations:
+    enable: false
+    type: local_file_csv
+    local_file_csv:
+      directory: null
+      file: null
+    ontology:
+      enable: false
+      obo_location: null
+
+  embeddings:
+    names: []
+    enable_reembedding: false

client/__tests__/reducers/cascade.test.js

@@ -26,7 +26,7 @@ describe("cascade", () => {
           expect(nextSharedState).toStrictEqual({});
           expect(prevSharedState).toBe(topLevelState);
           return 0;
-        }
+        },
       ],
       [
         "bar",
@@ -36,8 +36,8 @@ describe("cascade", () => {
           expect(nextSharedState).toStrictEqual({ foo: 0 });
           expect(prevSharedState).toBe(topLevelState);
           return 99;
-        }
-      ]
+        },
+      ],
     ]);
 
     const nextState = reducer(topLevelState, topLevelAction);

client/__tests__/reducers/undoable.test.js

@@ -23,7 +23,7 @@ describe("create", () => {
 describe("undo", () => {
   test("expected state modifications", () => {
     const initialState = { a: 0, b: 1000 };
-    const reducer = state => {
+    const reducer = (state) => {
       return { a: state.a + 1, b: state.b + 1 };
     };
     const undoableReducer = undoable(reducer, ["a"]);
@@ -43,7 +43,7 @@ describe("undo", () => {
 
 describe("redo", () => {
   const initialState = { a: 0, b: 1000 };
-  const reducer = state => {
+  const reducer = (state) => {
     return { a: state.a + 1, b: state.b + 1 };
   };
   let UR;
@@ -58,7 +58,7 @@ describe("redo", () => {
 
     // verify undo->redo reverts state.
     const s2 = UR(UR(s1, { type: "@@undoable/undo" }), {
-      type: "@@undoable/redo"
+      type: "@@undoable/redo",
     });
     expect(s2).toMatchObject({ a: 1, b: 1001 });
 

client/__tests__/screenshots/.gitignore

@@ -0,0 +1,4 @@
+# Ignore everything in this directory
+*
+# Except this file
+!.gitignore

client/__tests__/util/actionHelpers.test.js

@@ -13,16 +13,16 @@ describe("rangeEncodeIndices", () => {
     expect(rangeEncodeIndices([1, 9, 432], 10, true)).toMatchObject([
       1,
       9,
-      432
+      432,
     ]);
     expect(rangeEncodeIndices([1, 9, 432], 10, false)).toMatchObject([
       1,
       9,
-      432
+      432,
     ]);
-    expect(rangeEncodeIndices([0, 1, 2, 3, 9, 10, 432], 2, true)).toMatchObject(
-      [[0, 3], [9, 10], 432]
-    );
+    expect(
+      rangeEncodeIndices([0, 1, 2, 3, 9, 10, 432], 2, true)
+    ).toMatchObject([[0, 3], [9, 10], 432]);
     expect(
       rangeEncodeIndices([0, 1, 2, 3, 9, 10, 432], 2, false)
     ).toMatchObject([[0, 3], [9, 10], 432]);
@@ -34,7 +34,10 @@ describe("rangeEncodeIndices", () => {
     ).toMatchObject([[3, 7], [10, 12], 99]);
     expect(
       rangeEncodeIndices([3, 4, 5, 6, 7, 10, 11, 12], 3, false)
-    ).toMatchObject([[3, 7], [10, 12]]);
+    ).toMatchObject([
+      [3, 7],
+      [10, 12],
+    ]);
     expect(
       rangeEncodeIndices([0, 3, 4, 5, 6, 7, 10, 11, 12], 3, false)
     ).toMatchObject([0, [3, 7], [10, 12]]);

client/__tests__/util/annoMatrix/annoMatrix.test.js

@@ -0,0 +1,316 @@
+// these TWO statements MUST be first in the file, before any other imports
+import { enableFetchMocks } from "jest-fetch-mock";
+import * as serverMocks from "./serverMocks";
+// OK, continue on!
+
+import {
+  AnnoMatrixLoader,
+  clip,
+  isubset,
+  isubsetMask,
+} from "../../../src/annoMatrix";
+import { Dataframe } from "../../../src/util/dataframe";
+
+enableFetchMocks();
+
+describe("AnnoMatrix", () => {
+  let annoMatrix;
+
+  beforeEach(async () => {
+    fetch.resetMocks(); // reset all fetch mocking state
+    annoMatrix = new AnnoMatrixLoader(
+      serverMocks.baseDataURL,
+      serverMocks.schema.schema
+    );
+  });
+
+  describe("basics", () => {
+    test("annomatrix static checks", () => {
+      expect(annoMatrix).toBeDefined();
+      expect(annoMatrix.schema).toMatchObject(serverMocks.schema.schema);
+      expect(annoMatrix.nObs).toEqual(serverMocks.schema.schema.dataframe.nObs);
+      expect(annoMatrix.nVar).toEqual(serverMocks.schema.schema.dataframe.nVar);
+      expect(annoMatrix.isView).toBeFalsy();
+      expect(annoMatrix.viewOf).toBeUndefined();
+      expect(annoMatrix.rowIndex).toBeDefined();
+    });
+
+    test("simple single column fetch", async () => {
+      fetch.once(serverMocks.annotationsObs(["name_0"]));
+
+      const df = await annoMatrix.fetch("obs", "name_0");
+      expect(df).toBeInstanceOf(Dataframe);
+      expect(df.colIndex.labels()).toEqual(["name_0"]);
+      expect(df.dims).toEqual([annoMatrix.nObs, 1]);
+    });
+
+    test("simple multi column fetch", async () => {
+      fetch
+        .once(serverMocks.annotationsObs(["name_0"]))
+        .once(serverMocks.annotationsObs(["n_genes"]));
+
+      await expect(
+        annoMatrix.fetch("obs", ["name_0", "n_genes"])
+      ).resolves.toBeInstanceOf(Dataframe);
+    });
+
+    describe("fetch from field", () => {
+      const getLastTwo = async (field) => {
+        const columnNames = annoMatrix.getMatrixColumns(field).slice(-2);
+        fetch.mockResponses(...columnNames.map(() => serverMocks.responder));
+        await expect(
+          annoMatrix.fetch(field, columnNames)
+        ).resolves.toBeInstanceOf(Dataframe);
+      };
+
+      test("obs", async () => getLastTwo("obs"));
+      test("var", async () => getLastTwo("var"));
+      test("emb", async () => getLastTwo("emb"));
+    });
+
+    test("fetch - test all query forms", async () => {
+      // single string is a column name
+      fetch.once(serverMocks.annotationsObs(["n_genes"]));
+      await expect(annoMatrix.fetch("obs", "n_genes")).resolves.toBeInstanceOf(
+        Dataframe
+      );
+
+      // array of column names, expecting n_genes to be cached.
+      fetch.once(serverMocks.annotationsObs(["percent_mito"]));
+      await expect(
+        annoMatrix.fetch("obs", ["n_genes", "percent_mito"])
+      ).resolves.toBeInstanceOf(Dataframe);
+
+      // more complex value filter query, enumerated
+      fetch.once(serverMocks.responder);
+      await expect(
+        annoMatrix.fetch("X", {
+          field: "var",
+          column: annoMatrix.schema.annotations.var.index,
+          value: "TYMP",
+        })
+      ).resolves.toBeInstanceOf(Dataframe);
+
+      // more complex value filter query, range
+      const varIndex = annoMatrix.schema.annotations.var.index;
+      fetch
+        .once(
+          serverMocks.withExpected("/data/var", [[`var:${varIndex}`, "SUMO3"]])
+        )
+        .once(
+          serverMocks.withExpected("/data/var", [[`var:${varIndex}`, "TYMP"]])
+        );
+      await expect(
+        annoMatrix.fetch("X", [
+          {
+            field: "var",
+            column: varIndex,
+            value: "SUMO3",
+          },
+          {
+            field: "var",
+            column: varIndex,
+            value: "TYMP",
+          },
+        ])
+      ).resolves.toBeInstanceOf(Dataframe);
+      // XXX inspect the wherecache?
+    });
+
+    test("push and pop views", async () => {
+      const am1 = clip(annoMatrix, 0.1, 0.9);
+      expect(am1.viewOf).toBe(annoMatrix);
+      expect(am1.nObs).toEqual(annoMatrix.nObs);
+      expect(am1.nVar).toEqual(annoMatrix.nVar);
+      expect(am1.rowIndex).toBe(annoMatrix.rowIndex);
+
+      const am2 = clip(annoMatrix, 0.1, 0.9);
+      expect(am2.viewOf).toBe(annoMatrix);
+      expect(am2).not.toBe(am1);
+      expect(am2.rowIndex).toBe(annoMatrix.rowIndex);
+    });
+
+    test("schema accessors", () => {
+      expect(annoMatrix.getMatrixFields()).toEqual(
+        expect.arrayContaining(["X", "obs", "emb", "var"])
+      );
+      expect(annoMatrix.getMatrixColumns("obs")).toEqual(
+        expect.arrayContaining(["name_0", "n_genes", "louvain"])
+      );
+      expect(annoMatrix.getColumnSchema("emb", "umap")).toEqual({
+        name: "umap",
+        dims: ["umap_0", "umap_1"],
+        type: "float32",
+      });
+      expect(annoMatrix.getColumnDimensions("emb", "umap")).toEqual([
+        "umap_0",
+        "umap_1",
+      ]);
+    });
+
+    /*
+  	test the mask & label access to subset via isubset and isubsetMask
+  	*/
+    test("isubset", async () => {
+      const rowList = [0, 10];
+      const rowMask = new Uint8Array(annoMatrix.nObs);
+      for (let i = 0; i < rowList.length; i += 1) {
+        rowMask[rowList[i]] = 1;
+      }
+
+      const am1 = isubset(annoMatrix, rowList);
+      const am2 = isubsetMask(annoMatrix, rowMask);
+      expect(am1).not.toBe(am2);
+      expect(am1.nObs).toEqual(2);
+      expect(am1.nObs).toEqual(am2.nObs);
+      expect(am1.nVar).toEqual(am2.nVar);
+
+      fetch
+        .once(serverMocks.annotationsObs(["n_genes"]))
+        .once(serverMocks.annotationsObs(["n_genes"]));
+      const ng1 = await am1.fetch("obs", "n_genes");
+      const ng2 = await am2.fetch("obs", "n_genes");
+      expect(ng1).toHaveLength(ng2.length);
+      expect(ng1.colIndex.labels()).toEqual(ng2.colIndex.labels());
+      expect(ng1.col("n_genes").asArray()).toEqual(
+        ng2.col("n_genes").asArray()
+      );
+    });
+  });
+
+  describe("add/drop column", () => {
+    async function addDrop(base) {
+      expect(base.getMatrixColumns("obs")).not.toContain("foo");
+      fetch.mockRejectOnce(new Error("unknown column name"));
+      await expect(base.fetch("obs", "foo")).rejects.toThrow(
+        "unknown column name"
+      );
+
+      /* add */
+      const am1 = base.addObsColumn(
+        { name: "foo", type: "float32", writable: true },
+        Float32Array,
+        0
+      );
+      expect(base.getMatrixColumns("obs")).not.toContain("foo");
+      expect(am1.getMatrixColumns("obs")).toContain("foo");
+      const foo = await am1.fetch("obs", "foo");
+      expect(foo).toBeDefined();
+      expect(foo).toBeInstanceOf(Dataframe);
+      expect(foo).toHaveLength(am1.nObs);
+      expect(foo.col("foo").asArray()).toEqual(
+        new Float32Array(am1.nObs).fill(0)
+      );
+
+      /* drop */
+      const am2 = am1.dropObsColumn("foo");
+      expect(base.getMatrixColumns("obs")).not.toContain("foo");
+      expect(am2.getMatrixColumns("obs")).not.toContain("foo");
+      fetch.mockRejectOnce(new Error("unknown column name"));
+      await expect(am2.fetch("obs", "foo")).rejects.toThrow(
+        "unknown column name"
+      );
+    }
+
+    test("add/drop column, without view", async () => {
+      await addDrop(annoMatrix);
+    });
+
+    test("add/drop column, with view", async () => {
+      const am1 = clip(annoMatrix, 0.1, 0.9);
+      await addDrop(am1);
+
+      const am2 = isubset(am1, [0, 1, 2, 20, 30, 400]);
+      await addDrop(am2);
+
+      const am3 = isubset(annoMatrix, [10, 0, 7, 3]);
+      await addDrop(am3);
+
+      const am4 = clip(am3, 0, 1);
+      await addDrop(am4);
+
+      fetch.mockResponse(serverMocks.responder);
+
+      await am1.fetch("obs", am1.getMatrixColumns("obs"));
+      await am2.fetch("obs", am2.getMatrixColumns("obs"));
+      await am3.fetch("obs", am3.getMatrixColumns("obs"));
+      await am4.fetch("obs", am4.getMatrixColumns("obs"));
+
+      fetch.resetMocks();
+
+      await addDrop(am1);
+      await addDrop(am2);
+      await addDrop(am3);
+      await addDrop(am4);
+    });
+  });
+
+  describe("setObsColumnValues", () => {
+    async function addSetDrop(base) {
+      /* add column */
+      let am = base.addObsColumn(
+        {
+          name: "test",
+          type: "categorical",
+          categories: ["unassigned", "red", "green"],
+          writable: true,
+        },
+        Array,
+        "unassigned"
+      );
+
+      const testVal = await am.fetch("obs", "test");
+      expect(testVal.col("test").asArray()).toEqual(
+        new Array(am.nObs).fill("unassigned")
+      );
+
+      /* set values in column */
+      const whichRows = [1, 2, 10];
+      const am1 = await am.setObsColumnValues("test", whichRows, "yo");
+      const testVal1 = await am1.fetch("obs", "test");
+      const expt = new Array(am1.nObs).fill("unassigned");
+      for (let i = 0; i < whichRows.length; i += 1) {
+        const offset = am1.rowIndex.getOffset(whichRows[i]);
+        expt[offset] = "yo";
+      }
+      expect(testVal1).not.toBe(testVal);
+      expect(testVal1.col("test").asArray()).toEqual(expt);
+      expect(am1.getColumnSchema("obs", "test").type).toBe("categorical");
+      expect(am1.getColumnSchema("obs", "test").categories).toEqual(
+        expect.arrayContaining(["unassigned", "red", "green", "yo"])
+      );
+
+      /* drop column */
+      fetch.mockRejectOnce(new Error("unknown column name"));
+      am = am1.dropObsColumn("test");
+      await expect(am.fetch("obs", "test")).rejects.toThrow(
+        "unknown column name"
+      );
+    }
+
+    test("set, without a view", async () => {
+      await addSetDrop(annoMatrix);
+    });
+
+    test("set, with a view", async () => {
+      const am1 = clip(annoMatrix, 0.1, 0.9);
+      await addSetDrop(am1);
+
+      const am2 = isubset(am1, [0, 1, 2, 10, 20, 30, 400]);
+      await addSetDrop(am2);
+
+      const am3 = isubset(annoMatrix, [10, 1, 0, 30, 2]);
+      await addSetDrop(am3);
+
+      fetch.mockResponse(serverMocks.responder);
+
+      await am1.fetch("obs", am1.getMatrixColumns("obs"));
+      await am2.fetch("obs", am2.getMatrixColumns("obs"));
+      await am3.fetch("obs", am3.getMatrixColumns("obs"));
+
+      await addSetDrop(am1);
+      await addSetDrop(am2);
+      await addSetDrop(am3);
+    });
+  });
+});

client/__tests__/util/annoMatrix/crossfilter.test.js

@@ -0,0 +1,688 @@
+// these TWO statements MUST be first in the file, before any other imports
+import { enableFetchMocks } from "jest-fetch-mock";
+import * as serverMocks from "./serverMocks";
+// OK, continue on!
+
+import obsLouvain from "./louvain.json";
+import obsNGenes from "./n_genes.json";
+import embUmap from "./umap.json";
+
+import {
+  AnnoMatrixLoader,
+  AnnoMatrixObsCrossfilter,
+  isubsetMask,
+} from "../../../src/annoMatrix";
+import { rangeFill } from "../../../src/util/range";
+
+enableFetchMocks();
+
+describe("AnnoMatrixCrossfilter", () => {
+  let annoMatrix;
+  let crossfilter;
+
+  beforeEach(async () => {
+    fetch.resetMocks(); // reset all fetch mocking state
+    annoMatrix = new AnnoMatrixLoader(
+      serverMocks.baseDataURL,
+      serverMocks.schema.schema
+    );
+    crossfilter = new AnnoMatrixObsCrossfilter(annoMatrix);
+  });
+
+  test("initial state of crossfilter", () => {
+    const { nObs } = annoMatrix;
+
+    expect(crossfilter).toBeDefined();
+    expect(crossfilter.size()).toEqual(nObs);
+    expect(crossfilter.annoMatrix).toBe(annoMatrix);
+
+    /* by default, everything should be selected, even if no data in cache */
+    expect(crossfilter.countSelected()).toEqual(nObs);
+    expect(crossfilter.allSelectedLabels()).toEqual(
+      rangeFill(new Int32Array(nObs))
+    );
+    expect(crossfilter.allSelectedMask()).toEqual(new Uint8Array(nObs).fill(1));
+    expect(crossfilter.fillByIsSelected(new Uint8Array(nObs), 2, 1)).toEqual(
+      new Uint8Array(nObs).fill(2)
+    );
+  });
+
+  describe("select", () => {
+    /*
+    test the selection state via crossfilter proxy
+    */
+
+    test("select loads index", async () => {
+      /*
+      Select should transparently load/create dimension index.
+
+      Internal dimension names are field/col:col:col..., eg,
+
+        obs:louvain
+        emb:umap_0:umap_1
+
+      */
+      expect(crossfilter.obsCrossfilter.dimensionNames()).toEqual([]);
+      expect(
+        crossfilter.obsCrossfilter.hasDimension("obs/louvain")
+      ).toBeFalsy();
+
+      fetch.once(serverMocks.dataframeResponse(["louvain"], [obsLouvain]));
+      let newCrossfilter = await crossfilter.select("obs", "louvain", {
+        mode: "none",
+      });
+
+      expect(newCrossfilter.countSelected()).toEqual(0);
+      expect(
+        newCrossfilter.obsCrossfilter.hasDimension("obs/louvain")
+      ).toBeTruthy();
+      expect(fetch.mock.calls).toHaveLength(1);
+
+      newCrossfilter = await crossfilter.select("obs", "louvain", {
+        mode: "all",
+      });
+      expect(newCrossfilter.countSelected()).toEqual(annoMatrix.nObs);
+    });
+
+    test("simple column select", async () => {
+      let xfltr;
+
+      fetch.once(serverMocks.dataframeResponse(["louvain"], [obsLouvain]));
+      xfltr = await crossfilter.select("obs", "louvain", {
+        mode: "exact",
+        values: ["NK cells", "B cells"],
+      });
+
+      expect(xfltr).toBeDefined();
+      expect(xfltr.countSelected()).toEqual(496);
+      expect(xfltr.allSelectedMask()).toEqual(
+        Uint8Array.from(
+          obsLouvain.map((val) =>
+            val === "NK cells" || val === "B cells" ? 1 : 0
+          )
+        )
+      );
+      expect(xfltr.allSelectedLabels()).toEqual(
+        Int32Array.from(
+          obsLouvain.reduce((acc, val, idx) => {
+            if (val === "NK cells" || val === "B cells") acc.push(idx);
+            return acc;
+          }, [])
+        )
+      );
+      expect(
+        xfltr.fillByIsSelected(new Uint8Array(annoMatrix.nObs), 3, 1)
+      ).toEqual(
+        Uint8Array.from(
+          obsLouvain.map((val) =>
+            val === "NK cells" || val === "B cells" ? 3 : 1
+          )
+        )
+      );
+
+      const df = await annoMatrix.fetch("obs", "louvain");
+      const values = df.col("louvain").asArray();
+      const selected = xfltr.allSelectedMask();
+      values.every(
+        (val, idx) => !["NK cells", "B cells"].includes(val) !== !selected[idx]
+      );
+
+      fetch.once(
+        serverMocks.dataframeResponse(["n_genes"], [new Int32Array(obsNGenes)])
+      );
+      xfltr = await xfltr.select("obs", "n_genes", {
+        mode: "range",
+        lo: 0,
+        hi: 500,
+        inclusive: false,
+      });
+      expect(xfltr.countSelected()).toEqual(33);
+      expect(xfltr.allSelectedLabels()).toEqual(
+        Int32Array.from(
+          obsNGenes.reduce((acc, val, idx) => {
+            const louvain = obsLouvain[idx];
+            if (
+              val >= 0 &&
+              val < 500 &&
+              (louvain === "NK cells" || louvain === "B cells")
+            )
+              acc.push(idx);
+            return acc;
+          }, [])
+        )
+      );
+
+      xfltr = await xfltr.selectAll();
+      expect(xfltr.countSelected()).toEqual(annoMatrix.nObs);
+    });
+
+    test("join column select", async () => {
+      const varIndex = annoMatrix.schema.annotations.var.index;
+
+      const { nObs } = annoMatrix.schema.dataframe;
+      fetch.once(
+        serverMocks.dataframeResponse(
+          ["TEST"],
+          [rangeFill(new Float32Array(nObs), 0, 0.1)]
+        )
+      );
+
+      const xfltr = await crossfilter.select(
+        "X",
+        {
+          field: "var",
+          column: varIndex,
+          value: "TYMP",
+        },
+        {
+          mode: "range",
+          lo: 0,
+          hi: 50,
+          inclusive: true,
+        }
+      );
+
+      expect(xfltr).toBeDefined();
+      expect(xfltr.countSelected()).toEqual(501);
+
+      const df = await annoMatrix.fetch("X", {
+        field: "var",
+        column: varIndex,
+        value: "TYMP",
+      });
+      const values = df.icol(0).asArray();
+      const selected = xfltr.allSelectedMask();
+      values.every((val, idx) => !(val >= 0 && val <= 50) !== !selected[idx]);
+      expect(selected.reduce((acc, val) => (val ? acc + 1 : acc), 0)).toEqual(
+        xfltr.countSelected()
+      );
+    });
+
+    test("spatial column select", async () => {
+      fetch.once(
+        serverMocks.dataframeResponse(
+          ["umap_0", "umap_1"],
+          [Float32Array.from(embUmap[0]), Float32Array.from(embUmap[1])]
+        )
+      );
+      const xfltr = await crossfilter.select("emb", "umap", {
+        mode: "within-rect",
+        minX: 0,
+        minY: 0,
+        maxX: 0.5,
+        maxY: 0.5,
+      });
+      expect(xfltr.countSelected()).toEqual(16);
+    });
+
+    test("select on subset", async () => {
+      const mask = new Uint8Array(annoMatrix.nObs).fill(0);
+      for (let i = 0; i < mask.length; i += 2) {
+        mask[i] = true;
+      }
+      const annoMatrixSubset = isubsetMask(annoMatrix, mask);
+      expect(annoMatrixSubset.nObs).toEqual(Math.floor(annoMatrix.nObs / 2));
+
+      let xfltr = new AnnoMatrixObsCrossfilter(annoMatrixSubset);
+      expect(xfltr.countSelected()).toEqual(annoMatrixSubset.nObs);
+
+      fetch.once(serverMocks.dataframeResponse(["louvain"], [obsLouvain]));
+      xfltr = await xfltr.select("obs", "louvain", {
+        mode: "exact",
+        values: ["NK cells", "B cells"],
+      });
+
+      expect(xfltr).toBeDefined();
+      expect(xfltr.countSelected()).toEqual(240);
+
+      const df = await annoMatrixSubset.fetch("obs", "louvain");
+      const values = df.col("louvain").asArray();
+      const selected = xfltr.allSelectedMask();
+      values.every(
+        (val, idx) => !["NK cells", "B cells"].includes(val) !== !selected[idx]
+      );
+    });
+
+    test("select catches errors", async () => {
+      await expect(crossfilter.select("NADA", "foo")).rejects.toThrow(
+        "Unknown field name"
+      );
+
+      await expect(crossfilter.select("var", "foo")).rejects.toThrow(
+        "unable to obsSelect upon the var dimension"
+      );
+
+      fetch.mockRejectOnce(new Error("unknown column name"));
+      await expect(crossfilter.select("obs", "foo")).rejects.toThrow(
+        "unknown column name"
+      );
+    });
+  });
+
+  describe("mutate matrix", () => {
+    /*
+    test the matrix mutators via crossfilter proxy
+    */
+    async function helperAddTestCol(cf, colName, colSchema = null) {
+      expect(
+        cf.annoMatrix.getMatrixColumns("obs").includes(colName)
+      ).toBeFalsy();
+
+      if (colSchema === null) {
+        colSchema = {
+          name: colName,
+          type: "categorical",
+          categories: ["toasty"],
+        };
+      }
+      colSchema.name = colName;
+      const initValue = colSchema.categories[0];
+      const xfltr = cf.addObsColumn(colSchema, Array, initValue);
+      expect(
+        xfltr.annoMatrix.schema.annotations.obs.columns.filter(
+          (v) => v.name === colName
+        )
+      ).toHaveLength(1);
+      const df = await xfltr.annoMatrix.fetch("obs", colName);
+      expect(df.hasCol(colName)).toBeTruthy();
+      return xfltr;
+    }
+
+    test("addObsColumn", async () => {
+      expect(crossfilter.countSelected()).toBe(annoMatrix.nObs);
+      expect(
+        crossfilter.annoMatrix.getMatrixColumns("obs").includes("foo")
+      ).toBeFalsy();
+      const xfltr = crossfilter.addObsColumn(
+        { name: "foo", type: "categorical", categories: ["A"] },
+        Array,
+        "A"
+      );
+
+      // check schema updates correctly.
+      expect(xfltr.countSelected()).toBe(annoMatrix.nObs);
+      expect(
+        xfltr.annoMatrix.getMatrixColumns("obs").includes("foo")
+      ).toBeTruthy();
+      expect(xfltr.annoMatrix.schema.annotations.obsByName.foo).toMatchObject({
+        name: "foo",
+        type: "categorical",
+      });
+      expect(
+        xfltr.annoMatrix.schema.annotations.obs.columns.filter(
+          (v) => v.name === "foo"
+        )
+      ).toHaveLength(1);
+
+      // check data update.
+      const df = await xfltr.annoMatrix.fetch("obs", "foo");
+      expect(
+        df
+          .col("foo")
+          .asArray()
+          .every((v) => v === "A")
+      ).toBeTruthy();
+
+      // check that we catch dups
+      expect(() =>
+        xfltr.addObsColumn(
+          { name: "foo", type: "categorical" },
+          Array,
+          "toasty"
+        )
+      ).toThrow("column already exists");
+      expect(() =>
+        xfltr.addObsColumn(
+          { name: "louvain", type: "categorical" },
+          Array,
+          "toasty"
+        )
+      ).toThrow("column already exists");
+    });
+
+    test("dropObsColumn", async () => {
+      let xfltr;
+
+      /* check that we catch attempt to drop readonly dimension */
+      expect(() => crossfilter.dropObsColumn("louvain")).toThrow(
+        "Unknown or readonly obs column"
+      );
+      /* non-existent column */
+      expect(() => crossfilter.dropObsColumn("does-not-exist")).toThrow(
+        "Unknown or readonly obs column"
+      );
+
+      // add a column, then drop it.
+      xfltr = await helperAddTestCol(crossfilter, "foo");
+      xfltr = xfltr.dropObsColumn("foo");
+      expect(
+        xfltr.annoMatrix.schema.annotations.obs.columns.filter(
+          (v) => v.name === "foo"
+        )
+      ).toHaveLength(0);
+      expect(xfltr.annoMatrix.schema.annotations.obsByName.foo).toBeUndefined();
+      fetch.mockRejectOnce(new Error("unknown column name"));
+      await expect(xfltr.annoMatrix.fetch("obs", "foo")).rejects.toThrow(
+        "unknown column name"
+      );
+
+      // now same, but ensure we have built an index before doing the drop
+      xfltr = await helperAddTestCol(crossfilter, "bar");
+      xfltr = await xfltr.select("obs", "bar", {
+        mode: "exact",
+        values: "whatever",
+      });
+      xfltr = xfltr.dropObsColumn("bar");
+
+      fetch.mockRejectOnce(new Error("unknown column name"));
+      await expect(xfltr.select("obs", "bar", { mode: "all" })).rejects.toThrow(
+        "unknown column name"
+      );
+    });
+
+    test("renameObsColumn", async () => {
+      let xfltr;
+
+      /* catch attempts to rename non-existent or readonly columns */
+      expect(() =>
+        crossfilter.renameObsColumn("does-not-exist", "foo")
+      ).toThrow("Unknown or readonly obs column");
+      expect(() => crossfilter.renameObsColumn("louvain", "foo")).toThrow(
+        "Unknown or readonly obs column"
+      );
+
+      // add a column, then rename it.
+      xfltr = await helperAddTestCol(crossfilter, "foo");
+      xfltr = xfltr.renameObsColumn("foo", "bar");
+      expect(xfltr.annoMatrix.getColumnSchema("obs", "foo")).toBeUndefined();
+      expect(xfltr.annoMatrix.getColumnSchema("obs", "bar")).toMatchObject({
+        name: "bar",
+        type: "categorical",
+      });
+
+      fetch.mockRejectOnce(new Error("unknown column name"));
+      await expect(xfltr.annoMatrix.fetch("obs", "foo")).rejects.toThrow(
+        "unknown column name"
+      );
+      const df = await xfltr.annoMatrix.fetch("obs", "bar");
+      expect(df.hasCol("bar")).toBeTruthy();
+
+      // now same, but ensure we have built an index before doing the rename
+      xfltr = await helperAddTestCol(crossfilter, "bar");
+      xfltr = await xfltr.select("obs", "bar", {
+        mode: "exact",
+        values: "whatever",
+      });
+      xfltr = xfltr.renameObsColumn("bar", "xyz");
+
+      fetch.mockRejectOnce(new Error("unknown column name"));
+      await expect(xfltr.select("obs", "bar", { mode: "all" })).rejects.toThrow(
+        "unknown column name"
+      );
+      await expect(
+        xfltr.select("obs", "xyz", { mode: "none" })
+      ).resolves.toBeInstanceOf(AnnoMatrixObsCrossfilter);
+    });
+
+    test("addObsAnnoCategory", async () => {
+      let xfltr;
+
+      // catch unknown or readonly columns
+      expect(() => crossfilter.addObsAnnoCategory("louvain", "mumble")).toThrow(
+        "Unknown or readonly obs column"
+      );
+      expect(() =>
+        crossfilter.addObsAnnoCategory("undefined-name", "mumble")
+      ).toThrow("Unknown or readonly obs column");
+
+      // add a column and then add category to it
+      xfltr = await helperAddTestCol(crossfilter, "foo", {
+        name: "foo",
+        type: "categorical",
+        categories: ["unassigned"],
+      });
+      xfltr = xfltr.addObsAnnoCategory("foo", "a-new-label");
+      expect(xfltr.annoMatrix.getColumnSchema("obs", "foo")).toMatchObject({
+        name: "foo",
+        type: "categorical",
+        categories: expect.arrayContaining(["a-new-label", "unassigned"]),
+      });
+
+      // do it again, dup; should throw
+      expect(() => xfltr.addObsAnnoCategory("foo", "a-new-label")).toThrow(
+        "category already exists"
+      );
+
+      // now same, but ensure we have built an index before doing the operation
+      xfltr = await helperAddTestCol(crossfilter, "bar", {
+        name: "bar",
+        type: "categorical",
+        categories: ["unassigned"],
+      });
+      xfltr = await xfltr.select("obs", "bar", {
+        mode: "exact",
+        values: "something",
+      });
+      xfltr = xfltr.addObsAnnoCategory("bar", "a-new-label");
+      expect(xfltr.annoMatrix.getColumnSchema("obs", "bar")).toMatchObject({
+        name: "bar",
+        type: "categorical",
+        categories: expect.arrayContaining(["a-new-label", "unassigned"]),
+      });
+    });
+
+    test("removeObsAnnoCategory", async () => {
+      let xfltr;
+
+      // catch unknown or readonly categories
+      await expect(() =>
+        crossfilter.removeObsAnnoCategory("louvain", "mumble", "unassigned")
+      ).rejects.toThrow("Unknown or readonly obs column");
+      await expect(() =>
+        crossfilter.removeObsAnnoCategory("undefined-name", "mumble")
+      ).rejects.toThrow("Unknown or readonly obs column");
+
+      xfltr = await helperAddTestCol(crossfilter, "foo", {
+        name: "foo",
+        type: "categorical",
+        categories: ["unassigned", "red", "green", "blue"],
+      });
+      xfltr = await xfltr.select("obs", "foo", { mode: "all" });
+      expect(
+        (await xfltr.annoMatrix.fetch("obs", "foo"))
+          .col("foo")
+          .asArray()
+          .every((v) => v === "unassigned")
+      ).toBeTruthy();
+      expect(xfltr.annoMatrix.getColumnSchema("obs", "foo")).toMatchObject({
+        name: "foo",
+        type: "categorical",
+        categories: expect.arrayContaining([
+          "unassigned",
+          "red",
+          "green",
+          "blue",
+        ]),
+      });
+
+      // remove an unused category
+      const xfltr1 = await xfltr.removeObsAnnoCategory("foo", "red", "mumble");
+      expect(
+        (await xfltr1.annoMatrix.fetch("obs", "foo"))
+          .col("foo")
+          .asArray()
+          .every((v) => v === "unassigned")
+      ).toBeTruthy();
+      expect(xfltr1.annoMatrix.getColumnSchema("obs", "foo")).toMatchObject({
+        name: "foo",
+        type: "categorical",
+        categories: expect.arrayContaining([
+          "unassigned",
+          "green",
+          "blue",
+          "mumble",
+        ]),
+      });
+
+      // remove a used category
+      const xfltr2 = await xfltr.removeObsAnnoCategory(
+        "foo",
+        "unassigned",
+        "red"
+      );
+      expect(
+        (await xfltr2.annoMatrix.fetch("obs", "foo"))
+          .col("foo")
+          .asArray()
+          .every((v) => v === "red")
+      ).toBeTruthy();
+      expect(xfltr2.annoMatrix.getColumnSchema("obs", "foo")).toMatchObject({
+        name: "foo",
+        type: "categorical",
+        categories: expect.arrayContaining(["green", "blue", "red"]),
+      });
+    });
+
+    test("setObsColumnValues", async () => {
+      // catch unknown or readonly categories
+      await expect(() =>
+        crossfilter.setObsColumnValues("louvain", [0, 1], "unassigned")
+      ).rejects.toThrow("Unknown or readonly obs column");
+      await expect(() =>
+        crossfilter.setObsColumnValues("undefined-name", [0], "mumble")
+      ).rejects.toThrow("Unknown or readonly obs column");
+
+      let xfltr = await helperAddTestCol(crossfilter, "foo", {
+        name: "foo",
+        type: "categorical",
+        categories: ["unassigned", "red", "green", "blue"],
+      });
+      xfltr = await xfltr.select("obs", "foo", { mode: "all" });
+
+      // catch unknown row label
+      await expect(() =>
+        xfltr.setObsColumnValues("foo", [-1], "red")
+      ).rejects.toThrow("Unknown row label");
+
+      // set a few rows
+      expect(
+        (await xfltr.annoMatrix.fetch("obs", "foo"))
+          .col("foo")
+          .asArray()
+          .every((v) => v === "unassigned")
+      ).toBeTruthy();
+      const xfltr1 = await xfltr.setObsColumnValues("foo", [0, 10], "purple");
+      expect(
+        (await xfltr1.annoMatrix.fetch("obs", "foo"))
+          .col("foo")
+          .asArray()
+          .every(
+            (v, i) =>
+              v === "unassigned" || (v === "purple" && (i === 0 || i === 10))
+          )
+      ).toBeTruthy();
+      expect(xfltr1.annoMatrix.getColumnSchema("obs", "foo")).toMatchObject({
+        name: "foo",
+        type: "categorical",
+        categories: expect.arrayContaining([
+          "unassigned",
+          "red",
+          "green",
+          "blue",
+          "purple",
+        ]),
+      });
+
+      expect(xfltr1.countSelected()).toEqual(xfltr1.annoMatrix.nObs);
+      const xfltr2 = await xfltr1.select("obs", "foo", {
+        mode: "exact",
+        values: ["purple"],
+      });
+      expect(xfltr2.countSelected()).toEqual(2);
+      expect(xfltr2.allSelectedLabels()).toEqual(Int32Array.from([0, 10]));
+    });
+
+    test("resetObsColumnValues", async () => {
+      // catch unknown or readonly categories
+      await expect(() =>
+        crossfilter.resetObsColumnValues("louvain", "red", "blue")
+      ).rejects.toThrow("Unknown or readonly obs column");
+      await expect(() =>
+        crossfilter.resetObsColumnValues("undefined-name", "red", "blue")
+      ).rejects.toThrow("Unknown or readonly obs column");
+
+      let xfltr = await helperAddTestCol(crossfilter, "foo", {
+        name: "foo",
+        type: "categorical",
+        categories: ["unassigned", "red", "green", "blue"],
+      });
+      xfltr = await xfltr.select("obs", "foo", {
+        mode: "exact",
+        values: "red",
+      });
+
+      // catch unknown category name label
+      await expect(() =>
+        xfltr.resetObsColumnValues("foo", "unknown-label", "red")
+      ).rejects.toThrow("unknown category");
+
+      let xfltr1 = await xfltr.setObsColumnValues("foo", [0, 10], "purple");
+      xfltr1 = await xfltr1.select("obs", "foo", {
+        mode: "exact",
+        values: "purple",
+      });
+      expect(
+        (await xfltr1.annoMatrix.fetch("obs", "foo"))
+          .col("foo")
+          .asArray()
+          .filter((v) => v === "purple")
+      ).toHaveLength(2);
+
+      xfltr1 = await xfltr1.resetObsColumnValues("foo", "purple", "magenta");
+      expect(
+        (await xfltr1.annoMatrix.fetch("obs", "foo"))
+          .col("foo")
+          .asArray()
+          .filter((v) => v === "magenta")
+      ).toHaveLength(2);
+      expect(
+        (await xfltr1.annoMatrix.fetch("obs", "foo"))
+          .col("foo")
+          .asArray()
+          .filter((v) => v === "purple")
+      ).toHaveLength(0);
+      expect(xfltr1.annoMatrix.getColumnSchema("obs", "foo")).toMatchObject({
+        name: "foo",
+        type: "categorical",
+        categories: expect.arrayContaining([
+          "unassigned",
+          "red",
+          "green",
+          "blue",
+          "purple",
+          "magenta",
+        ]),
+      });
+    });
+  });
+
+  describe("edge cases", () => {
+    test("transition from empty annoMatrix", async () => {
+      // select before fetch needs to work
+      fetch.once(serverMocks.dataframeResponse(["louvain"], [obsLouvain]));
+      const xfltr = await crossfilter.select("obs", "louvain", {
+        mode: "exact",
+        values: "B cells",
+      });
+      expect(fetch.mock.calls).toHaveLength(1);
+      expect(xfltr.obsCrossfilter.hasDimension("obs/louvain")).toBeTruthy();
+      expect(xfltr.obsCrossfilter.all()).toBe(xfltr.annoMatrix._cache.obs);
+      expect(xfltr.countSelected()).toEqual(
+        obsLouvain.reduce(
+          (count, v) => (v === "B cells" ? count + 1 : count),
+          0
+        )
+      );
+    });
+  });
+});

client/__tests__/util/annoMatrix/serverMocks/index.js

@@ -0,0 +1,11 @@
+export const baseDataURL = "https://a.fake.url/api/v0.2";
+
+window.CELLXGENE = {
+  API: {
+    prefix: baseDataURL,
+    version: "v0.2/",
+  },
+};
+
+export { schema } from "./schema";
+export * from "./routes";

client/__tests__/util/annoMatrix/serverMocks/routes.js

@@ -0,0 +1,211 @@
+import { schema } from "./schema";
+import { Dataframe, KeyIndex } from "../../../../src/util/dataframe";
+import { encodeMatrixFBS } from "../../../../src/util/stateManager/matrix";
+
+const indexedSchema = {
+  obsByName: Object.fromEntries(
+    schema.schema.annotations.obs.columns.map((v) => [v.name, v]) ?? []
+  ),
+  varByName: Object.fromEntries(
+    schema.schema.annotations.var.columns.map((v) => [v.name, v]) ?? []
+  ),
+  embByName: Object.fromEntries(
+    schema.schema.layout.obs.map((v) => [v.name, v]) ?? []
+  ),
+};
+
+function makeMockColumn(s, length) {
+  const { type } = s;
+  switch (type) {
+    case "int32":
+      return new Int32Array(length).fill(Math.floor(99 * Math.random()));
+
+    case "string":
+      return new Array(length).fill("test");
+
+    case "float32":
+      return new Float32Array(length).fill(99 * Math.random());
+
+    case "boolean":
+      return new Array(length).fill(false);
+
+    case "categorical":
+      return new Array(length).fill(s.categories[0]);
+
+    default:
+      throw new Error("unkonwn type");
+  }
+}
+
+function getEncodedDataframe(colNames, length, colSchemas) {
+  const colIndex = new KeyIndex(colNames);
+  const columns = colSchemas.map((s) => makeMockColumn(s, length));
+  const df = new Dataframe([length, colNames.length], columns, null, colIndex);
+  const body = encodeMatrixFBS(df);
+  return body;
+}
+
+export function dataframeResponse(colNames, columns) {
+  const colIndex = new KeyIndex(colNames);
+  const df = new Dataframe(
+    [columns[0].length, colNames.length],
+    columns,
+    null,
+    colIndex
+  );
+  const body = encodeMatrixFBS(df);
+  const headers = new Headers({
+    "Content-Type": "application/octet-stream",
+  });
+  return () => Promise.resolve({ body, init: { status: 200, headers } });
+}
+
+function annotationObsResponse(request) {
+  const url = new URL(request.url);
+  const params = Array.from(url.searchParams.entries());
+  const names = params
+    .filter(([k]) => k === "annotation-name")
+    .map(([, v]) => v);
+  if (!names.every((n) => indexedSchema.obsByName[n])) {
+    return Promise.reject(new Error("bad obs annotation name in URL"));
+  }
+  const colSchemas = names.map((n) => indexedSchema.obsByName[n]);
+  const body = getEncodedDataframe(
+    names,
+    schema.schema.dataframe.nObs,
+    colSchemas
+  );
+
+  const headers = new Headers({
+    "Content-Type": "application/octet-stream",
+  });
+  return Promise.resolve({
+    body,
+    init: { status: 200, headers },
+  });
+}
+
+function annotationVarResponse(request) {
+  const url = new URL(request.url);
+  const params = Array.from(url.searchParams.entries());
+  const names = params
+    .filter(([k]) => k === "annotation-name")
+    .map(([, v]) => v);
+  if (!names.every((n) => indexedSchema.varByName[n])) {
+    return Promise.reject(new Error("bad var annotation name in URL"));
+  }
+  const colSchemas = names.map((n) => indexedSchema.varByName[n]);
+  const body = getEncodedDataframe(
+    names,
+    schema.schema.dataframe.nVar,
+    colSchemas
+  );
+
+  const headers = new Headers({
+    "Content-Type": "application/octet-stream",
+  });
+  return Promise.resolve({
+    body,
+    init: { status: 200, headers },
+  });
+}
+
+function layoutObsResponse(request) {
+  const url = new URL(request.url);
+  const params = Array.from(url.searchParams.entries());
+  const names = params.filter(([k]) => k === "layout-name").map(([, v]) => v);
+  if (!names.every((n) => indexedSchema.embByName[n])) {
+    return Promise.reject(new Error("bad layout name in URL"));
+  }
+  const dims = names.map((n) => indexedSchema.embByName[n].dims).flat();
+  const colSchemas = names
+    .map((n) => [indexedSchema.embByName[n], indexedSchema.embByName[n]])
+    .flat();
+  const body = getEncodedDataframe(
+    dims,
+    schema.schema.dataframe.nObs,
+    colSchemas
+  );
+
+  const headers = new Headers({
+    "Content-Type": "application/octet-stream",
+  });
+  return Promise.resolve({
+    body,
+    init: { status: 200, headers },
+  });
+}
+
+function dataVarResponse(request) {
+  const url = new URL(request.url);
+  const params = Array.from(url.searchParams.entries());
+
+  const colNames = params.map((v) => `${v[0]}/${v[1]}`);
+  const colSchemas = colNames.map(() => schema.schema.dataframe);
+  const body = getEncodedDataframe(
+    colNames,
+    schema.schema.dataframe.nObs,
+    colSchemas
+  );
+
+  const headers = new Headers({
+    "Content-Type": "application/octet-stream",
+  });
+  return Promise.resolve({
+    body,
+    init: { status: 200, headers },
+  });
+}
+
+export function responder(request) {
+  const url = new URL(request.url);
+  const { pathname } = url;
+  if (pathname.endsWith("/annotations/obs")) {
+    return annotationObsResponse(request);
+  }
+  if (pathname.endsWith("/annotations/var")) {
+    return annotationVarResponse(request);
+  }
+  if (pathname.endsWith("/layout/obs")) {
+    return layoutObsResponse(request);
+  }
+  if (pathname.endsWith("/data/var")) {
+    return dataVarResponse(request);
+  }
+  return Promise.reject(new Error("bad URL"));
+}
+
+export function withExpected(expectedURL, expectedParams) {
+  /*
+  Do some additional error checking
+  */
+  return (request) => {
+    // if URL is bogus, reject the promise
+    const url = new URL(request.url);
+    if (!url.pathname.endsWith(expectedURL)) {
+      return Promise.reject(new Error("Unexpected URL!"));
+    }
+    const params = Array.from(url.searchParams.entries()).sort(
+      (a, b) => a[0] < b[0]
+    );
+    expectedParams = expectedParams.slice().sort((a, b) => a[0] < b[0]);
+
+    if (
+      params.length !== expectedParams.length ||
+      !params.every(
+        (p, i) => p[0] === expectedParams[i][0] && p[1] === expectedParams[i][1]
+      )
+    ) {
+      return Promise.reject(new Error("unexpected name requested in URL"));
+    }
+
+    return responder(request);
+  };
+}
+
+export function annotationsObs(names) {
+  return withExpected(
+    "/annotations/obs",
+    names.map((name) => ["annotation-name", name])
+  );
+}

client/__tests__/util/annoMatrix/serverMocks/schema.js

@@ -0,0 +1,80 @@
+export const schema = {
+  schema: {
+    annotations: {
+      obs: {
+        columns: [
+          {
+            name: "name_0",
+            type: "string",
+            writable: false,
+          },
+          {
+            name: "n_genes",
+            type: "int32",
+            writable: false,
+          },
+          {
+            name: "percent_mito",
+            type: "float32",
+            writable: false,
+          },
+          {
+            name: "n_counts",
+            type: "float32",
+            writable: false,
+          },
+          {
+            name: "louvain",
+            type: "string",
+            writable: false,
+          },
+        ],
+        index: "name_0",
+      },
+      var: {
+        columns: [
+          {
+            name: "name_0",
+            type: "string",
+            writable: false,
+          },
+          {
+            name: "n_cells",
+            type: "int32",
+            writable: false,
+          },
+        ],
+        index: "name_0",
+      },
+    },
+    dataframe: {
+      nObs: 2638,
+      nVar: 1838,
+      type: "float32",
+    },
+    layout: {
+      obs: [
+        {
+          dims: ["draw_graph_fr_0", "draw_graph_fr_1"],
+          name: "draw_graph_fr",
+          type: "float32",
+        },
+        {
+          dims: ["pca_0", "pca_1"],
+          name: "pca",
+          type: "float32",
+        },
+        {
+          dims: ["tsne_0", "tsne_1"],
+          name: "tsne",
+          type: "float32",
+        },
+        {
+          dims: ["umap_0", "umap_1"],
+          name: "umap",
+          type: "float32",
+        },
+      ],
+    },
+  },
+};

client/__tests__/util/annoMatrix/whereCache.test.js

@@ -0,0 +1,184 @@
+import {
+  _whereCacheGet,
+  _whereCacheCreate,
+  _whereCacheMerge,
+} from "../../../src/annoMatrix/whereCache";
+
+const schema = {};
+
+describe("whereCache", () => {
+  test("whereCacheGet - missing cache values", () => {
+    expect(
+      _whereCacheGet({}, schema, "X", {
+        field: "var",
+        column: "foo",
+        value: "bar",
+      })
+    ).toEqual([undefined]);
+    expect(
+      _whereCacheGet({ X: {} }, schema, "X", {
+        field: "var",
+        column: "foo",
+        value: "bar",
+      })
+    ).toEqual([undefined]);
+    expect(
+      _whereCacheGet({ X: { var: new Map() } }, schema, "X", {
+        field: "var",
+        column: "foo",
+        value: "bar",
+      })
+    ).toEqual([undefined]);
+    expect(
+      _whereCacheGet(
+        { X: { var: new Map([["foo", new Map()]]) } },
+        schema,
+        "X",
+        {
+          field: "var",
+          column: "foo",
+          value: "bar",
+        }
+      )
+    ).toEqual([undefined]);
+  });
+
+  test("whereCacheGet - varied lookups", () => {
+    const whereCache = {
+      X: {
+        var: new Map([
+          [
+            "foo",
+            new Map([
+              ["bar", [0]],
+              ["baz", [1, 2]],
+            ]),
+          ],
+        ]),
+      },
+    };
+
+    expect(
+      _whereCacheGet(whereCache, schema, "X", {
+        field: "var",
+        column: "foo",
+        value: "bar",
+      })
+    ).toEqual([0]);
+    expect(
+      _whereCacheGet(whereCache, schema, "X", {
+        field: "var",
+        column: "foo",
+        value: "baz",
+      })
+    ).toEqual([1, 2]);
+    expect(_whereCacheGet(whereCache, schema, "Y", {})).toEqual([undefined]);
+    expect(
+      _whereCacheGet(whereCache, schema, "X", {
+        field: "whoknows",
+        column: "whatever",
+        value: "snork",
+      })
+    ).toEqual([undefined]);
+    expect(
+      _whereCacheGet(whereCache, schema, "X", {
+        field: "var",
+        column: "whatever",
+        value: "snork",
+      })
+    ).toEqual([undefined]);
+    expect(
+      _whereCacheGet(whereCache, schema, "X", {
+        field: "var",
+        column: "foo",
+        value: "snork",
+      })
+    ).toEqual([undefined]);
+  });
+
+  test("whereCacheCreate", () => {
+    const query = {
+      field: "queryField",
+      column: "queryColumn",
+      value: "queryValue",
+    };
+    const wc = _whereCacheCreate(
+      "field",
+      { field: "queryField", column: "queryColumn", value: "queryValue" },
+      [0, 1, 2]
+    );
+    expect(wc).toBeDefined();
+    expect(wc).toEqual(
+      expect.objectContaining({
+        field: {
+          queryField: expect.any(Map),
+        },
+      })
+    );
+    expect(wc.field.queryField.has("queryColumn")).toEqual(true);
+    expect(wc.field.queryField.get("queryColumn")).toBeInstanceOf(Map);
+    expect(wc.field.queryField.get("queryColumn").has("queryValue")).toEqual(
+      true
+    );
+    expect(_whereCacheGet(wc, schema, "field", query)).toEqual([0, 1, 2]);
+  });
+
+  test("whereCacheMerge", () => {
+    let wc;
+
+    // remember, will mutate dst
+    const src = _whereCacheCreate(
+      "field",
+      { field: "queryField", column: "queryColumn", value: "foo" },
+      ["foo"]
+    );
+
+    const dst1 = _whereCacheCreate(
+      "field",
+      { field: "queryField", column: "queryColumn", value: "bar" },
+      ["dst1"]
+    );
+    wc = _whereCacheMerge(dst1, src);
+    expect(
+      _whereCacheGet(wc, schema, "field", {
+        field: "queryField",
+        column: "queryColumn",
+        value: "foo",
+      })
+    ).toEqual(["foo"]);
+    expect(
+      _whereCacheGet(wc, schema, "field", {
+        field: "queryField",
+        column: "queryColumn",
+        value: "bar",
+      })
+    ).toEqual(["dst1"]);
+
+    const dst2 = _whereCacheCreate(
+      "field",
+      { field: "queryField", column: "queryColumn", value: "bar" },
+      ["dst2"]
+    );
+    wc = _whereCacheMerge(dst2, dst1, src);
+    expect(
+      _whereCacheGet(wc, schema, "field", {
+        field: "queryField",
+        column: "queryColumn",
+        value: "foo",
+      })
+    ).toEqual(["foo"]);
+    expect(
+      _whereCacheGet(wc, schema, "field", {
+        field: "queryField",
+        column: "queryColumn",
+        value: "bar",
+      })
+    ).toEqual(["dst1"]);
+
+    wc = _whereCacheMerge({}, src);
+    expect(wc).toEqual(src);
+
+    wc = _whereCacheMerge({ field: { queryField: new Map() } }, src);
+    expect(wc).toEqual(src);
+  });
+});

client/__tests__/util/centroid.test.js

@@ -1,55 +1,35 @@
 import _ from "lodash";
 
 import calcCentroid from "../../src/util/centroid";
-
 import quantile from "../../src/util/quantile";
-import * as Universe from "../../src/util/stateManager/universe";
-import * as World from "../../src/util/stateManager/world";
+import { matrixFBSToDataframe } from "../../src/util/stateManager/matrix";
 import * as REST from "./stateManager/sampleResponses";
-import { ControlsHelpers as CH } from "../../src/util/stateManager";
+import { indexEntireSchema } from "../../src/util/stateManager/schemaHelpers";
+import { _normalizeCategoricalSchema } from "../../src/annoMatrix/schema";
 
 describe("centroid", () => {
-  let world;
-  let categoricalSelection;
+  let schema;
+  let obsAnnotations;
+  let obsLayout;
+
   beforeAll(() => {
-    // Create world + universe
-    let universe = Universe.createUniverseFromResponse(
-      _.cloneDeep(REST.config),
-      _.cloneDeep(REST.schema)
-    );
+    schema = indexEntireSchema(_.cloneDeep(REST.schema.schema));
+    obsAnnotations = matrixFBSToDataframe(REST.annotationsObs);
+    obsLayout = matrixFBSToDataframe(REST.layoutObs);
 
-    universe = {
-      ...universe,
-      ...Universe.addObsAnnotations(
-        universe,
-        Universe.matrixFBSToDataframe(REST.annotationsObs)
-      ),
-      ...Universe.addVarAnnotations(
-        universe,
-        Universe.matrixFBSToDataframe(REST.annotationsVar)
-      ),
-      ...Universe.addObsLayout(
-        universe,
-        Universe.matrixFBSToDataframe(REST.layoutObs)
-      )
-    };
-    world = World.createWorldFromEntireUniverse(universe);
-
-    // Create categorical selection from world
-    categoricalSelection = CH.createCategoricalSelection(
-      world,
-      CH.selectableCategoryNames(world.schema, CH.maxCategoryItems(REST.config))
+    _normalizeCategoricalSchema(
+      schema.annotations.obsByName.field3,
+      obsAnnotations.col("field3")
     );
   });
 
   test("field4 (categorical obsAnnotation)", () => {
     const centroidResult = calcCentroid(
-      world.obsAnnotations,
-      world.obsLayout,
+      schema,
       "field4",
-      ["umap_0", "umap_1"],
-      categoricalSelection,
-      world.schema.annotations.obsByName
+      obsAnnotations,
+      { current: "umap", currentDimNames: ["umap_0", "umap_1"] },
+      obsLayout
     );
 
     // Check to see that a centroid has been calculated for every categorical value
@@ -60,23 +40,22 @@ describe("centroid", () => {
 
     // This expected result assumes that all cells belong in all categorical values inside of sample response
     const expectedResult = [
-      quantile([0.5], world.obsLayout.col("umap_0").asArray())[0],
-      quantile([0.5], world.obsLayout.col("umap_1").asArray())[0]
+      quantile([0.5], obsLayout.col("umap_0").asArray())[0],
+      quantile([0.5], obsLayout.col("umap_1").asArray())[0],
     ];
 
-    centroidResult.forEach(coordinate => {
+    centroidResult.forEach((coordinate) => {
       expect(coordinate).toEqual(expectedResult);
     });
   });
 
-  test("field3 (boolean obsAnnotation", () => {
+  test("field3 (boolean obsAnnotation)", () => {
     const centroidResult = calcCentroid(
-      world.obsAnnotations,
-      world.obsLayout,
+      schema,
       "field3",
-      ["umap_0", "umap_1"],
-      categoricalSelection,
-      world.schema.annotations.obsByName
+      obsAnnotations,
+      { current: "umap", currentDimNames: ["umap_0", "umap_1"] },
+      obsLayout
     );
 
     // Check to see that a centroid has been calculated for every categorical value
@@ -85,11 +64,11 @@ describe("centroid", () => {
 
     // This expected result assumes that all cells belong in all categorical values inside of sample response
     const expectedResult = [
-      quantile([0.5], world.obsLayout.col("umap_0").asArray())[0],
-      quantile([0.5], world.obsLayout.col("umap_1").asArray())[0]
+      quantile([0.5], obsLayout.col("umap_0").asArray())[0],
+      quantile([0.5], obsLayout.col("umap_1").asArray())[0],
     ];
 
-    centroidResult.forEach(coordinate => {
+    centroidResult.forEach((coordinate) => {
       expect(coordinate).toEqual(expectedResult);
     });
   });

client/__tests__/util/dataframe/histogram.test.js

@@ -14,7 +14,7 @@ describe("Dataframe column histogram", () => {
       new Map([
         ["n1", new Map([["c1", 1]])],
         ["n2", new Map([["c2", 1]])],
-        ["n3", new Map([["c3", 1]])]
+        ["n3", new Map([["c3", 1]])],
       ])
     );
     // memoized?
@@ -31,7 +31,11 @@ describe("Dataframe column histogram", () => {
 
     const h1 = df.col("value").histogram(3, [0, 2], df.col("name"));
     expect(h1).toMatchObject(
-      new Map([["n1", [1, 0, 0]], ["n2", [0, 1, 0]], ["n3", [0, 0, 1]]])
+      new Map([
+        ["n1", [1, 0, 0]],
+        ["n2", [0, 1, 0]],
+        ["n3", [0, 0, 1]],
+      ])
     );
     // memoized?
     expect(df.col("value").histogram(3, [0, 2], df.col("name"))).toMatchObject(
@@ -48,7 +52,13 @@ describe("Dataframe column histogram", () => {
     );
 
     const h1 = df.col("cat").histogram();
-    expect(h1).toMatchObject(new Map([["c1", 1], ["c2", 1], ["c3", 1]]));
+    expect(h1).toMatchObject(
+      new Map([
+        ["c1", 1],
+        ["c2", 1],
+        ["c3", 1],
+      ])
+    );
     // memoized?
     expect(df.col("value").histogram(3, [0, 2])).toMatchObject(h1);
   });
@@ -87,7 +97,7 @@ describe("Dataframe column histogram", () => {
       0,
       0,
       0,
-      2
+      2,
     ]);
   });
 });

client/__tests__/util/dataframe/summarize.test.js

@@ -13,7 +13,7 @@ describe("Dataframe column summary", () => {
         categorical: true,
         categories: [],
         categoryCounts: new Map(),
-        numCategories: 0
+        numCategories: 0,
       })
     );
   });
@@ -27,7 +27,7 @@ describe("Dataframe column summary", () => {
         [true],
         new Float32Array([39.3]),
         new Int32Array([99]),
-        [1]
+        [1],
       ],
       null,
       new Dataframe.KeyIndex([
@@ -36,7 +36,7 @@ describe("Dataframe column summary", () => {
         "nameBoolean",
         "nameFloat32",
         "nameInt32",
-        "nameCategorical"
+        "nameCategorical",
       ])
     );
 
@@ -45,7 +45,7 @@ describe("Dataframe column summary", () => {
         categorical: true,
         categories: ["n1"],
         categoryCounts: new Map([["n1", 1]]),
-        numCategories: 1
+        numCategories: 1,
       })
     );
     expect(df.icol(1).summarize()).toEqual(
@@ -53,7 +53,7 @@ describe("Dataframe column summary", () => {
         categorical: true,
         categories: ["hi"],
         categoryCounts: new Map([["hi", 1]]),
-        numCategories: 1
+        numCategories: 1,
       })
     );
     expect(df.icol(2).summarize()).toEqual(
@@ -61,7 +61,7 @@ describe("Dataframe column summary", () => {
         categorical: true,
         categories: [true],
         categoryCounts: new Map([[true, 1]]),
-        numCategories: 1
+        numCategories: 1,
       })
     );
     expect(df.icol(3).summarize()).toEqual(
@@ -71,7 +71,7 @@ describe("Dataframe column summary", () => {
         max: float32Conversion(39.3),
         nan: 0,
         ninf: 0,
-        pinf: 0
+        pinf: 0,
       })
     );
     expect(df.icol(4).summarize()).toEqual(
@@ -81,7 +81,7 @@ describe("Dataframe column summary", () => {
         max: 99,
         nan: 0,
         ninf: 0,
-        pinf: 0
+        pinf: 0,
       })
     );
     expect(df.icol(5).summarize()).toEqual(
@@ -89,7 +89,7 @@ describe("Dataframe column summary", () => {
         categorical: true,
         categories: [1],
         categoryCounts: new Map([[1, 1]]),
-        numCategories: 1
+        numCategories: 1,
       })
     );
   });
@@ -103,7 +103,7 @@ describe("Dataframe column summary", () => {
         [false, true, true],
         new Float32Array([39.3, 39.3, 0]),
         new Int32Array([99, 99, 99]),
-        [1, false, "0"]
+        [1, false, "0"],
       ],
       null,
       new Dataframe.KeyIndex([
@@ -112,7 +112,7 @@ describe("Dataframe column summary", () => {
         "nameBoolean",
         "nameFloat32",
         "nameInt32",
-        "nameCategorical"
+        "nameCategorical",
       ])
     );
 
@@ -120,24 +120,34 @@ describe("Dataframe column summary", () => {
       expect.objectContaining({
         categorical: true,
         categories: expect.arrayContaining(["n0", "n1", "n2"]),
-        categoryCounts: new Map([["n0", 1], ["n1", 1], ["n2", 1]]),
-        numCategories: 3
+        categoryCounts: new Map([
+          ["n0", 1],
+          ["n1", 1],
+          ["n2", 1],
+        ]),
+        numCategories: 3,
       })
     );
     expect(df.icol(1).summarize()).toEqual(
       expect.objectContaining({
         categorical: true,
         categories: expect.arrayContaining(["hi", "bye"]),
-        categoryCounts: new Map([["hi", 2], ["bye", 1]]),
-        numCategories: 2
+        categoryCounts: new Map([
+          ["hi", 2],
+          ["bye", 1],
+        ]),
+        numCategories: 2,
       })
     );
     expect(df.icol(2).summarize()).toEqual(
       expect.objectContaining({
         categorical: true,
         categories: expect.arrayContaining([true, false]),
-        categoryCounts: new Map([[true, 2], [false, 1]]),
-        numCategories: 2
+        categoryCounts: new Map([
+          [true, 2],
+          [false, 1],
+        ]),
+        numCategories: 2,
       })
     );
     expect(df.icol(3).summarize()).toEqual(
@@ -147,7 +157,7 @@ describe("Dataframe column summary", () => {
         max: float32Conversion(39.3),
         nan: 0,
         ninf: 0,
-        pinf: 0
+        pinf: 0,
       })
     );
     expect(df.icol(4).summarize()).toEqual(
@@ -157,15 +167,19 @@ describe("Dataframe column summary", () => {
         max: 99,
         nan: 0,
         ninf: 0,
-        pinf: 0
+        pinf: 0,
       })
     );
     expect(df.icol(5).summarize()).toEqual(
       expect.objectContaining({
         categorical: true,
         categories: expect.arrayContaining([1, false, "0"]),
-        categoryCounts: new Map([[1, 1], [false, 1], ["0", 1]]),
-        numCategories: 3
+        categoryCounts: new Map([
+          [1, 1],
+          [false, 1],
+          ["0", 1],
+        ]),
+        numCategories: 3,
       })
     );
   });
@@ -181,10 +195,10 @@ describe("Dataframe column summary", () => {
           39.3,
           Number.NEGATIVE_INFINITY,
           Number.NaN,
-          Number.POSITIVE_INFINITY
+          Number.POSITIVE_INFINITY,
         ]),
         new Int32Array([99, 99, 99, 99]),
-        [1, false, "0", "0"]
+        [1, false, "0", "0"],
       ],
       null,
       new Dataframe.KeyIndex([
@@ -193,7 +207,7 @@ describe("Dataframe column summary", () => {
         "nameBoolean",
         "nameFloat32",
         "nameInt32",
-        "nameCategorical"
+        "nameCategorical",
       ])
     );
 
@@ -201,24 +215,34 @@ describe("Dataframe column summary", () => {
       expect.objectContaining({
         categorical: true,
         categories: expect.arrayContaining(["n0", "n1", "n2"]),
-        categoryCounts: new Map([["n0", 1], ["n1", 1], ["n2", 2]]),
-        numCategories: 3
+        categoryCounts: new Map([
+          ["n0", 1],
+          ["n1", 1],
+          ["n2", 2],
+        ]),
+        numCategories: 3,
       })
     );
     expect(df.icol(1).summarize()).toEqual(
       expect.objectContaining({
         categorical: true,
         categories: expect.arrayContaining(["hi", "bye"]),
-        categoryCounts: new Map([["hi", 2], ["bye", 1]]),
-        numCategories: 2
+        categoryCounts: new Map([
+          ["hi", 2],
+          ["bye", 1],
+        ]),
+        numCategories: 2,
       })
     );
     expect(df.icol(2).summarize()).toEqual(
       expect.objectContaining({
         categorical: true,
         categories: expect.arrayContaining([true, false]),
-        categoryCounts: new Map([[true, 2], [false, 1]]),
-        numCategories: 2
+        categoryCounts: new Map([
+          [true, 2],
+          [false, 1],
+        ]),
+        numCategories: 2,
       })
     );
     expect(df.icol(3).summarize()).toEqual(
@@ -228,7 +252,7 @@ describe("Dataframe column summary", () => {
         max: float32Conversion(39.3),
         nan: 1,
         ninf: 1,
-        pinf: 1
+        pinf: 1,
       })
     );
     expect(df.icol(4).summarize()).toEqual(
@@ -238,15 +262,19 @@ describe("Dataframe column summary", () => {
         max: 99,
         nan: 0,
         ninf: 0,
-        pinf: 0
+        pinf: 0,
       })
     );
     expect(df.icol(5).summarize()).toEqual(
       expect.objectContaining({
         categorical: true,
         categories: expect.arrayContaining([1, false, "0"]),
-        categoryCounts: new Map([[1, 1], [false, 1], ["0", 1]]),
-        numCategories: 3
+        categoryCounts: new Map([
+          [1, 1],
+          [false, 1],
+          ["0", 1],
+        ]),
+        numCategories: 3,
       })
     );
   });

client/__tests__/util/nameCreators.test.js

@@ -6,7 +6,7 @@ import {
   obsAnnoDimensionName,
   diffexpDimensionName,
   userDefinedDimensionName,
-  makeContinuousDimensionName
+  makeContinuousDimensionName,
 } from "../../src/util/nameCreators";
 
 describe("nameCreators", () => {
@@ -14,25 +14,25 @@ describe("nameCreators", () => {
     layoutDimensionName,
     obsAnnoDimensionName,
     diffexpDimensionName,
-    userDefinedDimensionName
+    userDefinedDimensionName,
   ];
 
   test("check for namespace isolation", () => {
     const foo = "foo";
-    nameCreators.forEach(fn => expect(fn(foo)).not.toBe(foo));
+    nameCreators.forEach((fn) => expect(fn(foo)).not.toBe(foo));
 
     const bar = "bar";
-    nameCreators.forEach(fn => expect(fn(foo)).not.toBe(fn(bar)));
+    nameCreators.forEach((fn) => expect(fn(foo)).not.toBe(fn(bar)));
 
-    nameCreators.forEach(fn => {
-      const allOtherFn = nameCreators.filter(elmnt => elmnt !== fn);
-      allOtherFn.forEach(otherFn => expect(fn(foo)).not.toBe(otherFn(foo)));
+    nameCreators.forEach((fn) => {
+      const allOtherFn = nameCreators.filter((elmnt) => elmnt !== fn);
+      allOtherFn.forEach((otherFn) => expect(fn(foo)).not.toBe(otherFn(foo)));
     });
   });
 
   test("check for legal keys", () => {
     /* need namespace creators to return strings only */
-    nameCreators.forEach(fn => expect(fn("X")).toMatch(/X/));
+    nameCreators.forEach((fn) => expect(fn("X")).toMatch(/X/));
   });
 });
 

client/__tests__/util/promiseLimit.test.js

@@ -0,0 +1,96 @@
+import PromiseLimit from "../../src/util/promiseLimit";
+import { range } from "../../src/util/range";
+
+const delay = (t) => new Promise((resolve) => setTimeout(resolve, t));
+
+describe("PromiseLimit", () => {
+  test("simple evaluation, concurrency 1", async () => {
+    const plimit = new PromiseLimit(1);
+    const result = await Promise.all([
+      plimit.add(() => Promise.resolve(1)),
+      plimit.add(() => Promise.resolve(2)),
+      plimit.add(() => Promise.resolve(3)),
+      plimit.add(() => Promise.resolve(4)),
+    ]);
+    expect(result).toEqual([1, 2, 3, 4]);
+  });
+
+  test("simple evaluation, concurrency > 1", async () => {
+    const plimit = new PromiseLimit(100);
+    const result = await Promise.all([
+      plimit.add(() => Promise.resolve(1)),
+      plimit.add(() => Promise.resolve(2)),
+      plimit.add(() => Promise.resolve(3)),
+      plimit.add(() => Promise.resolve(4)),
+    ]);
+    expect(result).toEqual([1, 2, 3, 4]);
+  });
+
+  test("eval in order of insertion", async () => {
+    const plimit = new PromiseLimit(100);
+
+    const result = await Promise.all([
+      plimit.add(() => Promise.resolve(1)),
+      plimit.add(() => Promise.resolve(2)),
+      plimit.add(() => Promise.resolve(3)),
+      plimit.add(() => Promise.resolve(4)),
+    ]);
+
+    expect(result).toEqual([1, 2, 3, 4]);
+  });
+
+  test("obeys concurrency limit", async () => {
+    const plimit = new PromiseLimit(2);
+    let running = 0;
+    let maxRunning = 0;
+
+    const callback = async () => {
+      running += 1;
+      maxRunning = running > maxRunning ? running : maxRunning;
+      await delay(100);
+      running -= 1;
+    };
+
+    await Promise.all(range(10).map((i) => plimit.add(() => callback(i))));
+
+    expect(maxRunning).toEqual(2);
+  });
+
+  test("rejection", async () => {
+    const plimit = new PromiseLimit(2);
+    const result = await Promise.all([
+      plimit.add(() => Promise.resolve("OK")),
+      // eslint-disable-next-line prefer-promise-reject-errors -- unit test
+      plimit.add(() => Promise.reject("not OK")).catch((e) => e),
+      plimit.add(() => Promise.resolve("OK")),
+      plimit
+        .add(() => {
+          throw new Error("not OK");
+        })
+        .catch((e) => e.message),
+    ]);
+    expect(result).toEqual(["OK", "not OK", "OK", "not OK"]);
+  });
+
+  test("priority queue", async () => {
+    const plimit = new PromiseLimit(1);
+
+    let finishOrder = 0;
+    const callback = () => async () => {
+      await delay(100);
+      const result = finishOrder;
+      finishOrder += 1;
+      return result;
+    };
+
+    const result = await Promise.all([
+      plimit.add(callback()),
+      plimit.priorityAdd(4, callback()),
+      plimit.priorityAdd(0, callback()),
+      plimit.priorityAdd(1, callback()),
+      plimit.priorityAdd(-1, callback()),
+    ]);
+
+    expect(result).toEqual([0, 4, 2, 3, 1]);
+  });
+});

client/__tests__/util/quantile.test.js

@@ -1,29 +1,29 @@
 import quantile from "../../src/util/quantile";
 
 describe("quantile", () => {
-	test("single q", () => {
-		const arr = new Float32Array([9, 3, 5, 6, 0]);
-		expect(quantile([1.0], arr)).toMatchObject([9]);
-		expect(quantile([0.9], arr)).toMatchObject([9]);
-		expect(quantile([0.8], arr)).toMatchObject([9]);
-		expect(quantile([0.7], arr)).toMatchObject([6]);
-		expect(quantile([0.6], arr)).toMatchObject([6]);
-		expect(quantile([0.5], arr)).toMatchObject([5]);
-		expect(quantile([0.4], arr)).toMatchObject([5]);
-		expect(quantile([0.3], arr)).toMatchObject([3]);
-		expect(quantile([0.2], arr)).toMatchObject([3]);
-		expect(quantile([0.1], arr)).toMatchObject([0]);
-		expect(quantile([0], arr)).toMatchObject([0]);
-	});
+  test("single q", () => {
+    const arr = new Float32Array([9, 3, 5, 6, 0]);
+    expect(quantile([1.0], arr)).toMatchObject([9]);
+    expect(quantile([0.9], arr)).toMatchObject([9]);
+    expect(quantile([0.8], arr)).toMatchObject([9]);
+    expect(quantile([0.7], arr)).toMatchObject([6]);
+    expect(quantile([0.6], arr)).toMatchObject([6]);
+    expect(quantile([0.5], arr)).toMatchObject([5]);
+    expect(quantile([0.4], arr)).toMatchObject([5]);
+    expect(quantile([0.3], arr)).toMatchObject([3]);
+    expect(quantile([0.2], arr)).toMatchObject([3]);
+    expect(quantile([0.1], arr)).toMatchObject([0]);
+    expect(quantile([0], arr)).toMatchObject([0]);
+  });
 
-	test("multi q", () => {
-		const arr = new Float32Array([9, 3, 5, 6, 0]);
-		expect(quantile([0, 0.25, 0.5, 0.75, 1.0], arr)).toMatchObject([
-			0,
-			3,
-			5,
-			6,
-			9
-		]);
-	});
+  test("multi q", () => {
+    const arr = new Float32Array([9, 3, 5, 6, 0]);
+    expect(quantile([0, 0.25, 0.5, 0.75, 1.0], arr)).toMatchObject([
+      0,
+      3,
+      5,
+      6,
+      9,
+    ]);
+  });
 });

client/__tests__/util/range.test.js

@@ -1,50 +1,48 @@
 import { range, rangeFill, linspace } from "../../src/util/range";
 
 describe("range", () => {
-	test("no defaults", () => {
-		expect(range(0, 3, 1)).toMatchObject([0, 1, 2]);
-	});
+  test("no defaults", () => {
+    expect(range(0, 3, 1)).toMatchObject([0, 1, 2]);
+  });
 
-	test("range(stop)", () => {
-		expect(range(3)).toMatchObject([0, 1, 2]);
-		expect(range(0)).toMatchObject([]);
-		expect(range(1)).toMatchObject([0]);
-	});
+  test("range(stop)", () => {
+    expect(range(3)).toMatchObject([0, 1, 2]);
+    expect(range(0)).toMatchObject([]);
+    expect(range(1)).toMatchObject([0]);
+  });
 
-	test("range(start,stop)", () => {
-		expect(range(0, 0)).toMatchObject([]);
-		expect(range(0, 2)).toMatchObject([0, 1]);
-		expect(range(4, 8)).toMatchObject([4, 5, 6, 7]);
-	});
+  test("range(start,stop)", () => {
+    expect(range(0, 0)).toMatchObject([]);
+    expect(range(0, 2)).toMatchObject([0, 1]);
+    expect(range(4, 8)).toMatchObject([4, 5, 6, 7]);
+  });
 
-	test("range(start, stop, step", () => {
-		expect(range(4, 0, -1)).toMatchObject([4, 3, 2, 1]);
-		expect(range(0, 4, 2)).toMatchObject([0, 2]);
-	});
+  test("range(start, stop, step", () => {
+    expect(range(4, 0, -1)).toMatchObject([4, 3, 2, 1]);
+    expect(range(0, 4, 2)).toMatchObject([0, 2]);
+  });
 });
 
 describe("rangefill", () => {
-	test("rangeFill(arr)", () => {
-		expect(rangeFill(new Int32Array(3))).toMatchObject(
-			new Int32Array([0, 1, 2])
-		);
-	});
-	test("rangeFill(arr, start)", () => {
-		expect(rangeFill(new Int32Array(2), 1)).toMatchObject(
-			new Int32Array([1, 2])
-		);
-	});
-	test("rangeFill(arr, start, step)", () => {
-		expect(rangeFill(new Int32Array(3), 2, -1)).toMatchObject(
-			new Int32Array([2, 1, 0])
-		);
-	});
+  test("rangeFill(arr)", () => {
+    expect(rangeFill(new Int32Array(3))).toMatchObject(
+      new Int32Array([0, 1, 2])
+    );
+  });
+  test("rangeFill(arr, start)", () => {
+    expect(rangeFill(new Int32Array(2), 1)).toMatchObject(
+      new Int32Array([1, 2])
+    );
+  });
+  test("rangeFill(arr, start, step)", () => {
+    expect(rangeFill(new Int32Array(3), 2, -1)).toMatchObject(
+      new Int32Array([2, 1, 0])
+    );
+  });
 });
 
 describe("linspace", () => {
-	test("linspace(arr, start, step)", () => {
-		expect(linspace(0.0, 2.0, 5)).toMatchObject(
-		    [0.0, 0.5, 1.0, 1.5, 2.0]
-		);
-	});
+  test("linspace(arr, start, step)", () => {
+    expect(linspace(0.0, 2.0, 5)).toMatchObject([0.0, 0.5, 1.0, 1.5, 2.0]);
+  });
 });

client/__tests__/util/stateManager/colorHelpers.test.js

@@ -0,0 +1,198 @@
+/* eslint-disable no-bitwise -- unsigned right shift better than Math.round */
+
+/*
+test color helpers
+*/
+import {
+  createColorTable,
+  loadUserColorConfig,
+} from "../../../src/util/stateManager/colorHelpers";
+import * as Dataframe from "../../../src/util/dataframe";
+
+describe("categorical color helpers", () => {
+  /*
+  Primary test constraint for categorical colors is that they are ordered/identified
+  by schema order, NOT by value.  Ie,
+
+    scale(schemaIndex) should match rgb[obsOffset]
+  */
+
+  const schema = indexSchema({
+    annotations: {
+      obs: {
+        columns: [
+          {
+            name: "name_0",
+            type: "string",
+            writable: false,
+          },
+          {
+            name: "continuousColumn",
+            type: "float32",
+            writable: false,
+          },
+          {
+            categories: [
+              "CD4 T cells",
+              "CD14+ Monocytes",
+              "B cells",
+              "CD8 T cells",
+              "NK cells",
+              "FCGR3A+ Monocytes",
+              "Dendritic cells",
+              "Megakaryocytes",
+            ],
+            name: "categoricalColumn",
+            type: "categorical",
+            writable: false,
+          },
+        ],
+        index: "name_0",
+      },
+      var: {
+        columns: [
+          {
+            name: "name_0",
+            type: "string",
+            writable: false,
+          },
+        ],
+        index: "name_0",
+      },
+    },
+    dataframe: {
+      nObs: 2638,
+      nVar: 1838,
+      type: "float32",
+    },
+    layout: {},
+  });
+
+  const catColCategories = schema.annotations.obs.columns[2].categories;
+  const obsDataframe = new Dataframe.Dataframe(
+    [schema.dataframe.nObs, 2],
+    [
+      new Float32Array(schema.dataframe.nObs).map(() => Math.random()),
+      new Array(schema.dataframe.nObs)
+        .fill("")
+        .map(
+          () =>
+            catColCategories[(Math.random() * catColCategories.length) >>> 0]
+        ),
+    ],
+    null,
+    new Dataframe.KeyIndex(["continuousColumn", "categoricalColumn"])
+  );
+
+  test("default category order", () => {
+    const ct = createColorTable(
+      "color by categorical metadata",
+      "categoricalColumn",
+      obsDataframe,
+      schema
+    );
+    expect(ct).toBeDefined();
+    const data = obsDataframe.col("categoricalColumn").asArray();
+    const cats = schema.annotations.obsByName.categoricalColumn.categories;
+    for (let i = 0; i < schema.dataframe.nObs; i += 1) {
+      expect(makeScale(ct.rgb[i])).toEqual(ct.scale(cats.indexOf(data[i])));
+    }
+  });
+
+  test("shuffle category order", () => {
+    const schemaClone = indexSchema(JSON.parse(JSON.stringify(schema)));
+    shuffle(schemaClone.annotations.obsByName.categoricalColumn.categories);
+    const ct = createColorTable(
+      "color by categorical metadata",
+      "categoricalColumn",
+      obsDataframe,
+      schemaClone
+    );
+    expect(ct).toBeDefined();
+    const data = obsDataframe.col("categoricalColumn").asArray();
+    const cats = schemaClone.annotations.obsByName.categoricalColumn.categories;
+    for (let i = 0; i < schemaClone.dataframe.nObs; i += 1) {
+      expect(makeScale(ct.rgb[i])).toEqual(ct.scale(cats.indexOf(data[i])));
+    }
+  });
+
+  test("user defined color order", () => {
+    const cats = schema.annotations.obsByName.categoricalColumn.categories;
+    const shuffleCats = shuffle(
+      Array.from(schema.annotations.obsByName.categoricalColumn.categories)
+    );
+    const userDefinedColorTable = {
+      categoricalColumn: shuffleCats.reduce((acc, label) => {
+        acc[label] = randRGBColor();
+        return acc;
+      }, {}),
+    };
+
+    const userColors = loadUserColorConfig(userDefinedColorTable);
+    expect(userColors).toBeDefined();
+
+    const ct = createColorTable(
+      "color by categorical metadata",
+      "categoricalColumn",
+      obsDataframe,
+      schema,
+      userColors
+    );
+    expect(ct).toBeDefined();
+    const data = obsDataframe.col("categoricalColumn").asArray();
+    for (let i = 0; i < schema.dataframe.nObs; i += 1) {
+      expect(makeScale(ct.rgb[i])).toEqual(
+        ct.scale(cats.indexOf(data[i])).toString()
+      );
+    }
+  });
+});
+
+/*
+TODO:
+1. mix up category order in schema to make sure it works with varied order
+2. user defined colors
+*/
+
+function indexSchema(schema) {
+  schema.annotations.obsByName = Object.fromEntries(
+    schema.annotations?.obs?.columns?.map((v) => [v.name, v]) ?? []
+  );
+  schema.annotations.varByName = Object.fromEntries(
+    schema.annotations?.var?.columns?.map((v) => [v.name, v]) ?? []
+  );
+  schema.layout.obsByName = Object.fromEntries(
+    schema.layout?.obs?.map((v) => [v.name, v]) ?? []
+  );
+  schema.layout.varByName = Object.fromEntries(
+    schema.layout?.var?.map((v) => [v.name, v]) ?? []
+  );
+
+  return schema;
+}
+
+function makeScale(rgb) {
+  // make a scale string from a rgb float triple
+  return `rgb(${(rgb[0] * 255) >>> 0}, ${(rgb[1] * 255) >>> 0}, ${
+    (rgb[2] * 256) >>> 0
+  })`;
+}
+
+function shuffle(array) {
+  for (let i = array.length - 1; i > 0; i -= 1) {
+    const j = (Math.random() * (i + 1)) >>> 0;
+    [array[i], array[j]] = [array[j], array[i]];
+  }
+  return array;
+}
+
+function randHexColor() {
+  const hex = ((Math.random() * 255) >>> 0).toString(16);
+  return `0${hex}`.slice(-2);
+}
+
+function randRGBColor() {
+  return `#${randHexColor()}${randHexColor()}${randHexColor()}`;
+}
+
+/* eslint-enable no-bitwise -- unsigned right shift better than Math.round */

client/__tests__/util/stateManager/controlsHelpers.test.js

@@ -5,20 +5,34 @@ import { subsetAndResetGeneLists } from "../../../src/util/stateManager/controls
 import * as globals from "../../../src/globals";
 
 describe("controls helpers", () => {
-
   test("subsetAndResetGeneLists", () => {
-    const geneList = [...Array(150).keys()].map(() =>
-      Math.random().toString(36).substring(2, 6) // random string of 4 characters
-    );
+    const geneList = [];
+    const genRandGene = () => Math.random().toString(36).substring(2, 6);
+
+    // build a unique set of genes
+    for (let i = 0; i < 150; i += 1) {
+      let randGene = genRandGene();
+      while (geneList.includes(randGene)) randGene = genRandGene();
+      geneList.push(randGene);
+    }
+    // insert duplicates
+    geneList[0] = "dupl";
+    geneList[20] = "dupl";
     const state = {
       userDefinedGenes: geneList.slice(0, 20),
       diffexpGenes: geneList.slice(20),
     };
-    const [newUserDefinedGenes, newDiffExpGenes] = subsetAndResetGeneLists(state);
+    const [newUserDefinedGenes, newDiffExpGenes] = subsetAndResetGeneLists(
+      state
+    );
+    const expectedNewUserDefinedGenes = [
+      ...geneList.slice(0, 20),
+      ...geneList.slice(21),
+    ].slice(0, globals.maxGenes);
     expect(globals.maxUserDefinedGenes).toBeLessThan(globals.maxGenes);
     expect(geneList.length).toBeGreaterThan(globals.maxGenes);
     expect(newUserDefinedGenes).toHaveLength(globals.maxGenes);
-    expect(newUserDefinedGenes).toStrictEqual(geneList.slice(0, globals.maxGenes));
+    expect(newUserDefinedGenes).toStrictEqual(expectedNewUserDefinedGenes);
     expect(newDiffExpGenes).toStrictEqual([]);
   });
 });

client/__tests__/util/stateManager/fbs.test.js

@@ -3,32 +3,32 @@ test FBS encode/decode API
 */
 import { Dataframe, KeyIndex } from "../../../src/util/dataframe";
 import {
-	decodeMatrixFBS,
-	encodeMatrixFBS
+  decodeMatrixFBS,
+  encodeMatrixFBS,
 } from "../../../src/util/stateManager/matrix";
 
 describe("encode/decode", () => {
-	test("round trip", () => {
-		const columns = [
-			["red", "green", "blue"],
-			new Int32Array(3).fill(0),
-			new Uint32Array(3).fill(1),
-			new Float32Array(3).fill(2)
-		];
+  test("round trip", () => {
+    const columns = [
+      ["red", "green", "blue"],
+      new Int32Array(3).fill(0),
+      new Uint32Array(3).fill(1),
+      new Float32Array(3).fill(2),
+    ];
 
-		const dfNoColIdx = new Dataframe([3, 4], columns);
-		const dfA = decodeMatrixFBS(encodeMatrixFBS(dfNoColIdx));
-		expect([dfA.nRows, dfA.nCols]).toEqual(dfNoColIdx.dims);
-		expect(dfA.colIdx).toBeNull();
-		expect(dfA.rowIdx).toBeNull();
-		expect(dfA.columns).toEqual(columns);
+    const dfNoColIdx = new Dataframe([3, 4], columns);
+    const dfA = decodeMatrixFBS(encodeMatrixFBS(dfNoColIdx));
+    expect([dfA.nRows, dfA.nCols]).toEqual(dfNoColIdx.dims);
+    expect(dfA.colIdx).toBeNull();
+    expect(dfA.rowIdx).toBeNull();
+    expect(dfA.columns).toEqual(columns);
 
-		const colIndex = new KeyIndex(["a", "b", "c", "d"]);
-		const dfWithColIdx = new Dataframe([3, 4], columns, null, colIndex);
-		const dfB = decodeMatrixFBS(encodeMatrixFBS(dfWithColIdx));
-		expect([dfB.nRows, dfB.nCols]).toEqual(dfWithColIdx.dims);
-		expect(dfB.colIdx).toEqual(colIndex.keys());
-		expect(dfB.rowIdx).toBeNull();
-		expect(dfB.columns).toEqual(columns);
-	});
+    const colIndex = new KeyIndex(["a", "b", "c", "d"]);
+    const dfWithColIdx = new Dataframe([3, 4], columns, null, colIndex);
+    const dfB = decodeMatrixFBS(encodeMatrixFBS(dfWithColIdx));
+    expect([dfB.nRows, dfB.nCols]).toEqual(dfWithColIdx.dims);
+    expect(dfB.colIdx).toEqual(colIndex.labels());
+    expect(dfB.rowIdx).toBeNull();
+    expect(dfB.columns).toEqual(columns);
+  });
 });

client/__tests__/util/stateManager/sampleResponses.js

@@ -1,4 +1,3 @@
-/* eslint no-bitwise: "off" */
 import _ from "lodash";
 import { flatbuffers } from "flatbuffers";
 import { NetEncoding } from "../../../src/util/stateManager/matrix_generated";
@@ -17,13 +16,13 @@ const aConfigResponse = {
       { method: "POST", path: "/cluster/", available: false },
       { method: "POST", path: "/layout/", available: false },
       { method: "POST", path: "/diffexp/", available: false },
-      { method: "POST", path: "/saveLocal/", available: false }
+      { method: "POST", path: "/saveLocal/", available: false },
     ],
     displayNames: {
       engine: "the little engine that could",
-      dataset: "all your zeros are mine"
-    }
-  }
+      dataset: "all your zeros are mine",
+    },
+  },
 };
 
 const aSchemaResponse = {
@@ -31,7 +30,7 @@ const aSchemaResponse = {
     dataframe: {
       nObs,
       nVar,
-      type: "float32"
+      type: "float32",
     },
     annotations: {
       obs: {
@@ -44,9 +43,9 @@ const aSchemaResponse = {
           {
             name: "field4",
             type: "categorical",
-            categories: field4Categories
-          }
-        ]
+            categories: field4Categories,
+          },
+        ],
       },
       var: {
         index: "name",
@@ -58,46 +57,48 @@ const aSchemaResponse = {
           {
             name: "fieldD",
             type: "categorical",
-            categories: fieldDCategories
-          }
-        ]
-      }
+            categories: fieldDCategories,
+          },
+        ],
+      },
     },
     layout: {
       obs: [{ name: "umap", type: "float32", dims: ["umap_0", "umap_1"] }],
-      var: []
-    }
-  }
+      var: [],
+    },
+  },
 };
 
 const anAnnotationsObsJSONResponse = {
   names: ["name", "field1", "field2", "field3", "field4"],
   data: _()
     .range(nObs)
-    .map(idx => [
+    .map((idx) => [
       idx,
       `obs${idx}`,
       2 * idx,
       idx + 0.0133,
+      // eslint-disable-next-line no-bitwise -- idx & 1 to check for odd numbers
       !!(idx & 1),
-      field4Categories[idx % field4Categories.length]
+      field4Categories[idx % field4Categories.length],
     ])
-    .value()
+    .value(),
 };
 
 const anAnnotationsVarJSONResponse = {
   names: ["fieldA", "fieldB", "fieldC", "fieldD", "name"],
   data: _()
     .range(nVar)
-    .map(idx => [
+    .map((idx) => [
       idx,
       10 * idx,
       idx + 2.90143,
+      // eslint-disable-next-line no-bitwise -- idx & 1 to check for odd numbers
       !!(idx & 1),
       fieldDCategories[idx % fieldDCategories.length],
-      `var${idx}`
+      `var${idx}`,
     ])
-    .value()
+    .value(),
 };
 
 function encodeTypedArray(builder, uType, uData) {
@@ -112,7 +113,7 @@ function encodeTypedArray(builder, uType, uData) {
 function encodeMatrix(columns, colIndex = undefined) {
   /*
   IMPORTANT: this is not a general purpose encoder.  in particular,
-  it doesn't correctly handle all column index types, nor does it 
+  it doesn't correctly handle all column index types, nor does it
   handle all column typedarray types.
 
   encodeMatrixFBS in matrix.py is more general.  This is used only
@@ -120,7 +121,7 @@ function encodeMatrix(columns, colIndex = undefined) {
   */
   const utf8Encoder = new TextEncoder("utf-8");
   const builder = new flatbuffers.Builder(1024);
-  const cols = _.map(columns, carr => {
+  const cols = _.map(columns, (carr) => {
     let uType;
     let tarr;
     if (_.every(carr, _.isNumber)) {
@@ -178,7 +179,7 @@ const anAnnotationsVarFBSResponse = (() => {
 const aLayoutFBSResponse = (() => {
   const coords = [
     new Float32Array(nObs).fill(Math.random()),
-    new Float32Array(nObs).fill(Math.random())
+    new Float32Array(nObs).fill(Math.random()),
   ];
   return encodeMatrix(coords, ["umap_0", "umap_1"]);
 })();
@@ -187,8 +188,8 @@ const aDataObsResponse = {
   var: [2, 4, 29],
   obs: _()
     .range(nObs)
-    .map(idx => [idx, Math.random(), Math.random(), Math.random()])
-    .value()
+    .map((idx) => [idx, Math.random(), Math.random(), Math.random()])
+    .value(),
 };
 
 export {
@@ -197,5 +198,5 @@ export {
   anAnnotationsVarFBSResponse as annotationsVar,
   anAnnotationsObsFBSResponse as annotationsObs,
   aSchemaResponse as schema,
-  aConfigResponse as config
+  aConfigResponse as config,
 };

client/__tests__/util/stateManager/universe.test.js

@@ -1,92 +0,0 @@
-import * as Universe from "../../../src/util/stateManager/universe";
-import * as Dataframe from "../../../src/util/dataframe";
-import * as REST from "./sampleResponses";
-
-describe("createUniverseFromResponse", () => {
-  /*
-  test createUniverseFromResponse - this function converts
-  a set of REST 0.2 responses into a "new" Universe.
-
-  createUniverseFromResponse(
-    configResponse,
-    schemaResponse,
-    annotationsObsResponse,
-    annotationsVarResponse,
-    layoutObsResponse
-  ) --> Universe
-
-  where:
-  configResponse: GET /.../config
-  schemaResponse: GET /.../schema
-  annotationsObsResponse: GET /.../annotations/obs
-  annotationsVarResponse: GET /.../annotations/var
-  layoutObsResponse: GET /.../layout/obs
-
-  See spec in docs/REST_API.md.
-  */
-
-  test("create from test data", () => {
-    /*
-    create a universe from sample data nad validate its shape & contents
-    */
-    const { nObs, nVar } = REST.schema.schema.dataframe;
-    let universe = Universe.createUniverseFromResponse(
-      REST.config,
-      REST.schema
-    );
-    expect(universe).toBeDefined();
-    expect(universe).toMatchObject(
-      expect.objectContaining({
-        nObs,
-        nVar,
-        schema: REST.schema.schema,
-        obsAnnotations: expect.any(Dataframe.Dataframe),
-        varAnnotations: expect.any(Dataframe.Dataframe),
-        obsLayout: expect.any(Dataframe.Dataframe),
-        varData: expect.any(Dataframe.Dataframe)
-      })
-    );
-
-    universe = {
-      ...universe,
-      ...Universe.addObsAnnotations(
-        universe,
-        Universe.matrixFBSToDataframe(REST.annotationsObs)
-      ),
-      ...Universe.addVarAnnotations(
-        universe,
-        Universe.matrixFBSToDataframe(REST.annotationsVar)
-      ),
-      ...Universe.addObsLayout(
-        universe,
-        Universe.matrixFBSToDataframe(REST.layoutObs)
-      )
-    };
-
-    expect(universe).toMatchObject(
-      expect.objectContaining({
-        nObs,
-        nVar,
-        schema: REST.schema.schema,
-        obsAnnotations: expect.any(Dataframe.Dataframe),
-        varAnnotations: expect.any(Dataframe.Dataframe),
-        obsLayout: expect.any(Dataframe.Dataframe),
-        varData: expect.any(Dataframe.Dataframe)
-      })
-    );
-
-    expect(universe.obsAnnotations.dims).toEqual([
-      nObs,
-      REST.schema.schema.annotations.obs.columns.length
-    ]);
-    expect(universe.obsLayout.dims).toEqual([nObs, 2]);
-    expect(universe.obsLayout.colIndex.keys()).toEqual(
-      universe.schema.layout.obs[0].dims
-    );
-    expect(universe.varAnnotations.dims).toEqual([
-      nVar,
-      REST.schema.schema.annotations.var.columns.length
-    ]);
-    expect(universe.varData.isEmpty()).toBeTruthy();
-  });
-});

client/__tests__/util/stateManager/world.test.js

@@ -1,204 +0,0 @@
-import _ from "lodash";
-import * as Universe from "../../../src/util/stateManager/universe";
-import * as World from "../../../src/util/stateManager/world";
-import * as Dataframe from "../../../src/util/dataframe";
-import Crossfilter from "../../../src/util/typedCrossfilter";
-import { DimTypes } from "../../../src/util/typedCrossfilter/crossfilter";
-import * as REST from "./sampleResponses";
-import {
-  obsAnnoDimensionName,
-  layoutDimensionName
-} from "../../../src/util/nameCreators";
-
-/*
-Helper - creates universe, world, corssfilter and dimensionMap from
-the default REST test response.
-*/
-const defaultBigBang = () => {
-  /* create unverse, world, crossfilter and dimensionMap */
-  /* create universe */
-  let universe = Universe.createUniverseFromResponse(
-    _.cloneDeep(REST.config),
-    _.cloneDeep(REST.schema)
-  );
-
-  universe = {
-    ...universe,
-    ...Universe.addObsAnnotations(
-      universe,
-      Universe.matrixFBSToDataframe(REST.annotationsObs)
-    ),
-    ...Universe.addVarAnnotations(
-      universe,
-      Universe.matrixFBSToDataframe(REST.annotationsVar)
-    ),
-    ...Universe.addObsLayout(
-      universe,
-      Universe.matrixFBSToDataframe(REST.layoutObs)
-    )
-  };
-
-  /* create world */
-  const world = World.createWorldFromEntireUniverse(universe);
-  /* create crossfilter */
-  const crossfilter = World.createObsDimensions(
-    new Crossfilter(world.obsAnnotations),
-    world,
-    REST.schema.schema.layout.obs[0].dims
-  );
-
-  return {
-    universe,
-    world,
-    crossfilter
-  };
-};
-
-describe("createWorldFromEntireUniverse", () => {
-  test("create from REST sample", () => {
-    const universe = Universe.createUniverseFromResponse(
-      _.cloneDeep(REST.config),
-      _.cloneDeep(REST.schema),
-      Universe.matrixFBSToDataframe(_.cloneDeep(REST.annotationsObs)),
-      Universe.matrixFBSToDataframe(_.cloneDeep(REST.annotationsVar)),
-      Universe.matrixFBSToDataframe(_.cloneDeep(REST.layoutObs))
-    );
-    expect(universe).toBeDefined();
-
-    const world = World.createWorldFromEntireUniverse(universe);
-    expect(world).toBeDefined();
-
-    expect(world).toMatchObject(
-      expect.objectContaining({
-        nObs: universe.nObs,
-        nVar: universe.nVar,
-        schema: universe.schema,
-        obsAnnotations: expect.any(Dataframe.Dataframe),
-        varAnnotations: expect.any(Dataframe.Dataframe),
-        obsLayout: expect.any(Dataframe.Dataframe),
-        varData: expect.any(Dataframe.Dataframe),
-        clipQuantiles: { min: 0, max: 1 },
-        unclipped: {
-          obsAnnotations: expect.any(Dataframe.Dataframe),
-          varData: expect.any(Dataframe.Dataframe)
-        }
-      })
-    );
-  });
-});
-
-describe("createWorldFromCurrentSelection", () => {
-  test("create from REST sample", () => {
-    const {
-      universe,
-      world: originalWorld,
-      crossfilter: originalCrossfilter
-    } = defaultBigBang();
-
-    /* mock a selection */
-    const crossfilter = originalCrossfilter
-      .select(obsAnnoDimensionName("field1"), { mode: "range", lo: 0, hi: 5 })
-      .select(obsAnnoDimensionName("field3"), {
-        mode: "exact",
-        values: [false]
-      });
-
-    /* create the world from the selection */
-    const world = World.createWorldBySelection(
-      universe,
-      originalWorld,
-      crossfilter
-    );
-    expect(world).toBeDefined();
-    expect(world.nObs).toEqual(crossfilter.countSelected());
-
-    /*
-    calculate expected values and match against result
-    */
-
-    /* matchFilter must match the dimension filters above */
-    const matchFilter = (df, row) => {
-      const field1 = df.at(row, "field1");
-      const field3 = df.at(row, "field3");
-      return field1 >= 0 && field1 < 5 && !field3;
-    };
-    const matchingIndices = _()
-      .range(universe.nObs)
-      .filter(idx => matchFilter(universe.obsAnnotations, idx))
-      .value();
-
-    expect(world).toMatchObject(
-      expect.objectContaining({
-        nObs: matchingIndices.length,
-        nVar: universe.nVar,
-        schema: universe.schema,
-        clipQuantiles: { min: 0, max: 1 },
-        obsAnnotations: expect.any(Dataframe.Dataframe),
-        varAnnotations: expect.any(Dataframe.Dataframe),
-        obsLayout: expect.any(Dataframe.Dataframe),
-        varData: expect.any(Dataframe.Dataframe),
-        unclipped: {
-          obsAnnotations: expect.any(Dataframe.Dataframe),
-          varData: expect.any(Dataframe.Dataframe)
-        }
-      })
-    );
-
-    expect(world.obsAnnotations.rowIndex.keys()).toEqual(
-      new Int32Array(matchingIndices)
-    );
-    expect(world.obsAnnotations.colIndex.keys()).toEqual(
-      universe.obsAnnotations.colIndex.keys()
-    );
-    expect(world.obsLayout.rowIndex.keys()).toEqual(
-      new Int32Array(matchingIndices)
-    );
-    expect(world.obsLayout.colIndex.keys()).toEqual(
-      world.schema.layout.obs[0].dims
-    );
-  });
-});
-
-describe("createObsDimensionMap", () => {
-  test("when universe eq world", () => {
-    /*
-    check for:
-    - creates a dimension for all obsAnnotations, PLUS X/Y layout
-    - check that dimension typing is sane
-    */
-
-    const { crossfilter } = defaultBigBang();
-    const annotationNames = _.map(
-      REST.schema.schema.annotations.obs.columns,
-      c => c.name
-    );
-    const obsIndexColName = REST.schema.schema.annotations.obs.index;
-    const schemaByObsName = _.keyBy(
-      REST.schema.schema.annotations.obs.columns,
-      "name"
-    );
-    expect(crossfilter).toBeDefined();
-    annotationNames.forEach(name => {
-      const dim = crossfilter.dimensions[obsAnnoDimensionName(name)];
-      if (name === obsIndexColName) {
-        expect(dim).toBeUndefined();
-      } else {
-        const { type } = schemaByObsName[name];
-        if (type === "string" || type === "boolean" || type === "categorical") {
-          expect(dim.dim).toBeInstanceOf(DimTypes.enum);
-        } else {
-          expect(dim.dim).toBeInstanceOf(DimTypes.scalar);
-        }
-      }
-    });
-    expect(
-      crossfilter.dimensions[layoutDimensionName("XY")].dim
-    ).toBeInstanceOf(DimTypes.spatial);
-  });
-});
-
-describe("worldEqUniverse", () => {
-  const { universe, world } = defaultBigBang();
-  const result = World.worldEqUniverse(world, universe);
-  expect(result).toBe(true);
-});

client/__tests__/util/typedCrossfilter/bitArray.test.js

@@ -187,7 +187,7 @@ describe("fillBySelection", () => {
 describe("wide bitarray", () => {
   test.each([9, 30, 31, 32, 33, 54, 63, 64, 65, 127, 128, 129])(
     "more than %d dimensions",
-    ndim => {
+    (ndim) => {
       /* ensure we move across the uint boundary correctly */
 
       const ba = new BitArray(defaultTestLength);

client/__tests__/util/typedCrossfilter/crossfilter.test.js

@@ -11,7 +11,7 @@ const someData = [
     type: "tab",
     productIDs: ["001"],
     coords: [0, 0],
-    nonFinite: 0.0
+    nonFinite: 0.0,
   },
   {
     date: "2011-11-14T16:20:19Z",
@@ -21,7 +21,7 @@ const someData = [
     type: "tab",
     productIDs: ["001", "005"],
     coords: [0.4, 0.4],
-    nonFinite: Number.NaN
+    nonFinite: Number.NaN,
   },
   {
     date: "2011-11-14T16:28:54Z",
@@ -31,7 +31,7 @@ const someData = [
     type: "visa",
     productIDs: ["004", "005"],
     coords: [0.3, 0.1],
-    nonFinite: Number.POSITIVE_INFINITY
+    nonFinite: Number.POSITIVE_INFINITY,
   },
   {
     date: "2011-11-14T16:30:43Z",
@@ -41,7 +41,7 @@ const someData = [
     type: "tab",
     productIDs: ["001", "002"],
     coords: [0.392, 0.1],
-    nonFinite: Number.NEGATIVE_INFINITY
+    nonFinite: Number.NEGATIVE_INFINITY,
   },
   {
     date: "2011-11-14T16:48:46Z",
@@ -51,7 +51,7 @@ const someData = [
     type: "tab",
     productIDs: ["005"],
     coords: [0.7, 0.0482],
-    nonFinite: 1.0
+    nonFinite: 1.0,
   },
   {
     date: "2011-11-14T16:53:41Z",
@@ -61,7 +61,7 @@ const someData = [
     type: "tab",
     productIDs: ["001", "004", "005"],
     coords: [0.9999, 1.0],
-    nonFinite: Number.NaN
+    nonFinite: Number.NaN,
   },
   {
     date: "2011-11-14T16:54:06Z",
@@ -71,7 +71,7 @@ const someData = [
     type: "cash",
     productIDs: ["001", "002", "003", "004", "005"],
     coords: [0.384, 0.6938],
-    nonFinite: 99.0
+    nonFinite: 99.0,
   },
   {
     date: "2011-11-14T16:58:03Z",
@@ -81,7 +81,7 @@ const someData = [
     type: "tab",
     productIDs: ["001"],
     coords: [0.4822, 0.482],
-    nonFinite: Number.NaN
+    nonFinite: Number.NaN,
   },
   {
     date: "2011-11-14T17:07:21Z",
@@ -91,7 +91,7 @@ const someData = [
     type: "tab",
     productIDs: ["004", "005"],
     coords: [0.2234, 0],
-    nonFinite: Number.NaN
+    nonFinite: Number.NaN,
   },
   {
     date: "2011-11-14T17:22:59Z",
@@ -101,7 +101,7 @@ const someData = [
     type: "tab",
     productIDs: ["001", "002", "004", "005"],
     coords: [0.382, 0.38485],
-    nonFinite: -1
+    nonFinite: -1,
   },
   {
     date: "2011-11-14T17:25:45Z",
@@ -111,7 +111,7 @@ const someData = [
     type: "cash",
     productIDs: ["002"],
     coords: [0.998, 0.8472],
-    nonFinite: 0.0
+    nonFinite: 0.0,
   },
   {
     date: "2011-11-14T17:29:52Z",
@@ -121,8 +121,8 @@ const someData = [
     type: "visa",
     productIDs: ["004"],
     coords: [0.8273, 0.3384],
-    nonFinite: 0.0
-  }
+    nonFinite: 0.0,
+  },
 ];
 
 let payments = null;
@@ -248,16 +248,26 @@ describe("ImmutableTypedCrossfilter", () => {
     test("none", () => {
       expect(p.select("quantity", { mode: "none" }).countSelected()).toEqual(0);
     });
-    test.each([[[]], [[2]], [[2, 1]], [[9, 82]], [[0, 1]]])("exact: %p", v =>
+    test.each([[[]], [[2]], [[2, 1]], [[9, 82]], [[0, 1]]])("exact: %p", (v) =>
       expect(
         p.select("quantity", { mode: "exact", values: v }).countSelected()
-      ).toEqual(_.filter(someData, d => v.includes(d.quantity)).length)
+      ).toEqual(_.filter(someData, (d) => v.includes(d.quantity)).length)
     );
-    test.each([[0, 1], [1, 2], [0, 99], [99, 100000]])("range %p", (lo, hi) =>
+    test("single value exact", () => {
+      expect(
+        p.select("quantity", { mode: "exact", values: 2 }).countSelected()
+      ).toEqual(_.filter(someData, (d) => d.quantity === 2).length);
+    });
+    test.each([
+      [0, 1],
+      [1, 2],
+      [0, 99],
+      [99, 100000],
+    ])("range %p", (lo, hi) =>
       expect(
         p.select("quantity", { mode: "range", lo, hi }).countSelected()
       ).toEqual(
-        _.filter(someData, d => d.quantity >= lo && d.quantity < hi).length
+        _.filter(someData, (d) => d.quantity >= lo && d.quantity < hi).length
       )
     );
     test("bad mode", () => {
@@ -284,12 +294,17 @@ describe("ImmutableTypedCrossfilter", () => {
       [["tab"]],
       [["visa"]],
       [["visa", "tab"]],
-      [["cash", "tab", "visa"]]
-    ])("exact: %p", v =>
+      [["cash", "tab", "visa"]],
+    ])("exact: %p", (v) =>
       expect(
         p.select("type", { mode: "exact", values: v }).countSelected()
-      ).toEqual(_.filter(someData, d => v.includes(d.type)).length)
+      ).toEqual(_.filter(someData, (d) => v.includes(d.type)).length)
     );
+    test("single value exact", () => {
+      expect(
+        p.select("type", { mode: "exact", values: "tab" }).countSelected()
+      ).toEqual(_.filter(someData, (d) => d.type === "tab").length);
+    });
     test("range", () => {
       expect(() => p.select("type", { mode: "range", lo: 0, hi: 9 })).toThrow(
         Error
@@ -303,8 +318,8 @@ describe("ImmutableTypedCrossfilter", () => {
   describe("spatial dimension", () => {
     let p;
     beforeEach(() => {
-      const X = someData.map(r => r.coords[0]);
-      const Y = someData.map(r => r.coords[1]);
+      const X = someData.map((r) => r.coords[0]);
+      const Y = someData.map((r) => r.coords[1]);
       p = payments.addDimension("coords", "spatial", X, Y);
     });
 
@@ -316,34 +331,76 @@ describe("ImmutableTypedCrossfilter", () => {
     test("none", () => {
       expect(p.select("coords", { mode: "none" }).countSelected()).toEqual(0);
     });
-    test.each([[0, 0, 1, 1], [0, 0, 0.5, 0.5], [0.5, 0.5, 1, 1]])(
-      "within-rect %d %d %d %d",
-      (minX, minY, maxX, maxY) => {
-        expect(
-          p
-            .select("coords", { mode: "within-rect", minX, minY, maxX, maxY })
-            .allSelected()
-        ).toEqual(
-          _.filter(someData, d => {
-            const [x, y] = d.coords;
-            return minX <= x && x < maxX && minY <= y && y < maxY;
-          })
-        );
-      }
-    );
+    test.each([
+      [0, 0, 1, 1],
+      [0, 0, 0.5, 0.5],
+      [0.5, 0.5, 1, 1],
+    ])("within-rect %d %d %d %d", (minX, minY, maxX, maxY) => {
+      expect(
+        p
+          .select("coords", { mode: "within-rect", minX, minY, maxX, maxY })
+          .allSelected()
+      ).toEqual(
+        _.filter(someData, (d) => {
+          const [x, y] = d.coords;
+          return minX <= x && x < maxX && minY <= y && y < maxY;
+        })
+      );
+    });
 
     test.each([
       [
-          [[0, 0], [0, 1], [1, 1], [1, 0]],
-          [true, true, true, true, true, false, true, true, true, true, true, true]
+        [
+          [0, 0],
+          [0, 1],
+          [1, 1],
+          [1, 0],
+        ],
+        [
+          true,
+          true,
+          true,
+          true,
+          true,
+          false,
+          true,
+          true,
+          true,
+          true,
+          true,
+          true,
+        ],
       ],
       [
-          [[0, 0], [0, 0.5], [0.5, 0.5], [0.5, 0]],
-          [true, true, true, true, false, false, false, true, true, true, false, false]
-      ]
+        [
+          [0, 0],
+          [0, 0.5],
+          [0.5, 0.5],
+          [0.5, 0],
+        ],
+        [
+          true,
+          true,
+          true,
+          true,
+          false,
+          false,
+          false,
+          true,
+          true,
+          true,
+          false,
+          false,
+        ],
+      ],
     ])("within-polygon %p", (polygon, expected) => {
-      expect(p.select("coords", { mode: "within-polygon", polygon }).allSelected())
-          .toEqual(_.zip(someData, expected).filter(x => x[1]).map(x => x[0]));
+      expect(
+        p.select("coords", { mode: "within-polygon", polygon }).allSelected()
+      ).toEqual(
+        _.zip(someData, expected)
+          .filter((x) => x[1])
+          .map((x) => x[0])
+      );
     });
   });
 
@@ -381,7 +438,7 @@ describe("ImmutableTypedCrossfilter", () => {
         p
           .select("nonFinite", {
             mode: "exact",
-            values: [Number.POSITIVE_INFINITY]
+            values: [Number.POSITIVE_INFINITY],
           })
           .countSelected()
       ).toEqual(1);
@@ -389,7 +446,7 @@ describe("ImmutableTypedCrossfilter", () => {
         p
           .select("nonFinite", {
             mode: "exact",
-            values: [Number.NEGATIVE_INFINITY]
+            values: [Number.NEGATIVE_INFINITY],
           })
           .countSelected()
       ).toEqual(1);
@@ -402,7 +459,7 @@ describe("ImmutableTypedCrossfilter", () => {
         p
           .select("nonFinite", {
             mode: "exact",
-            values: [Number.POSITIVE_INFINITY, 0, 1, 99]
+            values: [Number.POSITIVE_INFINITY, 0, 1, 99],
           })
           .countSelected()
       ).toEqual(6);
@@ -414,7 +471,7 @@ describe("ImmutableTypedCrossfilter", () => {
           .select("nonFinite", {
             mode: "range",
             lo: 0,
-            hi: Number.POSITIVE_INFINITY
+            hi: Number.POSITIVE_INFINITY,
           })
           .countSelected()
       ).toEqual(5);
@@ -423,7 +480,7 @@ describe("ImmutableTypedCrossfilter", () => {
           .select("nonFinite", {
             mode: "range",
             lo: 0,
-            hi: Number.NaN
+            hi: Number.NaN,
           })
           .countSelected()
       ).toEqual(6);
@@ -432,7 +489,7 @@ describe("ImmutableTypedCrossfilter", () => {
           .select("nonFinite", {
             mode: "range",
             lo: Number.NEGATIVE_INFINITY,
-            hi: Number.POSITIVE_INFINITY
+            hi: Number.POSITIVE_INFINITY,
           })
           .countSelected()
       ).toEqual(7);

client/__tests__/util/typedCrossfilter/positiveInterval.test.js

@@ -10,15 +10,30 @@ describe("canonicalize", () => {
 
   test("simple, already correct", () => {
     expect(PositiveIntervals.canonicalize([[0, 1]])).toEqual([[0, 1]]);
-    expect(PositiveIntervals.canonicalize([[0, 1], [2, 3]])).toEqual([
+    expect(
+      PositiveIntervals.canonicalize([
+        [0, 1],
+        [2, 3],
+      ])
+    ).toEqual([
       [0, 1],
-      [2, 3]
+      [2, 3],
     ]);
   });
 
   test("non-canonical, need to be canonicalized", () => {
-    expect(PositiveIntervals.canonicalize([[0, 1], [1, 2]])).toEqual([[0, 2]]);
-    expect(PositiveIntervals.canonicalize([[1, 2], [2, 3]])).toEqual([[1, 3]]);
+    expect(
+      PositiveIntervals.canonicalize([
+        [0, 1],
+        [1, 2],
+      ])
+    ).toEqual([[0, 2]]);
+    expect(
+      PositiveIntervals.canonicalize([
+        [1, 2],
+        [2, 3],
+      ])
+    ).toEqual([[1, 3]]);
   });
 });
 
@@ -26,14 +41,30 @@ describe("union", () => {
   test("empty range", () => {
     expect(PositiveIntervals.union([], [])).toEqual([]);
     expect(PositiveIntervals.union([], [[1, 2]])).toEqual([[1, 2]]);
-    expect(PositiveIntervals.union([], [[1, 2], [3, 4]])).toEqual([
+    expect(
+      PositiveIntervals.union(
+        [],
+        [
+          [1, 2],
+          [3, 4],
+        ]
+      )
+    ).toEqual([
       [1, 2],
-      [3, 4]
+      [3, 4],
     ]);
     expect(PositiveIntervals.union([[3, 4]], [])).toEqual([[3, 4]]);
-    expect(PositiveIntervals.union([[1, 2], [3, 4]], [])).toEqual([
+    expect(
+      PositiveIntervals.union(
+        [
+          [1, 2],
+          [3, 4],
+        ],
+        []
+      )
+    ).toEqual([
       [1, 2],
-      [3, 4]
+      [3, 4],
     ]);
     expect(PositiveIntervals.union([[3, 3]], [])).toEqual([[3, 3]]);
     expect(PositiveIntervals.union([], [[3, 3]])).toEqual([[3, 3]]);
@@ -44,17 +75,37 @@ describe("union", () => {
     expect(PositiveIntervals.union([[2, 3]], [[1, 2]])).toEqual([[1, 3]]);
     expect(PositiveIntervals.union([[1, 2]], [[3, 4]])).toEqual([
       [1, 2],
-      [3, 4]
+      [3, 4],
     ]);
     expect(
-      PositiveIntervals.union([[1, 2], [3, 4]], [[6, 7], [19, 40]])
-    ).toEqual([[1, 2], [3, 4], [6, 7], [19, 40]]);
-    expect(PositiveIntervals.union([[1, 4]], [[1, 1], [3, 4]])).toEqual([
-      [1, 4]
+      PositiveIntervals.union(
+        [
+          [1, 2],
+          [3, 4],
+        ],
+        [
+          [6, 7],
+          [19, 40],
+        ]
+      )
+    ).toEqual([
+      [1, 2],
+      [3, 4],
+      [6, 7],
+      [19, 40],
     ]);
+    expect(
+      PositiveIntervals.union(
+        [[1, 4]],
+        [
+          [1, 1],
+          [3, 4],
+        ]
+      )
+    ).toEqual([[1, 4]]);
     expect(PositiveIntervals.union([[3, 3]], [[4, 4]])).toEqual([
       [3, 3],
-      [4, 4]
+      [4, 4],
     ]);
   });
 });
@@ -70,34 +121,43 @@ describe("intersection", () => {
     expect(PositiveIntervals.intersection([[1, 2]], [[2, 3]])).toEqual([]);
     expect(PositiveIntervals.intersection([[2, 3]], [[1, 2]])).toEqual([]);
     expect(PositiveIntervals.intersection([[1, 10]], [[1, 10]])).toEqual([
-      [1, 10]
+      [1, 10],
     ]);
     expect(PositiveIntervals.intersection([[1, 10]], [[2, 8]])).toEqual([
-      [2, 8]
+      [2, 8],
     ]);
     expect(PositiveIntervals.intersection([[2, 8]], [[1, 10]])).toEqual([
-      [2, 8]
+      [2, 8],
     ]);
     expect(PositiveIntervals.intersection([[1, 10]], [[2, 12]])).toEqual([
-      [2, 10]
+      [2, 10],
     ]);
     expect(PositiveIntervals.intersection([[2, 12]], [[1, 10]])).toEqual([
-      [2, 10]
+      [2, 10],
     ]);
     expect(PositiveIntervals.intersection([[1, 10]], [[1, 8]])).toEqual([
-      [1, 8]
+      [1, 8],
     ]);
     expect(PositiveIntervals.intersection([[1, 8]], [[1, 10]])).toEqual([
-      [1, 8]
+      [1, 8],
     ]);
-    expect(PositiveIntervals.intersection([[1, 10]], [[1, 2], [6, 9]])).toEqual(
-      [[1, 2], [6, 9]]
-    );
-    expect(PositiveIntervals.intersection([[0, 2638]], [[1363, 2638]])).toEqual(
-      [[1363, 2638]]
-    );
+    expect(
+      PositiveIntervals.intersection(
+        [[1, 10]],
+        [
+          [1, 2],
+          [6, 9],
+        ]
+      )
+    ).toEqual([
+      [1, 2],
+      [6, 9],
+    ]);
+    expect(
+      PositiveIntervals.intersection([[0, 2638]], [[1363, 2638]])
+    ).toEqual([[1363, 2638]]);
     expect(PositiveIntervals.intersection([[1, 2]], [[1, 2]])).toEqual([
-      [1, 2]
+      [1, 2],
     ]);
   });
 });
@@ -110,32 +170,66 @@ describe("difference", () => {
   });
 
   test("simple", () => {
-    expect(PositiveIntervals.difference([[1, 2], [3, 4]], [])).toEqual([
+    expect(
+      PositiveIntervals.difference(
+        [
+          [1, 2],
+          [3, 4],
+        ],
+        []
+      )
+    ).toEqual([
       [1, 2],
-      [3, 4]
-    ]);
-    expect(PositiveIntervals.difference([[1, 2], [3, 10]], [[5, 10]])).toEqual([
-      [1, 2],
-      [3, 5]
-    ]);
-    expect(PositiveIntervals.difference([[1, 2], [3, 10]], [[0, 5]])).toEqual([
-      [5, 10]
+      [3, 4],
     ]);
     expect(
-      PositiveIntervals.difference([[0, 2638]], [[0, 1363], [2055, 2638]])
+      PositiveIntervals.difference(
+        [
+          [1, 2],
+          [3, 10],
+        ],
+        [[5, 10]]
+      )
+    ).toEqual([
+      [1, 2],
+      [3, 5],
+    ]);
+    expect(
+      PositiveIntervals.difference(
+        [
+          [1, 2],
+          [3, 10],
+        ],
+        [[0, 5]]
+      )
+    ).toEqual([[5, 10]]);
+    expect(
+      PositiveIntervals.difference(
+        [[0, 2638]],
+        [
+          [0, 1363],
+          [2055, 2638],
+        ]
+      )
     ).toEqual([[1363, 2055]]);
     expect(
-      PositiveIntervals.difference([[0, 1363], [2055, 2638]], [[0, 2638]])
+      PositiveIntervals.difference(
+        [
+          [0, 1363],
+          [2055, 2638],
+        ],
+        [[0, 2638]]
+      )
     ).toEqual([]);
     expect(PositiveIntervals.difference([[0, 10]], [[0, 1]])).toEqual([
-      [1, 10]
+      [1, 10],
     ]);
     expect(PositiveIntervals.difference([[0, 10]], [[1, 2]])).toEqual([
       [0, 1],
-      [2, 10]
+      [2, 10],
     ]);
     expect(PositiveIntervals.difference([[0, 10]], [[9, 10]])).toEqual([
-      [0, 9]
+      [0, 9],
     ]);
   });
 });

client/__tests__/util/typedCrossfilter/sort.test.js

@@ -2,9 +2,6 @@ import {
   sortArray,
   sortIndex,
   lowerBound,
-  upperBound,
-  lowerBoundIndirect,
-  upperBoundIndirect
 } from "../../../src/util/typedCrossfilter/sort";
 
 /*
@@ -40,7 +37,7 @@ describe("sortArray", () => {
       ["a", "b", "0", "1"],
       [0, "a", true, null, undefined, 3.1415],
       fillRand(new Array(1000)),
-      ["a", NaN, null, pInf]
+      ["a", NaN, null, pInf],
     ].map((val, idx) =>
       test(`JS vals ${idx}`, () => {
         expect(sortArray(val)).toMatchObject(val.sort());
@@ -49,7 +46,7 @@ describe("sortArray", () => {
   });
 
   describe("finite numbers", () => {
-    [Array, Float32Array, Uint32Array, Int32Array, Float64Array].map(Type =>
+    [Array, Float32Array, Uint32Array, Int32Array, Float64Array].map((Type) =>
       test(Type.name, () => {
         expect(sortArray(Type.from([6, 5, 4, 3, 2, 1, 0]))).toMatchObject(
           Type.from([0, 1, 2, 3, 4, 5, 6])
@@ -68,7 +65,7 @@ describe("sortArray", () => {
   });
 
   describe("non-finite numbers", () => {
-    test("inifinity", () => {
+    test("infinity", () => {
       expect(sortArray(new Float32Array([pInf, nInf, 0, 1, 2]))).toMatchObject(
         new Float32Array([nInf, 0, 1, 2, pInf])
       );
@@ -131,7 +128,7 @@ describe("sortArray", () => {
 
 describe("sortIndex", () => {
   describe("finite numbers", () => {
-    [Array, Float32Array, Uint32Array, Int32Array, Float64Array].map(Type =>
+    [Array, Float32Array, Uint32Array, Int32Array, Float64Array].map((Type) =>
       test(Type.name, () => {
         const source1 = Type.from([6, 5, 4, 3, 2, 1, 0]);
         const index1 = fillRange(new Uint32Array(source1.length));

client/__tests__/util/typedCrossfilter/util.test.js

@@ -1,6 +1,6 @@
 import {
   sliceByIndex,
-  makeSortIndex
+  makeSortIndex,
 } from "../../../src/util/typedCrossfilter/util";
 import { rangeFill as fillRange } from "../../../src/util/range";
 

client/configuration/babel/babel.dev.js

@@ -2,8 +2,15 @@ module.exports = {
   babelrc: false,
   cacheDirectory: true,
   presets: [
-    ["modern-browsers", { loose: true, modules: false }],
-    "@babel/preset-react"
+    [
+      "@babel/preset-env",
+      {
+        useBuiltIns: "entry",
+        corejs: 3,
+        modules: false,
+      },
+    ],
+    "@babel/preset-react",
   ],
   plugins: [
     "@babel/plugin-proposal-function-bind",
@@ -11,6 +18,6 @@ module.exports = {
     ["@babel/plugin-proposal-class-properties", { loose: true }],
     "@babel/plugin-proposal-export-namespace-from",
     "@babel/plugin-proposal-optional-chaining",
-    "@babel/plugin-proposal-nullish-coalescing-operator"
-  ]
+    "@babel/plugin-proposal-nullish-coalescing-operator",
+  ],
 };

client/configuration/babel/babel.prod.js

@@ -1,8 +1,15 @@
 module.exports = {
   babelrc: false,
   presets: [
-    ["modern-browsers", { loose: true, modules: false }],
-    "@babel/preset-react"
+    [
+      "@babel/preset-env",
+      {
+        useBuiltIns: "entry",
+        corejs: 3,
+        modules: false,
+      },
+    ],
+    "@babel/preset-react",
   ],
   plugins: [
     "@babel/plugin-proposal-function-bind",
@@ -12,6 +19,6 @@ module.exports = {
     "@babel/plugin-transform-react-constant-elements",
     "@babel/plugin-transform-runtime",
     "@babel/plugin-proposal-optional-chaining",
-    "@babel/plugin-proposal-nullish-coalescing-operator"
-  ]
+    "@babel/plugin-proposal-nullish-coalescing-operator",
+  ],
 };

client/configuration/eslint/eslint.js

@@ -1,18 +1,47 @@
 module.exports = {
   root: true,
   parser: "babel-eslint",
-  extends: ["airbnb", "prettier", "prettier/react"],
+  extends: [
+    "airbnb",
+    "plugin:eslint-comments/recommended",
+    "plugin:@blueprintjs/recommended",
+    "plugin:compat/recommended",
+    "plugin:prettier/recommended",
+    "prettier/react",
+  ],
+  settings: {
+    polyfills: [
+      "TextDecoder",
+      "TextEncoder",
+      "fetch",
+      "Request",
+      "Response",
+      "Headers",
+      "AbortController",
+    ],
+  },
   env: { browser: true, commonjs: true, es6: true },
-  globals: { expect: true },
+  globals: {
+    expect: true,
+    jest: true,
+    jestPuppeteer: true,
+    it: true,
+    page: true,
+    browser: true,
+    context: true,
+    beforeEach: true,
+  },
   parserOptions: {
     ecmaVersion: 2017,
     sourceType: "module",
     ecmaFeatures: {
       jsx: true,
-      generators: true
-    }
+      generators: true,
+    },
   },
   rules: {
+    "react/jsx-no-target-blank": "off",
+    "eslint-comments/require-description": ["error"],
     "no-magic-numbers": "off",
     "no-nested-ternary": "off",
     "func-style": "off",
@@ -21,13 +50,7 @@ module.exports = {
     "react/jsx-filename-extension": "off",
     "comma-dangle": "off",
     "no-underscore-dangle": "off",
-    quotes: ["error", "double"],
     "implicit-arrow-linebreak": "off",
-    "operator-linebreak": [
-      "error",
-      "after",
-      { overrides: { "?": "before", ":": "before" } }
-    ],
     "no-console": "off",
     "spaced-comment": ["error", "always", { exceptions: ["*"] }],
     "no-param-reassign": "off",
@@ -35,13 +58,26 @@ module.exports = {
     "react/prop-types": [0],
     "space-before-function-paren": "off",
     "function-paren-newline": "off",
-    "prefer-destructuring": ["error", { object: true, array: false }]
+    "prefer-destructuring": ["error", { object: true, array: false }],
+    "import/prefer-default-export": "off",
+    "no-restricted-syntax": [
+      "error",
+      "ForInStatement",
+      "LabeledStatement",
+      "WithStatement",
+    ],
+    "import/no-extraneous-dependencies": [
+      "error",
+      {
+        devDependencies: true,
+      },
+    ],
   },
   overrides: [
     {
       files: ["**/*.test.js"],
       env: {
-        jest: true // now **/*.test.js files' env has both es6 *and* jest
+        jest: true, // now **/*.test.js files' env has both es6 *and* jest
       },
       // Can't extend in overrides: https://github.com/eslint/eslint/issues/8813
       // "extends": ["plugin:jest/recommended"]
@@ -51,8 +87,8 @@ module.exports = {
         "jest/no-focused-tests": "error",
         "jest/no-identical-title": "error",
         "jest/prefer-to-have-length": "warn",
-        "jest/valid-expect": "error"
-      }
-    }
-  ]
+        "jest/valid-expect": "error",
+      },
+    },
+  ],
 };

client/configuration/lint-staged/lint-staged.config.js

@@ -0,0 +1,3 @@
+module.exports = {
+  "*.js": "eslint --fix",
+};

client/configuration/webpack/cspHashPlugin.js

@@ -0,0 +1,53 @@
+/* eslint-disable import/no-extraneous-dependencies -- this file is a devDependency*/
+const cheerio = require("cheerio");
+const crypto = require("crypto");
+const HtmlWebpackPlugin = require("html-webpack-plugin");
+
+const digest = (str) => {
+  const hash = crypto.createHash("sha256").update(str, "utf8").digest("base64");
+  return `sha256-${hash}`;
+};
+class CspHashPlugin {
+  constructor(opts) {
+    this.opts = { ...opts };
+  }
+
+  apply(compiler) {
+    compiler.hooks.compilation.tap("CspHashPlugin", (compilation) => {
+      HtmlWebpackPlugin.getHooks(compilation).beforeEmit.tapAsync(
+        "CspHashPlugin",
+        (data, cb) => {
+          const { filename } = this.opts;
+
+          const $ = cheerio.load(data.html, { decodeEntities: false });
+
+          if (filename) {
+            const results = {};
+            results["script-hashes"] = $("script:not([src]):not([no-csp-hash])")
+              .map((i, elmt) => digest($(elmt).html()))
+              .get();
+
+            const json = JSON.stringify(results);
+            compilation.assets[filename] = {
+              source: () => json,
+              size: () => json.length,
+            };
+          }
+
+          // Remove no-csp-hash attributes. Cheerio does not parse Jinja templates
+          // correctly, so we brute force this with a regular expression.
+          data.html = data.html.replace(
+            /(<script .*)no-csp-hash(.*>)/,
+            (match, p1, p2) => [p1, p2].join("")
+          );
+
+          // Tell webpack to move on
+          cb(null, data);
+        }
+      );
+    });
+  }
+}
+
+module.exports = CspHashPlugin;
+/* eslint-enable import/no-extraneous-dependencies -- enable*/

client/configuration/webpack/obsoleteHTMLTemplate.html

@@ -0,0 +1,82 @@
+<script>
+  /** Reflect ANY changes to this script in the script hash in `server/eb/app.py` **/
+  var root = document.getElementById("root");
+  root.remove();
+  var portals = document.getElementsByClassName("bp3-portal");
+  for (var i = 0; i < portals.length; i += 1) {
+    portals[i].remove();
+  }
+</script>
+<div
+  style="
+    display: flex;
+    flex-direction: column;
+    width: 100vw;
+    height: 100vh;
+    text-align: center;
+    justify-content: center;
+    align-items: center;
+    background: #8080801a;
+    font-family: 'Roboto Condensed, sans serif';
+  "
+>
+  <img
+    src="https://cellxgene.cziscience.com/s3/cellxgene/static/images/cellxgene-logo.png"
+    style="width: 320px;"
+  />
+  <div
+    style="
+      margin-top: 16px;
+      background: white;
+      width: 40vw;
+      border-radius: 4px;
+      padding: 24px 64px;
+      -webkit-box-shadow: 0px 0px 3px 2px rgba(0, 0, 0, 0.38);
+      -moz-box-shadow: 0px 0px 3px 2px rgba(0, 0, 0, 0.38);
+      box-shadow: 0px 0px 3px 2px rgba(0, 0, 0, 0.38);
+      max-width: 550px;
+    "
+  >
+    <div style="margin-bottom: 0; font-weight: bolder; font-size: 1.2em;">
+      Unsupported Browser
+    </div>
+    <div style="margin-top: 0;">
+      cellxgene is currently supported on the following browsers
+    </div>
+    <div
+      style="display: flex; justify-content: space-around; margin-top: 16px;"
+    >
+      <a
+        href="https://www.google.com/chrome/?hl=en%22"
+        aria-label="Download Google Chrome"
+      >
+        <img
+          src="https://cellxgene.cziscience.com/s3/cellxgene/static/images/chrome.png"
+          style="width: 80px; height: 80px;"
+        />
+        <div>Chrome &gt; 60</div>
+      </a>
+      <a href="https://www.apple.com/safari/" aria-label="Download Safari">
+        <img
+          src="https://cellxgene.cziscience.com/s3/cellxgene/static/images/safari.png"
+          style="width: 80px; height: 80px;"
+        />
+        <div>Safari ≥ 10.1</div>
+      </a>
+      <a href="https://www.mozilla.com/firefox/" aria-label="Download Firefox">
+        <img
+          src="https://cellxgene.cziscience.com/s3/cellxgene/static/images/firefox.png"
+          style="width: 80px; height: 80px;"
+        />
+        <div>Firefox ≥ 60</div>
+      </a>
+      <a href="//www.microsoft.com/edge" aria-label="Download Edge">
+        <img
+          src="https://cellxgene.cziscience.com/s3/cellxgene/static/images/edge.png"
+          style="width: 80px; height: 80px;"
+        />
+        <div>Edge ≥ 79</div>
+      </a>
+    </div>
+  </div>
+</div>

client/configuration/webpack/webpack.config.dev.js

@@ -1,96 +1,80 @@
-// jshint esversion: 6
 const path = require("path");
 const webpack = require("webpack");
 const HtmlWebpackPlugin = require("html-webpack-plugin");
+const FaviconsWebpackPlugin = require("favicons-webpack-plugin");
+const ScriptExtHtmlWebpackPlugin = require("script-ext-html-webpack-plugin");
+const MiniCssExtractPlugin = require("mini-css-extract-plugin");
+
+const { merge } = require("webpack-merge");
+
+const sharedConfig = require("./webpack.config.shared");
+const babelOptions = require("../babel/babel.dev");
 
-const src = path.resolve("src");
 const fonts = path.resolve("src/fonts");
 const nodeModules = path.resolve("node_modules");
 
-const babelOptions = require("../babel/babel.dev");
-
-module.exports = {
+const devConfig = {
   mode: "development",
   devtool: "eval",
-  entry: ["./src/index"],
   output: {
-    path: path.resolve("build"),
     pathinfo: true,
     filename: "static/js/bundle.js",
-    publicPath: "/"
   },
   module: {
     rules: [
       {
-        test: /\.js$/,
-        include: src,
+        test: /\.jsx?$/,
         loader: "babel-loader",
-        options: babelOptions
+        options: babelOptions,
       },
       {
-        test: /\.css$/,
-        include: src,
-        exclude: [path.resolve(src, "index.css")],
-        loader: [
-          {
-            loader: "style-loader"
-          },
-          {
-            loader: "css-loader",
-            options: {
-              modules: {
-                localIdentName: "[name]__[local]___[hash:base64:5]"
-              }
-            }
-          }
-        ]
-      },
-      {
-        test: /index\.css$/,
-        include: [path.resolve(src, "index.css")],
-        loader: [
-          {
-            loader: "style-loader"
-          },
-          {
-            loader: "css-loader"
-          }
-        ]
-      },
-
-      { test: /\.json$/, include: [src, nodeModules], loader: "json-loader" },
-      {
-        test: /\.(jpg|png|gif|eot|svg|ttf|woff|woff2)(\?.*)?$/,
-        include: nodeModules,
+        test: /\.(jpg|png|gif|eot|svg|ttf|woff|woff2|otf)$/i,
         loader: "file-loader",
-        query: { name: "static/media/[name].[ext]" }
+        include: [nodeModules, fonts],
+        query: {
+          name: "static/assets/[name].[ext]",
+          // (thuang): This is needed to make sure @font url path is '/static/assets/'
+          publicPath: "/",
+        },
       },
-      {
-        test: /\.(woff|woff2|eot|ttf|otf)$/,
-        include: fonts,
-        loader: "file-loader",
-        query: { name: "static/fonts/[name].[ext]" }
-      },
-      {
-        test: /\.(mp4|webm)(\?.*)?$/,
-        include: [src, nodeModules],
-        loader: "url-loader",
-        query: { limit: 10000, name: "static/media/[name].[ext]" }
-      }
-    ]
+    ],
   },
   plugins: [
     new HtmlWebpackPlugin({
       inject: true,
       template: path.resolve("index.html"),
-      favicon: path.resolve("favicon.png")
+    }),
+    new FaviconsWebpackPlugin({
+      logo: "./favicon.png",
+      prefix: "static/img/",
+      favicons: {
+        icons: {
+          android: false,
+          appleIcon: false,
+          appleStartup: false,
+          coast: false,
+          firefox: false,
+          windows: false,
+          yandex: false,
+        },
+      },
+    }),
+    new MiniCssExtractPlugin({
+      filename: "static/[name].css",
     }),
     new webpack.NoEmitOnErrorsPlugin(),
     new webpack.DefinePlugin({
-      __REACT_DEVTOOLS_GLOBAL_HOOK__: "({ isDisabled: true })"
+      __REACT_DEVTOOLS_GLOBAL_HOOK__: "({ isDisabled: true })",
     }),
     new webpack.DefinePlugin({
-      "process.env.CXG_SERVER_PORT": JSON.stringify(process.env.CXG_SERVER_PORT),
-    })
-  ]
+      "process.env.CXG_SERVER_PORT": JSON.stringify(
+        process.env.CXG_SERVER_PORT
+      ),
+    }),
+    new ScriptExtHtmlWebpackPlugin({
+      async: "obsolete",
+    }),
+  ],
 };
+
+module.exports = merge(sharedConfig, devConfig);

client/configuration/webpack/webpack.config.prod.js

@@ -1,124 +1,96 @@
-// jshint esversion: 6
 const path = require("path");
 const HtmlWebpackPlugin = require("html-webpack-plugin");
+const { CleanWebpackPlugin } = require("clean-webpack-plugin");
+const TerserJSPlugin = require("terser-webpack-plugin");
+const CleanCss = require("clean-css");
+const OptimizeCSSAssetsPlugin = require("optimize-css-assets-webpack-plugin");
+const FaviconsWebpackPlugin = require("favicons-webpack-plugin");
 const MiniCssExtractPlugin = require("mini-css-extract-plugin");
-const SWPrecacheWebpackPlugin = require("sw-precache-webpack-plugin");
-const HtmlWebpackInlineSourcePlugin = require("html-webpack-inline-source-plugin");
 
-const src = path.resolve("src");
-const fonts = path.resolve("src/fonts");
-const nodeModules = path.resolve("node_modules");
+const { merge } = require("webpack-merge");
 
 const babelOptions = require("../babel/babel.prod");
 
-const publicPath = "";
+const CspHashPlugin = require("./cspHashPlugin");
+const sharedConfig = require("./webpack.config.shared");
 
-module.exports = {
+const fonts = path.resolve("src/fonts");
+const nodeModules = path.resolve("node_modules");
+
+const prodConfig = {
   mode: "production",
   bail: true,
   cache: false,
-  devtool: "cheap-source-map",
-  entry: ["./src/index.js"],
   output: {
-    path: path.resolve("build"),
-    filename: "static/js/[name].[chunkhash:8].js",
-    publicPath
+    filename: "static/[name]-[contenthash].js",
   },
+  optimization: {
+    minimize: true,
+    minimizer: [
+      new TerserJSPlugin({}),
+      new OptimizeCSSAssetsPlugin({
+        cssProcessor: CleanCss,
+      }),
+    ],
+  },
+  devtool: "source-map",
   module: {
     rules: [
       {
-        test: /\.js$/,
-        include: src,
+        test: /\.jsx?$/,
         loader: "babel-loader",
-        options: babelOptions
+        options: babelOptions,
       },
       {
-        test: /\.css$/,
-        include: src,
-        exclude: [path.resolve(src, "index.css")],
-        use: [
-          MiniCssExtractPlugin.loader,
-          {
-            loader: "css-loader",
-            options: {
-              modules: {
-                localIdentName: "[name]__[local]___[hash:base64:5]"
-              },
-              importLoaders: 1
-            }
-          }
-        ]
-      },
-      {
-        test: /index\.css$/,
-        include: [path.resolve(src, "index.css")],
-        use: [
-          MiniCssExtractPlugin.loader,
-          {
-            loader: "css-loader",
-            options: {
-              importLoaders: 1
-            }
-          }
-        ]
-      },
-      {
-        test: /\.json$/,
-        include: [src, nodeModules],
-        loader: "json-loader",
-        exclude: /manifest.json$/
-      },
-      {
-        test: /\.(jpg|png|gif|eot|svg|ttf|woff|woff2)(\?.*)?$/,
-        include: nodeModules,
+        test: /\.(jpg|png|gif|eot|svg|ttf|woff|woff2|otf)$/i,
         loader: "file-loader",
-        query: { name: "static/media/[name].[ext]" }
+        include: [nodeModules, fonts],
+        query: {
+          name: "static/assets/[name]-[contenthash].[ext]",
+          // (thuang): This is needed to make sure @font url path is '../static/assets/'
+          publicPath: "static/",
+        },
       },
-      {
-        test: /\.(woff|woff2|eot|ttf|otf)$/,
-        include: fonts,
-        loader: "file-loader",
-        query: { name: "static/fonts/[name].[ext]" }
-      },
-      {
-        test: /\.(mp4|webm)(\?.*)?$/,
-        include: [src, nodeModules],
-        loader: "url-loader",
-        query: { limit: 10000, name: "static/media/[name].[hash:8].[ext]" }
-      }
-    ]
+    ],
   },
   plugins: [
     new HtmlWebpackPlugin({
-      inject: "body",
       filename: "index.html",
       template: path.resolve("index_template.html"),
-      favicon: path.resolve("favicon.png"),
-      inlineSource: ".(js|css)$",
-      minify: {
-        removeComments: true,
-        collapseWhitespace: true,
-        removeRedundantAttributes: true,
-        useShortDoctype: true,
-        removeEmptyAttributes: true,
-        removeStyleLinkTypeAttributes: true,
-        keepClosingSlash: true,
-        minifyJS: true,
-        minifyCSS: true,
-        minifyURLs: true
-      }
+      decodeEntities: false,
+      minify: false,
+    }),
+    new CleanWebpackPlugin({
+      verbose: true,
+      protectWebpackAssets: false,
+      cleanAfterEveryBuildPatterns: ["main.js", "main.css"],
+    }),
+    new FaviconsWebpackPlugin({
+      logo: "./favicon.png",
+      prefix: "static/assets/",
+      favicons: {
+        icons: {
+          android: false,
+          appleIcon: false,
+          appleStartup: false,
+          coast: false,
+          firefox: false,
+          windows: false,
+          yandex: false,
+        },
+      },
     }),
-    new HtmlWebpackInlineSourcePlugin(),
     new MiniCssExtractPlugin({
-      filename: "static/css/[name].[contenthash:8].css"
+      filename: "static/[name]-[contenthash].css",
+    }),
+    new CspHashPlugin({
+      filename: "csp-hashes.json",
     }),
-    new SWPrecacheWebpackPlugin({
-      cacheId: "cellxgene",
-      filename: "service-worker.js"
-    })
   ],
   performance: {
     maxEntrypointSize: 2000000,
-    maxAssetSize: 2000000
-  }
+    maxAssetSize: 2000000,
+  },
 };
+
+module.exports = merge(sharedConfig, prodConfig);

client/configuration/webpack/webpack.config.shared.js

@@ -0,0 +1,74 @@
+const path = require("path");
+const fs = require("fs");
+const MiniCssExtractPlugin = require("mini-css-extract-plugin");
+const ObsoleteWebpackPlugin = require("obsolete-webpack-plugin");
+const ScriptExtHtmlWebpackPlugin = require("script-ext-html-webpack-plugin");
+
+const src = path.resolve("src");
+const nodeModules = path.resolve("node_modules");
+
+const publicPath = "";
+
+const rawObsoleteHTMLTemplate = fs.readFileSync(
+  `${__dirname}/obsoleteHTMLTemplate.html`,
+  "utf8"
+);
+
+const obsoleteHTMLTemplate = rawObsoleteHTMLTemplate.replace(/'/g, '"');
+
+module.exports = {
+  entry: [
+    "core-js",
+    "regenerator-runtime/runtime",
+    "fastestsmallesttextencoderdecoder",
+    "whatwg-fetch",
+    "abort-controller/polyfill",
+    "./src/index",
+  ],
+  output: {
+    path: path.resolve("build"),
+    publicPath,
+  },
+  module: {
+    rules: [
+      {
+        test: /\.css$/,
+        include: src,
+        exclude: [path.resolve(src, "index.css")],
+        use: [
+          MiniCssExtractPlugin.loader,
+          {
+            loader: "css-loader",
+            options: {
+              modules: {
+                localIdentName: "[name]__[local]___[hash:base64:5]",
+              },
+              importLoaders: 1,
+            },
+          },
+        ],
+      },
+      {
+        test: /index\.css$/,
+        include: [path.resolve(src, "index.css")],
+        use: [MiniCssExtractPlugin.loader, "css-loader"],
+      },
+      {
+        test: /\.json$/,
+        include: [src, nodeModules],
+        loader: "json-loader",
+        exclude: /manifest.json$/,
+      },
+    ],
+  },
+  plugins: [
+    new ObsoleteWebpackPlugin({
+      name: "obsolete",
+      template: obsoleteHTMLTemplate,
+      promptOnNonTargetBrowser: false,
+    }),
+    new ScriptExtHtmlWebpackPlugin({
+      async: "obsolete",
+    }),
+  ],
+};

client/index.html

@@ -1,12 +1,27 @@
 <!DOCTYPE html>
 <html lang="en">
   <head>
-    <meta charset="utf-8">
-    <meta name="viewport" content="width=device-width, initial-scale=1">
+    <meta charset="utf-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1" />
     <title>cell&times;gene</title>
     <style>
-      html, body, p, h1, h2, h3, h4, h5, h6, span, button, input, label, text, div {
-        font-family: 'Roboto Condensed','Helvetica Neue','Helvetica','Arial',sans-serif;
+      html,
+      body,
+      p,
+      h1,
+      h2,
+      h3,
+      h4,
+      h5,
+      h6,
+      span,
+      button,
+      input,
+      label,
+      text,
+      div {
+        font-family: "Roboto Condensed", "Helvetica Neue", "Helvetica", "Arial",
+          sans-serif;
         font-size: 14px;
       }
       body {
@@ -20,9 +35,12 @@
     </style>
   </head>
   <body>
-    <noscript>If you're seeing this message, that means <strong>JavaScript has been disabled on your browser</strong>, please <strong>enable JS</strong> to make this app work.</noscript>
+    <noscript
+      >If you're seeing this message, that means
+      <strong>JavaScript has been disabled on your browser</strong>, please
+      <strong>enable JS</strong> to make this app work.</noscript
+    >
 
     <div id="root"></div>
-
   </body>
 </html>

client/index_template.html

@@ -1,41 +1,65 @@
 <!DOCTYPE html>
 <html lang="en">
-<head>
-    <meta charset="utf-8">
-    <meta name="viewport" content="width=device-width, initial-scale=1">
+  <head>
+    <meta charset="utf-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1" />
     <title>cell&times;gene</title>
     <style>
-        html, body, p, h1, h2, h3, h4, h5, h6, span, button, input, label, text, div {
-            font-family: 'Roboto Condensed', 'Helvetica Neue', 'Helvetica', 'Arial', sans-serif;
-            font-size: 14px;
-        }
+      html,
+      body,
+      p,
+      h1,
+      h2,
+      h3,
+      h4,
+      h5,
+      h6,
+      span,
+      button,
+      input,
+      label,
+      text,
+      div {
+        font-family: "Roboto Condensed", "Helvetica Neue", "Helvetica", "Arial",
+          sans-serif;
+        font-size: 14px;
+      }
 
-        body {
-            margin: 0;
-            padding: 0;
-        }
+      body {
+        margin: 0;
+        padding: 0;
+      }
 
-        * {
-            box-sizing: border-box;
-        }
+      * {
+        box-sizing: border-box;
+      }
     </style>
+  </head>
+  <body>
+    <script type="text/javascript">
+      window.CELLXGENE = {};
+      window.CELLXGENE.API = {
+        prefix: `${location.origin}${location.pathname}api/`,
+        version: "v0.2/",
+      };
+    </script>
+    <noscript
+      >If you're seeing this message, that means
+      <strong>JavaScript has been disabled on your browser</strong>, please
+      <strong>enable JS</strong> to make this app work.
+    </noscript>
 
-</head>
-<body>
-<script type="text/javascript">
-  window.CELLXGENE = {};
-  window.CELLXGENE.API = {
-    prefix: window.location.href + "api/",
-    version: "v0.2/"
-  };
-</script>
-<noscript>If you're seeing this message, that means <strong>JavaScript has been disabled on your browser</strong>,
-    please <strong>enable JS</strong> to make this app work.
-</noscript>
+    <div id="root"></div>
+    {% for script in SCRIPTS -%}
+    <script type="text/javascript"
+      {{ ('integrity="%s"' % script.integrity) | safe if script.integrity }}
+      {{ ('crossorigin="%s"' % script.crossorigin) | safe if script.crossorigin }}
+      src="{{script.src | safe}}"
+    ></script>
+    {%- endfor %}
 
-<div id="root"></div>
-{% for script in SCRIPTS %}
-<script type="text/javascript" src="{{script | safe}}"></script>
-{% endfor %}
-</body>
+    {% for ils in INLINE_SCRIPTS %}
+    <script type="text/javascript" no-csp-hash>{% include ils %}</script>
+    {% endfor %}
+  </body>
 </html>

client/jest-puppeteer.config.js

@@ -0,0 +1,46 @@
+/**
+ * `client/jest-puppeteer.config.js` is for configuring Puppeteer's launch config options
+ * `client/__tests__/e2e/puppeteer.setup.js` is for configuring `jest`, `browser`,
+ * and `page` objects
+ */
+
+const ENV_DEFAULT = require("../environment.default.json");
+
+const jestEnv = process.env.JEST_ENV || ENV_DEFAULT.JEST_ENV;
+const isHeadful =
+  process.env.HEADFUL === "true" || process.env.HEADLESS === "false";
+
+const DEFAULT_LAUNCH_CONFIG = {
+  headless: !isHeadful,
+  args: ["--ignore-certificate-errors", "--ignore-ssl-errors"],
+  ignoreHTTPSErrors: true,
+  defaultViewport: {
+    width: 1280,
+    height: 960,
+  },
+};
+
+const LAUNCH_CONFIG_BY_ENV = {
+  [ENV_DEFAULT.DEBUG]: {
+    ...DEFAULT_LAUNCH_CONFIG,
+    headless: false,
+    slowMo: 100,
+    devtools: true,
+    defaultViewport: {
+      width: DEFAULT_LAUNCH_CONFIG.defaultViewport.width,
+      height: DEFAULT_LAUNCH_CONFIG.defaultViewport.height + 560,
+    },
+  },
+  [ENV_DEFAULT.DEV]: {
+    ...DEFAULT_LAUNCH_CONFIG,
+    headless: false,
+    slowMo: 5,
+  },
+};
+
+const launchConfig = LAUNCH_CONFIG_BY_ENV[jestEnv] || DEFAULT_LAUNCH_CONFIG;
+
+module.exports = {
+  browserContext: "incognito",
+  launch: launchConfig,
+};

client/package.json

@@ -1,15 +1,20 @@
 {
   "name": "cellxgene",
-  "version": "0.15.0",
+  "version": "0.16.8",
   "license": "MIT",
   "description": "cellxgene is a web application for the interactive exploration of single cell sequence data.",
   "repository": "https://github.com/chanzuckerberg/cellxgene",
   "scripts": {
+    "build": "npm run clean && webpack --config",
     "clean": "rimraf build",
-    "build": "npm run clean && webpack --config configuration/webpack/webpack.config.prod.js",
-    "dev": "npm run clean && webpack --config configuration/webpack/webpack.config.dev.js",
-    "fmt": "eslint --fix src",
-    "lint": "eslint src"
+    "dev": "npm run build -- configuration/webpack/webpack.config.dev.js",
+    "e2e": "jest --config __tests__/e2e/e2eJestConfig.json e2e/e2e.test.js",
+    "e2e-annotations": "jest --config __tests__/e2e/e2eJestConfig.json e2e/e2eAnnotations.test.js",
+    "e2e-prod": "CXG_URL_BASE='https://cellxgene.cziscience.com/d/pbmc3k.cxg/' jest --config __tests__/e2e/e2eJestConfig.json e2e/e2e.test.js",
+    "fmt": "eslint --fix src __tests__",
+    "lint": "eslint --fix src __tests__",
+    "prod": "npm run build -- configuration/webpack/webpack.config.prod.js",
+    "test": "jest --testPathIgnorePatterns e2e"
   },
   "engineStrict": true,
   "engines": {
@@ -24,87 +29,120 @@
   "resolutions": {
     "eslint-scope": "3.7.1"
   },
+  "browserslist": [
+    "Chrome > 60",
+    "Safari >= 10.1",
+    "iOS >= 10.3",
+    "Firefox >= 60",
+    "Edge >= 79",
+    "not Explorer > 0"
+  ],
   "dependencies": {
-    "@blueprintjs/core": "^3.24.0",
-    "@blueprintjs/icons": "^3.14.0",
-    "@blueprintjs/select": "^3.12.0",
-    "client": "0.0.1",
+    "@blueprintjs/core": "^3.30.0",
+    "@blueprintjs/icons": "^3.19.0",
+    "@blueprintjs/select": "^3.13.5",
+    "abort-controller": "^3.0.0",
+    "core-js": "^3.6.5",
     "d3": "^4.10.0",
     "d3-scale-chromatic": "^1.5.0",
+    "fastestsmallesttextencoderdecoder": "^1.0.22",
     "flatbuffers": "^1.11.0",
     "fuzzysort": "^1.1.4",
     "gl-mat4": "^1.2.0",
-    "gl-matrix": "^3.2.1",
+    "gl-matrix": "^3.3.0",
     "gl-vec3": "^1.1.3",
     "is-number": "^7.0.0",
-    "lodash": "^4.17.15",
+    "lodash": "^4.17.20",
     "memoize-one": "^5.1.1",
-    "react": "^16.13.0",
-    "react-dom": "^16.13.0",
-    "react-flip-toolkit": "7.0.6",
+    "pako": "^1.0.11",
+    "react": "^16.13.1",
+    "react-async": "^10.0.1",
+    "react-dom": "^16.13.1",
+    "react-flip-toolkit": "^7.0.12",
     "react-helmet": "^5.2.1",
-    "react-icons": "^3.9.0",
+    "react-icons": "^3.10.0",
     "react-redux": "^7.2.0",
     "redux": "^4.0.5",
     "redux-thunk": "^2.3.0",
-    "regl": "^1.4.0"
+    "regenerator-runtime": "^0.13.7",
+    "regl": "^1.6.1",
+    "script-ext-html-webpack-plugin": "^2.1.4",
+    "tinyqueue": "^2.0.3",
+    "webpack-merge": "^5.0.9",
+    "whatwg-fetch": "^3.2.0"
   },
   "devDependencies": {
-    "@babel/core": "^7.8.7",
-    "@babel/plugin-proposal-class-properties": "^7.8.3",
-    "@babel/plugin-proposal-decorators": "^7.8.3",
-    "@babel/plugin-proposal-export-namespace-from": "^7.8.3",
-    "@babel/plugin-proposal-function-bind": "^7.8.3",
-    "@babel/plugin-proposal-nullish-coalescing-operator": "^7.8.3",
-    "@babel/plugin-proposal-optional-chaining": "^7.8.3",
-    "@babel/plugin-transform-react-constant-elements": "^7.8.3",
-    "@babel/plugin-transform-runtime": "^7.8.3",
-    "@babel/preset-env": "^7.8.7",
-    "@babel/preset-react": "^7.8.3",
-    "@babel/register": "^7.8.6",
-    "@babel/runtime": "^7.8.7",
+    "@babel/core": "^7.10.5",
+    "@babel/plugin-proposal-class-properties": "^7.10.4",
+    "@babel/plugin-proposal-decorators": "^7.10.5",
+    "@babel/plugin-proposal-export-namespace-from": "^7.10.4",
+    "@babel/plugin-proposal-function-bind": "^7.10.5",
+    "@babel/plugin-proposal-nullish-coalescing-operator": "^7.10.4",
+    "@babel/plugin-proposal-optional-chaining": "^7.10.4",
+    "@babel/plugin-transform-react-constant-elements": "^7.10.4",
+    "@babel/plugin-transform-runtime": "^7.10.5",
+    "@babel/preset-env": "^7.10.4",
+    "@babel/preset-react": "^7.10.4",
+    "@babel/register": "^7.10.5",
+    "@babel/runtime": "^7.10.5",
+    "@blueprintjs/eslint-plugin": "^0.3.0",
+    "@sentry/webpack-plugin": "^1.12.0",
     "babel-eslint": "^10.1.0",
-    "babel-jest": "^25.1.0",
-    "babel-loader": "^8.0.6",
-    "babel-preset-modern-browsers": "^14.1.1",
-    "chalk": "^3.0.0",
-    "codecov": "^3.6.5",
+    "babel-jest": "^26.1.0",
+    "babel-loader": "^8.1.0",
+    "babel-preset-modern-browsers": "^14.2.1",
+    "chalk": "^4.1.0",
+    "cheerio": "^1.0.0-rc.3",
+    "clean-css": "^4.2.3",
+    "clean-webpack-plugin": "^3.0.0",
+    "codecov": "^3.7.1",
     "connect-history-api-fallback": "^1.6.0",
-    "copy-webpack-plugin": "^5.1.1",
-    "css-loader": "^3.4.2",
-    "eslint": "^6.8.0",
-    "eslint-config-airbnb": "^18.0.1",
-    "eslint-config-prettier": "^6.10.0",
-    "eslint-loader": "^3.0.3",
+    "css-loader": "^3.6.0",
+    "eslint": "^7.4.0",
+    "eslint-config-airbnb": "^18.2.0",
+    "eslint-config-prettier": "^6.11.0",
+    "eslint-loader": "^3.0.4",
+    "eslint-plugin-compat": "^3.8.0",
+    "eslint-plugin-eslint-comments": "^3.2.0",
     "eslint-plugin-filenames": "^1.3.2",
-    "eslint-plugin-import": "^2.20.1",
-    "eslint-plugin-jest": "^23.8.2",
-    "eslint-plugin-jsx-a11y": "^6.2.3",
-    "eslint-plugin-react": "^7.19.0",
-    "eslint-plugin-react-hooks": "^2.5.0",
+    "eslint-plugin-import": "^2.22.0",
+    "eslint-plugin-jest": "^23.18.0",
+    "eslint-plugin-jsx-a11y": "^6.3.1",
+    "eslint-plugin-prettier": "^3.1.4",
+    "eslint-plugin-react": "^7.20.3",
+    "eslint-plugin-react-hooks": "^4.0.8",
+    "expect-puppeteer": "^4.4.0",
     "express": "^4.17.1",
-    "file-loader": "^5.1.0",
-    "html-webpack-inline-source-plugin": "0.0.10",
-    "html-webpack-plugin": "^3.2.0",
-    "jest": "^25.1.0",
+    "favicons-webpack-plugin": "^3.0.1",
+    "file-loader": "^6.0.0",
+    "html-webpack-plugin": "^4.3.0",
+    "husky": "^4.2.5",
+    "jest": "^26.1.0",
+    "jest-circus": "^26.1.0",
+    "jest-environment-puppeteer": "^4.4.0",
+    "jest-fetch-mock": "^3.0.3",
     "jest-puppeteer": "^4.4.0",
     "json-loader": "^0.5.7",
+    "lint-staged": "^10.2.11",
     "mini-css-extract-plugin": "^0.9.0",
-    "puppeteer": "^2.1.1",
+    "obsolete-webpack-plugin": "^0.5.6",
+    "optimize-css-assets-webpack-plugin": "^5.0.3",
+    "prettier": "^2.0.5",
+    "puppeteer": "^3.3.0",
     "rimraf": "^3.0.2",
     "serve-favicon": "^2.5.0",
-    "style-loader": "^1.1.3",
+    "style-loader": "^1.2.1",
     "sw-precache-webpack-plugin": "^1.0.0",
-    "url-loader": "^3.0.0",
-    "webpack": "^4.42.0",
-    "webpack-cli": "^3.3.11",
+    "terser-webpack-plugin": "^3.0.7",
+    "url-loader": "^4.1.0",
+    "webpack": "^4.43.0",
+    "webpack-cli": "^3.3.12",
     "webpack-dev-middleware": "^3.7.2"
   },
   "jest": {
     "testMatch": [
       "**/__tests__/**/?(*.)(spec|test).js?(x)"
     ],
-    "testURL": "http://localhost/",
     "setupFiles": [
       "./__tests__/setupMissingGlobals.js"
     ],
@@ -140,5 +178,10 @@
         ]
       }
     }
+  },
+  "husky": {
+    "hooks": {
+      "pre-commit": "lint-staged --config \"./configuration/lint-staged/lint-staged.config.js\""
+    }
   }
 }

client/server/development.js

@@ -1,49 +1,62 @@
-/* eslint-disable */
-// jshint esversion: 6
-var path = require("path");
-var historyApiFallback = require("connect-history-api-fallback");
-var chalk = require("chalk");
-var express = require("express");
-var favicon = require("serve-favicon");
-var webpack = require("webpack");
-var config = require("../configuration/webpack/webpack.config.dev");
-var utils = require("./utils");
+const chalk = require("chalk");
+const express = require("express");
+const favicon = require("serve-favicon");
+const webpack = require("webpack");
+const devMiddleware = require("webpack-dev-middleware");
+const config = require("../configuration/webpack/webpack.config.dev");
+const utils = require("./utils");
 
 process.env.NODE_ENV = "development";
 
 const CLIENT_PORT = process.env.CXG_CLIENT_PORT;
+const { CXG_SERVER_PORT } = process.env;
+
+const API = {
+  prefix: `http://localhost:${CXG_SERVER_PORT}/`,
+};
 
 // Set up compiler
-var compiler = webpack(config);
+const compiler = webpack(config);
 
-compiler.plugin("invalid", () => {
+compiler.hooks.invalid.tap("invalid", () => {
   utils.clearConsole();
   console.log("Compiling...");
 });
 
-compiler.plugin("done", stats => {
+compiler.hooks.done.tap("done", (stats) => {
   utils.formatStats(stats, CLIENT_PORT);
 });
 
 // Launch server
-var app = express();
-
-app.use(historyApiFallback({ verbose: false }));
+const app = express();
 
 app.use(
-  require("webpack-dev-middleware")(compiler, {
+  devMiddleware(compiler, {
     logLevel: "warn",
-    publicPath: config.output.publicPath
+    publicPath: config.output.publicPath,
+    index: true,
   })
 );
 
 app.use(favicon("./favicon.png"));
 
-app.get("*", (req, res) => {
-  res.sendFile(path.resolve("index.html"));
+app.get("/login", async (req, res) => {
+  try {
+    res.redirect(`${API.prefix}login?dataset=http://localhost:${CLIENT_PORT}`);
+  } catch (err) {
+    console.error(err);
+  }
 });
 
-app.listen(CLIENT_PORT, err => {
+app.get("/logout", async (req, res) => {
+  try {
+    res.redirect(`${API.prefix}logout?dataset=http://localhost:${CLIENT_PORT}`);
+  } catch (err) {
+    console.error(err);
+  }
+});
+
+app.listen(CLIENT_PORT, (err) => {
   if (err) {
     console.log(err);
     return;

client/server/utils.js

@@ -36,10 +36,10 @@ var formatStats = (stats, port) => {
 
   var json = stats.toJson();
   var formattedErrors = json.errors.map(
-    message => "Error in " + formatMessage(message)
+    (message) => "Error in " + formatMessage(message)
   );
   var formattedWarnings = json.warnings.map(
-    message => "Warning in " + formatMessage(message)
+    (message) => "Warning in " + formatMessage(message)
   );
 
   if (hasErrors) {
@@ -48,7 +48,7 @@ var formatStats = (stats, port) => {
     if (formattedErrors.some(isLikelyASyntaxError)) {
       formattedErrors = formattedErrors.filter(isLikelyASyntaxError);
     }
-    formattedErrors.forEach(message => {
+    formattedErrors.forEach((message) => {
       console.log(message);
       console.log();
     });
@@ -58,7 +58,7 @@ var formatStats = (stats, port) => {
   if (hasWarnings) {
     console.log(chalk.yellow("Compiled with warnings."));
     console.log();
-    formattedWarnings.forEach(message => {
+    formattedWarnings.forEach((message) => {
       console.log(message);
       console.log();
     });

client/src/actions/annotation.js

@@ -0,0 +1,389 @@
+/*
+Action creators for user annotation
+*/
+import _ from "lodash";
+import pako from "pako";
+import * as globals from "../globals";
+import { MatrixFBS, AnnotationsHelpers } from "../util/stateManager";
+
+const { isUserAnnotation } = AnnotationsHelpers;
+
+export const annotationCreateCategoryAction = (
+  newCategoryName,
+  categoryToDuplicate
+) => async (dispatch, getState) => {
+  /*
+  Add a new user-created category to the obs annotations.
+
+  Arguments:
+    newCategoryName - string name for the category.
+    categoryToDuplicate - obs category to use for initial values, or null.
+  */
+  const {
+    annoMatrix: prevAnnoMatrix,
+    obsCrossfilter: prevObsCrossfilter,
+  } = getState();
+  if (!prevAnnoMatrix || !prevObsCrossfilter) return;
+  const { schema } = prevAnnoMatrix;
+
+  /* name must be a string,  non-zero length */
+  if (typeof newCategoryName !== "string" || newCategoryName.length === 0)
+    throw new Error("user annotations require string name");
+
+  /* ensure the name isn't already in use! */
+  if (schema.annotations.obsByName[newCategoryName])
+    throw new Error("name collision on annotation category create");
+
+  let initialValue;
+  let newSchema;
+  let ctor;
+  if (categoryToDuplicate) {
+    /* if we are duplicating a category, retrieve it */
+    const catDupSchema = schema.annotations.obsByName[categoryToDuplicate];
+    const catDupType = catDupSchema?.type;
+    if (catDupType !== "string" && catDupType !== "categorical")
+      throw new Error("categoryToDuplicate does not exist or has invalid type");
+
+    const catToDupDf = await prevAnnoMatrix
+      .base()
+      .fetch("obs", categoryToDuplicate);
+    const col = catToDupDf.col(categoryToDuplicate);
+    initialValue = col.asArray();
+    const { categories } = col.summarizeCategorical();
+    // all user-created annotations must have the unassigned category
+    if (!categories.includes(globals.unassignedCategoryLabel)) {
+      categories.push(globals.unassignedCategoryLabel);
+    }
+    ctor = initialValue.constructor;
+    newSchema = {
+      ...catDupSchema,
+      name: newCategoryName,
+      categories,
+      writable: true,
+    };
+  } else {
+    /* else assign to the standard default value */
+    initialValue = globals.unassignedCategoryLabel;
+    ctor = Array;
+    newSchema = {
+      name: newCategoryName,
+      type: "categorical",
+      categories: [globals.unassignedCategoryLabel],
+      writable: true,
+    };
+  }
+
+  const obsCrossfilter = prevObsCrossfilter.addObsColumn(
+    newSchema,
+    ctor,
+    initialValue
+  );
+
+  dispatch({
+    type: "annotation: create category",
+    data: newCategoryName,
+    categoryToDuplicate,
+    annoMatrix: obsCrossfilter.annoMatrix,
+    obsCrossfilter,
+  });
+};
+
+export const annotationRenameCategoryAction = (
+  oldCategoryName,
+  newCategoryName
+) => (dispatch, getState) => {
+  /*
+  Rename a user-created annotation category
+  */
+  const {
+    annoMatrix: prevAnnoMatrix,
+    obsCrossfilter: prevObsCrossfilter,
+  } = getState();
+  if (!prevAnnoMatrix || !prevObsCrossfilter) return;
+  if (!isUserAnnotation(prevAnnoMatrix, oldCategoryName))
+    throw new Error("not a user annotation");
+
+  /* name must be a string,  non-zero length */
+  if (typeof newCategoryName !== "string" || newCategoryName.length === 0)
+    throw new Error("user annotations require string name");
+
+  if (oldCategoryName === newCategoryName) return;
+
+  const obsCrossfilter = prevObsCrossfilter.renameObsColumn(
+    oldCategoryName,
+    newCategoryName
+  );
+
+  dispatch({
+    type: "annotation: category edited",
+    annoMatrix: obsCrossfilter.annoMatrix,
+    obsCrossfilter,
+    metadataField: oldCategoryName,
+    newCategoryText: newCategoryName,
+    data: newCategoryName,
+  });
+};
+
+export const annotationDeleteCategoryAction = (categoryName) => (
+  dispatch,
+  getState
+) => {
+  /*
+  Delete a user-created category
+  */
+  const {
+    annoMatrix: prevAnnoMatrix,
+    obsCrossfilter: prevObsCrossfilter,
+  } = getState();
+  if (!prevAnnoMatrix || !prevObsCrossfilter) return;
+  if (!isUserAnnotation(prevAnnoMatrix, categoryName))
+    throw new Error("not a user annotation");
+
+  const obsCrossfilter = prevObsCrossfilter.dropObsColumn(categoryName);
+  dispatch({
+    type: "annotation: delete category",
+    annoMatrix: obsCrossfilter.annoMatrix,
+    obsCrossfilter,
+    metadataField: categoryName,
+  });
+};
+
+export const annotationCreateLabelInCategory = (
+  categoryName,
+  labelName,
+  assignSelected
+) => async (dispatch, getState) => {
+  /*
+  Add a new label to a user-defined category.  If assignSelected is true, assign
+  the label to all currently selected cells.
+  */
+  const {
+    annoMatrix: prevAnnoMatrix,
+    obsCrossfilter: prevObsCrossfilter,
+  } = getState();
+  if (!prevAnnoMatrix || !prevObsCrossfilter) return;
+  if (!isUserAnnotation(prevAnnoMatrix, categoryName))
+    throw new Error("not a user annotation");
+
+  let obsCrossfilter = prevObsCrossfilter.addObsAnnoCategory(
+    categoryName,
+    labelName
+  );
+  if (assignSelected) {
+    obsCrossfilter = await obsCrossfilter.setObsColumnValues(
+      categoryName,
+      prevObsCrossfilter.allSelectedLabels(),
+      labelName
+    );
+  }
+
+  dispatch({
+    type: "annotation: add new label to category",
+    annoMatrix: obsCrossfilter.annoMatrix,
+    obsCrossfilter,
+    metadataField: categoryName,
+    newLabelText: labelName,
+    assignSelectedCells: assignSelected,
+  });
+};
+
+export const annotationDeleteLabelFromCategory = (
+  categoryName,
+  labelName
+) => async (dispatch, getState) => {
+  /*
+  delete a label from a user-defined category
+  */
+  const {
+    annoMatrix: prevAnnoMatrix,
+    obsCrossfilter: prevObsCrossfilter,
+  } = getState();
+  if (!prevAnnoMatrix || !prevObsCrossfilter) return;
+  if (!isUserAnnotation(prevAnnoMatrix, categoryName))
+    throw new Error("not a user annotation");
+
+  const obsCrossfilter = await prevObsCrossfilter.removeObsAnnoCategory(
+    categoryName,
+    labelName,
+    globals.unassignedCategoryLabel
+  );
+
+  dispatch({
+    type: "annotation: delete label",
+    metadataField: categoryName,
+    label: labelName,
+    annoMatrix: obsCrossfilter.annoMatrix,
+    obsCrossfilter,
+  });
+};
+
+export const annotationRenameLabelInCategory = (
+  categoryName,
+  oldLabelName,
+  newLabelName
+) => async (dispatch, getState) => {
+  /*
+  label name change
+  */
+  const {
+    annoMatrix: prevAnnoMatrix,
+    obsCrossfilter: prevObsCrossfilter,
+  } = getState();
+  if (!prevAnnoMatrix || !prevObsCrossfilter) return;
+  if (!isUserAnnotation(prevAnnoMatrix, categoryName))
+    throw new Error("not a user annotation");
+
+  let obsCrossfilter = await prevObsCrossfilter.resetObsColumnValues(
+    categoryName,
+    oldLabelName,
+    newLabelName
+  );
+  obsCrossfilter = await obsCrossfilter.removeObsAnnoCategory(
+    categoryName,
+    oldLabelName,
+    globals.unassignedCategoryLabel
+  );
+
+  dispatch({
+    type: "annotation: label edited",
+    editedLabel: newLabelName,
+    metadataField: categoryName,
+    label: oldLabelName,
+    annoMatrix: obsCrossfilter.annoMatrix,
+    obsCrossfilter,
+  });
+};
+
+export const annotationLabelCurrentSelection = (
+  categoryName,
+  labelName
+) => async (dispatch, getState) => {
+  /*
+  set the label on all currently selected
+  */
+  const {
+    annoMatrix: prevAnnoMatrix,
+    obsCrossfilter: prevObsCrossfilter,
+  } = getState();
+  if (!prevAnnoMatrix || !prevObsCrossfilter) return;
+  if (!isUserAnnotation(prevAnnoMatrix, categoryName))
+    throw new Error("not a user annotation");
+
+  const obsCrossfilter = await prevObsCrossfilter.setObsColumnValues(
+    categoryName,
+    prevObsCrossfilter.allSelectedLabels(),
+    labelName
+  );
+
+  dispatch({
+    type: "annotation: label current cell selection",
+    metadataField: categoryName,
+    label: labelName,
+    obsCrossfilter,
+    annoMatrix: obsCrossfilter.annoMatrix,
+  });
+};
+
+function writableAnnotations(annoMatrix) {
+  return annoMatrix.schema.annotations.obs.columns
+    .filter((s) => s.writable)
+    .map((s) => s.name);
+}
+
+export const needToSaveObsAnnotations = (annoMatrix, lastSavedAnnoMatrix) => {
+  /*
+  Return true if there are LIKELY user-defined annotation modifications between the two
+  annoMatrices.  Technically not an action creator, but intimately intertwined
+  with the save process.
+
+  Two conditions will trigger a need to save:
+    * the collection of user-defined columns have changed
+    * the contents of the user-defined columns have change
+  */
+
+  annoMatrix = annoMatrix.base();
+
+  // if the annoMatrix hasn't changed, we are guaranteed no changes to the matrix schema or contents.
+  if (annoMatrix === lastSavedAnnoMatrix) return false;
+
+  // if the schema has changed, we need to save
+  const currentWritable = writableAnnotations(annoMatrix);
+  if (_.difference(currentWritable, writableAnnotations(lastSavedAnnoMatrix))) {
+    return true;
+  }
+
+  // no schema changes; check for change in contents
+  return currentWritable.some(
+    (col) => annoMatrix.col(col) !== lastSavedAnnoMatrix.col(col)
+  );
+};
+
+export const saveObsAnnotationsAction = () => async (dispatch, getState) => {
+  /*
+  Save the user-created obs annotations IF any have changed.
+  */
+  const state = getState();
+  const { annotations, autosave } = state;
+  const { dataCollectionNameIsReadOnly, dataCollectionName } = annotations;
+  const { lastSavedAnnoMatrix, saveInProgress } = autosave;
+
+  const annoMatrix = state.annoMatrix.base();
+
+  if (saveInProgress || annoMatrix === lastSavedAnnoMatrix) return;
+  if (!needToSaveObsAnnotations(annoMatrix, lastSavedAnnoMatrix)) {
+    dispatch({
+      type: "writable obs annotations - save complete",
+      lastSavedAnnoMatrix: annoMatrix,
+    });
+    return;
+  }
+
+  /*
+  Else, we really do need to save
+  */
+
+  dispatch({
+    type: "writable obs annotations - save started",
+  });
+
+  const df = await annoMatrix.fetch("obs", writableAnnotations(annoMatrix));
+  const matrix = MatrixFBS.encodeMatrixFBS(df);
+  const compressedMatrix = pako.deflate(matrix);
+  try {
+    const queryString =
+      !dataCollectionNameIsReadOnly && !!dataCollectionName
+        ? `?annotation-collection-name=${encodeURIComponent(
+            dataCollectionName
+          )}`
+        : "";
+    const res = await fetch(
+      `${globals.API.prefix}${globals.API.version}annotations/obs${queryString}`,
+      {
+        method: "PUT",
+        body: compressedMatrix,
+        headers: new Headers({
+          "Content-Type": "application/octet-stream",
+        }),
+        credentials: "include",
+      }
+    );
+    if (res.ok) {
+      dispatch({
+        type: "writable obs annotations - save complete",
+        lastSavedAnnoMatrix: annoMatrix,
+      });
+    } else {
+      dispatch({
+        type: "writable obs annotations - save error",
+        message: `HTTP error ${res.status} - ${res.statusText}`,
+        res,
+      });
+    }
+  } catch (error) {
+    dispatch({
+      type: "writable obs annotations - save error",
+      message: error.toString(),
+      error,
+    });
+  }
+};

client/src/actions/embedding.js

@@ -0,0 +1,51 @@
+/*
+action creators related to embeddings choice
+*/
+
+import { AnnoMatrixObsCrossfilter } from "../annoMatrix";
+import { _setEmbeddingSubset } from "../util/stateManager/viewStackHelpers";
+
+export async function _switchEmbedding(
+  prevAnnoMatrix,
+  prevCrossfilter,
+  newEmbeddingName
+) {
+  /*
+  DRY helper used by this and reembedding action creators
+  */
+  const base = prevAnnoMatrix.base();
+  const embeddingDf = await base.fetch("emb", newEmbeddingName);
+  const annoMatrix = _setEmbeddingSubset(prevAnnoMatrix, embeddingDf);
+  const obsCrossfilter = await new AnnoMatrixObsCrossfilter(
+    annoMatrix,
+    prevCrossfilter.obsCrossfilter
+  ).select("emb", newEmbeddingName, {
+    mode: "all",
+  });
+  return [annoMatrix, obsCrossfilter];
+}
+
+export const layoutChoiceAction = (newLayoutChoice) => async (
+  dispatch,
+  getState
+) => {
+  /*
+  On layout choice, make sure we have selected all on the previous layout, AND the new
+  layout.
+  */
+  const {
+    annoMatrix: prevAnnoMatrix,
+    obsCrossfilter: prevCrossfilter,
+  } = getState();
+  const [annoMatrix, obsCrossfilter] = await _switchEmbedding(
+    prevAnnoMatrix,
+    prevCrossfilter,
+    newLayoutChoice
+  );
+  dispatch({
+    type: "set layout choice",
+    layoutChoice: newLayoutChoice,
+    obsCrossfilter,
+    annoMatrix,
+  });
+};

client/src/actions/index.js

@@ -1,294 +1,119 @@
-import _ from "lodash";
 import * as globals from "../globals";
-import { Universe, MatrixFBS } from "../util/stateManager";
+import { AnnoMatrixLoader, AnnoMatrixObsCrossfilter } from "../annoMatrix";
 import {
   catchErrorsWrap,
   doJsonRequest,
-  doBinaryRequest,
-  dispatchNetworkErrorMessageToUser
+  dispatchNetworkErrorMessageToUser,
 } from "../util/actionHelpers";
-import { requestReembed, reembedResetWorldToUniverse } from "./reembed";
+import {
+  requestReembed /* , reembedResetWorldToUniverse -- disabled temporarily, TODO issue #1606 */,
+} from "./reembed";
+import { loadUserColorConfig } from "../util/stateManager/colorHelpers";
+import * as selnActions from "./selection";
+import * as annoActions from "./annotation";
+import * as viewActions from "./viewStack";
+import * as embActions from "./embedding";
 
 /*
-return promise to fetch the OBS annotations we need to load.  Omit anything
-we don't need.
+return promise fetching user-configured colors
 */
-function obsAnnotationFetchAndLoad(dispatch, schema) {
-  const obsAnnotations = schema?.schema?.annotations?.obs ?? {};
-  const columns = obsAnnotations.columns ?? [];
-  const index = obsAnnotations.index ?? false;
-  return Promise.all(
-    columns
-      .filter(col => col.name !== index)
-      .map(col => {
-        const path = `annotations/obs?annotation-name=${encodeURIComponent(
-          col.name
-        )}`;
-        const url = `${globals.API.prefix}${globals.API.version}${path}`;
-        return doBinaryRequest(url);
-      })
-      .map(rqst => rqst.then(buffer => Universe.matrixFBSToDataframe(buffer)))
-      .map(resp =>
-        resp.then(df =>
-          dispatch({
-            type: "universe: column load success",
-            dim: "obsAnnotations",
-            dataframe: df
-          })
-        )
-      )
+async function userColorsFetchAndLoad(dispatch) {
+  return fetchJson("colors").then((response) =>
+    dispatch({
+      type: "universe: user color load success",
+      userColors: loadUserColorConfig(response),
+    })
   );
 }
 
-/*
-return promise fetching VAR annotations we need to load.  Only index is currently used.
-*/
-function varAnnotationFetchAndLoad(dispatch, schema) {
-  const varAnnotations = schema?.schema?.annotations?.var ?? {};
-  const index = varAnnotations.index ?? false;
-  const names = index ? [index] : [];
-  return Promise.all(
-    names
-      .map(name => {
-        const path = `annotations/var?annotation-name=${encodeURIComponent(
-          name
-        )}`;
-        const url = `${globals.API.prefix}${globals.API.version}${path}`;
-        return doBinaryRequest(url);
-      })
-      .map(rqst => rqst.then(buffer => Universe.matrixFBSToDataframe(buffer)))
-      .map(resp =>
-        resp.then(df =>
-          dispatch({
-            type: "universe: column load success",
-            dim: "varAnnotations",
-            dataframe: df
-          })
-        )
-      )
-  );
+async function schemaFetch() {
+  return fetchJson("schema");
+}
+
+async function configFetch(dispatch) {
+  return fetchJson("config").then((response) => {
+    const config = { ...globals.configDefaults, ...response.config };
+    dispatch({
+      type: "configuration load complete",
+      config,
+    });
+    return config;
+  });
+}
+
+async function userInfoFetch(dispatch) {
+  return fetchJson("userinfo").then((response) => {
+    const { userinfo: userInfo } = response || {};
+    dispatch({
+      type: "userInfo load complete",
+      userInfo,
+    });
+    return userInfo;
+  });
+}
+
+function prefetchEmbeddings(annoMatrix) {
+  /*
+  prefetch requests for all embeddings
+  */
+  const { schema } = annoMatrix;
+  const available = schema.layout.obs.map((v) => v.name);
+  available.forEach((embName) => annoMatrix.prefetch("emb", embName));
 }
 
 /*
-return promise fetching layout we need
-*/
-function layoutFetchAndLoad(dispatch) {
-  return Promise.all(
-    ["layout/obs"]
-      .map(path => {
-        const url = `${globals.API.prefix}${globals.API.version}${path}`;
-        return doBinaryRequest(url);
-      })
-      .map(rqst => rqst.then(buffer => Universe.matrixFBSToDataframe(buffer)))
-      .map(resp =>
-        resp.then(df =>
-          dispatch({
-            type: "universe: column load success",
-            dim: "obsLayout",
-            dataframe: df
-          })
-        )
-      )
-  );
-}
-
-/*
-Bootstrap application with the initial data loading.
-  * /config - application configuration
-  * /schema - schema of dataframe
-  * /annotations - all metadata annotation
-  * /layout - all default layout
+Application bootstrap
 */
 const doInitialDataLoad = () =>
-  catchErrorsWrap(async dispatch => {
+  catchErrorsWrap(async (dispatch) => {
     dispatch({ type: "initial data load start" });
 
     try {
-      /*
-      Step 1 - config & schema, all JSON
-      */
-      const requestJson = ["config", "schema"]
-        .map(r => `${globals.API.prefix}${globals.API.version}${r}`)
-        .map(url => doJsonRequest(url));
-      const stepOneResults = await Promise.all(requestJson);
-      /* set config defaults */
-      const config = { ...globals.configDefaults, ...stepOneResults[0].config };
-      const schema = stepOneResults[1];
-      const universe = Universe.createUniverseFromResponse(config, schema);
-      dispatch({
-        type: "universe exists, but loading is still in progress",
-        universe
-      });
-      dispatch({
-        type: "configuration load complete",
-        config
-      });
-
-      /*
-      Step 2 - load the minimum stuff required to display.
-      */
-      await Promise.all([
-        layoutFetchAndLoad(dispatch),
-        varAnnotationFetchAndLoad(dispatch, schema)
+      const [config, schema] = await Promise.all([
+        configFetch(dispatch),
+        schemaFetch(dispatch),
+        userColorsFetchAndLoad(dispatch),
+        userInfoFetch(dispatch),
       ]);
 
-      /*
-      Step 3 - load everything else
-      */
-      await obsAnnotationFetchAndLoad(dispatch, schema);
+      const baseDataUrl = `${globals.API.prefix}${globals.API.version}`;
+      const annoMatrix = new AnnoMatrixLoader(baseDataUrl, schema.schema);
+      const obsCrossfilter = new AnnoMatrixObsCrossfilter(annoMatrix);
+      prefetchEmbeddings(annoMatrix);
 
       dispatch({
-        type: "initial data load complete (universe exists)",
-        universe
+        type: "annoMatrix: init complete",
+        annoMatrix,
+        obsCrossfilter,
       });
+      dispatch({ type: "initial data load complete" });
+
+      const defaultEmbedding = config?.parameters?.["default_embedding"];
+      const layoutSchema = schema?.schema?.layout?.obs ?? [];
+      if (
+        defaultEmbedding &&
+        layoutSchema.some((s) => s.name === defaultEmbedding)
+      ) {
+        dispatch(embActions.layoutChoiceAction(defaultEmbedding));
+      }
     } catch (error) {
       dispatch({ type: "initial data load error", error });
     }
   }, true);
 
-/*
-Set the view (world) to current selection.   Placeholder for an async action
-which also does re-layout.
-*/
-const setWorldToSelection = () => (dispatch, getState) => {
-  const { universe, world, crossfilter } = getState();
-  dispatch({
-    type: "set World to current selection",
-    universe,
-    world,
-    crossfilter
-  });
-};
-
-// Throws
-const dispatchExpressionErrors = (dispatch, res) => {
-  const msg = `Unexpected HTTP response while fetching expression data ${res.status}, ${res.statusText}`;
-  dispatchNetworkErrorMessageToUser(msg);
-  throw new Error(msg);
-};
-
-/*
-Fetch expression vectors for each gene in genes.  This is NOT an action
-function, but rather a helper to be called from an action helper that
-needs expression data.
-
-Transparently utilizes cached data if it is already present.
-*/
-async function _doRequestExpressionData(dispatch, getState, genes) {
-  const state = getState();
-  const { universe } = state;
-  const varIndexName = universe.schema.annotations.var.index;
-
-  /* helper for this function only */
-  const fetchData = async geneNames => {
-    const res = await fetch(
-      `${globals.API.prefix}${globals.API.version}data/var`,
-      {
-        method: "PUT",
-        body: JSON.stringify({
-          filter: {
-            var: {
-              annotation_value: [{ name: varIndexName, values: geneNames }]
-            }
-          }
-        }),
-        headers: new Headers({
-          accept: "application/octet-stream",
-          "Content-Type": "application/json"
-        }),
-        credentials: "include"
-      }
-    );
-
-    if (
-      !res.ok ||
-      res.headers.get("Content-Type") !== "application/octet-stream"
-    ) {
-      // WILL throw
-      return dispatchExpressionErrors(dispatch, res);
-    }
-
-    const data = await res.arrayBuffer();
-    return Universe.convertDataFBStoObject(universe, data);
-  };
-
-  /* preload data already in cache */
-  let expressionData = _.transform(
-    genes,
-    (expData, g) => {
-      const data = universe.varData.col(g);
-      if (data) {
-        expData[g] = data.asArray();
-      }
-    },
-    {}
-  ); // --> { gene: data }
-  /* make a list of genes for which we do not have data */
-  const genesToFetch = _.filter(genes, g => expressionData[g] === undefined);
-
-  dispatch({ type: "expression load start" });
-
-  /* Fetch data for any genes not in cache */
-  if (genesToFetch.length) {
-    try {
-      const newExpressionData = await fetchData(genesToFetch);
-      expressionData = {
-        ...expressionData,
-        ...newExpressionData
-      };
-    } catch (error) {
-      dispatch({ type: "expression load error", error });
-      throw error; // rethrow
-    }
-  }
-
-  dispatch({ type: "expression load success", expressionData });
-  return expressionData;
-}
-
 function requestSingleGeneExpressionCountsForColoringPOST(gene) {
-  return async (dispatch, getState) => {
-    dispatch({ type: "get single gene expression for coloring started" });
-    try {
-      await _doRequestExpressionData(dispatch, getState, [gene]);
-      const { world } = getState();
-      dispatch({
-        type: "color by expression",
-        gene,
-        data: {
-          [gene]: world.varData.col(gene).asArray()
-        }
-      });
-    } catch (error) {
-      dispatch({
-        type: "get single gene expression for coloring error",
-        error
-      });
-    }
+  return {
+    type: "color by expression",
+    gene,
   };
 }
 
-const requestUserDefinedGene = gene => async (dispatch, getState) => {
-  dispatch({ type: "request user defined gene started" });
-  try {
-    await await _doRequestExpressionData(dispatch, getState, [gene]);
-    const { world } = getState();
-
-    /* then send the success case action through */
-    return dispatch({
-      type: "request user defined gene success",
-      data: {
-        genes: [gene],
-        expression: world.varData.col(gene).asArray()
-      }
-    });
-  } catch (error) {
-    return dispatch({
-      type: "request user defined gene error",
-      error
-    });
-  }
-};
+const requestUserDefinedGene = (gene) => ({
+  type: "request user defined gene success",
+  data: {
+    genes: [gene],
+  },
+});
 
 const dispatchDiffExpErrors = (dispatch, response) => {
   switch (response.status) {
@@ -307,7 +132,7 @@ const dispatchDiffExpErrors = (dispatch, response) => {
       dispatchNetworkErrorMessageToUser(msg);
       dispatch({
         type: "request differential expression error",
-        error: new Error(msg)
+        error: new Error(msg),
       });
     }
   }
@@ -324,9 +149,8 @@ const requestDifferentialExpression = (set1, set2, num_genes = 10) => async (
     1. get the most differentially expressed genes
     2. get expression data for each
     */
-    const state = getState();
-    const { universe } = state;
-    const varIndexName = universe.schema.annotations.var.index;
+    const { annoMatrix } = getState();
+    const varIndexName = annoMatrix.schema.annotations.var.index;
 
     // Legal values are null, Array or TypedArray.  Null is initial state.
     if (!set1) set1 = [];
@@ -345,15 +169,15 @@ const requestDifferentialExpression = (set1, set2, num_genes = 10) => async (
         method: "POST",
         headers: new Headers({
           Accept: "application/json",
-          "Content-Type": "application/json"
+          "Content-Type": "application/json",
         }),
         body: JSON.stringify({
           mode: "topN",
           count: num_genes,
           set1: { filter: { obs: { index: set1 } } },
-          set2: { filter: { obs: { index: set2 } } }
+          set2: { filter: { obs: { index: set2 } } },
         }),
-        credentials: "include"
+        credentials: "include",
       }
     );
 
@@ -361,92 +185,31 @@ const requestDifferentialExpression = (set1, set2, num_genes = 10) => async (
       return dispatchDiffExpErrors(dispatch, res);
     }
 
-    const data = await res.json();
-    // result is [ [varIdx, ...], ... ]
-    const topNGenes = _.map(data, r =>
-      universe.varAnnotations.at(r[0], varIndexName)
-    );
-
-    /*
-    Kick off secondary action to fetch all of the expression data for the
-    topN expressed genes.
-    */
-    await _doRequestExpressionData(dispatch, getState, topNGenes);
+    const response = await res.json();
+    const varIndex = await annoMatrix.fetch("var", varIndexName);
+    const data = response.map((v) => [
+      varIndex.at(v[0], varIndexName),
+      ...v.slice(1),
+    ]);
 
     /* then send the success case action through */
     return dispatch({
       type: "request differential expression success",
-      data
+      data,
     });
   } catch (error) {
     return dispatch({
       type: "request differential expression error",
-      error
+      error,
     });
   }
 };
 
-const resetWorldToUniverse = () => (dispatch, getState) => {
-  const { universe } = getState();
-  reembedResetWorldToUniverse(dispatch, getState);
-  dispatch({
-    type: "reset World to eq Universe",
-    universe
-  });
-};
-
-const saveObsAnnotations = () => async (dispatch, getState) => {
-  const { universe, annotations } = getState();
-  const { obsAnnotations, schema } = universe;
-  const { dataCollectionNameIsReadOnly, dataCollectionName } = annotations;
-
-  dispatch({
-    type: "writable obs annotations - save started"
-  });
-
-  const writableAnnotations = schema.annotations.obs.columns
-    .filter(s => s.writable)
-    .map(s => s.name);
-  const df = obsAnnotations.subset(null, writableAnnotations);
-  const matrix = MatrixFBS.encodeMatrixFBS(df);
-  try {
-    const queryString =
-      !dataCollectionNameIsReadOnly && !!dataCollectionName
-        ? `?annotation-collection-name=${encodeURIComponent(
-            dataCollectionName
-          )}`
-        : "";
-    const res = await fetch(
-      `${globals.API.prefix}${globals.API.version}annotations/obs${queryString}`,
-      {
-        method: "PUT",
-        body: matrix,
-        headers: new Headers({
-          "Content-Type": "application/octet-stream"
-        }),
-        credentials: "include"
-      }
-    );
-    if (res.ok) {
-      dispatch({
-        type: "writable obs annotations - save complete",
-        obsAnnotations
-      });
-    } else {
-      dispatch({
-        type: "writable obs annotations - save error",
-        message: `HTTP error ${res.status} - ${res.statusText}`,
-        res
-      });
-    }
-  } catch (error) {
-    dispatch({
-      type: "writable obs annotations - save error",
-      message: error.toString(),
-      error
-    });
-  }
-};
+function fetchJson(pathAndQuery) {
+  return doJsonRequest(
+    `${globals.API.prefix}${globals.API.version}${pathAndQuery}`
+  );
+}
 
 export default {
   doInitialDataLoad,
@@ -454,7 +217,32 @@ export default {
   requestSingleGeneExpressionCountsForColoringPOST,
   requestUserDefinedGene,
   requestReembed,
-  resetWorldToUniverse,
-  saveObsAnnotations,
-  setWorldToSelection
+  selectContinuousMetadataAction: selnActions.selectContinuousMetadataAction,
+  selectCategoricalMetadataAction: selnActions.selectCategoricalMetadataAction,
+  selectCategoricalAllMetadataAction:
+    selnActions.selectCategoricalAllMetadataAction,
+  graphBrushStartAction: selnActions.graphBrushStartAction,
+  graphBrushChangeAction: selnActions.graphBrushChangeAction,
+  graphBrushDeselectAction: selnActions.graphBrushDeselectAction,
+  graphBrushCancelAction: selnActions.graphBrushCancelAction,
+  graphBrushEndAction: selnActions.graphBrushEndAction,
+  graphLassoStartAction: selnActions.graphLassoStartAction,
+  graphLassoEndAction: selnActions.graphLassoEndAction,
+  graphLassoCancelAction: selnActions.graphLassoCancelAction,
+  graphLassoDeselectAction: selnActions.graphLassoDeselectAction,
+  clipAction: viewActions.clipAction,
+  subsetAction: viewActions.subsetAction,
+  resetSubsetAction: viewActions.resetSubsetAction,
+  annotationCreateCategoryAction: annoActions.annotationCreateCategoryAction,
+  annotationRenameCategoryAction: annoActions.annotationRenameCategoryAction,
+  annotationDeleteCategoryAction: annoActions.annotationDeleteCategoryAction,
+  annotationCreateLabelInCategory: annoActions.annotationCreateLabelInCategory,
+  annotationDeleteLabelFromCategory:
+    annoActions.annotationDeleteLabelFromCategory,
+  annotationRenameLabelInCategory: annoActions.annotationRenameLabelInCategory,
+  annotationLabelCurrentSelection: annoActions.annotationLabelCurrentSelection,
+  saveObsAnnotationsAction: annoActions.saveObsAnnotationsAction,
+  needToSaveObsAnnotations: annoActions.needToSaveObsAnnotations,
+  layoutChoiceAction: embActions.layoutChoiceAction,
+  setCellSetFromSelection: selnActions.setCellSetFromSelection,
 };

client/src/actions/reembed.js

@@ -1,10 +1,10 @@
 import { API } from "../globals";
-import { Universe } from "../util/stateManager";
 import {
   postNetworkErrorToast,
   postAsyncSuccessToast,
-  postAsyncFailureToast
+  postAsyncFailureToast,
 } from "../components/framework/toasters";
+import { _switchEmbedding } from "./embedding";
 
 function abortableFetch(request, opts, timeout = 0) {
   const controller = new AbortController();
@@ -18,13 +18,13 @@ function abortableFetch(request, opts, timeout = 0) {
         setTimeout(() => controller.abort(), timeout);
       }
       return fetch(request, { ...opts, signal });
-    }
+    },
   };
 }
 
 async function doReembedFetch(dispatch, getState) {
   const state = getState();
-  let cells = state.world.obsAnnotations.rowIndex.keys();
+  let cells = state.annoMatrix.rowIndex.labels();
 
   // These lines ensure that we convert any TypedArray to an Array.
   // This is necessary because JSON.stringify() does some very strange
@@ -38,26 +38,23 @@ async function doReembedFetch(dispatch, getState) {
       method: "PUT",
       headers: new Headers({
         Accept: "application/octet-stream",
-        "Content-Type": "application/json"
+        "Content-Type": "application/json",
       }),
       body: JSON.stringify({
         method: "umap",
-        filter: { obs: { index: cells } }
+        filter: { obs: { index: cells } },
       }),
-      credentials: "include"
+      credentials: "include",
     },
     60000 // 1 minute timeout
   );
   dispatch({
     type: "reembed: request start",
-    abortableFetch: af
+    abortableFetch: af,
   });
   const res = await af.ready();
 
-  if (
-    res.ok &&
-    res.headers.get("Content-Type").includes("application/octet-stream")
-  ) {
+  if (res.ok && res.headers.get("Content-Type").includes("application/json")) {
     return res;
   }
 
@@ -67,7 +64,6 @@ async function doReembedFetch(dispatch, getState) {
   if (body && body.length > 0) {
     msg = `${msg} -- ${body}`;
   }
-  postNetworkErrorToast(msg);
   throw new Error(msg);
 }
 
@@ -78,21 +74,29 @@ export function requestReembed() {
   return async (dispatch, getState) => {
     try {
       const res = await doReembedFetch(dispatch, getState);
-      const schema = JSON.parse(res.headers.get("CxG-Schema"));
-      const buffer = await res.arrayBuffer();
-      const df = Universe.matrixFBSToDataframe(buffer);
+      const schema = await res.json();
       dispatch({
-        type: "reembed: request completed"
+        type: "reembed: request completed",
       });
+
+      const { annoMatrix: prevAnnoMatrix, obsCrossfilter: prevCrossfilter } = getState();
+      const base = prevAnnoMatrix.base().addEmbedding(schema);
+      const [annoMatrix, obsCrossfilter] = await _switchEmbedding(
+        base,
+        prevCrossfilter,
+        schema.name
+      );
       dispatch({
         type: "reembed: add reembedding",
-        embedding: df,
-        schema
+        schema,
+        annoMatrix,
+        obsCrossfilter,
       });
+
       postAsyncSuccessToast("Re-embedding has completed.");
     } catch (error) {
       dispatch({
-        type: "reembed: request aborted"
+        type: "reembed: request aborted",
       });
       if (error.name === "AbortError") {
         postAsyncFailureToast("Re-embedding calculation was aborted.");
@@ -103,11 +107,3 @@ export function requestReembed() {
     }
   };
 }
-
-export function reembedResetWorldToUniverse(dispatch, getState) {
-  const { reembedController } = getState();
-  if (reembedController.pendingFetch) reembedController.pendingFetch.abort();
-  dispatch({
-    type: "reembed: clear all reembeddings"
-  });
-}

client/src/actions/selection.js

@@ -0,0 +1,199 @@
+/*
+Action creators for selection 
+*/
+export const selectContinuousMetadataAction = (
+  type,
+  query,
+  range,
+  oldProps = {}
+) => async (dispatch, getState) => {
+  const { obsCrossfilter: prevObsCrossfilter } = getState();
+
+  const selection = range
+    ? {
+        mode: "range",
+        lo: range[0],
+        hi: range[1],
+        inclusive: true, // [lo, hi] incluisve selection
+      }
+    : { mode: "all" };
+
+  const obsCrossfilter = await prevObsCrossfilter.select(...query, selection);
+
+  dispatch({
+    type,
+    obsCrossfilter,
+    range,
+    ...oldProps,
+  });
+};
+
+export const selectCategoricalMetadataAction = (
+  type, // action type
+  metadataField, // annotation category name
+  labels,
+  label, // the label being selected/deselected
+  isSelected, // bool
+  oldProps = {}
+) => async (dispatch, getState) => {
+  const {
+    obsCrossfilter: prevObsCrossfilter,
+    categoricalSelection,
+  } = getState();
+
+  const labelSelectionState = new Map(categoricalSelection[metadataField]);
+  labels.forEach(
+    (l) => labelSelectionState.has(l) || labelSelectionState.set(l, true)
+  );
+  labelSelectionState.set(label, isSelected);
+
+  const values = Array.from(labelSelectionState.keys()).filter((k) =>
+    labelSelectionState.get(k)
+  );
+  const selection = {
+    mode: "exact",
+    values,
+  };
+  const obsCrossfilter = await prevObsCrossfilter.select(
+    "obs",
+    metadataField,
+    selection
+  );
+
+  dispatch({
+    type,
+    obsCrossfilter,
+    metadataField,
+    labelSelectionState,
+    ...oldProps,
+  });
+};
+
+export const selectCategoricalAllMetadataAction = (
+  type, // action type
+  metadataField, // annotation category name
+  labels,
+  isSelected, // bool, select all or none
+  oldProps = {}
+) => async (dispatch, getState) => {
+  const {
+    obsCrossfilter: prevObsCrossfilter,
+    categoricalSelection,
+  } = getState();
+
+  const labelSelectionState = new Map(categoricalSelection[metadataField]);
+  labels.forEach((label) => labelSelectionState.set(label, isSelected));
+
+  const selection = { mode: isSelected ? "all" : "none" };
+  const obsCrossfilter = await prevObsCrossfilter.select(
+    "obs",
+    metadataField,
+    selection
+  );
+
+  dispatch({
+    type,
+    obsCrossfilter,
+    metadataField,
+    labelSelectionState,
+    ...oldProps,
+  });
+};
+
+/**
+ ** Graph selection-related actions
+ **/
+
+export const graphBrushStartAction = () =>
+  /* no change to crossfilter until a change fires */
+  ({ type: "graph brush start" });
+
+const _graphBrushWithinRectAction = (type, embName, brushCoords) => async (
+  dispatch,
+  getState
+) => {
+  const { obsCrossfilter: prevObsCrossfilter } = getState();
+
+  const selection = { mode: "within-rect", ...brushCoords };
+  const obsCrossfilter = await prevObsCrossfilter.select(
+    "emb",
+    embName,
+    selection
+  );
+
+  dispatch({
+    type,
+    obsCrossfilter,
+    brushCoords,
+  });
+};
+
+const _graphAllAction = (type, embName) => async (dispatch, getState) => {
+  const { obsCrossfilter: prevObsCrossfilter } = getState();
+
+  const obsCrossfilter = await prevObsCrossfilter.select("emb", embName, {
+    mode: "all",
+  });
+
+  dispatch({
+    type,
+    obsCrossfilter,
+  });
+};
+
+export const graphBrushChangeAction = (embName, brushCoords) =>
+  _graphBrushWithinRectAction("graph brush change", embName, brushCoords);
+
+export const graphBrushEndAction = (embName, brushCoords) =>
+  _graphBrushWithinRectAction("graph brush end", embName, brushCoords);
+
+export const graphBrushCancelAction = (embName) =>
+  _graphAllAction("graph brush cancel", embName);
+export const graphBrushDeselectAction = (embName) =>
+  _graphAllAction("graph brush deselect", embName);
+
+export const graphLassoStartAction = () =>
+  /* no change to crossfilter until a change fires */
+  ({ type: "graph lasso start" });
+
+export const graphLassoCancelAction = (embName) =>
+  _graphAllAction("graph lasso cancel", embName);
+
+export const graphLassoDeselectAction = (embName) =>
+  _graphAllAction("graph lasso cancel", embName);
+
+export const graphLassoEndAction = (embName, polygon) => async (
+  dispatch,
+  getState
+) => {
+  const { obsCrossfilter: prevObsCrossfilter } = getState();
+
+  const selection = {
+    mode: "within-polygon",
+    polygon,
+  };
+  const obsCrossfilter = await prevObsCrossfilter.select(
+    "emb",
+    embName,
+    selection
+  );
+
+  dispatch({
+    type: "graph lasso end",
+    obsCrossfilter,
+    polygon,
+  });
+};
+
+/*
+Differential expression set selection
+*/
+export const setCellSetFromSelection = (cellSetId) => (dispatch, getState) => {
+  const { obsCrossfilter } = getState();
+  const selected = obsCrossfilter.allSelectedLabels();
+
+  dispatch({
+    type: `store current cell selection as differential set ${cellSetId}`,
+    data: selected.length > 0 ? selected : null,
+  });
+};

client/src/actions/viewStack.js

@@ -0,0 +1,76 @@
+/*
+The following actions manage the view stack for annoMatrix.
+
+Conventions used and assumed elsewhere in the code base:
+  * there will be zero or one clip view, and it will be the TOP view always.
+  * there will be zero or more subset views
+
+In other words, in our current use, we do not stack multiple clip views but we do
+stack multiple subsets.
+
+If these conventions change, code elsewhere (eg. menubar/clip.js) will need to
+change as well.
+*/
+import { AnnoMatrixObsCrossfilter } from "../annoMatrix";
+import {
+  _clipAnnoMatrix,
+  _userSubsetAnnoMatrix,
+  _userResetSubsetAnnoMatrix,
+} from "../util/stateManager/viewStackHelpers";
+
+export const clipAction = (min, max) => (dispatch, getState) => {
+  /*
+  apply a clip to the current annoMatrix.  By convention, the clip
+  view is ALWAYS the top view.
+  */
+  const { annoMatrix: prevAnnoMatrix } = getState();
+  const annoMatrix = _clipAnnoMatrix(prevAnnoMatrix, min, max);
+  const obsCrossfilter = new AnnoMatrixObsCrossfilter(annoMatrix);
+  dispatch({
+    type: "set clip quantiles",
+    clipQuantiles: { min, max },
+    annoMatrix,
+    obsCrossfilter,
+  });
+};
+
+export const subsetAction = () => (dispatch, getState) => {
+  /*
+  Subset the annoMatrix to the current crossfilter selection by pushing a 
+  subset view.
+
+  By convention, a clip view is ALWAYS the top view, so if present, pop
+  off and re-apply
+  */
+  const {
+    annoMatrix: prevAnnoMatrix,
+    obsCrossfilter: prevObsCrossfilter,
+  } = getState();
+  const annoMatrix = _userSubsetAnnoMatrix(
+    prevAnnoMatrix,
+    prevObsCrossfilter.allSelectedMask()
+  );
+  const obsCrossfilter = new AnnoMatrixObsCrossfilter(annoMatrix);
+  dispatch({
+    type: "subset to selection",
+    annoMatrix,
+    obsCrossfilter,
+  });
+};
+
+export const resetSubsetAction = () => (dispatch, getState) => {
+  /*
+  Reset the annoMatrix to all data.  Because we may have multiple views
+  stacked, we pop them all.  By convention, any clip transformation will
+  be the top of the stack, and must be preserved.
+  */
+
+  const { annoMatrix: prevAnnoMatrix } = getState();
+  const annoMatrix = _userResetSubsetAnnoMatrix(prevAnnoMatrix);
+  const obsCrossfilter = new AnnoMatrixObsCrossfilter(annoMatrix);
+  dispatch({
+    type: "reset subset",
+    annoMatrix,
+    obsCrossfilter,
+  });
+};

client/src/annoMatrix/annoMatrix.js

@@ -0,0 +1,658 @@
+import {
+  Dataframe,
+  IdentityInt32Index,
+  dataframeMemo,
+} from "../util/dataframe";
+import {
+  _getColumnDimensionNames,
+  _getColumnSchema,
+  _schemaColumns,
+  _getWritableColumns,
+} from "./schema";
+import { indexEntireSchema } from "../util/stateManager/schemaHelpers";
+import { _whereCacheGet, _whereCacheMerge } from "./whereCache";
+import _shallowClone from "./clone";
+
+const _dataframeCache = dataframeMemo(128);
+
+export default class AnnoMatrix {
+  /*
+  Abstract base class for all AnnoMatrix objects.  This class provides a proxy
+  to the annotated matrix data authoritatively served by the server/back-end.
+
+  AnnoMatrix instances are immutable, meaning that their schema and dimensionality
+  will not change, and simple object equality can be used to detect structural
+  changes. The actual data is cached, and not guaranteed to be present -- any
+  request to access data must be resolved by a fetch() call, which is async, and
+  may involve a server round-trip.
+
+  Guarantees made by the immutabilty, ie, any of these can be detected by
+  simple annoMatrix compare:
+    * schema is the same, including all fields and columns
+    * dimensionality is the same (nObs, nVar)
+    * data mapping/transformation, such as clipping, are the same
+
+  AnnoMatrixes also "stack" like filters, allowing for the construction of
+  views which transform the data in some manner.
+
+  The bootstrap class is AnnoMatrixLoader, which is the caching server proxy, and
+  is bootstrapped with a API URL:
+    new AnnoMatirx(url, schema) -> annoMatrix
+
+  There are various "views", such as AnnoMatrixRowSubsetView, which provide
+  the same interface but with a transformed view of the server data.  Utilities in
+  viewCreators.js can be used to create these views:
+      clip(annoMatrix, min, max) -> annoMatrix
+      subset(annoMatrix, rowLabels) -> annoMatrix
+  etc.
+  */
+  static fields() {
+    /*
+    return the fields present in the AnnoMatrix instance.
+    */
+    return ["obs", "var", "emb", "X"];
+  }
+
+  constructor(schema, nObs, nVar, rowIndex = null) {
+    /*
+    Private constructor - this is an abstract base class.  Do not use.
+    */
+
+    /*
+    Public instance fields:
+      * schema - the matrix schema.  IMPORTANT: always the entire schema, for the
+        base (unfiltered, unclipped, unsubset) annotated matrix, as the server
+        presents it.
+      * nObs, nVar - size of each dimension.  These will accurately reflect the
+        size of the current annoMatrix view.  For example, if you subset the view,
+        the nObs will be smaller.
+      * rowIndex - a rowIndex shared by all data on this view (ie, the list of cells).
+        The row index labels are as defined by the base dataset from the server.
+      * isView - true if this is a view, false if not.
+      * viewOf - pointer to parent annomatrix if a view, undefined/null if not a view.
+      * userFlags - container for any additional state a user of this API wants to hang
+        off of an annoMatrix, and have propagated by the (shallow) cloning protocol.
+    */
+    this.schema = indexEntireSchema(schema);
+    this.nObs = nObs;
+    this.nVar = nVar;
+    this.rowIndex = rowIndex || new IdentityInt32Index(nObs);
+    this.isView = false;
+    this.viewOf = undefined;
+    this.userFlags = {};
+
+    /*
+		Private instance variables.
+
+		These are caches - lazily loaded. The only guarantee is that if they
+		are loaded, they will conform to the schema & dimensionality constraints.
+
+		Do NOT use directly - instead, use the fetch() and preload() API.
+		*/
+    this._cache = {
+      obs: Dataframe.empty(this.rowIndex),
+      var: Dataframe.empty(this.rowIndex),
+      emb: Dataframe.empty(this.rowIndex),
+      X: Dataframe.empty(this.rowIndex),
+    };
+    this._pendingLoad = {
+      obs: {},
+      var: {},
+      emb: {},
+      X: {},
+    };
+    this._whereCache = {};
+    this._gcInfo = new Map();
+  }
+
+  /**
+   ** Schema helper/accessors
+   **/
+  getMatrixColumns(field) {
+    /*
+    Return array of column names in the field.  ONLY supported on the
+    obs, var and emb fields.  X currently unimplemented and will throw.
+
+    For exmaple:
+
+      annoMatrix.getMatrixColumns("obs") -> ["louvain", "n_genes"]
+    */
+    return _schemaColumns(this.schema, field);
+  }
+
+  // eslint-disable-next-line class-methods-use-this -- need to be able to call this on instances
+  getMatrixFields() {
+    /*
+    Return array of fields in this annoMatrix.  Currently hard-wired to
+    return:  ["X", "obs", "var", "emb"].
+
+    These are the fields from data may be requested.
+    */
+    return AnnoMatrix.fields();
+  }
+
+  getColumnSchema(field, col) {
+    /*
+    Return the schema for the field & column ,eg,
+
+      anonMatrix.getColumnSchema("obs", "n_genes") -> { type: "int32", name: "n_genes" }
+
+    This is identical to the information in the annoMatrix.schema
+    instance variable.
+    */
+    return _getColumnSchema(this.schema, field, col);
+  }
+
+  getColumnDimensions(field, col) {
+    /*
+    Return the dimensions on this field / column.  For most fields, which are 1D,
+    this just return the column name.  Multi-dimensional columns, such as embeddings,
+    will return >1 name.
+
+    Examples:
+
+      getColumnDimensions("obs", "louvain") -> ["louvain"]
+      getColumnDimensions("emb", "umap") -> ["umap_0", "umap_1"]
+
+    */
+    return _getColumnDimensionNames(this.schema, field, col);
+  }
+
+  /**
+   ** General utility methods
+   **/
+  base() {
+    /*
+    return the base of view, or `this` if not a view.
+    */
+    let annoMatrix = this;
+    while (annoMatrix.isView) annoMatrix = annoMatrix.viewOf;
+    return annoMatrix;
+  }
+
+  /**
+   ** Load / read interfaces
+   **/
+  fetch(field, q) {
+    /*
+		Return the given query on a single matrix field as a single dataframe.
+		Currently supports ONLY full column query.
+
+		Returns a Promise for the query result, which will resolve to a dataframe.
+
+		Field must be one of the matrix fields: 'obs', 'var', 'X', 'emb'.  Value
+		represents the underlying object upon which the query is occuring.
+
+		Query is one of:
+			* a string, representing a single column name from the field, eg,
+				"n_genes"
+			* an object, containing an "value" query (see below).
+			* an array, containing one or more of the above.
+
+		Columns may have more than one dimension, and all will be fetched
+		and returned together.  This is most commonly seen in an embedding,
+    which usually has two dimensions.
+
+		A value query allows for fetching based upon the value in another 
+		field/column, similar to a join.  Currently only supported on the var
+		dimension, allowing query of X columns by var value (eg, gene name)
+
+    The query filter	is a single value filter:
+			{ "field name": [
+				{name: "column name", values: [ list of values ]}
+			]}
+		One and only one value filter is allowed in a value query.
+
+		Examples:
+
+    1. Fetch the "n_genes" column the "obs":
+
+			const df = await fetch("obs", "n_genes")
+      console.log("Largest number of genes is: ", df.summarize().max);
+
+    2. Fetch two separate columns from obs.  Returns a single dataframe containing
+       the columns:
+
+			const df = await fetch("obs", ["n_genes", "louvain"])
+      console.log("Cell 0 has category: ", df.at(0, "louvain"));
+
+    3. Fetch an entire X (expression counts) column that has a var annotation
+       value "TYMP" in the var index.
+
+			fetch("X", { 
+				where: {field: "var", column: this.schema.annotations.var.index, value: "TYMP"}
+			})
+
+      In AnnData & Pandas DataFrame API, this is equivalent to:
+        adata.X[:, adata.var.index.get_loc("SUMO3")]
+
+		The value query is a recodification and subset of the server REST API 
+		value filter JSON.  Range queries and multiple filters are not currently
+		supported.
+
+		*/
+    return this._fetch(field, q);
+  }
+
+  prefetch(field, q) {
+    /*
+		Start a data fetch & cache fill.  Identical to fetch() except it does
+		not return a value.
+
+    Primary use is to being a cache load as early as is possible, reducing
+    overall component rendering latency.
+		*/
+    this._fetch(field, q);
+    return undefined;
+  }
+
+  /**
+   ** Save / mutate interfaces - manipulation of "writable" OBS annotations.
+   **
+   ** These are all present to support client-side creation of OBS annotations, aka
+   ** "user annotations".
+   **
+   ** They implement common manipulations to the AnnoMatrix, maintaining the
+   ** norma guarantees around correctness of public API, eg,
+   **   - schema will be correct, including the "writable" attribute
+   **   - fetch() will return the latest data, even from views
+   **   - immutability guranteeds
+   **
+   ** As most of these interfaces mutate the annoMatrix, they return a new
+   ** annoMatrix
+   **
+   ** The actual implementation is in the sub-classes, which MUST override these.
+   **/
+
+  // eslint-disable-next-line class-methods-use-this, no-unused-vars -- make sure subclass implements
+  addObsAnnoCategory(col, category) {
+    /*
+    Add a new category value (aka "label") to a writable obs column, and return the new AnnoMatrix.
+    Typical use is to add a new user-created label to a user-created obs categorical
+    annotation.
+
+    Will throw column does not exist or is not writable.
+
+    Example:
+
+      addObsAnnoCategory("my cell type", "left toenail") -> AnnoMatrix
+
+    */
+    _subclassResponsibility();
+  }
+
+  // eslint-disable-next-line class-methods-use-this, no-unused-vars -- make sure subclass implements
+  async removeObsAnnoCategory(col, category, unassignedCategory) {
+    /*
+    Remove a category value from an obs column, reassign any obs having that value
+    to the 'unassignedCategory' value, and return a promise for a new AnnoMatrix.
+    Typical use is to remove a user-created label from a user-created obs categorical
+    annotation.
+
+    Will throw column does not exist or is not writable.
+
+    An `unassignedCategory` value must be provided, for assignment to any obs/cells
+    that had the now-delete category label as their value.
+
+    Example:
+      await removeObsAnnoCategory("my-tissue-type", "right earlobe", "unassigned") -> AnnoMatrix
+
+    NOTE: method is async as it may need to fetch data to provide the reassignment.
+    */
+    _subclassResponsibility();
+  }
+
+  // eslint-disable-next-line class-methods-use-this, no-unused-vars -- make sure subclass implements
+  dropObsColumn(col) {
+    /*
+    Drop an entire writable column, eg a user-created obs annotation.  Typical use
+    is to provide the "Delete Category" implementation.  Returns the new AnnoMatrix.
+    Will throw if not a writable annotation.
+
+    Will throw column does not exist or is not writable.
+
+    Example:
+
+      dropObsColumn("old annotations") ->  AnnoMatrix
+    */
+    _subclassResponsibility();
+  }
+
+  // eslint-disable-next-line class-methods-use-this, no-unused-vars -- make sure subclass implements
+  addObsColumn(colSchema, Ctor, value) {
+    /*
+    Add a new writable OBS annotation column, with the caller-specified schema, initial value
+    type and value.
+
+    Value may be any one of:
+      * an array of values
+      * a primitive type, including null or undefined.
+    If an array, length must be the same as 'this.nObs', and constructor must equal 'Ctor'.
+    If a primitive, 'Ctor' will be used to create the initial value, which will be filled
+    with 'value'.
+
+    Throws if the name specified in 'colSchema' duplicates an existing obs column.
+
+    Returns a new AnnoMatrix.
+
+    Examples:
+
+      addObsColumn(
+        { name: "foo", type: "categorical", categories: "unassigned" },
+        Array,
+        "unassigned"
+      ) -> AnnoMatrix
+
+    */
+    _subclassResponsibility();
+  }
+
+  // eslint-disable-next-line class-methods-use-this, no-unused-vars -- make sure subclass implements
+  renameObsColumn(oldCol, newCol) {
+    /*
+    Rename the obs column 'oldCol' to have name 'newCol' and returns new AnnoMatrix.
+
+    Will throw column does not exist or is not writable, or if 'newCol' is not unique.
+
+    Example:
+
+      renameObsColumn('cell type', 'old cell type') -> AnnoMatrix.
+
+    */
+    _subclassResponsibility();
+  }
+
+  // eslint-disable-next-line class-methods-use-this, no-unused-vars -- make sure subclass implements
+  async setObsColumnValues(col, obsLabels, value) {
+    /*
+    Set all obs with label in array 'obsLabels' to have 'value'.  Typical use would be
+    to set a group of cells to have a label on a user-created categorical anntoation
+    (eg set all selected cells to have a label).
+
+    NOTE: async method, as it may need to fetch.
+
+    Will throw column does not exist or is not writable.
+
+    Example:
+      await setObsColmnValues("flavor", [383, 400], "tasty") -> AnnoMtarix
+
+    */
+    _subclassResponsibility();
+  }
+
+  // eslint-disable-next-line class-methods-use-this, no-unused-vars -- make sure subclass implements
+  async resetObsColumnValues(col, oldValue, newValue) {
+    /*
+    Set by value - all elements in the column with value 'oldValue' are set to 'newValue'.
+    Async method - returns a promise for a new AnnoMatrix.
+
+    Typical use would be to set all labels of one value to another.
+
+    Will throw column does not exist or is not writable.
+
+    Example:
+      await resetObsColumnValues("my notes", "good", "not-good") -> AnnoMatrix
+
+    */
+    _subclassResponsibility();
+  }
+
+  // eslint-disable-next-line class-methods-use-this, no-unused-vars -- make sure subclass implements
+  addEmbedding(colSchema) {
+    /*
+    Add a new obs embedding to the AnnoMatrix, with provided schema.
+    Returns a new annomatrix.
+
+    Typical use will be to add a re-embedding that the server has calculated.
+
+    Will throw if the column schema is invalid (eg, duplicate name).
+    */
+    _subclassResponsibility();
+  }
+
+  /**
+   ** Private interfaces below.
+   **/
+  _resolveCachedQueries(field, queries) {
+    return queries
+      .map((query) =>
+        _whereCacheGet(this._whereCache, this.schema, field, query).filter(
+          (cacheKey) =>
+            cacheKey !== undefined && this._cache[field].hasCol(cacheKey)
+        )
+      )
+      .flat();
+  }
+
+  async _fetch(field, q) {
+    if (!AnnoMatrix.fields().includes(field)) return undefined;
+    const queries = Array.isArray(q) ? q : [q];
+
+    /* find cached columns we need, and GC the rest */
+    const cachedColumns = this._resolveCachedQueries(field, queries);
+    this._gcFetchCleanup(field, cachedColumns);
+
+    /* find any query not already cached */
+    const uncachedQueries = queries.filter((query) =>
+      _whereCacheGet(this._whereCache, this.schema, field, query).some(
+        (cacheKey) =>
+          cacheKey === undefined || !this._cache[field].hasCol(cacheKey)
+      )
+    );
+
+    /* load uncached queries */
+    if (uncachedQueries.length > 0) {
+      await Promise.all(
+        uncachedQueries.map((query) =>
+          this._getPendingLoad(field, query, async (_field, _query) => {
+            /* fetch, then index.  _doLoad is subclass interface */
+            const [whereCacheUpdate, df] = await this._doLoad(_field, _query);
+            this._cache[_field] = this._cache[_field].withColsFrom(df);
+            this._whereCache = _whereCacheMerge(
+              this._whereCache,
+              whereCacheUpdate
+            );
+          })
+        )
+      );
+    }
+
+    /* everything we need is in the cache, so just cherry-pick requested columns */
+    const requestedCacheKeys = this._resolveCachedQueries(field, queries);
+    const response = _dataframeCache(
+      this._cache[field].subset(null, requestedCacheKeys)
+    );
+    this._gcUpdateStats(field, response);
+    return response;
+  }
+
+  async _getPendingLoad(field, query, fetchFn) {
+    /*
+    Given a query on a field, ensure that we only have a single outstanding
+    fetch at any given time.  If multiple requests occur while a fetch is
+    outstanding, just wait for the original.
+
+    This is implemented by returning a promise that will await the singular
+    fetch promise.
+    */
+    const key = _queryCacheKey(field, query);
+    if (!this._pendingLoad[field][key]) {
+      this._pendingLoad[field][key] = fetchFn(field, query);
+      try {
+        await this._pendingLoad[field][key];
+      } finally {
+        delete this._pendingLoad[field][key];
+      }
+    }
+    return this._pendingLoad[field][key];
+  }
+
+  // eslint-disable-next-line class-methods-use-this -- make sure subclass implements
+  async _doLoad() {
+    _subclassResponsibility();
+  }
+
+  /**
+   ** Garbage collection of annomatrix cache to manage memory use.
+   **/
+
+  /*
+  These callbacks implement a GC policy for the cache.  Background:
+
+    * For the Loader (base) annomatrix, re-filling the cache is expensive as
+      it requires an HTTP fetch.
+    * user-defined / writable columns must not be GC'ed as they may be
+      still pending a save/commit.
+    * For views, cost is less and (roughly) proportional with nObs
+    * obs, var and emb do not grow without bounds, and are needed constantly 
+      for rendering.  
+      a) There is no upside to GC'ing these in the base (loader)
+      b) The undo/redo cache can hold a large number in views, which is worht GC'ing
+    * X is often much larger than memory, and the UI allows add/del from
+      this.  Most of the GC potential is here in both the base and views.
+
+  Current policy:
+    * if in active use ("hot") do not GC obs, var or emb.
+    * never, ever GC writable obs columns
+    * For base/loader set a numeric limit on maximum X column count
+    * For views, apply a fixed limit to the number of columns cached in any field.
+      Limit will be lower if not hot.
+
+  To be effective, the GC callback needs to be invoked from the undo/redo code,
+  as much of the cache is pinned by that data structure.
+  */
+  _gcField(field, isHot, pinnedColumns) {
+    const maxColumns = isHot ? 256 : 10; // maybe to aggessive?
+
+    const cache = this._cache[field];
+    if (cache.colIndex.size() < maxColumns) return; // trivial rejection
+
+    const candidates = cache.colIndex
+      .labels()
+      .filter((col) => !pinnedColumns.includes(col));
+
+    const excessCount = candidates.length + pinnedColumns.length - maxColumns;
+    if (excessCount > 0) {
+      const { _gcInfo } = this;
+      candidates.sort((a, b) => {
+        let atime = _gcInfo.get(_columnCacheKey(field, a));
+        if (atime === undefined) atime = 0;
+
+        let btime = _gcInfo.get(_columnCacheKey(field, b));
+        if (btime === undefined) btime = 0;
+
+        return atime - btime;
+      });
+
+      const toDrop = candidates.slice(0, excessCount);
+      // helpful debugging - please leave in place.
+      // console.log(
+      //   `GC: dropping from ${field} hot:${isHot}, columns [${toDrop.join(
+      //     ", "
+      //   )}]`
+      // );
+      this._cache[field] = toDrop.reduce(
+        (df, col) => df.dropCol(col),
+        this._cache[field]
+      );
+      toDrop.forEach((col) => _gcInfo.delete(_columnCacheKey(field, col)));
+    }
+  }
+
+  _gcFetchCleanup(field, pinnedColumns) {
+    /*
+    Called during data load/fetch.  By definition, this is 'hot', so we
+    only want to gc X.
+    */
+    if (field === "X") {
+      this._gcField(
+        field,
+        true,
+        pinnedColumns.concat(_getWritableColumns(this.schema, field))
+      );
+    }
+  }
+
+  _gc(hints) {
+    /*
+    Called from middleware, or elsewhere.  isHot is true if we are in the active store, 
+    or false if we are in some other context (eg, history state).
+    */
+    const { isHot } = hints;
+    const candidateFields = isHot ? ["X"] : ["X", "emb", "var", "obs"];
+    candidateFields.forEach((field) =>
+      this._gcField(field, isHot, _getWritableColumns(this.schema, field))
+    );
+  }
+
+  _gcUpdateStats(field, dataframe) {
+    /*
+    called each time a query is performed, allowing the gc to update any bookkeeping
+    information.  Currently, this is just a simple last-fetched timestamp, stored
+    in a Map.
+
+    Map objects preserve order of insertion. This is leveraged as a cheap way to
+    do LRU, by removing and re-inserting keys.  IMPORTANT: the cleanup code assumes
+    the map insertion order is least-recently-used first.
+    */
+    const cols = dataframe.colIndex.labels();
+    const { _gcInfo } = this;
+    const now = Date.now();
+    cols.forEach((c) => {
+      // gcInfo.delete(c);
+      _gcInfo.set(_columnCacheKey(field, c), now);
+    });
+  }
+
+  /**
+  Cloning sublcass protocol - we rely in cloning to preserve immutable
+  symantics while not causing races or other side effects in internal
+  cache management.
+
+  Subclasses must override _cloneDeeper() if they have state which requires 
+  something other than a shallow copy.  Overrides MUST call super()._cloneDeepr(), 
+  and return its result (after any required modification).  _cloneDeeper()
+  will be called on the OLD object, with the NEW object as an argument.
+
+  Do not override _clone();
+  **/
+  _cloneDeeper(clone) {
+    clone._cache = _shallowClone(this._cache);
+    clone._gcInfo = new Map();
+    clone._pendingLoad = {
+      obs: {},
+      var: {},
+      emb: {},
+      X: {},
+    };
+    return clone;
+  }
+
+  _clone() {
+    const clone = _shallowClone(this);
+    this._cloneDeeper(clone);
+    Object.seal(clone);
+    return clone;
+  }
+}
+
+/*
+private utility functions below
+*/
+
+function _queryCacheKey(field, query) {
+  if (typeof query === "object") {
+    const { field: queryField, column: queryColumn, value: queryValue } = query;
+    return `${field}/${queryField}/${queryColumn}/${queryValue}`;
+  }
+  return `${field}/${query}`;
+}
+
+function _columnCacheKey(field, column) {
+  return `${field}/${column}`;
+}
+
+function _subclassResponsibility() {
+  /* protect against bugs in subclass */
+  throw new Error("subclass failed to implement required method");
+}

client/src/annoMatrix/clone.js

@@ -0,0 +1,6 @@
+/*
+Shallow clone an object, correctly handling prototype
+*/
+export default function _shallowClone(orig) {
+  return Object.assign(Object.create(Object.getPrototypeOf(orig)), orig);
+}

client/src/annoMatrix/crossfilter.js

@@ -0,0 +1,268 @@
+/*
+Row crossfilter proxy for an AnnoMatrix.  This wraps Crossfilter,
+providing a number of services, and ensuring that the crossfilter and
+AnnoMatrix stay in sync:
+	- on-demand index creation as data is loaded
+	- transparently mapping between queries and crossfilter index names.
+  - for mutation of the matrix by user annotations, maintain synchronization
+    between Crossfilter and AnnoMatrix.
+*/
+import Crossfilter from "../util/typedCrossfilter";
+import { _getColumnSchema } from "./schema";
+
+function _dimensionNameFromDf(field, df) {
+  const colNames = df.colIndex.labels();
+  return _dimensionName(field, colNames);
+}
+
+function _dimensionName(field, colNames) {
+  if (!Array.isArray(colNames)) return `${field}/${colNames}`;
+  return `${field}/${colNames.join(":")}`;
+}
+
+export default class AnnoMatrixObsCrossfilter {
+  constructor(annoMatrix, _obsCrossfilter = null) {
+    this.annoMatrix = annoMatrix;
+    this.obsCrossfilter =
+      _obsCrossfilter || new Crossfilter(annoMatrix._cache.obs);
+    this.obsCrossfilter = this.obsCrossfilter.setData(annoMatrix._cache.obs);
+  }
+
+  size() {
+    return this.obsCrossfilter.size();
+  }
+
+  /**
+  Managing the associated annoMatrix.  These wrappers are necessary to 
+  make coordinated changes to BOTH the crossfilter and annoMatrix, and
+  ensure that all state stays synchronized.
+
+  See API documentation in annoMatrix.js.
+  **/
+  addObsColumn(colSchema, Ctor, value) {
+    const annoMatrix = this.annoMatrix.addObsColumn(colSchema, Ctor, value);
+    const obsCrossfilter = this.obsCrossfilter.setData(annoMatrix._cache.obs);
+    return new AnnoMatrixObsCrossfilter(annoMatrix, obsCrossfilter);
+  }
+
+  dropObsColumn(col) {
+    const annoMatrix = this.annoMatrix.dropObsColumn(col);
+    let { obsCrossfilter } = this;
+    const dimName = _dimensionName("obs", col);
+    if (obsCrossfilter.hasDimension(dimName)) {
+      obsCrossfilter = obsCrossfilter.delDimension(dimName);
+    }
+    return new AnnoMatrixObsCrossfilter(annoMatrix, obsCrossfilter);
+  }
+
+  renameObsColumn(oldCol, newCol) {
+    const annoMatrix = this.annoMatrix.renameObsColumn(oldCol, newCol);
+    const oldDimName = _dimensionName("obs", oldCol);
+    const newDimName = _dimensionName("obs", newCol);
+    let { obsCrossfilter } = this;
+    if (obsCrossfilter.hasDimension(oldDimName)) {
+      obsCrossfilter = obsCrossfilter.renameDimension(oldDimName, newDimName);
+    }
+    return new AnnoMatrixObsCrossfilter(annoMatrix, obsCrossfilter);
+  }
+
+  addObsAnnoCategory(col, category) {
+    const annoMatrix = this.annoMatrix.addObsAnnoCategory(col, category);
+    const dimName = _dimensionName("obs", col);
+    let { obsCrossfilter } = this;
+    if (obsCrossfilter.hasDimension(dimName)) {
+      obsCrossfilter = obsCrossfilter.delDimension(dimName);
+    }
+    return new AnnoMatrixObsCrossfilter(annoMatrix, obsCrossfilter);
+  }
+
+  async removeObsAnnoCategory(col, category, unassignedCategory) {
+    const annoMatrix = await this.annoMatrix.removeObsAnnoCategory(
+      col,
+      category,
+      unassignedCategory
+    );
+    const dimName = _dimensionName("obs", col);
+    let { obsCrossfilter } = this;
+    if (obsCrossfilter.hasDimension(dimName)) {
+      obsCrossfilter = obsCrossfilter.delDimension(dimName);
+    }
+    return new AnnoMatrixObsCrossfilter(annoMatrix, obsCrossfilter);
+  }
+
+  async setObsColumnValues(col, rowLabels, value) {
+    const annoMatrix = await this.annoMatrix.setObsColumnValues(
+      col,
+      rowLabels,
+      value
+    );
+    const dimName = _dimensionName("obs", col);
+    let { obsCrossfilter } = this;
+    if (obsCrossfilter.hasDimension(dimName)) {
+      obsCrossfilter = obsCrossfilter.delDimension(dimName);
+    }
+    return new AnnoMatrixObsCrossfilter(annoMatrix, obsCrossfilter);
+  }
+
+  async resetObsColumnValues(col, oldValue, newValue) {
+    const annoMatrix = await this.annoMatrix.resetObsColumnValues(
+      col,
+      oldValue,
+      newValue
+    );
+    const dimName = _dimensionName("obs", col);
+    let { obsCrossfilter } = this;
+    if (obsCrossfilter.hasDimension(dimName)) {
+      obsCrossfilter = obsCrossfilter.delDimension(dimName);
+    }
+    return new AnnoMatrixObsCrossfilter(annoMatrix, obsCrossfilter);
+  }
+
+  addEmbedding(colSchema) {
+    const annoMatrix = this.annoMatrix.addEmbedding(colSchema);
+    return new AnnoMatrixObsCrossfilter(annoMatrix, this.obsCrossfilter);
+  }
+
+  /**
+  Selection state - API is identical to ImmutableTypedCrossfilter, as these
+  are just wrappers to lazy create indices.
+  **/
+
+  async select(field, query, spec) {
+    const { annoMatrix } = this;
+    let { obsCrossfilter } = this;
+
+    if (!annoMatrix?._cache?.[field]) {
+      throw new Error("Unknown field name");
+    }
+    if (field === "var") {
+      throw new Error("unable to obsSelect upon the var dimension");
+    }
+
+    // grab the data, so we can grab the index.
+    const df = await annoMatrix.fetch(field, query);
+
+    const dimName = _dimensionNameFromDf(field, df);
+    if (!obsCrossfilter.hasDimension(dimName)) {
+      // lazy index generation - add dimension when first used
+      obsCrossfilter = this._addObsCrossfilterDimension(
+        annoMatrix,
+        obsCrossfilter,
+        field,
+        df
+      );
+    }
+
+    // select
+    obsCrossfilter = obsCrossfilter.select(dimName, spec);
+    return new AnnoMatrixObsCrossfilter(annoMatrix, obsCrossfilter);
+  }
+
+  selectAll() {
+    /*
+		Select all on any dimension in this field.
+		*/
+    const { annoMatrix } = this;
+    const currentDims = this.obsCrossfilter.dimensionNames();
+    const obsCrossfilter = currentDims.reduce((xfltr, dim) => {
+      return xfltr.select(dim, { mode: "all" });
+    }, this.obsCrossfilter);
+    return new AnnoMatrixObsCrossfilter(annoMatrix, obsCrossfilter);
+  }
+
+  countSelected() {
+    /* if no data yet indexed in the crossfilter, just say everything is selected */
+    if (this.obsCrossfilter.size() === 0) return this.annoMatrix.nObs;
+    return this.obsCrossfilter.countSelected();
+  }
+
+  allSelectedMask() {
+    /* if no data yet indexed in the crossfilter, just say everything is selected */
+    if (
+      this.obsCrossfilter.size() === 0 ||
+      this.obsCrossfilter.dimensionNames().length === 0
+    ) {
+      /* fake the mask */
+      return new Uint8Array(this.annoMatrix.nObs).fill(1);
+    }
+    return this.obsCrossfilter.allSelectedMask();
+  }
+
+  allSelectedLabels() {
+    /* if no data yet indexed in the crossfilter, just say everything is selected */
+    if (
+      this.obsCrossfilter.size() === 0 ||
+      this.obsCrossfilter.dimensionNames().length === 0
+    ) {
+      return this.annoMatrix.rowIndex.labels();
+    }
+
+    const mask = this.obsCrossfilter.allSelectedMask();
+    const index = this.annoMatrix.rowIndex.isubsetMask(mask);
+    return index.labels();
+  }
+
+  fillByIsSelected(array, selectedValue, deselectedValue) {
+    /* if no data yet indexed in the crossfilter, just say everything is selected */
+    if (
+      this.obsCrossfilter.size() === 0 ||
+      this.obsCrossfilter.dimensionNames().length === 0
+    ) {
+      return array.fill(selectedValue);
+    }
+    return this.obsCrossfilter.fillByIsSelected(
+      array,
+      selectedValue,
+      deselectedValue
+    );
+  }
+
+  /**
+   ** Private below
+   **/
+
+  _addObsCrossfilterDimension(annoMatrix, obsCrossfilter, field, df) {
+    if (field === "var") return obsCrossfilter;
+    const dimName = _dimensionNameFromDf(field, df);
+    const dimParams = this._getObsDimensionParams(field, df);
+    obsCrossfilter = obsCrossfilter.setData(annoMatrix._cache.obs);
+    obsCrossfilter = obsCrossfilter.addDimension(dimName, ...dimParams);
+    return obsCrossfilter;
+  }
+
+  _getColumnBaseType(field, col) {
+    /* Look up the primitive type for this field/col */
+    const colSchema = _getColumnSchema(this.annoMatrix.schema, field, col);
+    return colSchema.type;
+  }
+
+  _getObsDimensionParams(field, df) {
+    /* return the crossfilter dimensiontype type and params for this field/dataframe */
+
+    if (field === "emb") {
+      /* assumed to be 2D */
+      return ["spatial", df.icol(0).asArray(), df.icol(1).asArray()];
+    }
+
+    /* assumed to be 1D */
+    const col = df.icol(0);
+    const colName = df.colIndex.getLabel(0);
+    const type = this._getColumnBaseType(field, colName);
+    if (type === "string" || type === "categorical" || type === "boolean") {
+      return ["enum", col.asArray()];
+    }
+    if (type === "int32") {
+      return ["scalar", col.asArray(), Int32Array];
+    }
+    if (type === "float32") {
+      return ["scalar", col.asArray(), Float32Array];
+    }
+    // Currently not supporting boolean and categorical types.
+    console.error(
+      `Warning - unknown metadata schema (${type}) for field ${field} ${colName}.`
+    );
+    // skip it - we don't know what to do with this type
+
+    return undefined;
+  }
+}

client/src/annoMatrix/fetchHelpers.js

@@ -0,0 +1,27 @@
+export { doBinaryRequest } from "../util/actionHelpers";
+
+/* double URI encode - needed for query-param filters */
+export function _dubEncURIComp(s) {
+  return encodeURIComponent(encodeURIComponent(s));
+}
+
+/* currently unused, consider deleting */
+export function _fetchResult(promise) {
+  let _status = "pending";
+  const res = promise.then(
+    (r) => {
+      _status = "success";
+      return r;
+    },
+    (e) => {
+      _status = "error";
+      throw e;
+    }
+  );
+
+  res.status = () => {
+    return _status;
+  };
+
+  return res;
+}

client/src/annoMatrix/index.js

@@ -0,0 +1,15 @@
+/*
+AnnoMatrix -- Annotated Matrix exported interface
+
+Public API is defined in:
+
+	annoMatrix.js
+	viewCreators.js
+	crossfilter.js
+
+*/
+
+export { default as AnnoMatrixLoader } from "./loader";
+export * from "./viewCreators";
+export { default as AnnoMatrixObsCrossfilter } from "./crossfilter";
+export { default as gcMiddleware } from "./middleware";

client/src/annoMatrix/loader.js

@@ -0,0 +1,307 @@
+import { doBinaryRequest, _dubEncURIComp } from "./fetchHelpers";
+import { matrixFBSToDataframe } from "../util/stateManager/matrix";
+import { _getColumnSchema, _normalizeCategoricalSchema } from "./schema";
+import {
+  addObsAnnoColumn,
+  removeObsAnnoColumn,
+  addObsAnnoCategory,
+  removeObsAnnoCategory,
+  addObsLayout,
+} from "../util/stateManager/schemaHelpers";
+import { isArrayOrTypedArray } from "../util/typeHelpers";
+import { _whereCacheCreate } from "./whereCache";
+import AnnoMatrix from "./annoMatrix";
+import PromiseLimit from "../util/promiseLimit";
+
+const promiseThrottle = new PromiseLimit(5);
+
+export default class AnnoMatrixLoader extends AnnoMatrix {
+  /*
+  AnnoMatrix implementation which proxies to HTTP server using the CXG REST API.
+  Used as the base (non-view) instance.
+
+  Public API is same as AnnoMatrix class (refer there for API description),
+  with the addition of the constructor which bootstraps:
+
+    new AnnoMatrixLoader(serverBaseURL, schema) -> instance
+
+  */
+  constructor(baseURL, schema) {
+    const { nObs, nVar } = schema.dataframe;
+    super(schema, nObs, nVar);
+
+    if (baseURL[baseURL.length - 1] !== "/") {
+      // must have trailing slash
+      baseURL += "/";
+    }
+    this.baseURL = baseURL;
+    Object.seal(this);
+  }
+
+  /**
+   ** Public.  API described in base class.
+   **/
+  addObsAnnoCategory(col, category) {
+    /*
+    Add a new category (aka label) to the schema for an obs column.
+    */
+    const colSchema = _getColumnSchema(this.schema, "obs", col);
+    _writableCategoryTypeCheck(colSchema); // throws on error
+
+    const newAnnoMatrix = this._clone();
+    newAnnoMatrix.schema = addObsAnnoCategory(this.schema, col, category);
+    return newAnnoMatrix;
+  }
+
+  async removeObsAnnoCategory(col, category, unassignedCategory) {
+    /*
+    Remove a single "category" (aka "label") from the data & schema of an obs column.
+    */
+    const colSchema = _getColumnSchema(this.schema, "obs", col);
+    _writableCategoryTypeCheck(colSchema); // throws on error
+
+    const newAnnoMatrix = await this.resetObsColumnValues(
+      col,
+      category,
+      unassignedCategory
+    );
+    newAnnoMatrix.schema = removeObsAnnoCategory(
+      newAnnoMatrix.schema,
+      col,
+      category
+    );
+    return newAnnoMatrix;
+  }
+
+  dropObsColumn(col) {
+    /*
+		drop column from field
+		*/
+    const colSchema = _getColumnSchema(this.schema, "obs", col);
+    _writableCheck(colSchema); // throws on error
+
+    const newAnnoMatrix = this._clone();
+    newAnnoMatrix._cache.obs = this._cache.obs.dropCol(col);
+    newAnnoMatrix.schema = removeObsAnnoColumn(this.schema, col);
+    return newAnnoMatrix;
+  }
+
+  addObsColumn(colSchema, Ctor, value) {
+    /*
+		add a column to field, initializing with value.  Value may 
+    be one of:
+      * an array of values
+      * a primitive type, including null or undefined.
+    If an array, it must be of same size as nObs and same type as Ctor
+		*/
+    colSchema.writable = true;
+    const colName = colSchema.name;
+    if (
+      _getColumnSchema(this.schema, "obs", colName) ||
+      this._cache.obs.hasCol(colName)
+    ) {
+      throw new Error("column already exists");
+    }
+
+    const newAnnoMatrix = this._clone();
+    let data;
+    if (isArrayOrTypedArray(value)) {
+      if (value.constructor !== Ctor)
+        throw new Error("Mismatched value array type");
+      if (value.length !== this.nObs)
+        throw new Error("Value array has incorrect length");
+      data = value.slice();
+    } else {
+      data = new Ctor(this.nObs).fill(value);
+    }
+    newAnnoMatrix._cache.obs = this._cache.obs.withCol(colName, data);
+    _normalizeCategoricalSchema(
+      colSchema,
+      newAnnoMatrix._cache.obs.col(colName)
+    );
+    newAnnoMatrix.schema = addObsAnnoColumn(this.schema, colName, colSchema);
+    return newAnnoMatrix;
+  }
+
+  renameObsColumn(oldCol, newCol) {
+    /*
+    Rename the obs oldColName to newColName.  oldCol must be writable.
+    */
+    const oldColSchema = _getColumnSchema(this.schema, "obs", oldCol);
+    _writableCheck(oldColSchema); // throws on error
+
+    const value = this._cache.obs.hasCol(oldCol)
+      ? this._cache.obs.col(oldCol).asArray()
+      : undefined;
+    return this.dropObsColumn(oldCol).addObsColumn(
+      {
+        ...oldColSchema,
+        name: newCol,
+      },
+      value.constructor,
+      value
+    );
+  }
+
+  async setObsColumnValues(col, rowLabels, value) {
+    /*
+		Set all rows identified by rowLabels to value.
+		*/
+    const colSchema = _getColumnSchema(this.schema, "obs", col);
+    _writableCategoryTypeCheck(colSchema); // throws on error
+
+    // ensure that we have the data in cache before we manipulate it
+    await this.fetch("obs", col);
+    if (!this._cache.obs.hasCol(col))
+      throw new Error("Internal error - user annotation data missing");
+
+    const rowIndices = this.rowIndex.getOffsets(rowLabels);
+    const data = this._cache.obs.col(col).asArray().slice();
+    for (let i = 0, len = rowIndices.length; i < len; i += 1) {
+      const idx = rowIndices[i];
+      if (idx === undefined) throw new Error("Unknown row label");
+      data[idx] = value;
+    }
+
+    const newAnnoMatrix = this._clone();
+    newAnnoMatrix._cache.obs = this._cache.obs.replaceColData(col, data);
+    const { categories } = colSchema;
+    if (!categories?.includes(value)) {
+      newAnnoMatrix.schema = addObsAnnoCategory(this.schema, col, value);
+    }
+    return newAnnoMatrix;
+  }
+
+  async resetObsColumnValues(col, oldValue, newValue) {
+    /*
+    Set all rows with value 'oldValue' to 'newValue'.
+    */
+    const colSchema = _getColumnSchema(this.schema, "obs", col);
+    _writableCategoryTypeCheck(colSchema); // throws on error
+
+    if (!colSchema.categories.includes(oldValue)) {
+      throw new Error("unknown category");
+    }
+
+    // ensure that we have the data in cache before we manipulate it
+    await this.fetch("obs", col);
+    if (!this._cache.obs.hasCol(col))
+      throw new Error("Internal error - user annotation data missing");
+
+    const data = this._cache.obs.col(col).asArray().slice();
+    for (let i = 0, l = data.length; i < l; i += 1) {
+      if (data[i] === oldValue) data[i] = newValue;
+    }
+
+    const newAnnoMatrix = this._clone();
+    newAnnoMatrix._cache.obs = this._cache.obs.replaceColData(col, data);
+    const { categories } = colSchema;
+    if (!categories?.includes(newValue)) {
+      newAnnoMatrix.schema = addObsAnnoCategory(this.schema, col, newValue);
+    }
+    return newAnnoMatrix;
+  }
+
+  addEmbedding(colSchema) {
+    /*
+    add new layout to the obs embeddings
+    */
+    const { name: colName } = colSchema;
+    if (_getColumnSchema(this.schema, "emb", colName)) {
+      throw new Error("column already exists");
+    }
+
+    const newAnnoMatrix = this._clone();
+    newAnnoMatrix.schema = addObsLayout(this.schema, colSchema);
+    return newAnnoMatrix;
+  }
+
+  /**
+   ** Private below
+   **/
+  async _doLoad(field, query) {
+    /*
+    _doLoad - evaluates the query against the field. Returns:
+      * whereCache update: column query map mapping the query to the column labels
+      * Dataframe containing the new colums (one per dimension)
+    */
+    let urlQuery;
+    let urlBase;
+    let priority = 10; // default fetch priority
+
+    switch (field) {
+      case "obs":
+      case "var": {
+        urlBase = `${this.baseURL}annotations/${field}`;
+        urlQuery = _encodeQuery("annotation-name", query);
+        break;
+      }
+      case "X": {
+        urlBase = `${this.baseURL}data/var`;
+        urlQuery = _encodeQuery(undefined, query);
+        break;
+      }
+      case "emb": {
+        urlBase = `${this.baseURL}layout/obs`;
+        urlQuery = _encodeQuery("layout-name", query);
+        priority = 0; // high prio load for embeddings
+        break;
+      }
+      default:
+        throw new Error("Unknown field name");
+    }
+
+    const url = `${urlBase}?${urlQuery}`;
+    const buffer = await promiseThrottle.priorityAdd(
+      priority,
+      doBinaryRequest,
+      url
+    );
+    const result = matrixFBSToDataframe(buffer);
+    if (!result || result.isEmpty()) throw Error("Unknown field/col");
+
+    const whereCacheUpdate = _whereCacheCreate(
+      field,
+      query,
+      result.colIndex.labels()
+    );
+
+    if (field === "obs") {
+      /* cough, cough - see comment on method */
+      _normalizeCategoricalSchema(
+        this.schema.annotations.obsByName[query],
+        result.col(query)
+      );
+    }
+
+    return [whereCacheUpdate, result];
+  }
+}
+
+/*
+Utility functions below
+*/
+
+function _encodeQuery(colKey, q) {
+  if (typeof q === "object") {
+    const { field: queryField, column: queryColumn, value: queryValue } = q;
+    return `${_dubEncURIComp(queryField)}:${_dubEncURIComp(
+      queryColumn
+    )}=${_dubEncURIComp(queryValue)}`;
+  }
+  if (!colKey) throw new Error("Unsupported query by name");
+  return `${colKey}=${encodeURIComponent(q)}`;
+}
+
+function _writableCheck(colSchema) {
+  if (!colSchema?.writable) {
+    throw new Error("Unknown or readonly obs column");
+  }
+}
+
+function _writableCategoryTypeCheck(colSchema) {
+  _writableCheck(colSchema);
+  if (colSchema.type !== "categorical") {
+    throw new Error("column must be categorical");
+  }
+}

client/src/annoMatrix/middleware.js

@@ -0,0 +1,66 @@
+/*
+Garbage collection / cache management support
+
+Middleware that knows how to pull annoMatrix from the undoable state,
+and pass it along to the AnnoMatrix class for possible cache GC.
+
+Private interface.
+
+Future work item: this middleware knows internal details of both the
+Undoable metareducer and the AnnoMatrix private API.  It would be helpful
+to make the Undoable interface better factored.
+*/
+
+const annoMatrixGC = (store) => (next) => (action) => {
+  if (_itIsTimeForGC()) {
+    _doGC(store);
+  }
+  return next(action);
+};
+
+let lastGCTime = 0;
+const InterGCDelayMS = 30 * 1000; // 30 seconds
+function _itIsTimeForGC() {
+  /*
+  we don't want to run GC on every dispatch, so throttle it a bit.
+
+  Runs every InterGCDelay period
+  */
+  const now = Date.now();
+  if (now - lastGCTime > InterGCDelayMS) {
+    lastGCTime = now;
+    return true;
+  }
+  return false;
+}
+
+function _doGC(store) {
+  const state = store.getState();
+
+  // these should probably be a function imported from undoable.js, etc, as
+  // they have overly intimiate knowledge of our reducers.
+  const undoablePast = state["@@undoable/past"];
+  const undoableFuture = state["@@undoable/future"];
+  const undoableStack = undoablePast
+    .concat(undoableFuture)
+    .flatMap((snapshot) =>
+      snapshot.filter((v) => v[0] === "annoMatrix").map((v) => v[1])
+    );
+  const currentAnnoMatrix = state.annoMatrix;
+
+  /*
+  We want to identify those matrixes currently "hot", ie, linked from the current annoMatrix,
+  as our current gc algo is more aggressive with those not hot.
+  */
+  const allAnnoMatrices = new Map(
+    undoableStack.map((m) => [m, { isHot: false }])
+  );
+  let am = currentAnnoMatrix;
+  while (am) {
+    allAnnoMatrices.set(am, { isHot: true });
+    am = am.viewOf;
+  }
+  allAnnoMatrices.forEach((hints, annoMatrix) => annoMatrix._gc(hints));
+}
+
+export default annoMatrixGC;

client/src/annoMatrix/schema.js

@@ -0,0 +1,88 @@
+/*
+Private helper functions related to schema
+*/
+import catLabelSort from "../util/catLabelSort";
+import { unassignedCategoryLabel } from "../globals";
+
+export function _getColumnSchema(schema, field, col) {
+  /* look up the column definition */
+  switch (field) {
+    case "obs":
+      if (typeof col === "object")
+        throw new Error("unable to get column schema by query");
+      return schema.annotations.obsByName[col];
+    case "var":
+      if (typeof col === "object")
+        throw new Error("unable to get column schema by query");
+      return schema.annotations.varByName[col];
+    case "emb":
+      if (typeof col === "object")
+        throw new Error("unable to get column schema by query");
+      return schema.layout.obsByName[col];
+    case "X":
+      return schema.dataframe;
+    default:
+      throw new Error(`unknown field name: ${field}`);
+  }
+}
+
+export function _getColumnDimensionNames(schema, field, col) {
+  /*
+		field/col may be an alias for multiple columns. Currently used to map ND 
+		values to 1D dataframe columns for embeddings/layout. Signfied by the presence
+		of the "dims" value in the schema.
+		*/
+  const colSchema = _getColumnSchema(schema, field, col);
+  if (!colSchema) {
+    return undefined;
+  }
+  return colSchema.dims || [col];
+}
+
+export function _schemaColumns(schema, field) {
+  switch (field) {
+    case "obs":
+      return Object.keys(schema.annotations.obsByName);
+    case "var":
+      return Object.keys(schema.annotations.varByName);
+    case "emb":
+      return Object.keys(schema.layout.obsByName);
+    default:
+      throw new Error(`unknown field name: ${field}`);
+  }
+}
+
+export function _getWritableColumns(schema, field) {
+  if (field !== "obs") return [];
+  return schema.annotations.obs.columns
+    .filter((v) => v.writable)
+    .map((v) => v.name);
+}
+
+export function _isContinuousType(schema) {
+  const { type } = schema;
+  return !(type === "string" || type === "boolean" || type === "categorical");
+}
+
+export function _normalizeCategoricalSchema(colSchema, col) {
+  const { type, writable } = colSchema;
+  if (
+    type === "string" ||
+    type === "boolean" ||
+    type === "categorical" ||
+    writable
+  ) {
+    const categorySet = new Set(
+      col.summarizeCategorical().categories.concat(colSchema.categories ?? [])
+    );
+    if (writable && !categorySet.has(unassignedCategoryLabel)) {
+      categorySet.add(unassignedCategoryLabel);
+    }
+    colSchema.categories = Array.from(categorySet);
+  }
+
+  if (colSchema.categories) {
+    colSchema.categories = catLabelSort(writable, colSchema.categories);
+  }
+  return colSchema;
+}

client/src/annoMatrix/viewCreators.js

@@ -0,0 +1,70 @@
+/*
+View creators.  These are helper functions which create new views from existing
+instances of AnnoMatrix, implementing common UI functions.
+*/
+
+import { AnnoMatrixRowSubsetView, AnnoMatrixClipView } from "./views";
+
+export function isubsetMask(annoMatrix, obsMask) {
+  /*
+		Subset annomatrix to contain the rows which have truish value in the mask.
+    Maks length must equal annoMatrix.nObs (row count).
+	*/
+  return isubset(annoMatrix, _maskToList(obsMask));
+}
+
+export function isubset(annoMatrix, obsOffsets) {
+  /*
+		Subset annomatrix to contain the positions contained in the obsOffsets array
+
+    Example:
+
+      isubset(annoMatrix, [0, 1]) -> annoMatrix with only the first two rows
+	*/
+  const obsIndex = annoMatrix.rowIndex.isubset(obsOffsets);
+  return new AnnoMatrixRowSubsetView(annoMatrix, obsIndex);
+}
+
+export function subset(annoMatrix, obsLabels) {
+  /*
+		subset based on labels
+  */
+  const obsIndex = annoMatrix.rowIndex.subset(obsLabels);
+  return new AnnoMatrixRowSubsetView(annoMatrix, obsIndex);
+}
+
+export function subsetByIndex(annoMatrix, obsIndex) {
+  /*
+  subset based upon the new obs index.
+  */
+  return new AnnoMatrixRowSubsetView(annoMatrix, obsIndex);
+}
+
+export function clip(annoMatrix, qmin, qmax) {
+  /*
+		Create a view that clips all continuous data to the [min, max] range.
+		The matrix shape does not change, but the continuous values outside the
+		specified range will become a NaN.
+		*/
+  return new AnnoMatrixClipView(annoMatrix, qmin, qmax);
+}
+
+/*
+Private utility functions below
+*/
+
+function _maskToList(mask) {
+  /* convert masks to lists - method wastes space, but is fast */
+  if (!mask) {
+    return null;
+  }
+  const list = new Int32Array(mask.length);
+  let elems = 0;
+  for (let i = 0, l = mask.length; i < l; i += 1) {
+    if (mask[i]) {
+      list[elems] = i;
+      elems += 1;
+    }
+  }
+  return list.subarray(0, elems);
+}

client/src/annoMatrix/views.js

@@ -0,0 +1,176 @@
+/* eslint-disable max-classes-per-file -- Classes are interrelated*/
+
+/*
+Views on the annomatrix.  all API here is defined in viewCreators.js and annoMatrix.js.
+*/
+import clip from "../util/clip";
+import AnnoMatrix from "./annoMatrix";
+import { _whereCacheCreate } from "./whereCache";
+import { _isContinuousType, _getColumnSchema } from "./schema";
+
+class AnnoMatrixView extends AnnoMatrix {
+  constructor(viewOf, rowIndex = null) {
+    const nObs = rowIndex ? rowIndex.size() : viewOf.nObs;
+    super(viewOf.schema, nObs, viewOf.nVar, rowIndex || viewOf.rowIndex);
+    this.viewOf = viewOf;
+    this.isView = true;
+  }
+
+  addObsAnnoCategory(col, category) {
+    const newAnnoMatrix = this._clone();
+    newAnnoMatrix.viewOf = this.viewOf.addObsAnnoCategory(col, category);
+    newAnnoMatrix.schema = newAnnoMatrix.viewOf.schema;
+    return newAnnoMatrix;
+  }
+
+  async removeObsAnnoCategory(col, category, unassignedCategory) {
+    const newAnnoMatrix = this._clone();
+    newAnnoMatrix.viewOf = await this.viewOf.removeObsAnnoCategory(
+      col,
+      category,
+      unassignedCategory
+    );
+    newAnnoMatrix.schema = newAnnoMatrix.viewOf.schema;
+    return newAnnoMatrix;
+  }
+
+  dropObsColumn(col) {
+    const newAnnoMatrix = this._clone();
+    newAnnoMatrix.viewOf = this.viewOf.dropObsColumn(col);
+    newAnnoMatrix._cache.obs = this._cache.obs.dropCol(col);
+    newAnnoMatrix.schema = newAnnoMatrix.viewOf.schema;
+    return newAnnoMatrix;
+  }
+
+  addObsColumn(colSchema, Ctor, value) {
+    const newAnnoMatrix = this._clone();
+    newAnnoMatrix.viewOf = this.viewOf.addObsColumn(colSchema, Ctor, value);
+    newAnnoMatrix.schema = newAnnoMatrix.viewOf.schema;
+    return newAnnoMatrix;
+  }
+
+  renameObsColumn(oldCol, newCol) {
+    const newAnnoMatrix = this._clone();
+    newAnnoMatrix.viewOf = this.viewOf.renameObsColumn(oldCol, newCol);
+    newAnnoMatrix.schema = newAnnoMatrix.viewOf.schema;
+    return newAnnoMatrix;
+  }
+
+  async setObsColumnValues(col, rowLabels, value) {
+    const newAnnoMatrix = this._clone();
+    newAnnoMatrix.viewOf = await this.viewOf.setObsColumnValues(
+      col,
+      rowLabels,
+      value
+    );
+    newAnnoMatrix._cache.obs = this._cache.obs.dropCol(col);
+    newAnnoMatrix.schema = newAnnoMatrix.viewOf.schema;
+    return newAnnoMatrix;
+  }
+
+  async resetObsColumnValues(col, oldValue, newValue) {
+    const newAnnoMatrix = this._clone();
+    newAnnoMatrix.viewOf = await this.viewOf.resetObsColumnValues(
+      col,
+      oldValue,
+      newValue
+    );
+    newAnnoMatrix._cache.obs = this._cache.obs.dropCol(col);
+    newAnnoMatrix.schema = newAnnoMatrix.viewOf.schema;
+    return newAnnoMatrix;
+  }
+
+  addEmbedding(colSchema) {
+    const newAnnoMatrix = this._clone();
+    newAnnoMatrix.viewOf = this.viewOf.addEmbedding(colSchema);
+    newAnnoMatrix.schema = newAnnoMatrix.viewOf.schema;
+    return newAnnoMatrix;
+  }
+}
+
+class AnnoMatrixMapView extends AnnoMatrixView {
+  /*
+	A view which knows how to transform its data.
+	*/
+  constructor(viewOf, mapFn) {
+    super(viewOf);
+    this.mapFn = mapFn;
+  }
+
+  async _doLoad(field, query) {
+    const df = await this.viewOf._fetch(field, query);
+    const dfMapped = df.mapColumns((colData, colIdx) => {
+      const colLabel = df.colIndex.getLabel(colIdx);
+      const colSchema = _getColumnSchema(this.schema, field, colLabel);
+      return this.mapFn(field, colLabel, colSchema, colData, df);
+    });
+    const whereCacheUpdate = _whereCacheCreate(
+      field,
+      query,
+      dfMapped.colIndex.labels()
+    );
+    return [whereCacheUpdate, dfMapped];
+  }
+}
+
+export class AnnoMatrixClipView extends AnnoMatrixMapView {
+  /*
+	A view which is a clipped transformation of its parent
+	*/
+  constructor(viewOf, qmin, qmax) {
+    super(viewOf, (field, colLabel, colSchema, colData, df) =>
+      _clipAnnoMatrix(field, colLabel, colSchema, colData, df, qmin, qmax)
+    );
+    this.isClipped = true;
+    this.clipRange = [qmin, qmax];
+    Object.seal(this);
+  }
+}
+
+export class AnnoMatrixRowSubsetView extends AnnoMatrixView {
+  /*
+	A view which is a subset of total rows.
+	*/
+  constructor(viewOf, rowIndex) {
+    super(viewOf, rowIndex);
+    Object.seal(this);
+  }
+
+  async _doLoad(field, query) {
+    const df = await this.viewOf._fetch(field, query);
+
+    // don't try to row-subset the var dimension.
+    if (field === "var") {
+      return [null, df];
+    }
+
+    const dfSubset = df.subset(null, null, this.rowIndex);
+    const whereCacheUpdate = _whereCacheCreate(
+      field,
+      query,
+      dfSubset.colIndex.labels()
+    );
+    return [whereCacheUpdate, dfSubset];
+  }
+}
+
+/*
+Utility functions below
+*/
+
+function _clipAnnoMatrix(field, colLabel, colSchema, colData, df, qmin, qmax) {
+  /* only clip obs and var scalar columns */
+  if (field !== "obs" && field !== "X") return colData;
+  if (!_isContinuousType(colSchema)) return colData;
+  if (qmin < 0) qmin = 0;
+  if (qmax > 1) qmax = 1;
+  if (qmin === 0 && qmax === 1) return colData;
+
+  const quantiles = df.col(colLabel).summarize().percentiles;
+  const lower = quantiles[100 * qmin];
+  const upper = quantiles[100 * qmax];
+  const clippedData = clip(colData.slice(), lower, upper, Number.NaN);
+  return clippedData;
+}
+
+/* eslint-enable max-classes-per-file -- enable*/

client/src/annoMatrix/whereCache.js

@@ -0,0 +1,92 @@
+/*
+Private support functions.
+
+Support for a "where" query, eg,
+
+	{ where: { field: "var", column: "gene", value: "FOXP2" }}
+
+These evaluate to a given column label.
+
+The "where cache" is a map that saves evaluated queries and points 
+to the column label they resolve to.
+
+Data structure, using X as the example field being queried, and var as
+the index.
+
+{ 
+	X: {
+		var: Map(
+			column_label_in_var => Map(value_in_var_column => [column_label_in_X, ...])
+		)
+	}
+}
+*/
+import { _getColumnDimensionNames } from "./schema";
+
+export function _whereCacheGet(whereCache, schema, field, query) {
+  /* 
+	query will either be an where query (object) or a column name (string).
+
+	Return array of column labels or undefined.
+	*/
+
+  if (typeof query === "object") {
+    const { field: queryField, column: queryColumn, value: queryValue } = query;
+
+    const columnMap = whereCache?.[field]?.[queryField];
+    if (columnMap === undefined) return [undefined];
+
+    const valueMap = columnMap.get(queryColumn);
+    if (valueMap === undefined) return [undefined];
+
+    const columnLabels = valueMap.get(queryValue);
+    return columnLabels === undefined ? [undefined] : columnLabels;
+  }
+
+  const colDims = _getColumnDimensionNames(schema, field, query);
+  return colDims === undefined ? [undefined] : colDims;
+}
+
+export function _whereCacheCreate(field, query, columnLabels) {
+  /*
+	Create a new whereCache
+	*/
+  if (typeof query !== "object") return null;
+
+  const { field: queryField, column: queryColumn, value: queryValue } = query;
+  const whereCache = {
+    [field]: {
+      [queryField]: new Map([
+        [queryColumn, new Map([[queryValue, columnLabels]])],
+      ]),
+    },
+  };
+  return whereCache;
+}
+
+function __whereCacheMerge(dst, src) {
+  /*
+	merge src into dst (modifies dst)
+	*/
+  if (!dst) dst = {};
+  if (!src || typeof src !== "object") return dst;
+  Object.entries(src).forEach(([field, query]) => {
+    if (!Object.prototype.hasOwnProperty.call(dst, field)) dst[field] = {};
+    Object.entries(query).forEach(([queryField, columnMap]) => {
+      if (!Object.prototype.hasOwnProperty.call(dst[field], queryField))
+        dst[field][queryField] = new Map();
+      columnMap.forEach((valueMap, queryColumn) => {
+        if (!dst[field][queryField].has(queryColumn))
+          dst[field][queryField].set(queryColumn, new Map());
+        valueMap.forEach((columnLabels, queryValue) => {
+          dst[field][queryField].get(queryColumn).set(queryValue, columnLabels);
+        });
+      });
+    });
+  });
+  return dst;
+}
+
+export function _whereCacheMerge(...caches) {
+  return caches.reduce((dst, src) => __whereCacheMerge(dst, src), {});
+}

client/src/components/app.js

@@ -4,26 +4,24 @@ import Helmet from "react-helmet";
 import { connect } from "react-redux";
 
 import Container from "./framework/container";
+import Layout from "./framework/layout";
 import LeftSideBar from "./leftSidebar";
 import RightSideBar from "./rightSidebar";
 import Legend from "./continuousLegend";
 import Graph from "./graph/graph";
 import MenuBar from "./menubar";
 import Autosave from "./autosave";
+import Embedding from "./embedding";
+import TermsOfServicePrompt from "./termsPrompt";
 
 import actions from "../actions";
 
-@connect(state => ({
+@connect((state) => ({
   loading: state.controls.loading,
   error: state.controls.error,
-  graphRenderCounter: state.controls.graphRenderCounter
+  graphRenderCounter: state.controls.graphRenderCounter,
 }))
 class App extends React.Component {
-  constructor(props) {
-    super(props);
-    this.state = {};
-  }
-
   componentDidMount() {
     const { dispatch } = this.props;
 
@@ -32,24 +30,7 @@ class App extends React.Component {
     this._onURLChanged();
 
     dispatch(actions.doInitialDataLoad(window.location.search));
-
-    /* listen for resize events */
-    window.addEventListener("resize", () => {
-      dispatch({
-        type: "window resize",
-        data: {
-          height: window.innerHeight,
-          width: window.innerWidth
-        }
-      });
-    });
-    dispatch({
-      type: "window resize",
-      data: {
-        height: window.innerHeight,
-        width: window.innerWidth
-      }
-    });
+    this.forceUpdate();
   }
 
   _onURLChanged() {
@@ -69,7 +50,7 @@ class App extends React.Component {
               position: "fixed",
               fontWeight: 500,
               top: window.innerHeight / 2,
-              left: window.innerWidth / 2 - 50
+              left: window.innerWidth / 2 - 50,
             }}
           >
             loading cellxgene
@@ -81,20 +62,28 @@ class App extends React.Component {
               position: "fixed",
               fontWeight: 500,
               top: window.innerHeight / 2,
-              left: window.innerWidth / 2 - 50
+              left: window.innerWidth / 2 - 50,
             }}
           >
-            error loading
+            error loading cellxgene
           </div>
         ) : null}
-        <div>
-          {loading ? null : <LeftSideBar />}
-          {loading ? null : <RightSideBar />}
-          {loading ? null : <MenuBar />}
-          {loading ? null : <Graph key={graphRenderCounter} />}
-          {loading ? null : <Autosave />}
-          <Legend />
-        </div>
+        {loading || error ? null : (
+          <Layout>
+            <LeftSideBar />
+            {(viewportRef) => (
+              <>
+                <MenuBar />
+                <Embedding />
+                <Autosave />
+                <TermsOfServicePrompt />
+                <Legend viewportRef={viewportRef} />
+                <Graph key={graphRenderCounter} viewportRef={viewportRef} />
+              </>
+            )}
+            <RightSideBar />
+          </Layout>
+        )}
       </Container>
     );
   }

client/src/components/autosave/filenameDialog.js

@@ -3,28 +3,26 @@ import { connect } from "react-redux";
 
 import {
   Button,
-  Tooltip,
-  InputGroup,
-  Dialog,
   Classes,
-  Colors
+  Code,
+  Colors,
+  Dialog,
+  InputGroup,
+  Tooltip,
 } from "@blueprintjs/core";
 
-@connect(state => ({
-  universe: state.universe,
+@connect((state) => ({
   idhash: state.config?.parameters?.["annotations-user-data-idhash"] ?? null,
   annotations: state.annotations,
-  obsAnnotations: state.universe.obsAnnotations,
-  saveInProgress: state.autosave?.saveInProgress ?? false,
-  lastSavedObsAnnotations: state.autosave?.lastSavedObsAnnotations,
-  error: state.autosave?.error,
-  writableCategoriesEnabled: state.config?.parameters?.["annotations"] ?? false
+  auth: state.config?.authentication,
+  userInfo: state.userInfo,
+  writableCategoriesEnabled: state.config?.parameters?.annotations ?? false,
 }))
 class FilenameDialog extends React.Component {
   constructor(props) {
     super(props);
     this.state = {
-      filenameText: ""
+      filenameText: "",
     };
   }
 
@@ -36,7 +34,7 @@ class FilenameDialog extends React.Component {
 
     dispatch({
       type: "set annotations collection name",
-      data: filenameText
+      data: filenameText,
     });
   };
 
@@ -71,7 +69,7 @@ class FilenameDialog extends React.Component {
             fontStyle: "italic",
             fontSize: 12,
             marginTop: 5,
-            color: Colors.ORANGE3
+            color: Colors.ORANGE3,
           }}
         >
           Name cannot be blank
@@ -84,7 +82,7 @@ class FilenameDialog extends React.Component {
             fontStyle: "italic",
             fontSize: 12,
             marginTop: 5,
-            color: Colors.ORANGE3
+            color: Colors.ORANGE3,
           }}
         >
           Only alphanumeric and underscore allowed
@@ -95,12 +93,19 @@ class FilenameDialog extends React.Component {
   };
 
   render() {
-    const { writableCategoriesEnabled, annotations, idhash } = this.props;
+    const {
+      writableCategoriesEnabled,
+      annotations,
+      idhash,
+      userInfo,
+    } = this.props;
     const { filenameText } = this.state;
 
     return writableCategoriesEnabled &&
+      annotations.promptForFilename &&
       !annotations.dataCollectionNameIsReadOnly &&
-      !annotations.dataCollectionName ? (
+      !annotations.dataCollectionName &&
+      userInfo.is_authenticated ? (
       <Dialog
         icon="tag"
         title="Annotations Collection"
@@ -108,20 +113,23 @@ class FilenameDialog extends React.Component {
         onClose={this.dismissFilenameDialog}
       >
         <form
-          onSubmit={e => {
+          onSubmit={(e) => {
             e.preventDefault();
             this.handleCreateFilename();
           }}
         >
-          <div className={Classes.DIALOG_BODY}>
+          <div className={Classes.DIALOG_BODY} data-testid="annotation-dialog">
             <div style={{ marginBottom: 20 }}>
               <p>Name your annotations collection:</p>
               <InputGroup
                 autoFocus
                 value={filenameText}
                 intent={this.filenameError(filenameText) ? "warning" : "none"}
-                onChange={e => this.setState({ filenameText: e.target.value })}
+                onChange={(e) =>
+                  this.setState({ filenameText: e.target.value })
+                }
                 leftIcon="tag"
+                data-testid="new-annotation-name"
               />
               <p
                 style={{
@@ -129,7 +137,7 @@ class FilenameDialog extends React.Component {
                   visibility: this.filenameError(filenameText)
                     ? "visible"
                     : "hidden",
-                  color: Colors.ORANGE3
+                  color: Colors.ORANGE3,
                 }}
               >
                 {this.filenameErrorMessage(filenameText)}
@@ -138,9 +146,9 @@ class FilenameDialog extends React.Component {
             <div>
               <p>
                 Your annotations are stored in this file:
-                <code className="bp3-code">
+                <Code>
                   {filenameText}-{idhash}.csv
-                </code>
+                </Code>
               </p>
               <p style={{ fontStyle: "italic" }}>
                 (We added a unique ID to your filename)
@@ -157,6 +165,7 @@ class FilenameDialog extends React.Component {
                 onClick={this.handleCreateFilename}
                 intent="primary"
                 type="submit"
+                data-testid="submit-annotation"
               >
                 Create annotations collection
               </Button>