server update

* Bump version: 0.19.0 → 1.0.0-rc.0

* Bump version: 1.0.0-rc.0 → 1.0.0

.bumpversion.cfg

@@ -1,5 +1,16 @@
 [bumpversion]
-current_version = 0.15.0
+current_version = 1.0.0
+commit = True
+parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(?:-(?P<prerel>rc)\.(?P<prerelversion>\d+))?
+serialize = 
+	{major}.{minor}.{patch}-{prerel}.{prerelversion}
+	{major}.{minor}.{patch}
+
+[bumpversion:part:prerel]
+optional_value = release
+values = 
+	rc
+	release
 
 [bumpversion:file:setup.py]
 search = version="{current_version}"
@@ -12,4 +23,3 @@ replace = "version": "{new_version}"
 [bumpversion:file:server/__init__.py]
 search = __version__ = "{current_version}"
 replace = __version__ = "{new_version}"
-

.github/ISSUE_TEMPLATE/---bug-report.md

@@ -0,0 +1,32 @@
+---
+name: "\U0001F41E Bug report"
+about: Create a report to identify a bug in cellxgene
+title: "[BUG]"
+labels: bug
+assignees: ''
+
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behavior:
+1. Go to '...'
+2. Click on '....'
+3. Scroll down to '....'
+4. See error
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Screenshots**
+If applicable, add screenshots to help explain your problem.
+
+**Version (please complete the following information):**
+ - Desktop or hosted?:
+ - Browser (if hosted) [e.g. chrome, safari]:
+ - Version [e.g. 0.13.0]:
+
+**Additional context**
+Add any other context about the problem here.

.github/ISSUE_TEMPLATE/---feature-request.md

@@ -0,0 +1,20 @@
+---
+name: "\U0001F4A1 Feature request"
+about: Suggest an idea for this project
+title: "[FEATURE REQUEST]"
+labels: user request
+assignees: ''
+
+---
+
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Describe alternatives you've considered**
+A clear and concise description of any alternative solutions or features you've considered.
+
+**Additional context**
+Add any other context or screenshots about the feature request here.

.github/ISSUE_TEMPLATE/---question-clarification.md

@@ -0,0 +1,10 @@
+---
+name: "\U0001F9D0 Question/Clarification"
+about: Ask a question or for a clarification
+title: "[QUESTION]"
+labels: question
+assignees: ''
+
+---
+
+

.github/workflows/codeql-analysis.yml

@@ -0,0 +1,67 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL Scan"
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [ main ]
+  schedule:
+    - cron: '0 8 * * *'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'javascript', 'python' ]
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ]
+        # Learn more:
+        # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v1
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+        # queries: ./path/to/local/query, your-org/your-repo/queries@main
+
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v1
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 https://git.io/JvXDl
+
+    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
+    #    and modify them (or add more) to build your code if your project
+    #    uses a compiled language
+
+    #- run: |
+    #   make bootstrap
+    #   make release
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v1

.github/workflows/compatibility_tests.yml

@@ -5,7 +5,7 @@ on:
     - cron: '0 8 7 * 2'
   push:
     branches:
-      - master
+      - main
 
 env:
   JEST_ENV: prod
@@ -22,97 +22,102 @@ jobs:
       - name: Build docker image
         run: docker build .
 
-  cellxgene-master-with-python-and-anndata-versions:
-    name: python versions x anndata versions
-    runs-on: ubuntu-latest
+  matrix-compatibility-test:
+    name: cxg:${{ matrix.cellxgene_build }} os:${{ matrix.os }} py:${{ matrix.python-version }} anndata:${{ matrix.anndata_version || 'latest' }}
+    runs-on: ${{ matrix.os }}
     strategy:
+      fail-fast: false
       matrix:
-        python-version: [3.6, 3.7, 3.8]
-        anndata-version: [0.6.22.post1, 0.7.1]
-        test-suite: [smoke-test, smoke-test-annotations]
+        # note: The `macos-latest` is latest Catalina version, and not Big Sur. So we explicitly ask for Big Sur (`macos-11`)
+        os: [ubuntu-latest, macos-latest, macos-11]
+        python-version: [3.6, 3.7, 3.8, 3.9]
+        cellxgene_build: [main, latest]
+        exclude:
+          # 3.6 no longer avail on Big Sur (`macos-11`)
+          - os: macos-11
+            python-version: 3.6
+          # no pypi build exists for macos+py3.9 and source install fails to
+          # install `tables` py pkg (a `scanpy` dependency), so we test py3.9
+          # only on ubuntu
+          - os: macos-11
+            python-version: 3.9
+          - os: macos-latest
+            python-version: 3.9
+        # add anndata pinned version test for subset of matrix configurations,
+        # in order to reduce matrix cross-product explosion
+        include:
+          - python-version: 3.8
+            cellxgene_build: latest
+            # TODO: dynamically use the literal version in requirements.txt,
+            # to avoid having to update this in manually in the future
+            # TODO: Do not bother running this if anndata latest version
+            # matches this pinned version, to avoid a redundant test
+            anndata_version: '==0.7.6'
     steps:
     - uses: actions/checkout@v2
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v1
       with:
         python-version: ${{ matrix.python-version }}
-    - name: Install dependencies
+    - name: Cache env vars
+      run: echo "PIP_CACHE=`python -m pip cache dir`" >> $GITHUB_ENV
+    - name: Cache env vars (MacOS)
+      if: startsWith(matrix.os, 'macos')
+      run: echo "BREW_CACHE=`brew --cache`" >> $GITHUB_ENV
+    # FIXME: Only working for Linux
+    - name: Python cache
+      uses: actions/cache@v1
+      with:
+        path: ${{ env.PIP_CACHE }}
+        key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}
+        restore-keys: |
+          ${{ runner.os }}-pip-
+    - name: Node cache
+      uses: actions/cache@v1
+      with:
+        path: ~/.npm
+        key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
+        restore-keys: |
+          ${{ runner.os }}-node-
+    - name: Brew cache (MacOS)
+      if: startsWith(matrix.os, 'macos')
+      uses: actions/cache@v1
+      with:
+        path: ${{ env.BREW_CACHE }}
+        key: ${{ runner.os }}-brew-
+    - name: Install dependencies (Ubuntu Linux)
+      if: startsWith(matrix.os, 'ubuntu')
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y libhdf5-serial-dev
+    - name: Install dependencies (MacOS)
+      if: startsWith(matrix.os, 'macos')
+      run: brew install hdf5
+    - name: Install cellxgene from `main` branch
+      if: matrix.cellxgene_build == 'main'
       run: |
-        # 1. only install the dev requirements on top of what is in the cellxgene pip package
-        sudo apt-get update && sudo apt-get install -y libhdf5-serial-dev
-        sed -i 's/-r requirements.txt//' server/requirements-dev.txt
         pip install -r server/requirements-dev.txt
-        # 2. install cellxgene
         make pydist install-dist
-        # 3. install anndata
-        pip install anndata==${{ matrix.anndata-version }}
-        # workaround for anndata 0.6.22.post1 bug
-        [[ "0.6.22.post1" = "${{ matrix.anndata-version }}" ]] && pip install h5py==2.9.0 || true
-    - name: Tests
-      run: make unit-test ${{ matrix.test-suite }}
-
-  cellxgene-release-with-anndata-master:
-    name: cellxgene release with anndata master
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        test-suite: [smoke-test, smoke-test-annotations]
-    steps:
-      - uses: actions/checkout@v2
-      - name: Set up Python 3.7
-        uses: actions/setup-python@v1
-        with:
-          python-version: 3.7
-      - name: Checkout
-        uses: actions/checkout@v2
-        with:
-          path: cellxgene
-      - name: Checkout tools repo
-        uses: actions/checkout@v2
-        with:
-          repository: theislab/anndata
-          path: anndata
-      - name: Install dependencies
-        run: |
-          cd cellxgene
-          # 1. only install the dev requirements on top of what is in the cellxgene pip package
-          make dev-env-client
-          sed -i 's/-r requirements.txt//' server/requirements-dev.txt
-          pip install -r server/requirements-dev.txt
-          # 2. install cellxgene
-          pip install --upgrade cellxgene
-          # 3. install anndata
-          cd ../anndata && pip install -e .
-      - name: Tests
-        run: cd cellxgene && make unit-test ${{ matrix.test-suite }}
-
-  cellxgene-master-with-anndata-master:
-    name: cellxgene master with anndata master
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        test-suite: [smoke-test, smoke-test-annotations]
-    steps:
-      - uses: actions/checkout@v2
-      - name: Set up Python 3.7
-        uses: actions/setup-python@v1
-        with:
-          python-version: 3.7
-      - name: Checkout
-        uses: actions/checkout@v2
-        with:
-          path: cellxgene
-      - name: Checkout tools repo
-        uses: actions/checkout@v2
-        with:
-          repository: theislab/anndata
-          path: anndata
-      - name: Install dependencies
-        run: |
-          cd cellxgene
-          sed -i -E 's/^anndata[>=]=[0-9]+.[0-9]+.[0-9]+$/anndata/g' server/requirements.txt
-          make pydist install-dist dev-env
-          cd ../anndata
-          pip install -e .
-      - name: Tests
-        run: cd cellxgene && make unit-test ${{ matrix.test-suite }}
+    - name: Install cellxgene from latest release (pypi.org)
+      if: matrix.cellxgene_build == 'latest'
+      run: |
+        pip install --upgrade cellxgene
+        # install the additional dev requirements on top of what is in the
+        # cellxgene pip package, which are needed for testing, but otherwise
+        # keep same pip pkg versions as in the cxg release
+        sed -i'' -e 's/-r requirements.txt//' server/requirements-dev.txt
+        pip install -r server/requirements-dev.txt
+    - name: Install anndata version per matrix variable
+      run: pip install anndata${{ matrix.anndata_version }}
+    - name: Install node
+      run: make dev-env-client
+    # Run different types of test separately, to facilitate troubleshooting
+    - name: Unit Tests - client
+      run: make unit-test-client
+    - name: Unit Tests - server
+      run: make unit-test-server
+    - name: Smoke Tests
+      run: make smoke-test
+    # FIXME: Fails intermittently. See https://app.zenhub.com/workspaces/single-cell-5e2a191dad828d52cc78b028/issues/chanzuckerberg/cellxgene/2415
+    # - name: Smoke Tests with Annotations
+    #   run: make smoke-test-annotations

.github/workflows/push_tests.yml

@@ -1,6 +1,10 @@
 name: Push Tests
 
-on: [push]
+on:
+  push:
+    branches: main
+  pull_request:
+    branches: "*"
 
 env:
   JEST_ENV: prod
@@ -10,100 +14,120 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
-    - name: Set up Python 3.7
-      uses: actions/setup-python@v1
-      with:
-        python-version: 3.7
-    - name: Lint with flake8
-      run: |
-        pip install flake8
-        make lint
+      - uses: actions/checkout@v2
+      - run: |
+          git fetch --depth=1 origin +${{github.base_ref}}
+      - name: Set up Python 3.7
+        uses: actions/setup-python@v1
+        with:
+          python-version: 3.7
+      - name: Node cache
+        uses: actions/cache@v1
+        with:
+          path: ~/.npm
+          key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-node-
+      - name: Install dependencies
+        run: |
+          pip install flake8
+          pip install black
+          cd client
+          npm install
+      - name: Format with black and lint with flake8
+        run: |
+          make lint-server
+      - name: Lint src with eslint
+        working-directory: ./client
+        run: |
+          npx eslint src __tests__
 
   unit-test:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
-    - name: Set up Python 3.7
-      uses: actions/setup-python@v1
-      with:
-        python-version: 3.7
-    - name: Python cache
-      uses: actions/cache@v1
-      with:
-        path: ~/.cache/pip
-        key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}
-        restore-keys: |
-          ${{ runner.os }}-pip-
-    - name: Node cache
-      uses: actions/cache@v1
-      with:
-        path: ~/.npm
-        key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
-        restore-keys: |
-          ${{ runner.os }}-node-
-    - name: Install dependencies
-      run: make pydist install-dist dev-env-server
-    - name: Unit tests
-      run: |
-        make unit-test
-        bash <(curl -s https://codecov.io/bash) -y .codecov.yml -k server -cF backend,python,unitTest
-        cd client && ./node_modules/codecov/bin/codecov --yml=../.codecov.yml --root=../ --gcov-root=../ -C -F frontend,javascript,unitTest
+      - uses: actions/checkout@v2
+      - name: Set up Python 3.7
+        uses: actions/setup-python@v1
+        with:
+          python-version: 3.7
+      - name: Python cache
+        uses: actions/cache@v1
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+      - name: Node cache
+        uses: actions/cache@v1
+        with:
+          path: ~/.npm
+          key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-node-
+      - name: Install dependencies
+        run: make pydist install-dist dev-env-server
+      - name: Unit tests
+        run: |
+          make unit-test-server unit-test-client
+          bash <(curl -s https://codecov.io/bash) -y .codecov.yml -k server -cF server,python,unitTest
+          cd client && ./node_modules/codecov/bin/codecov --yml=../.codecov.yml --root=../ --gcov-root=../ -C -F frontend,javascript,unitTest
 
   smoke-tests:
-    runs-on: ubuntu-latest
+    runs-on: macos-latest
+    timeout-minutes: 20
     steps:
-    - uses: actions/checkout@v2
-    - name: Set up Python 3.7
-      uses: actions/setup-python@v1
-      with:
-        python-version: 3.7
-    - name: Python cache
-      uses: actions/cache@v1
-      with:
-        path: ~/.cache/pip
-        key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}
-        restore-keys: |
-          ${{ runner.os }}-pip-
-    - name: Node cache
-      uses: actions/cache@v1
-      with:
-        path: ~/.npm
-        key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
-        restore-keys: |
-          ${{ runner.os }}-node-
-    - name: Install dependencies
-      run: make pydist install-dist
-    - name: Smoke tests (without annotations feature)
-      run: |
-        cd client && make smoke-test
-        ./node_modules/codecov/bin/codecov --yml=../.codecov.yml --root=../ --gcov-root=../ -C -F frontend,javascript,smokeTest
+      - uses: actions/checkout@v2
+      - name: Set up Python 3.7
+        uses: actions/setup-python@v1
+        with:
+          python-version: 3.7
+      - name: Python cache
+        uses: actions/cache@v1
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+      - name: Node cache
+        uses: actions/cache@v1
+        with:
+          path: ~/.npm
+          key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-node-
+      - name: Install dependencies
+        run: make pydist install-dist
+      - name: Smoke tests (without annotations feature)
+        run: |
+          cd client && make smoke-test
+          ./node_modules/codecov/bin/codecov --yml=../.codecov.yml --root=../ --gcov-root=../ -C -F frontend,javascript,smokeTest
 
   smoke-tests-annotations:
     runs-on: ubuntu-latest
+    timeout-minutes: 20
     steps:
-    - uses: actions/checkout@v2
-    - name: Set up Python 3.7
-      uses: actions/setup-python@v1
-      with:
-        python-version: 3.7
-    - name: Python cache
-      uses: actions/cache@v1
-      with:
-        path: ~/.cache/pip
-        key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}
-        restore-keys: |
-          ${{ runner.os }}-pip-
-    - name: Node cache
-      uses: actions/cache@v1
-      with:
-        path: ~/.npm
-        key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
-        restore-keys: |
-          ${{ runner.os }}-node-
-    - name: Install dependencies
-      run: make pydist install-dist
-    - name: Smoke tests (with annotations feature)
-      run: |
-        cd client && make smoke-test-annotations
-        ./node_modules/codecov/bin/codecov --yml=../.codecov.yml --root=../ --gcov-root=../ -C -F frontend,javascript,smokeTestAnnotations
+      - uses: actions/checkout@v2
+      - name: Set up Python 3.7
+        uses: actions/setup-python@v1
+        with:
+          python-version: 3.7
+      - name: Python cache
+        uses: actions/cache@v1
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+      - name: Node cache
+        uses: actions/cache@v1
+        with:
+          path: ~/.npm
+          key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-node-
+      - name: Install dependencies
+        run: make pydist install-dist
+      - name: Smoke tests (with annotations feature)
+        run: |
+          cd client && make smoke-test-annotations
+          ./node_modules/codecov/bin/codecov --yml=../.codecov.yml --root=../ --gcov-root=../ -C -F frontend,javascript,smokeTestAnnotations

.gitignore

@@ -15,16 +15,18 @@ dist/
 *.egg-info
 
 # Environments
-venv/
+venv*/
 cellxgene/
 
 # client build
-server/common/web/static/css/
-server/common/web/static/img/
-server/common/web/static/media/
-server/common/web/static/fonts/
-server/common/web/static/js/
-server/common/web/templates/index\.html
+server/common/web/static/*
+server/common/web/templates/
+server/common/web/csp-hashes.json
+
+# eb build
+artifact.dir
+artifact.zip
+customize
 
 # Jupyter Notebook
 .ipynb_checkpoints
@@ -41,7 +43,14 @@ npm-debug.log
 __pycache__
 *.DS_Store*
 data
+tags
+myconfig.yaml
 
 # Jekyll
 docs/_site/
 docs/Gemfile.lock
+
+client/.eslintcache
+
+# E2E Testing
+ignoreE2E*

CONTRIBUTING.md

@@ -1,3 +1,3 @@
 We warmly welcome contributions from the community!
 
-Whether you want to contribute ideas, requests, documentation, or code, you can get started by visiting our [contribution guide](https://chanzuckerberg.github.io/cellxgene/posts/contribute).
+Whether you want to contribute ideas, requests, documentation, or code, you can get started by visiting our [contribution guide](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/contribute.md).

Dockerfile

@@ -4,7 +4,8 @@ ENV LC_ALL=C.UTF-8
 ENV LANG=C.UTF-8
 
 RUN apt-get update && \
-    apt-get install -y build-essential libxml2-dev python3-dev python3-pip zlib1g-dev python3-requests && \
+    apt-get install -y build-essential libxml2-dev python3-dev python3-pip zlib1g-dev python3-requests python3-aiohttp && \
+    python3 -m pip install --upgrade pip && \
     pip3 install cellxgene
 
 ENTRYPOINT ["cellxgene"]

LICENSE.txt

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2013
+Copyright (c) 2017-2021 Chan Zuckerberg Initiative
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of
 this software and associated documentation files (the "Software"), to deal in
@@ -17,4 +17,4 @@ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
 FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
 COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
 IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
-CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

MANIFEST.in

@@ -3,3 +3,5 @@ recursive-include server/common/web/static *
 
 include server/requirements.txt
 include server/requirements-prepare.txt
+include server/converters/schema/hgnc_complete_set.txt.gz
+include server/converters/schema/schema_definitions/*

Makefile

@@ -16,22 +16,23 @@ clean: clean-lite clean-server clean-client
 clean-lite:
 	rm -rf $(CLEANFILES)
 
-clean-%:
-	cd $(*) && $(MAKE) clean
+.PHONY: clean-client
+clean-client:
+	cd client && $(MAKE) clean
+
+.PHONY: clean-server
+clean-server:
+	cd server && $(MAKE) clean
 
 
 # BUILDING PACKAGE
 
-.PHONY: build
-build: clean build-cli
-	@echo "done"
-
 .PHONY: build-client
 build-client:
 	cd client && $(MAKE) ci build
 
-.PHONY: build-cli
-build-cli: build-client
+.PHONY: build
+build: clean build-client
 	git ls-files server/ | cpio -pdm $(BUILDDIR)
 	cp -r client/build/  $(CLIENTBUILD)
 	$(call copy_client_assets,$(CLIENTBUILD),$(SERVERBUILD))
@@ -39,9 +40,13 @@ build-cli: build-client
 
 # If you are actively developing in the server folder use this, dirties the source tree
 .PHONY: build-for-server-dev
-build-for-server-dev: clean-server build-client
+build-for-server-dev: clean-server build-client copy-client-assets
+
+.PHONY: copy-client-assets
+copy-client-assets:
 	$(call copy_client_assets,client/build,server)
 
+
 # TESTING
 .PHONY: test
 test: unit-test smoke-test
@@ -49,8 +54,22 @@ test: unit-test smoke-test
 .PHONY: unit-test
 unit-test: unit-test-server unit-test-client
 
-unit-test-%:
-	cd $(*) && $(MAKE) unit-test
+.PHONY: test-server
+test-server: unit-test-server smoke-test
+
+.PHONY: unit-test-client
+unit-test-client:
+	cd client && $(MAKE) unit-test
+
+.PHONY: unit-test-server
+unit-test-server:
+	PYTHONWARNINGS=ignore:ResourceWarning coverage run \
+		--source=server \
+		--omit=.coverage,venv \
+		-m unittest discover \
+		--start-directory test/unit \
+		--verbose; test_result=$$?; \
+	exit $$test_result \
 
 .PHONY: smoke-test
 smoke-test:
@@ -62,7 +81,7 @@ smoke-test-annotations:
 
 # FORMATTING CODE
 
-.PHOHY: fmt
+.PHONY: fmt
 fmt: fmt-client fmt-py
 
 .PHONY: fmt-client
@@ -74,10 +93,17 @@ fmt-py:
 	black .
 
 .PHONY: lint
-lint:
-	flake8 server
+lint: lint-server lint-client
 
 
+.PHONY: lint-server
+lint-server: fmt-py
+	flake8 server --per-file-ignores='test/fixtures/dataset_config_outline.py:F821 test/fixtures/server_config_outline.py:F821 test/performance/scale_test_annotations.py:E501'
+
+.PHONY: lint-client
+lint-client:
+	cd client && $(MAKE) lint
+
 # CREATING DISTRIBUTION RELEASE
 
 .PHONY: pydist
@@ -85,31 +111,46 @@ pydist: build
 	cd $(BUILDDIR); python setup.py sdist -d ../dist
 	@echo "done"
 
-
 # RELEASE HELPERS
 
-# create new version to commit to master
-.PHONY: release-stage-1
-release-stage-1: dev-env bump clean-lite gen-package-lock
+# Set PART=[major, minor, patch] as param to make bump.
+# This will create a release candidate. (i.e. 0.16.1 -> 0.16.2-rc.0 for a patch bump)
+.PHONY: bump-version
+bump-version:
+	bumpversion --config-file .bumpversion.cfg $(PART)
+
+# Create new version to commit to main
+.PHONY: create-release-candidate
+create-release-candidate: bump-version clean-lite gen-package-lock
 	@echo "Version bumped part:$(PART) and client built. Ready to commit and push"
 
-# build dist and release to dev pypi
-.PHONY: release-stage-2
-release-stage-2: dev-env pydist twine
+# Bump the release candidate version if needed (i.e. the previous release candidate had errors).
+.PHONY: recreate-release-candidate
+recreate-release-candidate: bump-release-candidate clean-lite gen-package-lock
+	@echo "Version bumped part:$(PART) and client built. Ready to commit and push"
+
+# Build dist and release to Test PyPI
+.PHONY: release-candidate-to-test-pypi
+release-candidate-to-test-pypi: pydist twine
 	@echo "Dist built and uploaded to test.pypi.org"
 	@echo "Test the install:"
 	@echo "    make install-release-test"
-	@echo "Then upload to Pypi prod:"
-	@echo "    make twine-prod"
 
-.PHONY: release-stage-final
-release-stage-final: twine-prod
+# Build final dist (gets rid of the rc tag) and release final candidate to TestPyPI
+.PHONY: release-final-to-test-pypi
+release-final-to-test-pypi: bump-release clean-lite gen-package-lock pydist twine
+	@echo "Final release dist built and uploaded to test.pypi.org"
+	@echo "Test the install:"
+	@echo "    make install-release-test"
+
+.PHONY: release-final
+release-final: twine-prod
 	@echo "Release uploaded to pypi.org"
 
 # DANGER: releases directly to prod
-# use this if you accidently burned a test release version number,
+# use this if you accidentally burned a test release version number,
 .PHONY: release-directly-to-prod
-release-directly-to-prod: dev-env pydist twine-prod
+release-directly-to-prod: pydist twine-prod
 	@echo "Dist built and uploaded to pypi.org"
 	@echo "Test the install:"
 	@echo "    make install-release"
@@ -125,10 +166,15 @@ dev-env-client:
 dev-env-server:
 	pip install -r server/requirements-dev.txt
 
-# give PART=[major, minor, part] as param to make bump
-.PHONY: bump
-bump:
-	bumpversion --config-file .bumpversion.cfg $(PART)
+# Increments the release candidate version (i.e. 0.16.2-rc.1 -> 0.16.2-rc.2)
+.PHONY: bump-release-candidate
+bump-release-candidate:
+	bumpversion --config-file .bumpversion.cfg prerelversion --allow-dirty
+
+# Finalizes the release candidate by removing the release candidate tag (i.e. 0.16.2-rc.2 -> 0.16.2).
+.PHONY: bump-release
+bump-release:
+	bumpversion --config-file .bumpversion.cfg prerel --allow-dirty
 
 .PHONY: twine
 twine:
@@ -146,13 +192,6 @@ gen-package-lock:
 
 # INSTALL
 
-# setup.py sucks when you have your library in a separate folder, adding these in to help setup envs
-
-# install from build directory
-.PHONY: install
-install: uninstall
-	cd $(BUILDDIR); pip install -e .
-
 # install from source tree for development
 .PHONY: install-dev
 install-dev: uninstall
@@ -161,7 +200,7 @@ install-dev: uninstall
 # install from test.pypi to test your release
 .PHONY: install-release-test
 install-release-test: uninstall
-	pip install --no-cache-dir --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple cellxgene
+	pip install --no-cache-dir --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple cellxgene==$(VERSION)
 	@echo "Installed cellxgene from test.pypi.org, now run and smoke test"
 
 # install from pypi to test your release

PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,11 @@
+#### Reviewers
+**Functional:** 
+
+**Readability:** 
+
+---
+
+## Changes
+- add
+- remove
+- modify

README-visium.md

@@ -0,0 +1,20 @@
+# Cellxgene Visium Beta
+
+## How it works
+1. Launch `cellxgene` as normal. 
+1. If the loaded dataset has spatial information available, the image data will be loaded on startup.
+1. On the toolbar, next to the Zoom icon, a `Toggle image` button will now appear. Click on it and the image will be added as an underlay.
+1. You can now use any `cellxgene` functionality and the image will still be present. If you pan and zoom, the image will also be panned and zoomed.
+1. If you want to hide the image, you can click on `Toggle image` again
+
+In order for the image to be displayed with the correct size and alignment, the H5AD needs to have a few requirements. See the following section to learn more.
+
+## h5ad requirements
+1. The spatial embedding layer should be contained in `obsm` and be named `X_spatial`. Other layers can exist, but only this one will have the spatial feature enabled.
+2. A `spatial` dict needs to be defined in the `uns` dictionary.
+3. Inside the `spatial` dict, an `images` dict must be defined. 
+4. The `images` dict must contain a `hires` key, which should reference an image encoded as an RGB matrix (i.e., a three-dimensional matrix of size `height x width x 3` where the final dimension has the RGB values for each pixel)
+5. The `images` dict must contain a `scalefactors` dict. This should in turn contain a `tissue_hires_scalef` key, which should reference a floating point number.
+
+Moreover, in order to have the image correctly aligned with the dots, the following must be true:
+1. `tissue_hires_scalef` should represent the ratio between the embedding layer `X_spatial` and the image matrix. In particular, if you multiply `X_spatial` by `tissue_hires_scalef`, you should obtain an array of points that ovelap the tissue image if you plot them in a plane.

README.md

@@ -5,41 +5,55 @@ _an interactive explorer for single-cell transcriptomics data_
 [![DOI](https://zenodo.org/badge/105615409.svg)](https://zenodo.org/badge/latestdoi/105615409) [![PyPI](https://img.shields.io/pypi/v/cellxgene)](https://pypi.org/project/cellxgene/) [![PyPI - Downloads](https://img.shields.io/pypi/dm/cellxgene)](https://pypistats.org/packages/cellxgene) [![GitHub last commit](https://img.shields.io/github/last-commit/chanzuckerberg/cellxgene)](https://github.com/chanzuckerberg/cellxgene/pulse)
 [![Push Tests](https://github.com/chanzuckerberg/cellxgene/workflows/Push%20Tests/badge.svg)](https://github.com/chanzuckerberg/cellxgene/actions?query=workflow%3A%22Push+Tests%22)
 [![Compatibility Tests](https://github.com/chanzuckerberg/cellxgene/workflows/Compatibility%20Tests/badge.svg)](https://github.com/chanzuckerberg/cellxgene/actions?query=workflow%3A%22Compatibility+Tests%22)
-![Code Coverage](https://codecov.io/gh/chanzuckerberg/cellxgene/branch/master/graph/badge.svg)
+![Code Coverage](https://codecov.io/gh/chanzuckerberg/cellxgene/branch/main/graph/badge.svg)
 
-cellxgene (pronounced "cell-by-gene") is an interactive data explorer for single-cell transcriptomics datasets, such as those coming from the [Human Cell Atlas](https://humancellatlas.org). Leveraging modern web development techniques to enable fast visualizations of at least 1 million cells, we hope to enable biologists and computational researchers to explore their data.
+cellxgene Desktop (pronounced "cell-by-gene") is an interactive data explorer for single-cell datasets, such as those coming from the [Human Cell Atlas](https://humancellatlas.org). Leveraging modern web development techniques to enable fast visualizations of at least 1 million cells, we hope to enable biologists and computational researchers to explore their data.
 
-Whether you need to visualize one thousand cells or one million, cellxgene helps you gain insight into your single-cell data.
+Whether you need to visualize one thousand cells or one million, cellxgene Desktop helps you gain insight into your single-cell data.
 
-<img src="https://github.com/chanzuckerberg/cellxgene/raw/master/docs/images/crossfilter.gif" width="350" height="200" hspace="30"><img src="https://github.com/chanzuckerberg/cellxgene/raw/master/docs/images/category-breakdown.gif" width="350" height="200" hspace="30">
+<img src="https://github.com/chanzuckerberg/cellxgene/raw/main/docs/images/crossfilter.gif" width="350" height="200" hspace="30"><img src="https://github.com/chanzuckerberg/cellxgene/raw/main/docs/images/category-breakdown.gif" width="350" height="200" hspace="30">
 
 # Getting started
-### The comprehensive guide to cellxgene
-[The cellxgene documentation is your one-stop-shop for information about cellxgene](https://chanzuckerberg.github.io/cellxgene/)! You may be particularly interested in:
-* Seeing [what cellxgene can do](https://chanzuckerberg.github.io/cellxgene/posts/gallery)  
-* Learning more about cellxgene [installation](https://chanzuckerberg.github.io/cellxgene/posts/install) and [usage](https://chanzuckerberg.github.io/cellxgene/posts/launch)
-* [Preparing your own data](https://chanzuckerberg.github.io/cellxgene/posts/prepare) for use in cellxgene
-* Checking out [our roadmap](https://chanzuckerberg.github.io/cellxgene/posts/roadmap) for future development
-* [Contributing](https://chanzuckerberg.github.io/cellxgene/posts/contribute) to cellxgene
+
+### The comprehensive guide to cellxgene Desktop
+
+[The cellxgene documentation is your one-stop-shop for information about cellxgene Desktop](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/README.md)! You may be particularly interested in:
+
+- Seeing [what cellxgene Desktop can do](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/explore-data/explorer-tutorials.md)
+- Learning more about cellxgene [installation](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/desktop/install.md) and [usage](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/desktop/quick-start.md#quick-start-1)
+- [Preparing your own data](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/desktop/data-reqs.md) for use in cellxgene Desktop
+- Checking out [our roadmap](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/roadmap.md) for future development
+- [Contributing](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/contribute.md) to cellxgene Desktop
 
 ### Quick start
 
-To install cellxgene you need Python 3.6+. We recommend [installing cellxgene into a conda or virtual environment.](https://chanzuckerberg.github.io/cellxgene/posts/install)
+To install cellxgene Desktop you need Python 3.6+. We recommend [installing cellxgene Desktop into a conda or virtual environment.](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/desktop/install.md)
 
 Install the package.
-``` bash
+
+```bash
 pip install cellxgene
 ```
 
-Launch cellxgene with an example [anndata](https://anndata.readthedocs.io/en/latest/) file
+Launch cellxgene Desktop with an example [anndata](https://anndata.readthedocs.io/en/latest/) file
 
-``` bash
+```bash
 cellxgene launch https://cellxgene-example-data.czi.technology/pbmc3k.h5ad
 ```
 
-To explore more datasets already formatted for cellxgene, check out the [Demo data](https://chanzuckerberg.github.io/cellxgene/posts/demo-data) or
-see [Preparing your data](https://chanzuckerberg.github.io/cellxgene/posts/prepare) to learn more about formatting your own
-data for cellxgene.
+To explore more datasets already formatted for cellxgene Desktop, check out the [Demo data](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/desktop/quick-start.md#example-datasets) or
+see [Preparing your data](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/desktop/data-reqs.md) to learn more about formatting your own
+data for cellxgene Desktop.
+
+### Supported browsers
+
+cellxgene Desktop currently supports the following browsers:
+
+- Google Chrome 61+
+- Edge 15+
+- Firefox 60+
+
+Please [file an issue](https://github.com/chanzuckerberg/cellxgene/issues/new/choose) if you would like us to add support for an unsupported browser.
 
 ### Finding help
 
@@ -48,39 +62,34 @@ For questions, suggestions, or accolades, [join the `#cellxgene-users` channel o
 
 For any errors, [report bugs on Github](https://github.com/chanzuckerberg/cellxgene/issues).
 
-# Developing with cellxgene
+# Developing with cellxgene Desktop
 
 ### Contributing
-We warmly welcome contributions from the community! Please see our [contributing guide](https://chanzuckerberg.github.io/cellxgene/posts/contribute) and don't hesitate to open an issue or send a pull request to improve cellxgene.
+
+We warmly welcome contributions from the community! Please see our [contributing guide](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/contribute.md) and don't hesitate to open an issue or send a pull request to improve cellxgene Desktop. Please see the [dev_docs](https://github.com/chanzuckerberg/cellxgene/tree/main/dev_docs) for pull request suggestions, unit test details, local documentation preview, and other development specifics. 
 
 This project adheres to the Contributor Covenant [code of conduct](https://github.com/chanzuckerberg/.github/blob/master/CODE_OF_CONDUCT.md). By participating, you are expected to uphold this code. Please report unacceptable behavior to opensource@chanzuckerberg.com.
 
 ### Reuse
 
-This project was started with the sole goal of empowering the scientific community to explore and understand their data. As such, we encourage other scientific tool builders in academia or industry to adopt the patterns, tools, and code from this project, and reach out to us with ideas or questions. All code is freely available for reuse under the [MIT license](https://opensource.org/licenses/MIT).
+This project was started with the sole goal of empowering the scientific community to explore and understand their data. 
+As such, we encourage other scientific tool builders in academia or industry to adopt the patterns, tools, and code from 
+this project. All code is freely available for reuse under the [MIT license](https://opensource.org/licenses/MIT).
+
+
+Before extending cellxgene, we encourage you to reach out to us with ideas or questions. It might be possible that an 
+extension could be directly contributed, which would make it available for a wider audience, or that it's on our 
+[roadmap](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/roadmap.md) and under active development. 
+
+See the [cellxgene extensions](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/community-extensions.md) section of our documentation for examples of community use and cellxgene extensions. 
 
 ### Security
 
 If you believe you have found a security issue, we would appreciate notification. Please send email to <security@chanzuckerberg.com>.
 
-# About
-### Core team
+# Inspiration
 
-The current core team:
-
-- Colin Megill, frontend & product design
-- Bruce Martin, software engineer
-- Sidney Bell, computational biologist
-- Lia Prins, designer
-- Severiano Badajoz, software engineer
-
-We would also like to gratefully acknowledge contributions from past core team members:
-
-- Charlotte Weaver, software engineer
-
-### Inspiration
-
-We've been heavily inspired by several other related single-cell visualization projects, including the [UCSC Cell Browswer](http://cells.ucsc.edu/), [Cytoscape](http://www.cytoscape.org/), [Xena](https://xena.ucsc.edu/), [ASAP](https://asap.epfl.ch/), [Gene Pattern](http://genepattern-notebook.org/), and many others. We hope to explore collaborations where useful as this community works together on improving interactive visualization for single-cell data.
+We've been heavily inspired by several other related single-cell visualization projects, including the [UCSC Cell Browser](http://cells.ucsc.edu/), [Cytoscape](http://www.cytoscape.org/), [Xena](https://xena.ucsc.edu/), [ASAP](https://asap.epfl.ch/), [GenePattern](http://genepattern-notebook.org/), and many others. We hope to explore collaborations where useful as this community works together on improving interactive visualization for single-cell data.
 
 We were inspired by Mike Bostock and the [crossfilter](https://github.com/crossfilter) team for the design of our filtering implementation.
 

app.json

@@ -1,21 +0,0 @@
-{
-  "name": "cellxgene",
-  "description": "An interactive explorer for single-cell transcriptomics data",
-  "repository": "https://github.com/chanzuckerberg/cellxgene",
-  "logo": "https://cellxgene-example-data.czi.technology/favicon.png",
-  "keywords": [
-    "scientific",
-    "visualization",
-    "scrna-seq",
-    "transcriptomics",
-    "dataviz"
-  ],
-  "stack": "container",
-  "env": {
-    "DATASET": {
-      "description": "Link to dataset",
-      "value": "https://cellxgene-example-data.czi.technology/pbmc3k.h5ad",
-      "required": "true"
-    }
-  }
-}

client/.husky/pre-commit

@@ -0,0 +1,5 @@
+#!/bin/sh
+. "$(dirname "$0")/_/husky.sh"
+
+cd client
+npx --no-install lint-staged --config "./configuration/lint-staged/lint-staged.config.js"

client/Makefile

@@ -1,94 +1,61 @@
 include ../common.mk
 
-DATASET := $(if $(DATASET),$(DATASET),../example-dataset/pbmc3k.h5ad)
-ANNOTATIONS := $(if $(ANNOTATIONS),$(ANNOTATIONS),../server/test/test_datasets/pbmc3k-annotations.csv)
+ANNOTATIONS := $(if $(ANNOTATIONS),$(ANNOTATIONS),../test/fixtures/pbmc3k-annotations.csv)
+GENE_SETS := $(if $(GENE_SETS),$(GENE_SETS),../test/fixtures/pbmc3k-genesets.csv)
 ANNOTATIONS_FILENAME := $(shell basename $(ANNOTATIONS))
+GENE_SETS_FILENAME := $(shell basename $(GENE_SETS))
+
+CXG_CONFIG := $(if $(CXG_CONFIG),$(CXG_CONFIG),./__tests__/e2e/test_config.yaml)
+
+
 
 # Packaging
 .PHONY: clean
 clean:
 	rm -rf node_modules
+	rm -f __tests__/screenshots/*.png
 
 .PHONY: ci
 ci:
-	npm ci client
+	npm ci
 
 .PHONY: install
 install:
-	npm install client
+	npm install
 
 .PHONY: build
+WEBPACK_CONFIG ?= configuration/webpack/webpack.config.prod.js
 build:
-	npm run build
-
+	npm run build $(WEBPACK_CONFIG)
 
 # Development convenience methods
-
-.PHONY: server-requirements
-server-requirements:
-	virtualenv -p python3 ../venv
-	source ../venv/bin/activate && \
-		pip install -r ../server/requirements-dev.txt && \
-		yes | pip uninstall cellxgene || true && \
-		pip install -e ..
-
 .PHONY: start-frontend
 start-frontend:
 	node server/development.js
 
-.PHONY: start-server
-start-server:
-	cellxgene launch -p $(CXG_SERVER_PORT) $(CXG_OPTIONS) $(DATASET)
-
-.PHONY: backend-dev
-backend-dev: server-requirements
-	source ../venv/bin/activate && $(MAKE) start-server
-
-.PHONY: backend-dev-anno-ontology
-backend-dev-anno-ontology: server-requirements
-	CXG_OPTIONS='--experimental-annotations-ontology' \
-		$(MAKE) backend-dev
-
-.PHONY: test
-test:
-	node node_modules/jest/bin/jest.js
-
-.PHONY: e2e
-e2e:
-	node node_modules/jest/bin/jest.js \
-		--verbose false \
-		--config __tests__/e2e/e2eJestConfig.json \
-		e2e/e2e.test.js
-
-.PHONY: e2e-annotations
-e2e-annotations:
-	node node_modules/jest/bin/jest.js \
-		--verbose false \
-		--config __tests__/e2e/e2eJestConfig.json \
-		e2e/e2eAnnotations.test.js
-
+# start an instance of cellxgene and run the end-to-end tests
 .PHONY: smoke-test
 smoke-test:
 	start_server_and_test \
-		'CXG_OPTIONS="--disable-annotations" $(MAKE) start-server' \
+		'CXG_OPTIONS="--config-file $(CXG_CONFIG)" $(MAKE) start-server' \
 		$(CXG_SERVER_PORT) \
-		'$(MAKE) e2e'
-
+		'CXG_URL_BASE="http://localhost:$(CXG_SERVER_PORT)" npm run e2e -- --verbose false'
 
+# start an instance of cellxgene and run the end-to-end annotations tests
 .PHONY: smoke-test-annotations
 smoke-test-annotations:
 	$(eval TMP_DIR := $(shell  mktemp -d /tmp/cellxgene_XXXXXX))
 	cp $(ANNOTATIONS) $(TMP_DIR)/ && \
+	cp $(GENE_SETS) $(TMP_DIR)/ && \
 	start_server_and_test \
-		'CXG_OPTIONS="--annotations-file $(TMP_DIR)/$(ANNOTATIONS_FILENAME)" $(MAKE) start-server' \
+		'CXG_OPTIONS="--annotations-file $(TMP_DIR)/$(ANNOTATIONS_FILENAME) --gene-sets-file $(TMP_DIR)/$(GENE_SETS_FILENAME)" $(MAKE) start-server' \
 		$(CXG_SERVER_PORT) \
-		'$(MAKE) e2e-annotations'
+		'CXG_URL_BASE="http://localhost:$(CXG_SERVER_PORT)" npm run e2e-annotations -- --verbose false'
 	rm -rf $(TMP_DIR)
 
 .PHONY: unit-test
 unit-test:
-	node node_modules/jest/bin/jest.js \
-	    --testPathIgnorePatterns e2e
+	node node_modules/jest/bin/jest.js --testPathIgnorePatterns e2e
 
 # pass remaining commands through to npm run
 %:

client/__tests__/e2e/__snapshots__/e2e.test.js.snap

@@ -0,0 +1,5 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`did launch page launched 1`] = `"<span style=\\"max-width: 155px; display: flex; overflow: hidden; justify-content: flex-start; width: 100%; padding: 0px;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">pbm</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">c3k</span><span style=\\"position: absolute; right: 0px; color: inherit;\\">c3k</span></span></span>"`;
+
+exports[`metadata loads categories and values from dataset appear 1`] = `"<div style=\\"display: flex; justify-content: space-between; align-items: baseline;\\"><div style=\\"display: flex; justify-content: flex-start; align-items: flex-start;\\"><label class=\\"bp3-control bp3-checkbox\\" for=\\"category-select-louvain\\"><input id=\\"category-select-louvain\\" data-testclass=\\"category-select\\" data-testid=\\"louvain:category-select\\" type=\\"checkbox\\" checked=\\"\\"><span class=\\"bp3-control-indicator\\"></span></label><span role=\\"menuitem\\" tabindex=\\"0\\" data-testclass=\\"category-expand\\" data-testid=\\"louvain:category-expand\\" style=\\"cursor: pointer;\\"><span aria-haspopup=\\"true\\" class=\\"bp3-popover2-target\\"><span data-testid=\\"louvain:category-label\\" tabindex=\\"-1\\" aria-label=\\"louvain\\" class=\\"\\" style=\\"max-width: 265px;\\"><span style=\\"max-width: 265px; display: flex; overflow: hidden; justify-content: flex-start; width: 100%; padding: 0px;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">lou</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">vain</span><span style=\\"position: absolute; right: 0px; color: inherit;\\">vain</span></span></span></span></span><svg stroke=\\"currentColor\\" fill=\\"currentColor\\" stroke-width=\\"0\\" viewBox=\\"0 0 320 512\\" data-testclass=\\"category-expand-is-not-expanded\\" height=\\"1em\\" width=\\"1em\\" xmlns=\\"http://www.w3.org/2000/svg\\" style=\\"font-size: 10px; margin-left: 5px;\\"><path d=\\"M285.476 272.971L91.132 467.314c-9.373 9.373-24.569 9.373-33.941 0l-22.667-22.667c-9.357-9.357-9.375-24.522-.04-33.901L188.505 256 34.484 101.255c-9.335-9.379-9.317-24.544.04-33.901l22.667-22.667c9.373-9.373 24.569-9.373 33.941 0L285.475 239.03c9.373 9.372 9.373 24.568.001 33.941z\\"></path></svg></span></div><div><span class=\\"bp3-popover-wrapper\\"><span aria-haspopup=\\"true\\" class=\\"bp3-popover-target\\"><a role=\\"button\\" data-testclass=\\"colorby\\" data-testid=\\"colorby-louvain\\" class=\\"bp3-button\\" tabindex=\\"0\\"><span icon=\\"tint\\" class=\\"bp3-icon bp3-icon-tint\\"><svg data-icon=\\"tint\\" width=\\"16\\" height=\\"16\\" viewBox=\\"0 0 16 16\\"><desc>tint</desc><path d=\\"M7.88 1s-4.9 6.28-4.9 8.9c.01 2.82 2.34 5.1 4.99 5.1 2.65-.01 5.03-2.3 5.03-5.13C12.99 7.17 7.88 1 7.88 1z\\" fill-rule=\\"evenodd\\"></path></svg></span></a></span></span></div></div><div style=\\"margin-left: 26px;\\"></div>"`;

client/__tests__/e2e/cellxgeneActions.js

@@ -1,196 +1,445 @@
+/* eslint-disable no-await-in-loop -- await in loop is needed to emulate sequential user actions  */
 import { strict as assert } from "assert";
+import {
+  clearInputAndTypeInto,
+  clickOn,
+  getAllByClass,
+  getOneElementInnerText,
+  typeInto,
+  waitByID,
+  waitByClass,
+  waitForAllByIds,
+  clickOnUntil,
+  getTestClass,
+  getTestId,
+  isElementPresent,
+} from "./puppeteerUtils";
 
-export const cellxgeneActions = (page, utils) => ({
-
-  async drag(testId, start, end, lasso = false) {
-    const layout = await utils.waitByID(testId);
-    const elBox = await layout.boxModel();
-    const x1 = elBox.content[0].x + start.x;
-    const x2 = elBox.content[0].x + end.x;
-    const y1 = elBox.content[0].y + start.y;
-    const y2 = elBox.content[0].y + end.y;
+export async function drag(testId, start, end, lasso = false) {
+  const layout = await waitByID(testId);
+  const elBox = await layout.boxModel();
+  const x1 = elBox.content[0].x + start.x;
+  const x2 = elBox.content[0].x + end.x;
+  const y1 = elBox.content[0].y + start.y;
+  const y2 = elBox.content[0].y + end.y;
+  await page.mouse.move(x1, y1);
+  await page.mouse.down();
+  if (lasso) {
+    await page.mouse.move(x2, y1);
+    await page.mouse.move(x2, y2);
+    await page.mouse.move(x1, y2);
     await page.mouse.move(x1, y1);
-    await page.mouse.down();
-    if (lasso) {
-      await page.mouse.move(x2, y1);
-      await page.mouse.move(x2, y2);
-      await page.mouse.move(x1, y2);
-      await page.mouse.move(x1, y1);
-    } else {
-      await page.mouse.move(x2, y2);
-    }
-    await page.mouse.up();
-  },
-
-  async clickOnCoordinate(testId, coord) {
-    const layout = await utils.waitByID(testId);
-    const elBox = await layout.boxModel();
-    const x = elBox.content[0].x + coord.x;
-    const y = elBox.content[0].y + coord.y;
-    await page.mouse.click(x, y);
-  },
-
-  async getAllHistograms(testclass, testIds) {
-    const histTestIds = testIds.map(tid => `histogram-${tid}`);
-    // these load asynchronously, so we need to wait for each histogram individually
-    await utils.waitForAllByIds(histTestIds);
-    const allHistograms = await utils.getAllByClass(testclass);
-    return allHistograms.map(hist => hist.replace(/^histogram-/, ""));
-  },
-
-  async getAllCategoriesAndCounts(category) {
-    await utils.waitByClass("categorical-row");
-    return page.$$eval(
-      `[data-testid="category-${category}"] [data-testclass='categorical-row']`,
-      rows => Object.fromEntries(rows.map(row => {
-        const cat = row.querySelector("[data-testclass='categorical-value']").innerText;
-        const count = row.querySelector("[data-testclass='categorical-value-count']").innerText;
-        return [cat, count];
-      }))
-    );
-  },
-
-  async cellSet(num) {
-    await utils.clickOn(`cellset-button-${num}`);
-    return utils.getOneElementInnerText(`[data-testid='cellset-count-${num}']`);
-  },
-
-  async resetCategory(category) {
-    const checkboxId = `${category}:category-select`;
-    await utils.waitByID(checkboxId);
-    const checkedPseudoclass = await page.$eval(
-      `[data-testid='${checkboxId}']`,
-      el => el.matches(":checked")
-    );
-    if (!checkedPseudoclass) await utils.clickOn(checkboxId);
-    try {
-      const categoryRow = await utils.waitByID(`${category}:category-expand`);
-      const isExpanded = await categoryRow.$("[data-testclass='category-expand-is-expanded']");
-      if (isExpanded) await utils.clickOn(`${category}:category-expand`);
-    } catch {}
-  },
-
-  async calcCoordinate(testId, xAsPercent, yAsPercent) {
-    const el = await utils.waitByID(testId);
-    const size = await el.boxModel();
-    return {
-      x: Math.floor(size.width * xAsPercent),
-      y: Math.floor(size.height * yAsPercent)
-    }
-  },
-
-  async calcDragCoordinates(testId, coordinateAsPercent) {
-    return {
-      start: await this.calcCoordinate(testId, coordinateAsPercent.x1, coordinateAsPercent.y1),
-      end: await this.calcCoordinate(testId, coordinateAsPercent.x2, coordinateAsPercent.y2)
-    };
-  },
-
-  async selectCategory(category, values, reset = true) {
-    if (reset) await this.resetCategory(category);
-    await utils.clickOn(`${category}:category-expand`);
-    await utils.clickOn(`${category}:category-select`);
-    for (const val of values) {
-      await utils.clickOn(`categorical-value-select-${category}-${val}`);
-    }
-  },
-
-  async expandCategory(category) {
-    const expand = await utils.waitByID(`${category}:category-expand`);
-    const notExpanded = await expand.$("[data-testclass='category-expand-is-not-expanded']");
-    if (notExpanded) await utils.clickOn(`${category}:category-expand`);
-  },
-
-  async clip(min = 0, max = 100) {
-    await utils.clickOn("visualization-settings");
-    await utils.clearInputAndTypeInto("clip-min-input", min);
-    await utils.clearInputAndTypeInto("clip-max-input", max);
-    await utils.clickOn("clip-commit");
-  },
-
-  async createCategory(categoryName) {
-    await utils.clickOn("open-annotation-dialog");
-    await utils.typeInto("new-category-name", categoryName);
-    await utils.clickOn("submit-category");
-  },
-
-  async renameCategory(oldCatgoryName, newCategoryName) {
-    await utils.clickOn(`${oldCatgoryName}:see-actions`);
-    await utils.clickOn(`${oldCatgoryName}:edit-category-mode`);
-    await utils.clearInputAndTypeInto(`${oldCatgoryName}:edit-category-name-text`, newCategoryName);
-    await utils.clickOn(`${oldCatgoryName}:submit-category-edit`);
-  },
-
-  async deleteCategory(categoryName) {
-    await utils.clickOn(`${categoryName}:see-actions`);
-    await utils.clickOn(`${categoryName}:delete-category`);
-  },
-
-  async createLabel(categoryName, labelName) {
-    await utils.clickOn(`${categoryName}:see-actions`);
-    await utils.clickOn(`${categoryName}:add-new-label-to-category`);
-    await utils.typeInto(`${categoryName}:new-label-name`, labelName);
-    await utils.clickOn(`${categoryName}:submit-label`);
-  },
-
-  async deleteLabel(categoryName, labelName) {
-    await this.expandCategory(categoryName);
-    await utils.clickOn(`${categoryName}:${labelName}:see-actions`);
-    await utils.clickOn( `${categoryName}:${labelName}:delete-label`);
-  },
-
-  async renameLabel(categoryName, oldLabelName, newLabelName) {
-    await this.expandCategory(categoryName);
-    await utils.clickOn(`${categoryName}:${oldLabelName}:see-actions`);
-    await utils.clickOn(`${categoryName}:${oldLabelName}:edit-label`);
-    await utils.clearInputAndTypeInto(
-      `${categoryName}:${oldLabelName}:edit-label-name`,
-      newLabelName
-    );
-    await utils.clickOn(`${categoryName}:${oldLabelName}:submit-label-edit`);
-  },
-
-  async addGeneToSearch(geneName) {
-    await utils.typeInto("gene-search", geneName);
-    await page.keyboard.press("Enter");
-    await page.waitForSelector(
-      `[data-testid='histogram-${geneName}']`
-    );
-  },
-
-  async subset(coordinatesAsPercent) {
-    // In order to deselect the selection after the subset, make sure we have some clear part
-    // of the scatterplot we can click on
-    assert(coordinatesAsPercent.x2 < 0.99 || coordinatesAsPercent.y2 < 0.99);
-    const lassoSelection = await this.calcDragCoordinates( "layout-graph", coordinatesAsPercent);
-    await this.drag("layout-graph", lassoSelection.start, lassoSelection.end, true );
-    await utils.clickOn("subset-button");
-    const clearCoordinate = await this.calcCoordinate(
-      "layout-graph",
-      0.5,
-      0.99
-    );
-    await this.clickOnCoordinate("layout-graph", clearCoordinate);
-  },
-
-  async setSellSet(cellSet, cellSetNum) {
-    for (const selection of cellSet.filter(sel => sel.kind === "categorical")) {
-      await this.selectCategory(selection.metadata, selection.values, true);
-    }
-    await this.cellSet(cellSetNum);
-  },
-
-  async runDiffExp(cellSet1, cellSet2) {
-    await this.setSellSet(cellSet1, 1);
-    await this.setSellSet(cellSet2, 2);
-    await utils.clickOn("diffexp-button");
-  },
-
-  async bulkAddGenes(geneNames) {
-    await utils.clickOn("section-bulk-add");
-    await utils.typeInto("input-bulk-add", geneNames.join(","));
-    await page.keyboard.press("Enter");
+  } else {
+    await page.mouse.move(x2, y2);
   }
-});
+  await page.mouse.up();
+}
 
+export async function clickOnCoordinate(testId, coord) {
+  const layout = await expect(page).toMatchElement(getTestId(testId));
+  const elBox = await layout.boxModel();
 
+  if (!elBox) {
+    throw Error("Layout's boxModel is not available!");
+  }
+
+  const x = elBox.content[0].x + coord.x;
+  const y = elBox.content[0].y + coord.y;
+  await page.mouse.click(x, y);
+}
+
+export async function getAllHistograms(testclass, testIds) {
+  const histTestIds = testIds.map((tid) => `histogram-${tid}`);
+
+  // these load asynchronously, so we need to wait for each histogram individually,
+  // and they may be quite slow in some cases.
+  await waitForAllByIds(histTestIds, { timeout: 4 * 60 * 1000 });
+
+  const allHistograms = await getAllByClass(testclass);
+
+  const testIDs = await Promise.all(
+    allHistograms.map((hist) =>
+      page.evaluate((elem) => elem.dataset.testid, hist)
+    )
+  );
+
+  return testIDs.map((id) => id.replace(/^histogram-/, ""));
+}
+
+export async function getAllCategoriesAndCounts(category) {
+  // these load asynchronously, so we have to wait for the specific category.
+  await waitByID(`category-${category}`);
+
+  return page.$$eval(
+    `[data-testid="category-${category}"] [data-testclass='categorical-row']`,
+    (rows) =>
+      Object.fromEntries(
+        rows.map((row) => {
+          const cat = row
+            .querySelector("[data-testclass='categorical-value']")
+            .getAttribute("aria-label");
+
+          const count = row.querySelector(
+            "[data-testclass='categorical-value-count']"
+          ).innerText;
+
+          return [cat, count];
+        })
+      )
+  );
+}
+
+export async function getCellSetCount(num) {
+  await clickOn(`cellset-button-${num}`);
+  return getOneElementInnerText(`[data-testid='cellset-count-${num}']`);
+}
+
+export async function resetCategory(category) {
+  const checkboxId = `${category}:category-select`;
+  await waitByID(checkboxId);
+  const checkedPseudoclass = await page.$eval(
+    `[data-testid='${checkboxId}']`,
+    (el) => el.matches(":checked")
+  );
+  if (!checkedPseudoclass) await clickOn(checkboxId);
+
+  const categoryRow = await waitByID(`${category}:category-expand`);
+
+  const isExpanded = await categoryRow.$(
+    "[data-testclass='category-expand-is-expanded']"
+  );
+
+  if (isExpanded) await clickOn(`${category}:category-expand`);
+}
+
+export async function calcCoordinate(testId, xAsPercent, yAsPercent) {
+  const el = await waitByID(testId);
+  const size = await el.boxModel();
+  return {
+    x: Math.floor(size.width * xAsPercent),
+    y: Math.floor(size.height * yAsPercent),
+  };
+}
+
+export async function calcDragCoordinates(testId, coordinateAsPercent) {
+  return {
+    start: await calcCoordinate(
+      testId,
+      coordinateAsPercent.x1,
+      coordinateAsPercent.y1
+    ),
+    end: await calcCoordinate(
+      testId,
+      coordinateAsPercent.x2,
+      coordinateAsPercent.y2
+    ),
+  };
+}
+
+export async function selectCategory(category, values, reset = true) {
+  if (reset) await resetCategory(category);
+
+  await clickOn(`${category}:category-expand`);
+  await clickOn(`${category}:category-select`);
+
+  for (const value of values) {
+    await clickOn(`categorical-value-select-${category}-${value}`);
+  }
+}
+
+export async function expandCategory(category) {
+  const expand = await waitByID(`${category}:category-expand`);
+  const notExpanded = await expand.$(
+    "[data-testclass='category-expand-is-not-expanded']"
+  );
+  if (notExpanded) await clickOn(`${category}:category-expand`);
+}
+
+export async function clip(min = 0, max = 100) {
+  await clickOn("visualization-settings");
+  await clearInputAndTypeInto("clip-min-input", min);
+  await clearInputAndTypeInto("clip-max-input", max);
+  await clickOn("clip-commit");
+}
+
+export async function createCategory(categoryName) {
+  await clickOnUntil("open-annotation-dialog", async () => {
+    await expect(page).toMatchElement(getTestId("new-category-name"));
+  });
+
+  await typeInto("new-category-name", categoryName);
+  await clickOn("submit-category");
+}
+
+/*
+
+  GENESET
+
+*/
+
+export async function colorByGeneset(genesetName) {
+  await clickOn(`${genesetName}:colorby-entire-geneset`);
+}
+
+export async function colorByGene(gene) {
+  await clickOn(`colorby-${gene}`);
+}
+
+export async function assertColorLegendLabel(label) {
+  const handle = await waitByID("continuous_legend_color_by_label");
+
+  const result = await handle.evaluate((node) =>
+    node.getAttribute("aria-label")
+  );
+
+  return expect(result).toBe(label);
+}
+
+export async function expandGeneset(genesetName) {
+  const expand = await waitByID(`${genesetName}:geneset-expand`);
+  const notExpanded = await expand.$(
+    "[data-testclass='geneset-expand-is-not-expanded']"
+  );
+  if (notExpanded) await clickOn(`${genesetName}:geneset-expand`);
+}
+
+export async function createGeneset(genesetName) {
+  await clickOnUntil("open-create-geneset-dialog", async () => {
+    await expect(page).toMatchElement(getTestId("create-geneset-input"));
+  });
+
+  await typeInto("create-geneset-input", genesetName);
+  await clickOn("submit-geneset");
+  await waitByClass("autosave-complete");
+}
+
+export async function editGenesetName(genesetName, editText) {
+  const editButton = `${genesetName}:edit-genesetName-mode`;
+  const submitButton = `${genesetName}:submit-geneset`;
+  await clickOnUntil(`${genesetName}:see-actions`, async () => {
+    await expect(page).toMatchElement(getTestId(editButton));
+  });
+  await clickOn(editButton);
+  await typeInto("rename-geneset-modal", editText);
+  await clickOn(submitButton);
+}
+
+export async function deleteGeneset(genesetName) {
+  const targetId = `${genesetName}:delete-geneset`;
+
+  await clickOnUntil(`${genesetName}:see-actions`, async () => {
+    await expect(page).toMatchElement(getTestId(targetId));
+  });
+
+  await clickOn(targetId);
+
+  await assertGenesetDoesNotExist(genesetName);
+  await waitByClass("autosave-complete");
+}
+
+export async function assertGenesetDoesNotExist(genesetName) {
+  const result = await isElementPresent(
+    getTestId(`${genesetName}:geneset-name`)
+  );
+  await expect(result).toBe(false);
+}
+
+export async function assertGenesetExists(genesetName) {
+  const handle = await waitByID(`${genesetName}:geneset-name`);
+
+  const result = await handle.evaluate((node) =>
+    node.getAttribute("aria-label")
+  );
+
+  return expect(result).toBe(genesetName);
+}
+
+/*
+
+  GENE
+
+*/
+
+export async function addGeneToSet(genesetName, geneToAddToSet) {
+  const submitButton = `${genesetName}:submit-gene`;
+
+  await clickOn(`${genesetName}:add-new-gene-to-geneset`);
+  await typeInto("add-genes", geneToAddToSet);
+  await clickOn(submitButton);
+}
+
+export async function removeGene(geneSymbol) {
+  const targetId = `delete-from-geneset:${geneSymbol}`;
+
+  await clickOn(targetId);
+
+  await waitByClass("autosave-complete");
+}
+
+export async function assertGeneExistsInGeneset(geneSymbol) {
+  const handle = await waitByID(`${geneSymbol}:gene-label`);
+
+  const result = await handle.evaluate((node) =>
+    node.getAttribute("aria-label")
+  );
+
+  return expect(result).toBe(geneSymbol);
+}
+
+export async function assertGeneDoesNotExist(geneSymbol) {
+  const result = await isElementPresent(getTestId(`${geneSymbol}:gene-label`));
+
+  await expect(result).toBe(false);
+}
+
+export async function expandGene(geneSymbol) {
+  await clickOn(`maximize-${geneSymbol}`);
+}
+
+/*
+
+  CATEGORY
+
+*/
+
+export async function duplicateCategory(categoryName) {
+  await clickOn("open-annotation-dialog");
+
+  await typeInto("new-category-name", categoryName);
+
+  const dropdownOptionClass = "duplicate-category-dropdown-option";
+
+  await clickOnUntil("duplicate-category-dropdown", async () => {
+    await expect(page).toMatchElement(getTestClass(dropdownOptionClass));
+  });
+
+  const option = await expect(page).toMatchElement(
+    getTestClass(dropdownOptionClass)
+  );
+
+  await option.click();
+
+  await clickOnUntil("submit-category", async () => {
+    await expect(page).toMatchElement(
+      getTestId(`${categoryName}:category-expand`)
+    );
+  });
+
+  await waitByClass("autosave-complete");
+}
+
+export async function renameCategory(oldCategoryName, newCategoryName) {
+  await clickOn(`${oldCategoryName}:see-actions`);
+  await clickOn(`${oldCategoryName}:edit-category-mode`);
+  await clearInputAndTypeInto(
+    `${oldCategoryName}:edit-category-name-text`,
+    newCategoryName
+  );
+  await clickOn(`${oldCategoryName}:submit-category-edit`);
+}
+
+export async function deleteCategory(categoryName) {
+  const targetId = `${categoryName}:delete-category`;
+
+  await clickOnUntil(`${categoryName}:see-actions`, async () => {
+    await expect(page).toMatchElement(getTestId(targetId));
+  });
+
+  await clickOn(targetId);
+
+  await assertCategoryDoesNotExist();
+}
+
+export async function createLabel(categoryName, labelName) {
+  /**
+   * (thuang): This explicit wait is needed, since currently showing
+   * the modal again quickly after the previous action dismissing the
+   * modal will persist the input value from the previous action.
+   *
+   * To reproduce:
+   * 1. Click on the plus sign to show the modal to add a new label to the category
+   * 2. Type `123` in the input box
+   * 3. Hover over your mouse over the plus sign and double click to quickly dismiss and
+   * invoke the modal again
+   * 4. You will see `123` is persisted in the input box
+   * 5. Expected behavior is to get an empty input box
+   */
+  await page.waitForTimeout(500);
+
+  await clickOn(`${categoryName}:see-actions`);
+
+  await clickOn(`${categoryName}:add-new-label-to-category`);
+
+  await typeInto(`${categoryName}:new-label-name`, labelName);
+
+  await clickOn(`${categoryName}:submit-label`);
+}
+
+export async function deleteLabel(categoryName, labelName) {
+  await expandCategory(categoryName);
+  await clickOn(`${categoryName}:${labelName}:see-actions`);
+  await clickOn(`${categoryName}:${labelName}:delete-label`);
+}
+
+export async function renameLabel(categoryName, oldLabelName, newLabelName) {
+  await expandCategory(categoryName);
+  await clickOn(`${categoryName}:${oldLabelName}:see-actions`);
+  await clickOn(`${categoryName}:${oldLabelName}:edit-label`);
+  await clearInputAndTypeInto(
+    `${categoryName}:${oldLabelName}:edit-label-name`,
+    newLabelName
+  );
+  await clickOn(`${categoryName}:${oldLabelName}:submit-label-edit`);
+}
+
+export async function addGeneToSearch(geneName) {
+  await typeInto("gene-search", geneName);
+  await page.keyboard.press("Enter");
+  await page.waitForSelector(`[data-testid='histogram-${geneName}']`);
+}
+
+export async function subset(coordinatesAsPercent) {
+  // In order to deselect the selection after the subset, make sure we have some clear part
+  // of the scatterplot we can click on
+  assert(coordinatesAsPercent.x2 < 0.99 || coordinatesAsPercent.y2 < 0.99);
+  const lassoSelection = await calcDragCoordinates(
+    "layout-graph",
+    coordinatesAsPercent
+  );
+  await drag("layout-graph", lassoSelection.start, lassoSelection.end, true);
+  await clickOn("subset-button");
+  const clearCoordinate = await calcCoordinate("layout-graph", 0.5, 0.99);
+  await clickOnCoordinate("layout-graph", clearCoordinate);
+}
+
+export async function setSellSet(cellSet, cellSetNum) {
+  const selections = cellSet.filter((sel) => sel.kind === "categorical");
+
+  for (const selection of selections) {
+    await selectCategory(selection.metadata, selection.values, true);
+  }
+
+  await getCellSetCount(cellSetNum);
+}
+
+export async function runDiffExp(cellSet1, cellSet2) {
+  await setSellSet(cellSet1, 1);
+  await setSellSet(cellSet2, 2);
+  await clickOn("diffexp-button");
+}
+
+export async function bulkAddGenes(geneNames) {
+  await clickOn("section-bulk-add");
+  await typeInto("input-bulk-add", geneNames.join(","));
+  await page.keyboard.press("Enter");
+}
+
+export async function assertCategoryDoesNotExist(categoryName) {
+  const result = await isElementPresent(
+    getTestId(`${categoryName}:category-label`)
+  );
+
+  await expect(result).toBe(false);
+}
+
+/* eslint-enable no-await-in-loop -- await in loop is needed to emulate sequential user actions */

client/__tests__/e2e/config.js

@@ -1,9 +1,10 @@
-export const jest_env = process.env.JEST_ENV;
-export const appPort = process.env.CXG_SERVER_PORT;
-export const appUrlBase = `http://localhost:${appPort}`;
-export const DEV = jest_env === "dev";
-export const DEBUG = jest_env === "debug";
-export const DATASET = "pbmc3k";
+import * as ENV_DEFAULT from "../../../environment.default.json";
 
-if (DEBUG) jest.setTimeout(100000);
-if (DEV) jest.setTimeout(10000);
+export const jestEnv = process.env.JEST_ENV || ENV_DEFAULT.JEST_ENV;
+export const appUrlBase =
+  process.env.CXG_URL_BASE || `http://localhost:${ENV_DEFAULT.CXG_CLIENT_PORT}`;
+export const DATASET = "pbmc3k";
+export const isDev = jestEnv === ENV_DEFAULT.DEV;
+export const isDebug = jestEnv === ENV_DEFAULT.DEBUG;
+export const TEST_EMAIL = "user@example.com";
+export const TEST_PASSWORD = process.env.TEST_ACCOUNT_PASS ?? "";

client/__tests__/e2e/data.js

@@ -4,7 +4,7 @@ export const datasets = {
     dataframe: {
       nObs: "2638",
       nVar: "1838",
-      type: "float32"
+      type: "float32",
     },
     categorical: {
       louvain: {
@@ -15,47 +15,47 @@ export const datasets = {
         "Dendritic cells": "37",
         "FCGR3A+ Monocytes": "150",
         Megakaryocytes: "15",
-        "NK cells": "154"
-      }
+        "NK cells": "154",
+      },
     },
     continuous: {
       n_genes: "int32",
       percent_mito: "float32",
-      n_counts: "float32"
+      n_counts: "float32",
     },
     cellsets: {
       lasso: [
         {
           "coordinates-as-percent": { x1: 0.1, y1: 0.25, x2: 0.7, y2: 0.75 },
-          count: "1181"
-        }
+          count: "1131",
+        },
       ],
       categorical: [
         {
           metadata: "louvain",
           values: ["B cells", "Megakaryocytes"],
-          count: "357"
-        }
+          count: "357",
+        },
       ],
       continuous: [
         {
           metadata: "n_genes",
           "coordinates-as-percent": { x1: 0.25, y1: 0.5, x2: 0.55, y2: 0.5 },
-          count: "1552"
-        }
-      ]
+          count: "1537",
+        },
+      ],
     },
 
     diffexp: {
       cellset1: [
-        { kind: "categorical", metadata: "louvain", values: ["B cells"] }
+        { kind: "categorical", metadata: "louvain", values: ["B cells"] },
       ],
       cellset2: [
         {
           kind: "categorical",
           metadata: "louvain",
-          values: ["CD4 T cells", "NK cells"]
-        }
+          values: ["CD4 T cells", "NK cells"],
+        },
       ],
       "gene-results": [
         "HLA-DRB1",
@@ -67,21 +67,21 @@ export const datasets = {
         "HLA-DQB1",
         "MS4A1",
         "IL32",
-        "CD37"
-      ]
+        "CD37",
+      ],
     },
 
     genes: {
       bulkadd: ["S100A8", "FCGR3A", "LGALS2", "GSTP1"],
-      search: "ACD"
+      search: "ACD",
     },
     subset: {
       cellset1: [
         {
           kind: "categorical",
           metadata: "louvain",
-          values: ["B cells", "Megakaryocytes"]
-        }
+          values: ["B cells", "Megakaryocytes"],
+        },
       ],
       count: "357",
       categorical: {
@@ -93,27 +93,27 @@ export const datasets = {
           "Dendritic cells": "0",
           "FCGR3A+ Monocytes": "0",
           Megakaryocytes: "15",
-          "NK cells": "0"
-        }
+          "NK cells": "0",
+        },
       },
       lasso: {
         "coordinates-as-percent": { x1: 0.25, y1: 0.05, x2: 0.75, y2: 0.55 },
-        count: "329"
-      }
+        count: "331",
+      },
     },
     scatter: {
-      genes: { x: "S100A8", y: "FCGR3A" }
+      genes: { x: "S100A8", y: "FCGR3A" },
     },
     pan: {
-      "coordinates-as-percent": { x1: 0.75, y1: 0.75, x2: 0.35, y2: 0.35 }
+      "coordinates-as-percent": { x1: 0.75, y1: 0.75, x2: 0.35, y2: 0.35 },
     },
     features: {
       panzoom: {
         lasso: {
           "coordinates-as-percent": { x1: 0.3, y1: 0.3, x2: 0.5, y2: 0.5 },
-          count: "24"
-        }
-      }
+          count: "38",
+        },
+      },
     },
     categoryLabel: {
       lasso: {
@@ -121,18 +121,18 @@ export const datasets = {
       },
       newCount: {
         bySubsetConfig: {
-          false: "600",
-          true: "591"
-        }
-      }
+          false: "668",
+          true: "659",
+        },
+      },
     },
     annotationsFromFile: {
       count: {
         bySubsetConfig: {
           false: "1161",
-          true: "856"
-        }
-      }
+          true: "852",
+        },
+      },
     },
     clip: {
       min: "30",
@@ -140,8 +140,8 @@ export const datasets = {
       metadata: "n_genes",
       gene: "S100A8",
       "coordinates-as-percent": { x1: 0.25, y1: 0.5, x2: 0.55, y2: 0.5 },
-      count: "392",
-      "gene-cell-count": "421"
-    }
-  }
+      count: "386",
+      "gene-cell-count": "416",
+    },
+  },
 };

client/__tests__/e2e/diffexpGeneSets.js

@@ -0,0 +1,105 @@
+export const diffexpPop1Genes = [
+  "CD79A",
+  "HLA-DRB1",
+  "HLA-DQA1",
+  "HLA-DPB1",
+  "HLA-DQB1",
+  "HLA-DPA1",
+  "MS4A1",
+  "LTB",
+  "CD79B",
+  "CD37",
+  "HLA-DMA",
+  "TCL1A",
+  "LINC00926",
+  "HLA-DMB",
+  "HVCN1",
+  "EAF2",
+  "FCRLA",
+  "IRF8",
+  "PKIG",
+  "P2RX5",
+  "HLA-DOB",
+  "SPIB",
+  "BLNK",
+  "SWAP70",
+  "PNOC",
+  "CD19",
+  "SMIM14",
+  "CD72",
+  "KIAA0125",
+  "IGLL5",
+  "ARHGAP24",
+  "COTL1",
+  "C16orf74",
+  "BTK",
+  "SNX29P2",
+  "ADAM28",
+  "FCGR2B",
+  "PLD4",
+  "PPP1R14A",
+  "MZB1",
+  "KIAA0040",
+  "PHACTR1",
+  "FCRL2",
+  "RIC3",
+  "P2RY10",
+  "SCPEP1",
+  "DRAM2",
+  "RP5-887A10.1",
+  "CD82",
+  "GPX1",
+];
+
+export const diffexpPop2Genes = [
+  "NKG7",
+  "GZMB",
+  "CTSW",
+  "PRF1",
+  "GNLY",
+  "GZMA",
+  "CST7",
+  "FGFBP2",
+  "SRGN",
+  "CD247",
+  "FCGR3A",
+  "TYROBP",
+  "FCER1G",
+  "ID2",
+  "SPON2",
+  "CCL4",
+  "CCL5",
+  "GZMH",
+  "GIMAP7",
+  "CLIC3",
+  "HOPX",
+  "XCL2",
+  "LGALS1",
+  "IGFBP7",
+  "AKR1C3",
+  "IL32",
+  "EFHD2",
+  "PRSS23",
+  "TTC38",
+  "ZAP70",
+  "S1PR5",
+  "SAMD3",
+  "GIMAP4",
+  "CCL3",
+  "ABI3",
+  "XCL1",
+  "S100A6",
+  "UBB",
+  "GPR56",
+  "PDIA3",
+  "S100A11",
+  "APOBEC3G",
+  "HAVCR2",
+  "PLEKHF1",
+  "LITAF",
+  "ARPC5L",
+  "PTGDR",
+  "PRMT2",
+  "GSTP1",
+  "FCRL6",
+];

client/__tests__/e2e/e2e.test.js

@@ -1,46 +1,63 @@
-/*
-Smoke test suite that will be run in Travis CI
-
-Tests included in this file are expected to be relatively stable and test core features
+/**
+ * Smoke test suite that will be run in Travis CI
+ * Tests included in this file are expected to be relatively stable and test core features
  */
+
+/* eslint-disable no-await-in-loop -- await in loop is needed to emulate sequential user actions  */
 import { appUrlBase, DATASET } from "./config";
-import { setupTestBrowser } from "./testBrowser";
+
 import { datasets } from "./data";
 
-let browser, page, utils, cxgActions;
+import {
+  clickOn,
+  getAllByClass,
+  getElementCoordinates,
+  getOneElementInnerHTML,
+  getTestId,
+  goToPage,
+  waitByID,
+} from "./puppeteerUtils";
+
+import {
+  calcDragCoordinates,
+  clip,
+  drag,
+  getAllCategoriesAndCounts,
+  getCellSetCount,
+  selectCategory,
+} from "./cellxgeneActions";
+
 const data = datasets[DATASET];
 
-beforeAll(async () => {
-  const browserViewport = { width: 1280, height: 960 };
-  [browser, page, utils, cxgActions] = await setupTestBrowser(browserViewport);
-});
-
-beforeEach(async () => {
-  await page.goto(appUrlBase);
-});
-
-afterAll(() => {
-  if (browser !== undefined) browser.close();
-});
-
 describe("did launch", () => {
   test("page launched", async () => {
-    const element = await utils.getOneElementInnerHTML("[data-testid='header']");
-    expect(element).toBe(data.title);
+    await goToPage(appUrlBase);
+
+    const element = await getOneElementInnerHTML(getTestId("header"));
+
+    expect(element).toMatchSnapshot();
   });
 });
 
 describe("metadata loads", () => {
   test("categories and values from dataset appear", async () => {
-    for (const label in data.categorical) {
-      await utils.waitByID(`category-${label}`);
-      const categoryName = await utils.getOneElementInnerText(`[data-testid="category-${label}"]`);
-      expect(categoryName).toMatch(label);
-      await utils.clickOn(`${label}:category-expand`);
-      const categories = await cxgActions.getAllCategoriesAndCounts(label);
+    await goToPage(appUrlBase);
+
+    for (const label of Object.keys(data.categorical)) {
+      const element = await getOneElementInnerHTML(
+        getTestId(`category-${label}`)
+      );
+
+      expect(element).toMatchSnapshot();
+
+      await clickOn(`${label}:category-expand`);
+
+      const categories = await getAllCategoriesAndCounts(label);
+
       expect(Object.keys(categories)).toMatchObject(
         Object.keys(data.categorical[label])
       );
+
       expect(Object.values(categories)).toMatchObject(
         Object.values(data.categorical[label])
       );
@@ -48,100 +65,100 @@ describe("metadata loads", () => {
   });
 
   test("continuous data appears", async () => {
-    for (const label in data.continuous) {
-      await utils.waitByID(`histogram-${label}`);
+    await goToPage(appUrlBase);
+
+    for (const label of Object.keys(data.continuous)) {
+      await waitByID(`histogram-${label}`);
     }
   });
 });
 
 describe("cell selection", () => {
   test("selects all cells cellset 1", async () => {
-    const cellCount = await cxgActions.cellSet(1);
+    await goToPage(appUrlBase);
+
+    const cellCount = await getCellSetCount(1);
     expect(cellCount).toBe(data.dataframe.nObs);
   });
 
   test("selects all cells cellset 2", async () => {
-    const cellCount = await cxgActions.cellSet(2);
+    await goToPage(appUrlBase);
+
+    const cellCount = await getCellSetCount(2);
     expect(cellCount).toBe(data.dataframe.nObs);
   });
 
   test("selects cells via lasso", async () => {
+    await goToPage(appUrlBase);
+
     for (const cellset of data.cellsets.lasso) {
-      const cellset1 = await cxgActions.calcDragCoordinates(
+      const cellset1 = await calcDragCoordinates(
         "layout-graph",
         cellset["coordinates-as-percent"]
       );
-      await cxgActions.drag("layout-graph", cellset1.start, cellset1.end, true);
-      const cellCount = await cxgActions.cellSet(1);
+
+      await drag("layout-graph", cellset1.start, cellset1.end, true);
+      const cellCount = await getCellSetCount(1);
       expect(cellCount).toBe(cellset.count);
     }
   });
 
   test("selects cells via categorical", async () => {
+    await goToPage(appUrlBase);
+
     for (const cellset of data.cellsets.categorical) {
-      await utils.clickOn(`${cellset.metadata}:category-expand`);
-      await utils.clickOn(`${cellset.metadata}:category-select`);
-      for (const val of cellset.values) {
-        await utils.clickOn(`categorical-value-select-${cellset.metadata}-${val}`);
+      await clickOn(`${cellset.metadata}:category-expand`);
+      await clickOn(`${cellset.metadata}:category-select`);
+
+      for (const value of cellset.values) {
+        await clickOn(`categorical-value-select-${cellset.metadata}-${value}`);
       }
-      const cellCount = await cxgActions.cellSet(1);
+
+      const cellCount = await getCellSetCount(1);
+
       expect(cellCount).toBe(cellset.count);
     }
   });
 
   test("selects cells via continuous", async () => {
+    await goToPage(appUrlBase);
+
     for (const cellset of data.cellsets.continuous) {
       const histBrushableAreaId = `histogram-${cellset.metadata}-plot-brushable-area`;
-      const coords = await cxgActions.calcDragCoordinates(
+
+      const coords = await calcDragCoordinates(
         histBrushableAreaId,
         cellset["coordinates-as-percent"]
       );
-      await cxgActions.drag(histBrushableAreaId, coords.start, coords.end);
-      const cellCount = await cxgActions.cellSet(1);
+
+      await drag(histBrushableAreaId, coords.start, coords.end);
+
+      const cellCount = await getCellSetCount(1);
+
       expect(cellCount).toBe(cellset.count);
     }
   });
 });
 
-describe("gene entry", () => {
-  test("search for single gene", async () => cxgActions.addGeneToSearch(data.genes.search));
-
-  test("bulk add genes", async () => {
-    const testGenes = data.genes.bulkadd;
-    await cxgActions.bulkAddGenes(testGenes);
-    const allHistograms = await cxgActions.getAllHistograms("histogram-user-gene", testGenes);
-    expect(allHistograms).toEqual(expect.arrayContaining(testGenes));
-    expect(allHistograms.length).toEqual(testGenes.length);
-  });
-});
-
-describe("differential expression", () => {
-  test("selects cells, saves them and performs diffexp", async () => {
-    await cxgActions.runDiffExp(data.diffexp.cellset1, data.diffexp.cellset2);
-    const allHistograms = await cxgActions.getAllHistograms(
-      "histogram-diffexp",
-      data.diffexp["gene-results"]
-    );
-    expect(allHistograms).toEqual(
-      expect.arrayContaining(data.diffexp["gene-results"])
-    );
-    expect(allHistograms.length).toEqual(data.diffexp["gene-results"].length);
-  });
-});
-
 describe("subset", () => {
   test("subset - cell count matches", async () => {
+    await goToPage(appUrlBase);
+
     for (const select of data.subset.cellset1) {
       if (select.kind === "categorical") {
-        await cxgActions.selectCategory(select.metadata, select.values, true);
+        await selectCategory(select.metadata, select.values, true);
       }
     }
-    await utils.clickOn("subset-button");
-    for (const label in data.subset.categorical) {
-      const categories = await cxgActions.getAllCategoriesAndCounts(label);
+
+    await clickOn("subset-button");
+
+    for (const label of Object.keys(data.subset.categorical)) {
+      const categories = await getAllCategoriesAndCounts(label);
+
       expect(Object.keys(categories)).toMatchObject(
         Object.keys(data.subset.categorical[label])
       );
+
       expect(Object.values(categories)).toMatchObject(
         Object.values(data.subset.categorical[label])
       );
@@ -149,182 +166,122 @@ describe("subset", () => {
   });
 
   test("lasso after subset", async () => {
+    await goToPage(appUrlBase);
+
     for (const select of data.subset.cellset1) {
       if (select.kind === "categorical") {
-        await cxgActions.selectCategory(select.metadata, select.values, true);
+        await selectCategory(select.metadata, select.values, true);
       }
     }
-    await utils.clickOn("subset-button");
-    const lassoSelection = await cxgActions.calcDragCoordinates(
+
+    await clickOn("subset-button");
+
+    const lassoSelection = await calcDragCoordinates(
       "layout-graph",
       data.subset.lasso["coordinates-as-percent"]
     );
-    await cxgActions.drag(
-      "layout-graph",
-      lassoSelection.start,
-      lassoSelection.end,
-      true
-    );
-    const cellCount = await cxgActions.cellSet(1);
+
+    await drag("layout-graph", lassoSelection.start, lassoSelection.end, true);
+
+    const cellCount = await getCellSetCount(1);
     expect(cellCount).toBe(data.subset.lasso.count);
   });
-
-  test("undo selection appends the top diff exp genes to user defined genes", async () => {
-    const userDefinedGenes = data.genes.bulkadd;
-    const diffExpGenes = data.diffexp["gene-results"];
-    await cxgActions.bulkAddGenes(userDefinedGenes);
-    const userDefinedHistograms = await cxgActions.getAllHistograms("histogram-user-gene", userDefinedGenes);
-    expect(userDefinedHistograms).toEqual(expect.arrayContaining(userDefinedGenes));
-    await cxgActions.subset({x1: 0.15, y1: 0.10, x2: 0.98, y2: 0.98});
-    await cxgActions.runDiffExp(data.diffexp.cellset1, data.diffexp.cellset2);
-    const diffExpHistograms = await cxgActions.getAllHistograms("histogram-diffexp", diffExpGenes);
-    expect(diffExpHistograms).toEqual(expect.arrayContaining(diffExpGenes));
-    await utils.clickOn("reset-subset-button");
-    const expected = [].concat(userDefinedGenes, diffExpGenes);
-    const userDefinedHistogramsAfterSubset = await cxgActions.getAllHistograms(
-      "histogram-user-gene",
-      expected
-    );
-    expect(userDefinedHistogramsAfterSubset).toEqual(expect.arrayContaining(expected));
-  });
-
-  test("subset selection appends the top diff exp genes to user defined genes", async () => {
-    const userDefinedGenes = data.genes.bulkadd;
-    const diffExpGenes = data.diffexp["gene-results"];
-    await cxgActions.bulkAddGenes(userDefinedGenes);
-    const userDefinedHistograms = await cxgActions.getAllHistograms("histogram-user-gene", userDefinedGenes);
-    expect(userDefinedHistograms).toEqual(expect.arrayContaining(userDefinedGenes));
-    await cxgActions.subset({x1: 0.15, y1: 0.10, x2: 0.98, y2: 0.98});
-    await cxgActions.runDiffExp(data.diffexp.cellset1, data.diffexp.cellset2);
-    const diffExpHistograms = await cxgActions.getAllHistograms("histogram-diffexp", diffExpGenes);
-    expect(diffExpHistograms).toEqual(expect.arrayContaining(diffExpGenes));
-    await cxgActions.subset({x1: 0.16, y1: 0.11, x2: 0.97, y2: 0.97});
-    const expected = [].concat(userDefinedGenes, diffExpGenes);
-    const userDefinedHistogramsAfterSubset = await cxgActions.getAllHistograms(
-      "histogram-user-gene",
-      expected
-    );
-    expect(userDefinedHistogramsAfterSubset).toEqual(expect.arrayContaining(expected));
-  });
-});
-
-describe("scatter plot", () => {
-  test("scatter plot appears", async () => {
-    await cxgActions.bulkAddGenes(Object.values(data.scatter.genes));
-    await utils.clickOn(`plot-x-${data.scatter.genes.x}`);
-    await utils.clickOn(`plot-y-${data.scatter.genes.y}`);
-    await utils.waitByID("scatterplot");
-  });
 });
 
 describe("clipping", () => {
   test("clip continuous", async () => {
-    await cxgActions.clip(data.clip.min, data.clip.max);
-    const histBrushableAreaId = `histogram-${data.clip.metadata}-plot-brushable-area`;
-    const coords = await cxgActions.calcDragCoordinates(
-      histBrushableAreaId,
-      data.clip["coordinates-as-percent"]
-    );
-    await cxgActions.drag(histBrushableAreaId, coords.start, coords.end);
-    const cellCount = await cxgActions.cellSet(1);
-    expect(cellCount).toBe(data.clip.count);
-  });
+    await goToPage(appUrlBase);
 
-  test("clip gene", async () => {
-    await utils.typeInto("gene-search", data.clip.gene);
-    await page.keyboard.press("Enter");
-    await page.waitForSelector(`[data-testid='histogram-${data.clip.gene}']`);
-    await cxgActions.clip(data.clip.min, data.clip.max);
-    const histBrushableAreaId = `histogram-${data.clip.gene}-plot-brushable-area`;
-    const coords = await cxgActions.calcDragCoordinates(
+    await clip(data.clip.min, data.clip.max);
+    const histBrushableAreaId = `histogram-${data.clip.metadata}-plot-brushable-area`;
+    const coords = await calcDragCoordinates(
       histBrushableAreaId,
       data.clip["coordinates-as-percent"]
     );
-    await cxgActions.drag(histBrushableAreaId, coords.start, coords.end);
-    const cellCount = await cxgActions.cellSet(1);
-    expect(cellCount).toBe(data.clip["gene-cell-count"]);
+    await drag(histBrushableAreaId, coords.start, coords.end);
+    const cellCount = await getCellSetCount(1);
+    expect(cellCount).toBe(data.clip.count);
   });
 });
 
 // interact with UI elements just that they do not break
 describe("ui elements don't error", () => {
   test("color by", async () => {
-    for (const label in data.categorical) {
-      await utils.clickOn(`colorby-${label}`);
-    }
-    for (const label in data.continuous) {
-      await utils.clickOn(`colorby-${label}`);
-    }
-  });
+    await goToPage(appUrlBase);
 
-  test("color by for gene", async () => {
-    await utils.typeInto("gene-search", data.genes.search);
-    await page.keyboard.press("Enter");
-    await page.waitForSelector(
-      `[data-testid='histogram-${data.genes.search}']`
-    );
-    await utils.clickOn(`colorby-${data.genes.search}`);
+    const allLabels = [
+      ...Object.keys(data.categorical),
+      ...Object.keys(data.continuous),
+    ];
+
+    for (const label of allLabels) {
+      await clickOn(`colorby-${label}`);
+    }
   });
 
   test("pan and zoom", async () => {
-    await utils.clickOn("mode-pan-zoom");
-    const panCoords = await cxgActions.calcDragCoordinates(
+    await goToPage(appUrlBase);
+
+    await clickOn("mode-pan-zoom");
+    const panCoords = await calcDragCoordinates(
       "layout-graph",
       data.pan["coordinates-as-percent"]
     );
-    await cxgActions.drag(
-      "layout-graph",
-      panCoords.start,
-      panCoords.end,
-      false
-    );
+
+    await drag("layout-graph", panCoords.start, panCoords.end, false);
+
     await page.evaluate("window.scrollBy(0, 1000);");
   });
 });
 
 describe("centroid labels", () => {
   test("labels are created", async () => {
-    await utils.clickOn("centroid-label-toggle");
+    await goToPage(appUrlBase);
+
     const labels = Object.keys(data.categorical);
-    /* eslint-disable no-await-in-loop */
+
+    await clickOn(`colorby-${labels[0]}`);
+    await clickOn("centroid-label-toggle");
+
     // Toggle colorby for each category and check to see if labels are generated
     for (let i = 0, { length } = labels; i < length; i += 1) {
       const label = labels[i];
-      await utils.clickOn(`colorby-${label}`);
-      const generatedLabels = await utils.getAllByClass("centroid-label");
+      // first label is already enabled
+      if (i !== 0) await clickOn(`colorby-${label}`);
+      const generatedLabels = await getAllByClass("centroid-label");
       // Number of labels generated should be equal to size of the object
       expect(generatedLabels).toHaveLength(
         Object.keys(data.categorical[label]).length
       );
     }
-    /* eslint-enable no-await-in-loop */
   });
 });
 
 describe("graph overlay", () => {
   test("transform centroids correctly", async () => {
+    await goToPage(appUrlBase);
+
     const category = Object.keys(data.categorical)[0];
-    await utils.clickOn("centroid-label-toggle");
-    await utils.clickOn(`colorby-${category}`);
-    await utils.clickOn("mode-pan-zoom");
-    const panCoords = await cxgActions.calcDragCoordinates(
+
+    await clickOn(`colorby-${category}`);
+    await clickOn("centroid-label-toggle");
+    await clickOn("mode-pan-zoom");
+
+    const panCoords = await calcDragCoordinates(
       "layout-graph",
       data.pan["coordinates-as-percent"]
     );
 
     const categoryValue = Object.keys(data.categorical[category])[0];
-    const initialCoordinates = await utils.getElementCoordinates(
+    const initialCoordinates = await getElementCoordinates(
       `${categoryValue}-centroid-label`
     );
-    await cxgActions.drag(
-      "layout-graph",
-      panCoords.start,
-      panCoords.end,
-      false
-    );
-    const terminalCoordinates = await utils.getElementCoordinates(
+
+    await drag("layout-graph", panCoords.start, panCoords.end, false);
+    const terminalCoordinates = await getElementCoordinates(
       `${categoryValue}-centroid-label`
     );
+
     expect(terminalCoordinates[0] - initialCoordinates[0]).toBeCloseTo(
       panCoords.end.x - panCoords.start.x
     );
@@ -333,3 +290,62 @@ describe("graph overlay", () => {
     );
   });
 });
+
+test("pan zoom mode resets lasso selection", async () => {
+  await goToPage(appUrlBase);
+
+  const panzoomLasso = data.features.panzoom.lasso;
+
+  const lassoSelection = await calcDragCoordinates(
+    "layout-graph",
+    panzoomLasso["coordinates-as-percent"]
+  );
+
+  await drag("layout-graph", lassoSelection.start, lassoSelection.end, true);
+  await waitByID("lasso-element", { visible: true });
+
+  const initialCount = await getCellSetCount(1);
+
+  expect(initialCount).toBe(panzoomLasso.count);
+
+  await clickOn("mode-pan-zoom");
+  await clickOn("mode-lasso");
+
+  const modeSwitchCount = await getCellSetCount(1);
+
+  expect(modeSwitchCount).toBe(initialCount);
+});
+
+test("lasso moves after pan", async () => {
+  await goToPage(appUrlBase);
+
+  const panzoomLasso = data.features.panzoom.lasso;
+  const coordinatesAsPercent = panzoomLasso["coordinates-as-percent"];
+
+  const lassoSelection = await calcDragCoordinates(
+    "layout-graph",
+    coordinatesAsPercent
+  );
+
+  await drag("layout-graph", lassoSelection.start, lassoSelection.end, true);
+  await waitByID("lasso-element", { visible: true });
+
+  const initialCount = await getCellSetCount(1);
+
+  expect(initialCount).toBe(panzoomLasso.count);
+
+  await clickOn("mode-pan-zoom");
+
+  const panCoords = await calcDragCoordinates(
+    "layout-graph",
+    coordinatesAsPercent
+  );
+
+  await drag("layout-graph", panCoords.start, panCoords.end, false);
+  await clickOn("mode-lasso");
+
+  const panCount = await getCellSetCount(2);
+
+  expect(panCount).toBe(initialCount);
+});
+/* eslint-enable no-await-in-loop -- await in loop is needed to emulate sequential user actions */

client/__tests__/e2e/e2eAnnotations.test.js

@@ -2,221 +2,594 @@
 Tests included in this file are specific to annotation features
 */
 import { appUrlBase, DATASET } from "./config";
-import { setupTestBrowser } from "./testBrowser";
 import { datasets } from "./data";
 
-let browser, page, utils, actions;
+import {
+  clickOn,
+  goToPage,
+  waitByClass,
+  waitByID,
+  getTestId,
+  getTestClass,
+  getAllByClass,
+  clickOnUntil,
+  getOneElementInnerHTML,
+} from "./puppeteerUtils";
+
+import {
+  assertCategoryDoesNotExist,
+  calcDragCoordinates,
+  createCategory,
+  createLabel,
+  deleteCategory,
+  deleteLabel,
+  drag,
+  expandCategory,
+  renameCategory,
+  renameLabel,
+  subset,
+  duplicateCategory,
+  createGeneset,
+  deleteGeneset,
+  assertGenesetExists,
+  assertGenesetDoesNotExist,
+  getCellSetCount,
+  expandGeneset,
+  editGenesetName,
+  addGeneToSet,
+  assertGeneExistsInGeneset,
+  removeGene,
+  assertGeneDoesNotExist,
+  expandGene,
+  colorByGeneset,
+  assertColorLegendLabel,
+  colorByGene,
+} from "./cellxgeneActions";
+
 const data = datasets[DATASET];
 
-beforeAll(async () => {
-  const browserViewport = {width: 1280, height: 960};
-  [browser, page, utils, actions] = await setupTestBrowser(browserViewport);
-});
+const perTestCategoryName = "TEST-CATEGORY";
+const perTestLabelName = "TEST-LABEL";
 
-afterAll(() => {
-  if (browser !== undefined) browser.close()
+// geneset CRUD
+const genesetToDeleteName = "geneset_to_delete";
+const preExistingGenesetName = "fifth_dataset";
+const meanExpressionBrushGenesetName = "second_gene_set";
+const meanExpressionBrushCellsSelected = "557";
+const subsetMeanExpressionBrushCellsSelected = "452";
+
+// initial text, the text we type in, the result
+const editableGenesetName = "geneset_to_edit";
+const editText = "_111";
+const newGenesetName = "geneset_to_edit_111";
+
+// add gene to set
+const geneToAddToSet = "RER1";
+const setToAddGeneTo = "fill_this_geneset";
+
+// remove gene from set
+const geneToRemove = "SIK1";
+const setToRemoveFrom = "empty_this_geneset";
+
+// brush a gene
+const geneToBrushAndColorBy = "SIK1";
+const brushThisGeneGeneset = "brush_this_gene";
+const geneBrushedCellCount = "109";
+const subsetGeneBrushedCellCount = "96";
+
+const genesetDescriptionID =
+  "geneset-description-tooltip-fourth_gene_set: fourth description";
+const genesetDescriptionString = "fourth_gene_set: fourth description";
+const genesetToCheckForDescription = "fourth_gene_set";
+
+async function setup(config) {
+  await goToPage(appUrlBase);
+
+  if (config.categoricalAnno) {
+    // setup the test fixtures
+    await createCategory(perTestCategoryName);
+    await createLabel(perTestCategoryName, perTestLabelName);
+  }
+
+  if (config.withSubset) {
+    await subset({ x1: 0.1, y1: 0.1, x2: 0.8, y2: 0.8 });
+  }
+
+  await waitByClass("autosave-complete");
+}
+
+describe.each([
+  { withSubset: true, tag: "subset" },
+  { withSubset: false, tag: "whole" },
+])("geneSET crud operations and interactions", (config) => {
+  test("genesets load from csv", async () => {
+    await setup(config);
+
+    await assertGenesetExists(preExistingGenesetName);
+  });
+  test("brush on geneset mean", async () => {
+    await setup(config);
+
+    await expandGeneset(meanExpressionBrushGenesetName);
+
+    const histBrushableAreaId = `histogram-${meanExpressionBrushGenesetName}-plot-brushable-area`;
+
+    const coords = await calcDragCoordinates(histBrushableAreaId, {
+      x1: 0.25,
+      y1: 0.5,
+      x2: 0.55,
+      y2: 0.5,
+    });
+
+    await drag(histBrushableAreaId, coords.start, coords.end);
+
+    const cellCount = await getCellSetCount(1);
+    if (config.withSubset) {
+      expect(cellCount).toBe(subsetMeanExpressionBrushCellsSelected);
+    } else {
+      expect(cellCount).toBe(meanExpressionBrushCellsSelected);
+    }
+  });
+  test("color by mean expression", async () => {
+    await setup(config);
+
+    await colorByGeneset(meanExpressionBrushGenesetName);
+    await assertColorLegendLabel(meanExpressionBrushGenesetName);
+  });
+  test("diffexp", async () => {
+    if (config.withSubset) return;
+
+    await setup(config);
+
+    // set the two cell sets to b cells vs nk cells
+    await expandCategory(`louvain`);
+    await clickOn(`louvain:category-select`);
+    await clickOn(`categorical-value-select-louvain-B cells`);
+    await clickOn(`cellset-button-1`);
+    await clickOn(`categorical-value-select-louvain-B cells`);
+    await clickOn(`categorical-value-select-louvain-NK cells`);
+    await clickOn(`cellset-button-2`);
+
+    // run diffexp
+    await clickOn(`diffexp-button`);
+    await waitByClass("pop-1-geneset-expand");
+    await expect(page).toClick(getTestClass("pop-1-geneset-expand"));
+
+    await page.waitForFunction(
+      (selector) => !document.querySelector(selector),
+      {},
+      getTestClass("gene-loading-spinner")
+    );
+
+    let genesHTML = await getOneElementInnerHTML(
+      getTestClass("gene-set-genes")
+    );
+
+    expect(genesHTML).toMatchSnapshot();
+
+    await expect(page).toClick(getTestClass("pop-1-geneset-expand"));
+    await expect(page).toClick(getTestClass("pop-2-geneset-expand"));
+
+    await page.waitForFunction(
+      (selector) => !document.querySelector(selector),
+      {},
+      getTestClass("gene-loading-spinner")
+    );
+
+    genesHTML = await getOneElementInnerHTML(getTestClass("gene-set-genes"));
+
+    expect(genesHTML).toMatchSnapshot();
+  });
+  test("create a new geneset and undo/redo", async () => {
+    if (config.withSubset) return;
+
+    await setup(config);
+
+    const genesetName = `test-geneset-foo-123`;
+    await assertGenesetDoesNotExist(genesetName);
+    await createGeneset(genesetName);
+    /* note: as of June 2021, the aria label is in the truncate component which clones the element */
+    await assertGenesetExists(genesetName);
+    await clickOn("undo");
+    await assertGenesetDoesNotExist(genesetName);
+    await clickOn("redo");
+    await assertGenesetExists(genesetName);
+  });
+  test("edit geneset name and undo/redo", async () => {
+    await setup(config);
+
+    await editGenesetName(editableGenesetName, editText);
+    await assertGenesetExists(newGenesetName);
+    await clickOn("undo");
+    await assertGenesetExists(editableGenesetName);
+    await clickOn("redo");
+    await assertGenesetExists(newGenesetName);
+  });
+  test("delete a geneset and undo/redo", async () => {
+    if (config.withSubset) return;
+
+    await setup(config);
+
+    await deleteGeneset(genesetToDeleteName);
+    await clickOn("undo");
+    await assertGenesetExists(genesetToDeleteName);
+    await clickOn("redo");
+    await assertGenesetDoesNotExist(genesetToDeleteName);
+  });
+  test("geneset description", async () => {
+    if (config.withSubset) return;
+
+    await setup(config);
+
+    await clickOnUntil(
+      `${genesetToCheckForDescription}:geneset-expand`,
+      async () => {
+        expect(page).toMatchElement(getTestId(genesetDescriptionID), {
+          text: genesetDescriptionString,
+        });
+      }
+    );
+  });
 });
 
 describe.each([
-  {withSubset: true, tag: "subset"},
-  {withSubset: false, tag: "whole"}
+  { withSubset: true, tag: "subset" },
+  { withSubset: false, tag: "whole" },
+])("GENE crud operations and interactions", (config) => {
+  test("add a gene to geneset and undo/redo", async () => {
+    await setup(config);
+
+    await addGeneToSet(setToAddGeneTo, geneToAddToSet);
+    await expandGeneset(setToAddGeneTo);
+    await assertGeneExistsInGeneset(geneToAddToSet);
+    await clickOn("undo");
+    await assertGeneDoesNotExist(geneToAddToSet);
+    await clickOn("redo");
+    await assertGeneExistsInGeneset(geneToAddToSet);
+  });
+  test("expand gene and brush", async () => {
+    await setup(config);
+
+    await expandGeneset(brushThisGeneGeneset);
+    await expandGene(geneToBrushAndColorBy);
+    const histBrushableAreaId = `histogram-${geneToBrushAndColorBy}-plot-brushable-area`;
+
+    const coords = await calcDragCoordinates(histBrushableAreaId, {
+      x1: 0.25,
+      y1: 0.5,
+      x2: 0.55,
+      y2: 0.5,
+    });
+    await drag(histBrushableAreaId, coords.start, coords.end);
+    const cellCount = await getCellSetCount(1);
+    if (config.withSubset) {
+      expect(cellCount).toBe(subsetGeneBrushedCellCount);
+    } else {
+      expect(cellCount).toBe(geneBrushedCellCount);
+    }
+  });
+  test("color by gene in geneset", async () => {
+    await setup(config);
+
+    await expandGeneset(meanExpressionBrushGenesetName);
+
+    await colorByGene(geneToBrushAndColorBy);
+    await assertColorLegendLabel(geneToBrushAndColorBy);
+  });
+  test("delete gene from geneset and undo/redo", async () => {
+    // We've already deleted the gene
+    if (config.withSubset) return;
+
+    await setup(config);
+
+    await expandGeneset(setToRemoveFrom);
+    await removeGene(geneToRemove);
+    await assertGeneDoesNotExist(geneToRemove);
+    await clickOn("undo");
+    await assertGeneExistsInGeneset(geneToRemove);
+    await clickOn("redo");
+    await assertGeneDoesNotExist(geneToRemove);
+  });
+});
+
+describe.each([
+  { withSubset: true, tag: "subset", categoricalAnno: true },
+  { withSubset: false, tag: "whole", categoricalAnno: true },
 ])("annotations", (config) => {
-
-  const perTestCategoryName = "per-test-category";
-  const perTestLabelName = "per-test-label";
-
-  beforeEach(async () => {
-    await page.goto(appUrlBase);
-    // wait for the page to load
-    await utils.waitByClass("autosave-complete");
-    // setup the test fixtures
-    await actions.createCategory(perTestCategoryName);
-    await actions.createLabel(perTestCategoryName, perTestLabelName);
-    if (config.withSubset) await actions.subset({x1: 0.10, y1: 0.10, x2: 0.80, y2: 0.80});
-    await utils.waitByClass("autosave-complete");
-  });
-
-  afterEach(async () => {
-    await deleteCategoryIfExists(perTestCategoryName);
-    await utils.waitByClass("autosave-complete");
-  });
-
   test("create a category", async () => {
+    await setup(config);
+
     const categoryName = `category-created-${config.tag}`;
+
     await assertCategoryDoesNotExist(categoryName);
-    await actions.createCategory(categoryName);
+
+    await createCategory(categoryName);
+
     await assertCategoryExists(categoryName);
   });
 
   test("delete a category", async () => {
-    await actions.deleteCategory(perTestCategoryName);
+    await setup(config);
+
+    await deleteCategory(perTestCategoryName);
     await assertCategoryDoesNotExist(perTestCategoryName);
   });
 
   test("rename a category", async () => {
-    const newCategoryName = `cluster-for-real-${config.tag}`;
-    await actions.renameCategory(perTestCategoryName, newCategoryName);
+    await setup(config);
+
+    const newCategoryName = `NEW-${config.tag}`;
+
+    await renameCategory(perTestCategoryName, newCategoryName);
     await assertCategoryDoesNotExist(perTestCategoryName);
     await assertCategoryExists(newCategoryName);
   });
 
   test("create a label", async () => {
+    await setup(config);
+
     const labelName = `new-label-${config.tag}`;
+
     await assertLabelDoesNotExist(perTestCategoryName, labelName);
-    await actions.createLabel(perTestCategoryName, labelName);
+
+    await createLabel(perTestCategoryName, labelName);
+
     await assertLabelExists(perTestCategoryName, labelName);
   });
 
   test("delete a label", async () => {
-    await actions.deleteLabel(perTestCategoryName, perTestLabelName);
+    await setup(config);
+
+    await deleteLabel(perTestCategoryName, perTestLabelName);
     await assertLabelDoesNotExist(perTestCategoryName, perTestLabelName);
   });
 
   test("rename a label", async () => {
+    await setup(config);
+
     const newLabelName = "my-cool-new-label";
+
     await assertLabelDoesNotExist(perTestCategoryName, newLabelName);
-    await actions.renameLabel(perTestCategoryName, perTestLabelName, newLabelName);
+    await renameLabel(perTestCategoryName, perTestLabelName, newLabelName);
     await assertLabelDoesNotExist(perTestCategoryName, perTestLabelName);
     await assertLabelExists(perTestCategoryName, newLabelName);
   });
 
   test("check cell count for a label loaded from file", async () => {
-    const categoryName = "cluster-test";
-    const labelName = "four";
-    await actions.expandCategory(categoryName);
-    const result = await utils.waitByID(`categorical-value-count-${categoryName}-${labelName}`);
-    expect(await result.evaluate(node => node.innerText)).toBe(
-      data.annotationsFromFile.count.bySubsetConfig[config.withSubset]
+    await setup(config);
+
+    const duplicateCategoryName = "duplicate";
+    await duplicateCategory(duplicateCategoryName);
+
+    await page.reload({ waitUntil: ["networkidle0", "domcontentloaded"] });
+
+    const firstCategoryExpandIcon = await expect(page).toMatchElement(
+      getTestClass("category-expand")
     );
+
+    await firstCategoryExpandIcon.click();
+
+    const expectedCategoryRow = await expect(page).toMatchElement(
+      getTestClass("categorical-row")
+    );
+    const expectedLabelName = await getInnerText(
+      expectedCategoryRow,
+      "categorical-value"
+    );
+    const expectedLabelCount = await getInnerText(
+      expectedCategoryRow,
+      "categorical-value-count"
+    );
+
+    await expandCategory(duplicateCategoryName);
+
+    const expectedCategory = await expect(page).toMatchElement(
+      getTestClass("category")
+    );
+
+    const actualCategoryRow = await expect(expectedCategory).toMatchElement(
+      getTestClass("categorical-row")
+    );
+    const actualLabelName = await getInnerText(
+      actualCategoryRow,
+      "categorical-value"
+    );
+    const actualLabelCount = await getInnerText(
+      actualCategoryRow,
+      "categorical-value-count"
+    );
+
+    expect(actualLabelName).toBe(expectedLabelName);
+    expect(actualLabelCount).toBe(expectedLabelCount);
+
+    async function getInnerText(element, className) {
+      return element.$eval(getTestClass(className), (node) => node?.innerText);
+    }
   });
 
   test("assign cells to a label", async () => {
-    await actions.expandCategory(perTestCategoryName);
-    const lassoSelection = await actions.calcDragCoordinates(
+    await setup(config);
+
+    await expandCategory(perTestCategoryName);
+
+    const lassoSelection = await calcDragCoordinates(
       "layout-graph",
       data.categoryLabel.lasso["coordinates-as-percent"]
     );
-    await actions.drag(
-      "layout-graph",
-      lassoSelection.start,
-      lassoSelection.end,
-      true
+
+    await drag("layout-graph", lassoSelection.start, lassoSelection.end, true);
+    await waitByID("lasso-element", { visible: true });
+    await clickOn(`${perTestCategoryName}:${perTestLabelName}:see-actions`);
+    await clickOn(
+      `${perTestCategoryName}:${perTestLabelName}:add-current-selection-to-this-label`
     );
-    await utils.waitByID("lasso-element", {visible: true});
-    await utils.clickOn(`${perTestCategoryName}:${perTestLabelName}:see-actions`);
-    await utils.clickOn(`${perTestCategoryName}:${perTestLabelName}:add-current-selection-to-this-label`);
-    const result = await utils.waitByID(`categorical-value-count-${perTestCategoryName}-${perTestLabelName}`);
-    expect(await result.evaluate(node => node.innerText)).toBe(
+
+    const result = await waitByID(
+      `categorical-value-count-${perTestCategoryName}-${perTestLabelName}`
+    );
+
+    expect(await result.evaluate((node) => node.innerText)).toBe(
       data.categoryLabel.newCount.bySubsetConfig[config.withSubset]
     );
   });
 
   test("undo/redo category creation", async () => {
+    await setup(config);
+
     const categoryName = `category-created-undo-${config.tag}`;
+
     await assertCategoryDoesNotExist(categoryName);
-    await actions.createCategory(categoryName);
+    await createCategory(categoryName);
     await assertCategoryExists(categoryName);
-    await utils.clickOn("undo");
+    await clickOn("undo");
     await assertCategoryDoesNotExist(categoryName);
-    await utils.clickOn("redo");
+    await clickOn("redo");
     await assertCategoryExists(categoryName);
   });
 
   test("undo/redo category deletion", async () => {
+    await setup(config);
+
     const categoryName = `category-deleted-undo-${config.tag}`;
-    await actions.createCategory(categoryName);
+
+    await createCategory(categoryName);
     await assertCategoryExists(categoryName);
-    await actions.deleteCategory(categoryName);
+    await deleteCategory(categoryName);
     await assertCategoryDoesNotExist(categoryName);
-    await utils.clickOn("undo");
+    await clickOn("undo");
     await assertCategoryExists(categoryName);
-    await utils.clickOn("redo");
+    await clickOn("redo");
     await assertCategoryDoesNotExist(categoryName);
   });
 
   test("undo/redo category rename", async () => {
+    await setup(config);
+
     const newCategoryName = `category-renamed-undo-${config.tag}`;
+
     await assertCategoryDoesNotExist(newCategoryName);
-    await actions.renameCategory(perTestCategoryName, newCategoryName);
+    await renameCategory(perTestCategoryName, newCategoryName);
     await assertCategoryExists(newCategoryName);
     await assertCategoryDoesNotExist(perTestCategoryName);
-    await utils.clickOn("undo");
+    await clickOn("undo");
     await assertCategoryExists(perTestCategoryName);
     await assertCategoryDoesNotExist(newCategoryName);
-    await utils.clickOn("redo");
+    await clickOn("redo");
     await assertCategoryExists(newCategoryName);
     await assertCategoryDoesNotExist(perTestCategoryName);
   });
 
   test("undo/redo label creation", async () => {
+    await setup(config);
+
     const labelName = `label-created-undo-${config.tag}`;
+
     await assertLabelDoesNotExist(perTestCategoryName, labelName);
-    await actions.createLabel(perTestCategoryName, labelName);
+    await createLabel(perTestCategoryName, labelName);
     await assertLabelExists(perTestCategoryName, labelName);
-    await utils.clickOn("undo");
+    await clickOn("undo");
     await assertLabelDoesNotExist(perTestCategoryName);
-    await utils.clickOn("redo");
+    await clickOn("redo");
     await assertLabelExists(perTestCategoryName, labelName);
   });
 
   test("undo/redo label deletion", async () => {
-    await actions.deleteLabel(perTestCategoryName, perTestLabelName);
+    await setup(config);
+
+    await deleteLabel(perTestCategoryName, perTestLabelName);
     await assertLabelDoesNotExist(perTestCategoryName);
-    await utils.clickOn("undo");
+    await clickOn("undo");
     await assertLabelExists(perTestCategoryName, perTestLabelName);
-    await utils.clickOn("redo");
+    await clickOn("redo");
     await assertLabelDoesNotExist(perTestCategoryName);
   });
 
   test("undo/redo label rename", async () => {
+    await setup(config);
+
     const newLabelName = `label-renamed-undo-${config.tag}`;
+
     await assertLabelDoesNotExist(perTestCategoryName, newLabelName);
-    await actions.renameLabel(perTestCategoryName, perTestLabelName, newLabelName);
+    await renameLabel(perTestCategoryName, perTestLabelName, newLabelName);
     await assertLabelExists(perTestCategoryName, newLabelName);
     await assertLabelDoesNotExist(perTestCategoryName, perTestLabelName);
-    await utils.clickOn("undo");
+    await clickOn("undo");
     await assertLabelExists(perTestCategoryName, perTestLabelName);
     await assertLabelDoesNotExist(perTestCategoryName, newLabelName);
-    await utils.clickOn("redo");
+    await clickOn("redo");
     await assertLabelExists(perTestCategoryName, newLabelName);
     await assertLabelDoesNotExist(perTestCategoryName, perTestLabelName);
   });
 
-  async function assertCategoryExists(categoryName) {
-    const handle = await utils.waitByID(`${categoryName}:category-expand`);
-    const result = await handle.evaluate(node => node.innerText);
-    // slice beginning and end of category name result to account for truncation of long names
-    expect(result.slice(0, 10)).toBe(categoryName.slice(0, 10));
-    expect(result.slice(-10)).toBe(categoryName.slice(-10));
-  }
+  test("stacked bar graph renders", async () => {
+    await setup(config);
 
-  async function assertCategoryDoesNotExist(categoryName) {
-    const result = await page.$(`[data-testid='${categoryName}:category-expand']`);
-    expect(result).toBeNull();
+    await expandCategory(perTestCategoryName);
+
+    await clickOn(`colorby-louvain`);
+
+    const labels = await getAllByClass("categorical-row");
+
+    const result = await Promise.all(
+      labels.map((label) => page.evaluate((element) => element.outerHTML, label))
+    );
+
+    expect(result).toMatchSnapshot();
+  });
+
+  test("truncate midpoint whitespace", async () => {
+    await setup(config);
+    const newLabelName = "123 456";
+    await renameLabel(perTestCategoryName, perTestLabelName, newLabelName);
+    const value = await waitByID(
+      `categorical-value-${perTestCategoryName}-${newLabelName}`
+    );
+    const result = await page.evaluate((elem) => elem.outerHTML, value);
+    expect(result).toMatchSnapshot();
+  });
+
+  test("truncate single character", async () => {
+    await setup(config);
+    const newLabelName = "T";
+    await renameLabel(perTestCategoryName, perTestLabelName, newLabelName);
+    const value = await waitByID(
+      `categorical-value-${perTestCategoryName}-${newLabelName}`
+    );
+    const result = await page.evaluate((elem) => elem.outerHTML, value);
+    expect(result).toMatchSnapshot();
+  });
+
+  async function assertCategoryExists(categoryName) {
+    const handle = await waitByID(`${categoryName}:category-label`);
+
+    const result = await handle.evaluate((node) =>
+      node.getAttribute("aria-label")
+    );
+
+    return expect(result).toBe(categoryName);
   }
 
   async function assertLabelExists(categoryName, labelName) {
-    const category = await utils.waitByID(`${categoryName}:category-expand`);
-    expect(category).not.toBeNull();
-    await actions.expandCategory(categoryName);
-    const previous = await utils.waitByID(`categorical-value-${categoryName}-${labelName}`);
-    expect(await previous.evaluate(node => node.innerText)).toBe(labelName);
+    await expect(page).toMatchElement(
+      getTestId(`${categoryName}:category-expand`)
+    );
+
+    await expandCategory(categoryName);
+
+    const previous = await waitByID(
+      `categorical-value-${categoryName}-${labelName}`
+    );
+
+    expect(
+      await previous.evaluate((node) => node.getAttribute("aria-label"))
+    ).toBe(labelName);
   }
 
   async function assertLabelDoesNotExist(categoryName, labelName) {
-    await actions.expandCategory(categoryName);
-    const result = await page.$(`[data-testid='categorical-value-${categoryName}-${labelName}']`);
+    await expandCategory(categoryName);
+    const result = await page.$(
+      `[data-testid='categorical-value-${categoryName}-${labelName}']`
+    );
     expect(result).toBeNull();
   }
-
-  async function deleteCategoryIfExists(categoryName) {
-    try {
-      const category = await page.waitForSelector(
-        `[data-testid='${categoryName}:category-expand']`,
-        {timeout: 200}
-      );
-      if (category !== null) return await actions.deleteCategory(categoryName);
-    } catch {}
-    return null
-  }
 });

client/__tests__/e2e/e2eJestConfig.json

@@ -1,10 +1,10 @@
 {
+  "testRunner": "jest-circus/runner",
   "preset": "jest-puppeteer",
-  "testMatch": [
-    "**/__tests__/**/?(*.)(spec|test).js?(x)"
-  ],
-  "testURL": "http://localhost/",
-  "setupFiles": [
-    "../setupMissingGlobals.js"
-  ]
+  "testMatch": ["**/__tests__/**/?(*.)(spec|test).js?(x)"],
+  "setupFiles": ["../setupMissingGlobals.js"],
+  "setupFilesAfterEnv": ["expect-puppeteer", "./puppeteer.setup.js"],
+  "globalSetup": "../globalSetup.js",
+  "globalTeardown": "jest-environment-puppeteer/teardown",
+  "testEnvironment": "./screenshot_env.js"
 }

client/__tests__/e2e/feature.test.js

@@ -1,118 +0,0 @@
-/*
-NOT run in Travis CI
-
-UX tests using puppeteer to be run locally.
-
-To run locally, ensure you are running the client is running on port 3000.
-Then run jest --verbose false --config __tests__/e2e/e2eJestConfig.json feature.
- */
-
-import puppeteer from "puppeteer";
-import { appUrlBase, DEBUG, DEV, DATASET } from "./config";
-import { puppeteerUtils, cellxgeneActions } from "./puppeteerUtils";
-import { datasets } from "./data";
-
-let browser, page, utils, cxgActions, spy;
-const browserViewport = { width: 1280, height: 960 };
-let data = datasets[DATASET].features;
-
-if (DEBUG) jest.setTimeout(100000);
-if (DEV) jest.setTimeout(10000);
-
-beforeAll(async () => {
-  const browserParams = DEV
-    ? { headless: false, slowMo: 5 }
-    : DEBUG
-    ? { headless: false, slowMo: 100, devtools: true }
-    : {};
-  browser = await puppeteer.launch(browserParams);
-  page = await browser.newPage();
-  await page.setViewport(browserViewport);
-  if (DEV || DEBUG) {
-    page.on("console", msg => console.log(`PAGE LOG: ${msg.text()}`));
-  }
-  page.on("pageerror", err => {
-    throw new Error(`Console error: ${err}`);
-  });
-  utils = puppeteerUtils(page);
-  cxgActions = cellxgeneActions(page);
-});
-
-beforeEach(async () => {
-  await page.goto(appUrlBase);
-});
-
-afterAll(() => {
-  if (!DEBUG) {
-    browser.close();
-  }
-});
-
-describe("zoom interaction", async () => {
-  // Skip this test since UI is to hide lasso path when switching modes
-  test.skip("lasso visible after switching modes to pan/zoom", async () => {
-    const lassoSelection = await cxgActions.calcDragCoordinates(
-      "layout-graph",
-      data.panzoom.lasso["coordinates-as-percent"]
-    );
-    await cxgActions.drag(
-      "layout-graph",
-      lassoSelection.start,
-      lassoSelection.end,
-      true
-    );
-    await utils.waitByID("lasso-element", { visible: true });
-    await utils.clickOn("mode-pan-zoom");
-    await utils.waitByID("lasso-element", { visible: true });
-  });
-
-  test("pan zoom mode resets lasso selection", async () => {
-    const lassoSelection = await cxgActions.calcDragCoordinates(
-      "layout-graph",
-      data.panzoom.lasso["coordinates-as-percent"]
-    );
-    await cxgActions.drag(
-      "layout-graph",
-      lassoSelection.start,
-      lassoSelection.end,
-      true
-    );
-    await utils.waitByID("lasso-element", { visible: true });
-    const initialCount = await cxgActions.cellSet(1);
-    expect(initialCount).toBe(data.panzoom.lasso.count);
-    await utils.clickOn("mode-pan-zoom");
-    await utils.clickOn("mode-lasso");
-    const modeSwitchCount = await cxgActions.cellSet(1);
-    expect(modeSwitchCount).toBe(initialCount);
-  });
-
-  test("lasso moves after pan", async () => {
-    const lassoSelection = await cxgActions.calcDragCoordinates(
-      "layout-graph",
-      data.panzoom.lasso["coordinates-as-percent"]
-    );
-    await cxgActions.drag(
-      "layout-graph",
-      lassoSelection.start,
-      lassoSelection.end,
-      true
-    );
-    await utils.waitByID("lasso-element", { visible: true });
-    const initialCount = await cxgActions.cellSet(1);
-    expect(initialCount).toBe(data.panzoom.lasso.count);
-    await utils.clickOn("mode-pan-zoom");
-    const panCoords = await cxgActions.calcDragCoordinates(
-      "layout-graph",
-      data.panzoom.lasso["coordinates-as-percent"]
-    );
-    await cxgActions.drag(
-      "layout-graph",
-      panCoords.start,
-      panCoords.end,
-      false
-    );
-    await utils.clickOn("mode-lasso");
-    const panCount = await cxgActions.cellSet(2);
-    expect(panCount).toBe(initialCount);
-  });
-});

client/__tests__/e2e/puppeteer.setup.js

@@ -0,0 +1,58 @@
+/**
+ * `client/jest-puppeteer.config.js` is for configuring Puppeteer's launch config options
+ * `client/__tests__/e2e/puppeteer.setup.js` is for configuring `jest`, `browser`,
+ * and `page` objects
+ */
+
+import { setDefaultOptions } from "expect-puppeteer";
+import { isDebug, isDev } from "./config";
+import * as ENV_DEFAULT from "../../../environment.default.json";
+
+// (thuang): This is the max time a test can take to run.
+// Since when debugging, we run slowMo and !headless, this means
+// a test can take more time to finish, so we don't want
+// jest to shut off the test too soon
+jest.setTimeout(2 * 60 * 1000);
+setDefaultOptions({ timeout: 20 * 1000 });
+
+jest.retryTimes(ENV_DEFAULT.RETRY_ATTEMPTS);
+
+beforeEach(async () => {
+  await jestPuppeteer.resetBrowser();
+
+  const userAgent = await browser.userAgent();
+  await page.setUserAgent(`${userAgent}bot`);
+
+  await page._client.send("Animation.setPlaybackRate", { playbackRate: 12 });
+
+  page.on("pageerror", (err) => {
+    throw new Error(`Console error: ${err}`);
+  });
+
+  page.on("error", (err) => {
+    throw new Error(`Console error: ${err}`);
+  });
+
+  page.on("console", async (msg) => {
+    if (isDev || isDebug) {
+      // If there is a console.error but an error is not thrown, this will ensure the test fails
+      console.log(`PAGE LOG: ${msg.text()}`);
+      if (msg.type() === "error") {
+        // TODO: chromium does not currently support the CSP directive on the
+        // line below, so we swallow this error. Remove this when the test
+        // suite uses a browser version that supports this directive.
+        if (
+          msg.text() ===
+          "Unrecognized Content-Security-Policy directive 'require-trusted-types-for'.\n"
+        ) {
+          return;
+        }
+        const errorMsgText = await Promise.all(
+          // TODO can we do this without internal properties?
+          msg.args().map((arg) => arg._remoteObject.description)
+        );
+        throw new Error(`Console error: ${errorMsgText}`);
+      }
+    }
+  });
+});

client/__tests__/e2e/puppeteerUtils.js

@@ -1,69 +1,124 @@
-export const puppeteerUtils = page => ({
-  async waitByID(testId, props = {}) {
-    return page.waitForSelector(`[data-testid='${testId}']`, props);
-  },
+/* eslint-disable no-await-in-loop -- await in loop is needed to emulate sequential user actions */
+export function getTestId(id) {
+  return `[data-testid='${id}']`;
+}
 
-  async waitByClass(testClass, props = {}) {
-    return page.waitForSelector(`[data-testclass='${testClass}']`, props);
-  },
+export function getTestClass(className) {
+  return `[data-testclass='${className}']`;
+}
 
-  async waitForAllByIds(testIds) {
-    await Promise.all(
-      testIds.map(testId => page.waitForSelector(`[data-testid='${testId}']`))
-    );
-  },
+export async function waitByID(testId, props = {}) {
+  return page.waitForSelector(getTestId(testId), props);
+}
 
-  async getAllByClass(testClass) {
-    return page.$$eval(`[data-testclass=${testClass}]`, eles =>
-      eles.map(ele => ele.dataset.testid)
-    );
-  },
+export async function waitByClass(testClass, props = {}) {
+  return page.waitForSelector(`[data-testclass='${testClass}']`, props);
+}
 
-  async typeInto(testId, text) {
-    // blueprint's  typeahead is treating typing weird, clicking & waiting first solves this
-    // only works for text without special characters
-    await this.waitByID(testId);
-    const selector = `[data-testid='${testId}']`;
-    // type ahead can be annoying if you don't pause before you type
-    await page.click(selector);
-    await page.waitFor(200);
-    await page.type(selector, text);
-  },
+export async function waitForAllByIds(testIds) {
+  await Promise.all(
+    testIds.map((testId) => page.waitForSelector(getTestId(testId)))
+  );
+}
 
-  async clearInputAndTypeInto(testId, text) {
-    await this.waitByID(testId);
-    const selector = `[data-testid='${testId}']`;
-    // only works for text without special characters
-    // type ahead can be annoying if you don't pause before you type
-    await page.click(selector);
-    await page.waitFor(200);
-    // select all
-    await page.click(selector, { clickCount: 3 });
-    await page.keyboard.press("Backspace");
-    await page.type(selector, text);
-  },
+export async function getAllByClass(testClass) {
+  return page.$$(`[data-testclass=${testClass}]`);
+}
 
-  async clickOn(testid, options = {}) {
-    await this.waitByID(testid);
-    const click = await page.click(`[data-testid='${testid}']`, options);
-    await page.waitFor(50);
-    return click;
-  },
+export async function typeInto(testId, text) {
+  // blueprint's  typeahead is treating typing weird, clicking & waiting first solves this
+  // only works for text without special characters
+  await waitByID(testId);
+  const selector = getTestId(testId);
+  // type ahead can be annoying if you don't pause before you type
+  await page.click(selector);
+  await page.waitForTimeout(200);
+  await page.type(selector, text);
+}
 
-  async getOneElementInnerHTML(selector) {
-    await page.waitForSelector(selector);
-    return page.$eval(selector, el => el.innerHTML);
-  },
+export async function clearInputAndTypeInto(testId, text) {
+  await waitByID(testId);
+  const selector = getTestId(testId);
+  // only works for text without special characters
+  // type ahead can be annoying if you don't pause before you type
+  await page.click(selector);
+  await page.waitForTimeout(200);
+  // select all
+  await page.click(selector, { clickCount: 3 });
+  await page.keyboard.press("Backspace");
+  await page.type(selector, text);
+}
 
-  async getOneElementInnerText(selector) {
-    await page.waitForSelector(selector);
-    return page.$eval(selector, el => el.innerText);
-  },
+export async function clickOn(testId, options = {}) {
+  await expect(page).toClick(getTestId(testId), options);
+}
 
-  async getElementCoordinates(testid) {
-    return page.$eval(`[data-testid='${testid}']`, elem => {
-      const { left, top } = elem.getBoundingClientRect();
-      return [left, top];
-    });
+/**
+ * (thuang): There are times when Puppeteer clicks on a button and the page doesn't respond.
+ * So I added clickOnUntil() to retry clicking until a given condition is met.
+ */
+export async function clickOnUntil(testId, assert) {
+  const MAX_RETRY = 10;
+  const WAIT_FOR_MS = 200;
+
+  let retry = 0;
+
+  while (retry < MAX_RETRY) {
+    try {
+      await clickOn(testId);
+      await assert();
+
+      break;
+    } catch (error) {
+      retry += 1;
+
+      await page.waitForTimeout(WAIT_FOR_MS);
+    }
   }
-});
+
+  if (retry === MAX_RETRY) {
+    throw Error("clickOnUntil() assertion failed!");
+  }
+}
+
+export async function getOneElementInnerHTML(selector, options = {}) {
+  await page.waitForSelector(selector, options);
+
+  return page.$eval(selector, (el) => el.innerHTML);
+}
+
+export async function getOneElementInnerText(selector) {
+  expect(page).toMatchElement(selector);
+
+  return page.$eval(selector, (el) => el.innerText);
+}
+
+export async function getElementCoordinates(testId) {
+  return page.$eval(getTestId(testId), (elem) => {
+    const { left, top } = elem.getBoundingClientRect();
+    return [left, top];
+  });
+}
+
+async function nameNewAnnotation() {
+  if (await isElementPresent(getTestId("annotation-dialog"))) {
+    await typeInto("new-annotation-name", "ignoreE2E");
+    await clickOn("submit-annotation");
+
+    // wait for the page to load
+    await waitByClass("autosave-complete");
+  }
+}
+
+export async function goToPage(url) {
+  await page.goto(url, {
+    waitUntil: "networkidle0",
+  });
+
+  await nameNewAnnotation();
+}
+
+export async function isElementPresent(selector, options) {
+  return Boolean(await page.$(selector, options));
+}
+/* eslint-enable no-await-in-loop -- await in loop is needed to emulate sequential user actions */

client/__tests__/e2e/screenshot_env.js

@@ -0,0 +1,36 @@
+const PuppeteerEnvironment = require("jest-environment-puppeteer");
+require("jest-circus");
+const ENV_DEFAULT = require("../../../environment.default.json");
+
+const takeScreenshot = require("./takeScreenshot");
+
+class ScreenshotEnvironment extends PuppeteerEnvironment {
+  async handleTestEvent(event, state) {
+    if (["test_start", "test_done"].includes(event.name)) {
+      console.log("------------------event name:\n", event.name);
+      console.log("~~~~ Current test errors\n", new Date(), event.test.errors);
+      console.log("~~~~ Current test\n", new Date(), event.test);
+    }
+
+    if (event.name === "error") {
+      console.log("error event:", JSON.stringify(event));
+    }
+
+    if (event.name === "test_fn_failure" || event.name === "hook_failure") {
+      console.log("------------------event name:\n", event.name);
+      console.log(">>>> Current state\n", new Date(), state);
+      console.log("===> Failure event\n", new Date(), event);
+
+      // (thuang): We only want to take screenshot on the last try
+      if (
+        state.currentlyRunningTest.invocations <= ENV_DEFAULT.RETRY_ATTEMPTS
+      ) {
+        return;
+      }
+
+      await takeScreenshot(state.currentlyRunningTest.name, this.global.page);
+    }
+  }
+}
+
+module.exports = ScreenshotEnvironment;

client/__tests__/e2e/takeScreenshot.js

@@ -0,0 +1,17 @@
+function toFilename(name) {
+  return name.replace(/[^a-z0-9.-]+/gi, "-");
+}
+
+async function takeScreenshot(currentTestName, page) {
+  const testName = toFilename(currentTestName);
+
+  // Take a screenshot at the point of failure
+  const date = new Date().toISOString();
+  const screenshotName = `${date}-${testName}.png`;
+
+  await page.screenshot({
+    path: `./__tests__/screenshots/ignoreE2E-screenshot-${screenshotName}`,
+  });
+}
+
+module.exports = takeScreenshot;

client/__tests__/e2e/testBrowser.js

@@ -1,35 +0,0 @@
-import puppeteer from "puppeteer";
-import { DEBUG, DEV } from "./config";
-import { puppeteerUtils } from "./puppeteerUtils";
-import { cellxgeneActions } from "./cellxgeneActions";
-
-export async function setupTestBrowser(browserViewport) {
-  const browserParams = DEV
-    ? { headless: false, slowMo: 5 }
-    : DEBUG
-      ? { headless: false, slowMo: 100, devtools: true }
-      : {};
-  const browser = await puppeteer.launch(browserParams);
-  const page = await browser.pages().then(pages => pages[0]);
-  await page.setViewport(browserViewport);
-  if (DEV || DEBUG) {
-    page.on("console", async msg => {
-      // If there is a console.error but an error is not thrown, this will ensure the test fails
-      if (msg.type() === "error") {
-        const errorMsgText = await Promise.all(
-          // TODO can we do this without internal properties?
-          msg.args().map(arg => arg._remoteObject.description)
-        );
-        throw new Error(`Console error: ${errorMsgText}`);
-      }
-      console.log(`PAGE LOG: ${msg.text()}`);
-    });
-  }
-  page.on("pageerror", err => {
-    throw new Error(`Console error: ${err}`);
-  });
-  const utils = puppeteerUtils(page);
-  const cxgActions = cellxgeneActions(page, utils);
-  return [browser, page, utils, cxgActions];
-}
-

client/__tests__/e2e/test_config.yaml

@@ -0,0 +1,19 @@
+server:
+  app:
+    force_https: true
+    port: 5005
+
+dataset:
+  presentation:
+    max_categories: 1000
+    custom_colors: true
+
+  user_annotations:
+    enable: false
+    type: local_file_csv
+    local_file_csv:
+      directory: null
+      file: null
+
+  embeddings:
+    names: []

client/__tests__/globalSetup.js

@@ -0,0 +1,5 @@
+const { setup } = require("jest-environment-puppeteer");
+
+module.exports = async () => {
+  await setup();
+};

client/__tests__/reducers/cascade.test.js

@@ -26,7 +26,7 @@ describe("cascade", () => {
           expect(nextSharedState).toStrictEqual({});
           expect(prevSharedState).toBe(topLevelState);
           return 0;
-        }
+        },
       ],
       [
         "bar",
@@ -36,8 +36,8 @@ describe("cascade", () => {
           expect(nextSharedState).toStrictEqual({ foo: 0 });
           expect(prevSharedState).toBe(topLevelState);
           return 99;
-        }
-      ]
+        },
+      ],
     ]);
 
     const nextState = reducer(topLevelState, topLevelAction);

client/__tests__/reducers/genesets.test.js

@@ -0,0 +1,524 @@
+import genesetsReducer from "../../src/reducers/genesets";
+
+describe("initial reducer state", () => {
+  test("some other action", () => {
+    expect(genesetsReducer(undefined, { type: "foo" })).toMatchObject({
+      initialized: false,
+      lastTid: undefined,
+      genesets: new Map(),
+    });
+  });
+});
+
+describe("geneset: initial load", () => {
+  test("missing JSON response", () => {
+    expect(() =>
+      genesetsReducer(undefined, {
+        type: "geneset: initial load",
+      })
+    ).toThrow("missing or malformed JSON response");
+  });
+
+  test("empty geneset", () => {
+    expect(
+      genesetsReducer(undefined, {
+        type: "geneset: initial load",
+        data: {
+          tid: 0,
+          genesets: [],
+        },
+      })
+    ).toMatchObject({
+      initialized: true,
+      lastTid: 0,
+      genesets: new Map(),
+    });
+  });
+
+  test("non-empty geneset", () => {
+    expect(
+      genesetsReducer(undefined, {
+        type: "geneset: initial load",
+        data: {
+          tid: 99,
+          genesets: [
+            {
+              geneset_name: "G1",
+              genes: [{ gene_symbol: "F5" }],
+            },
+            {
+              geneset_name: "G2",
+              geneset_description: "G2 desc",
+              genes: [{ gene_symbol: "F6" }],
+            },
+            {
+              geneset_name: "G3",
+              geneset_description: "G3 desc",
+              genes: [{ gene_symbol: "F7", gene_description: "gene desc" }],
+            },
+          ],
+        },
+      })
+    ).toMatchObject({
+      initialized: true,
+      lastTid: 99,
+      genesets: new Map([
+        [
+          "G1",
+          {
+            genesetName: "G1",
+            genesetDescription: "",
+            genes: new Map([["F5", { geneSymbol: "F5", geneDescription: "" }]]),
+          },
+        ],
+        [
+          "G2",
+          {
+            genesetName: "G2",
+            genesetDescription: "G2 desc",
+            genes: new Map([["F6", { geneSymbol: "F6", geneDescription: "" }]]),
+          },
+        ],
+        [
+          "G3",
+          {
+            genesetName: "G3",
+            genesetDescription: "G3 desc",
+            genes: new Map([
+              ["F7", { geneSymbol: "F7", geneDescription: "gene desc" }],
+            ]),
+          },
+        ],
+      ]),
+    });
+  });
+});
+
+describe("geneset: create", () => {
+  const initialState = genesetsReducer(undefined, {
+    type: "geneset: initial load",
+    data: {
+      tid: 0,
+      genesets: [],
+    },
+  });
+
+  test("simple create", () => {
+    expect(
+      genesetsReducer(initialState, {
+        type: "geneset: create",
+        genesetName: "a geneset",
+        genesetDescription: "",
+      })
+    ).toMatchObject({
+      ...initialState,
+      genesets: new Map([
+        [
+          "a geneset",
+          {
+            genesetName: "a geneset",
+            genesetDescription: "",
+            genes: new Map(),
+          },
+        ],
+      ]),
+    });
+  });
+
+  test("error - duplicate name", () => {
+    expect(() => {
+      genesetsReducer(
+        genesetsReducer(initialState, {
+          type: "geneset: create",
+          genesetName: "foo",
+          genesetDescription: "foo",
+        }),
+        {
+          type: "geneset: create",
+          genesetName: "foo",
+          genesetDescription: "bar",
+        }
+      );
+    }).toThrow("name already defined");
+  });
+
+  test("error - missing required action values", () => {
+    expect(() => {
+      genesetsReducer(initialState, {
+        type: "geneset: create",
+        genesetDescription: "foo",
+      });
+    }).toThrow();
+    expect(() => {
+      genesetsReducer(initialState, {
+        type: "geneset: create",
+        genesetName: "foo",
+      });
+    }).toThrow("name or description unspecified");
+  });
+});
+
+describe("geneset: delete", () => {
+  const initialState = genesetsReducer(undefined, {
+    type: "geneset: initial load",
+    data: {
+      tid: 0,
+      genesets: [],
+    },
+  });
+
+  test("simple delete", () => {
+    expect(
+      genesetsReducer(
+        genesetsReducer(initialState, {
+          type: "geneset: create",
+          genesetName: "foo",
+          genesetDescription: "foo",
+        }),
+        {
+          type: "geneset: delete",
+          genesetName: "foo",
+        }
+      )
+    ).toMatchObject({
+      initialized: true,
+      lastTid: 0,
+      genesets: new Map(),
+    });
+  });
+
+  test("error - missing name", () => {
+    expect(() => {
+      genesetsReducer(initialState, {
+        type: "geneset: delete",
+        genesetName: "foo",
+      });
+    }).toThrow("name does not exist");
+  });
+});
+
+describe("geneset: update", () => {
+  const initialState = genesetsReducer(undefined, {
+    type: "geneset: initial load",
+    data: {
+      tid: 0,
+      genesets: [],
+    },
+  });
+
+  test("simple update", () => {
+    expect(
+      genesetsReducer(
+        genesetsReducer(
+          genesetsReducer(initialState, {
+            type: "geneset: create",
+            genesetName: "foo1",
+            genesetDescription: "foo1",
+          }),
+          {
+            type: "geneset: create",
+            genesetName: "foo2",
+            genesetDescription: "foo2",
+          }
+        ),
+        {
+          type: "geneset: update",
+          genesetName: "foo1",
+          update: {
+            genesetName: "bar",
+            genesetDescription: "bar",
+          },
+        }
+      )
+    ).toMatchObject({
+      initialized: true,
+      lastTid: 0,
+      genesets: new Map([
+        [
+          "bar",
+          { genesetName: "bar", genesetDescription: "bar", genes: new Map() },
+        ],
+        [
+          "foo2",
+          { genesetName: "foo2", genesetDescription: "foo2", genes: new Map() },
+        ],
+      ]),
+    });
+  });
+
+  test("error - unknown name", () => {
+    expect(() => {
+      genesetsReducer(initialState, {
+        type: "geneset: update",
+        genesetName: "foo",
+        update: {
+          genesetName: "foo",
+          genesetDescription: "bar",
+        },
+      });
+    }).toThrow("name unspecified or does not exist");
+  });
+
+  test("error - duplicate name", () => {
+    expect(() => {
+      genesetsReducer(
+        genesetsReducer(initialState, {
+          type: "geneset: create",
+          genesetName: "foo",
+          genesetDescription: "foo",
+        }),
+        {
+          type: "geneset: update",
+          genesetName: "foo",
+          update: {
+            genesetName: "foo",
+            genesetDescription: "foo",
+          },
+        }
+      );
+    }).toThrow("update specified existing name and description");
+  });
+});
+
+describe("geneset: add genes", () => {
+  const initialState = genesetsReducer(
+    genesetsReducer(undefined, {
+      type: "geneset: initial load",
+      data: {
+        tid: 0,
+        genesets: [],
+      },
+    }),
+    {
+      type: "geneset: create",
+      genesetName: "test",
+      genesetDescription: "",
+    }
+  );
+
+  test("add a gene", () => {
+    expect(
+      genesetsReducer(initialState, {
+        type: "geneset: add genes",
+        genesetName: "test",
+        genes: [{ geneSymbol: "F5" }],
+      })
+    ).toMatchObject({
+      ...initialState,
+      genesets: new Map([
+        [
+          "test",
+          {
+            genesetName: "test",
+            genesetDescription: "",
+            genes: new Map([["F5", { geneSymbol: "F5", geneDescription: "" }]]),
+          },
+        ],
+      ]),
+    });
+
+    expect(
+      genesetsReducer(initialState, {
+        type: "geneset: add genes",
+        genesetName: "test",
+        genes: [
+          { geneSymbol: "F5", geneDescription: "desc" },
+          { geneSymbol: "SET1", geneDescription: "" },
+        ],
+      })
+    ).toMatchObject({
+      ...initialState,
+      genesets: new Map([
+        [
+          "test",
+          {
+            genesetName: "test",
+            genesetDescription: "",
+            genes: new Map([
+              ["F5", { geneSymbol: "F5", geneDescription: "desc" }],
+              ["SET1", { geneSymbol: "SET1", geneDescription: "" }],
+            ]),
+          },
+        ],
+      ]),
+    });
+  });
+
+  test("no such geneset error", () => {
+    expect(() => {
+      genesetsReducer(initialState, {
+        type: "geneset: add genes",
+        genesetName: "mumble",
+        genes: [],
+      });
+    }).toThrow("geneset name does not exist");
+  });
+});
+
+describe("geneset: delete genes", () => {
+  const initialState = genesetsReducer(
+    genesetsReducer(
+      genesetsReducer(undefined, {
+        type: "geneset: initial load",
+        data: {
+          tid: 0,
+          genesets: [],
+        },
+      }),
+      {
+        type: "geneset: create",
+        genesetName: "test",
+        genesetDescription: "",
+      }
+    ),
+    {
+      type: "geneset: add genes",
+      genesetName: "test",
+      genes: [{ geneSymbol: "F5" }],
+    }
+  );
+
+  test("simple", () => {
+    expect(
+      genesetsReducer(initialState, {
+        type: "geneset: delete genes",
+        genesetName: "test",
+        geneSymbols: ["F5"],
+      })
+    ).toMatchObject({
+      ...initialState,
+      genesets: new Map([
+        [
+          "test",
+          {
+            genesetName: "test",
+            genesetDescription: "",
+            genes: new Map(),
+          },
+        ],
+      ]),
+    });
+  });
+
+  test("no such geneset error", () => {
+    expect(() => {
+      genesetsReducer(initialState, {
+        type: "geneset: delete genes",
+        genesetName: "mumble",
+        geneSymbols: [],
+      });
+    }).toThrow("name does not exist");
+  });
+});
+
+describe("geneset: set gene description", () => {
+  const initialState = genesetsReducer(
+    genesetsReducer(
+      genesetsReducer(undefined, {
+        type: "geneset: initial load",
+        data: {
+          tid: 0,
+          genesets: [],
+        },
+      }),
+      {
+        type: "geneset: create",
+        genesetName: "test",
+        genesetDescription: "",
+      }
+    ),
+    {
+      type: "geneset: add genes",
+      genesetName: "test",
+      genes: [{ geneSymbol: "F5" }],
+    }
+  );
+
+  test("simple set", () => {
+    expect(
+      genesetsReducer(initialState, {
+        type: "geneset: set gene description",
+        genesetName: "test",
+        update: {
+          geneSymbol: "F5",
+          geneDescription: "mumble",
+        },
+      })
+    ).toMatchObject({
+      ...initialState,
+      genesets: new Map([
+        [
+          "test",
+          {
+            genesetName: "test",
+            genesetDescription: "",
+            genes: new Map([
+              ["F5", { geneSymbol: "F5", geneDescription: "mumble" }],
+            ]),
+          },
+        ],
+      ]),
+    });
+  });
+
+  test("no such geneset error", () => {
+    expect(() => {
+      genesetsReducer(initialState, {
+        type: "geneset: set gene description",
+        genesetName: "does not exist",
+        update: {
+          geneSymbol: "F5",
+          geneDescription: "mumble",
+        },
+      });
+    }).toThrow("geneset name does not exist");
+  });
+
+  test("no such gene error", () => {
+    expect(() => {
+      genesetsReducer(initialState, {
+        type: "geneset: set gene description",
+        genesetName: "test",
+        update: {
+          geneSymbol: "NO SUCH GENE",
+          geneDescription: "mumble",
+        },
+      });
+    }).toThrow("no such gene");
+  });
+});
+
+describe("geneset: set tid", () => {
+  test("simple set", () => {
+    expect(
+      genesetsReducer(undefined, {
+        type: "geneset: set tid",
+        tid: 1,
+      })
+    ).toMatchObject({ lastTid: 1 });
+  });
+
+  test("not a number error", () => {
+    expect(() => {
+      genesetsReducer(
+        { lastTid: 1 },
+        {
+          type: "geneset: set tid",
+          tid: "0",
+        }
+      );
+    }).toThrow("must be a positive integer");
+  });
+
+  test("decrement error", () => {
+    expect(() => {
+      genesetsReducer(
+        { lastTid: 1 },
+        {
+          type: "geneset: set tid",
+          tid: 0,
+        }
+      );
+    }).toThrow("may not be decremented");
+  });
+});

client/__tests__/reducers/genesetsUI.test.js

@@ -0,0 +1,75 @@
+import genesetsUIReducer from "../../src/reducers/genesetsUI";
+
+// Format: GeneSetsUI(state,action)
+
+const initialState = {
+  createGenesetModeActive: false,
+  isEditingGenesetName: false,
+  isAddingGenesToGeneset: false,
+};
+
+/* initial */
+describe("geneset UI states", () => {
+  test("initial state, some other action", () => {
+    expect(
+      genesetsUIReducer(undefined, {
+        type: "foo",
+      })
+    ).toMatchObject(initialState);
+  });
+  test("geneset: activate add new geneset mode", () => {
+    expect(
+      genesetsUIReducer(initialState, {
+        type: "geneset: activate add new geneset mode",
+      })
+    ).toMatchObject({
+      createGenesetModeActive: true,
+      isEditingGenesetName: false,
+      isAddingGenesToGeneset: false,
+    });
+  });
+  test("geneset: disable create geneset mode", () => {
+    expect(
+      genesetsUIReducer(undefined, { isEditingGenesetName: false })
+    ).toMatchObject(initialState);
+  });
+
+  test("activate add new genes mode", () => {
+    expect(
+      genesetsUIReducer(undefined, {
+        type: "geneset: activate add new genes mode",
+        geneset: "a geneset name",
+      })
+    ).toMatchObject({
+      createGenesetModeActive: false,
+      isEditingGenesetName: false,
+      isAddingGenesToGeneset: "a geneset name",
+    });
+  });
+  test("disable create geneset mode", () => {
+    expect(
+      genesetsUIReducer(undefined, {
+        type: "geneset: disable create geneset mode",
+      })
+    ).toMatchObject(initialState);
+  });
+  test("activate rename geneset mode", () => {
+    expect(
+      genesetsUIReducer(undefined, {
+        type: "geneset: activate rename geneset mode",
+        data: "a geneset name",
+      })
+    ).toMatchObject({
+      createGenesetModeActive: false,
+      isEditingGenesetName: "a geneset name",
+      isAddingGenesToGeneset: false,
+    });
+  });
+  test("disable rename geneset mode", () => {
+    expect(
+      genesetsUIReducer(undefined, {
+        type: "geneset: disable rename geneset mode",
+      })
+    ).toMatchObject(initialState);
+  });
+});

client/__tests__/reducers/undoable.test.js

@@ -23,9 +23,7 @@ describe("create", () => {
 describe("undo", () => {
   test("expected state modifications", () => {
     const initialState = { a: 0, b: 1000 };
-    const reducer = state => {
-      return { a: state.a + 1, b: state.b + 1 };
-    };
+    const reducer = (state) => ({ a: state.a + 1, b: state.b + 1 });
     const undoableReducer = undoable(reducer, ["a"]);
 
     const s1 = undoableReducer(initialState, { type: "test" });
@@ -43,9 +41,7 @@ describe("undo", () => {
 
 describe("redo", () => {
   const initialState = { a: 0, b: 1000 };
-  const reducer = state => {
-    return { a: state.a + 1, b: state.b + 1 };
-  };
+  const reducer = (state) => ({ a: state.a + 1, b: state.b + 1 });
   let UR;
 
   beforeEach(() => {
@@ -58,7 +54,7 @@ describe("redo", () => {
 
     // verify undo->redo reverts state.
     const s2 = UR(UR(s1, { type: "@@undoable/undo" }), {
-      type: "@@undoable/redo"
+      type: "@@undoable/redo",
     });
     expect(s2).toMatchObject({ a: 1, b: 1001 });
 

client/__tests__/screenshots/.gitignore

@@ -0,0 +1,4 @@
+# Ignore everything in this directory
+*
+# Except this file
+!.gitignore

client/__tests__/util/actionHelpers.test.js

@@ -11,14 +11,10 @@ describe("rangeEncodeIndices", () => {
 
   test("sorted flag", () => {
     expect(rangeEncodeIndices([1, 9, 432], 10, true)).toMatchObject([
-      1,
-      9,
-      432
+      1, 9, 432,
     ]);
     expect(rangeEncodeIndices([1, 9, 432], 10, false)).toMatchObject([
-      1,
-      9,
-      432
+      1, 9, 432,
     ]);
     expect(rangeEncodeIndices([0, 1, 2, 3, 9, 10, 432], 2, true)).toMatchObject(
       [[0, 3], [9, 10], 432]
@@ -34,7 +30,10 @@ describe("rangeEncodeIndices", () => {
     ).toMatchObject([[3, 7], [10, 12], 99]);
     expect(
       rangeEncodeIndices([3, 4, 5, 6, 7, 10, 11, 12], 3, false)
-    ).toMatchObject([[3, 7], [10, 12]]);
+    ).toMatchObject([
+      [3, 7],
+      [10, 12],
+    ]);
     expect(
       rangeEncodeIndices([0, 3, 4, 5, 6, 7, 10, 11, 12], 3, false)
     ).toMatchObject([0, [3, 7], [10, 12]]);

client/__tests__/util/annoMatrix/annoMatrix.test.js

@@ -0,0 +1,322 @@
+// these TWO statements MUST be first in the file, before any other imports
+import { enableFetchMocks } from "jest-fetch-mock";
+import * as serverMocks from "./serverMocks";
+// OK, continue on!
+
+import {
+  AnnoMatrixLoader,
+  clip,
+  isubset,
+  isubsetMask,
+} from "../../../src/annoMatrix";
+import { Dataframe } from "../../../src/util/dataframe";
+
+enableFetchMocks();
+
+describe("AnnoMatrix", () => {
+  let annoMatrix;
+
+  beforeEach(async () => {
+    fetch.resetMocks(); // reset all fetch mocking state
+    annoMatrix = new AnnoMatrixLoader(
+      serverMocks.baseDataURL,
+      serverMocks.schema.schema
+    );
+  });
+
+  describe("basics", () => {
+    test("annomatrix static checks", () => {
+      expect(annoMatrix).toBeDefined();
+      expect(annoMatrix.schema).toMatchObject(serverMocks.schema.schema);
+      expect(annoMatrix.nObs).toEqual(serverMocks.schema.schema.dataframe.nObs);
+      expect(annoMatrix.nVar).toEqual(serverMocks.schema.schema.dataframe.nVar);
+      expect(annoMatrix.isView).toBeFalsy();
+      expect(annoMatrix.viewOf).toBeUndefined();
+      expect(annoMatrix.rowIndex).toBeDefined();
+    });
+
+    test("simple single column fetch", async () => {
+      fetch.once(serverMocks.annotationsObs(["name_0"]));
+
+      const df = await annoMatrix.fetch("obs", "name_0");
+      expect(df).toBeInstanceOf(Dataframe);
+      expect(df.colIndex.labels()).toEqual(["name_0"]);
+      expect(df.dims).toEqual([annoMatrix.nObs, 1]);
+    });
+
+    test("simple multi column fetch", async () => {
+      fetch
+        .once(serverMocks.annotationsObs(["name_0"]))
+        .once(serverMocks.annotationsObs(["n_genes"]));
+
+      await expect(
+        annoMatrix.fetch("obs", ["name_0", "n_genes"])
+      ).resolves.toBeInstanceOf(Dataframe);
+    });
+
+    describe("fetch from field", () => {
+      const getLastTwo = async (field) => {
+        const columnNames = annoMatrix.getMatrixColumns(field).slice(-2);
+        fetch.mockResponses(...columnNames.map(() => serverMocks.responder));
+        await expect(
+          annoMatrix.fetch(field, columnNames)
+        ).resolves.toBeInstanceOf(Dataframe);
+      };
+
+      test("obs", async () => getLastTwo("obs"));
+      test("var", async () => getLastTwo("var"));
+      test("emb", async () => getLastTwo("emb"));
+    });
+
+    test("fetch - test all query forms", async () => {
+      // single string is a column name
+      fetch.once(serverMocks.annotationsObs(["n_genes"]));
+      await expect(annoMatrix.fetch("obs", "n_genes")).resolves.toBeInstanceOf(
+        Dataframe
+      );
+
+      // array of column names, expecting n_genes to be cached.
+      fetch.once(serverMocks.annotationsObs(["percent_mito"]));
+      await expect(
+        annoMatrix.fetch("obs", ["n_genes", "percent_mito"])
+      ).resolves.toBeInstanceOf(Dataframe);
+
+      // more complex value filter query, enumerated
+      fetch.once(serverMocks.responder);
+      await expect(
+        annoMatrix.fetch("X", {
+          where: {
+            field: "var",
+            column: annoMatrix.schema.annotations.var.index,
+            value: "TYMP",
+          },
+        })
+      ).resolves.toBeInstanceOf(Dataframe);
+
+      // more complex value filter query, range
+      const varIndex = annoMatrix.schema.annotations.var.index;
+      fetch
+        .once(
+          serverMocks.withExpected("/data/var", [[`var:${varIndex}`, "SUMO3"]])
+        )
+        .once(
+          serverMocks.withExpected("/data/var", [[`var:${varIndex}`, "TYMP"]])
+        );
+      await expect(
+        annoMatrix.fetch("X", [
+          {
+            where: {
+              field: "var",
+              column: varIndex,
+              value: "SUMO3",
+            },
+          },
+          {
+            where: {
+              field: "var",
+              column: varIndex,
+              value: "TYMP",
+            },
+          },
+        ])
+      ).resolves.toBeInstanceOf(Dataframe);
+      // XXX inspect the wherecache?
+    });
+
+    test("push and pop views", async () => {
+      const am1 = clip(annoMatrix, 0.1, 0.9);
+      expect(am1.viewOf).toBe(annoMatrix);
+      expect(am1.nObs).toEqual(annoMatrix.nObs);
+      expect(am1.nVar).toEqual(annoMatrix.nVar);
+      expect(am1.rowIndex).toBe(annoMatrix.rowIndex);
+
+      const am2 = clip(annoMatrix, 0.1, 0.9);
+      expect(am2.viewOf).toBe(annoMatrix);
+      expect(am2).not.toBe(am1);
+      expect(am2.rowIndex).toBe(annoMatrix.rowIndex);
+    });
+
+    test("schema accessors", () => {
+      expect(annoMatrix.getMatrixFields()).toEqual(
+        expect.arrayContaining(["X", "obs", "emb", "var"])
+      );
+      expect(annoMatrix.getMatrixColumns("obs")).toEqual(
+        expect.arrayContaining(["name_0", "n_genes", "louvain"])
+      );
+      expect(annoMatrix.getColumnSchema("emb", "umap")).toEqual({
+        name: "umap",
+        dims: ["umap_0", "umap_1"],
+        type: "float32",
+      });
+      expect(annoMatrix.getColumnDimensions("emb", "umap")).toEqual([
+        "umap_0",
+        "umap_1",
+      ]);
+    });
+
+    /*
+  	test the mask & label access to subset via isubset and isubsetMask
+  	*/
+    test("isubset", async () => {
+      const rowList = [0, 10];
+      const rowMask = new Uint8Array(annoMatrix.nObs);
+      for (let i = 0; i < rowList.length; i += 1) {
+        rowMask[rowList[i]] = 1;
+      }
+
+      const am1 = isubset(annoMatrix, rowList);
+      const am2 = isubsetMask(annoMatrix, rowMask);
+      expect(am1).not.toBe(am2);
+      expect(am1.nObs).toEqual(2);
+      expect(am1.nObs).toEqual(am2.nObs);
+      expect(am1.nVar).toEqual(am2.nVar);
+
+      fetch
+        .once(serverMocks.annotationsObs(["n_genes"]))
+        .once(serverMocks.annotationsObs(["n_genes"]));
+      const ng1 = await am1.fetch("obs", "n_genes");
+      const ng2 = await am2.fetch("obs", "n_genes");
+      expect(ng1).toHaveLength(ng2.length);
+      expect(ng1.colIndex.labels()).toEqual(ng2.colIndex.labels());
+      expect(ng1.col("n_genes").asArray()).toEqual(
+        ng2.col("n_genes").asArray()
+      );
+    });
+  });
+
+  describe("add/drop column", () => {
+    async function addDrop(base) {
+      expect(base.getMatrixColumns("obs")).not.toContain("foo");
+      fetch.mockRejectOnce(new Error("unknown column name"));
+      await expect(base.fetch("obs", "foo")).rejects.toThrow(
+        "unknown column name"
+      );
+
+      /* add */
+      const am1 = base.addObsColumn(
+        { name: "foo", type: "float32", writable: true },
+        Float32Array,
+        0
+      );
+      expect(base.getMatrixColumns("obs")).not.toContain("foo");
+      expect(am1.getMatrixColumns("obs")).toContain("foo");
+      const foo = await am1.fetch("obs", "foo");
+      expect(foo).toBeDefined();
+      expect(foo).toBeInstanceOf(Dataframe);
+      expect(foo).toHaveLength(am1.nObs);
+      expect(foo.col("foo").asArray()).toEqual(
+        new Float32Array(am1.nObs).fill(0)
+      );
+
+      /* drop */
+      const am2 = am1.dropObsColumn("foo");
+      expect(base.getMatrixColumns("obs")).not.toContain("foo");
+      expect(am2.getMatrixColumns("obs")).not.toContain("foo");
+      fetch.mockRejectOnce(new Error("unknown column name"));
+      await expect(am2.fetch("obs", "foo")).rejects.toThrow(
+        "unknown column name"
+      );
+    }
+
+    test("add/drop column, without view", async () => {
+      await addDrop(annoMatrix);
+    });
+
+    test("add/drop column, with view", async () => {
+      const am1 = clip(annoMatrix, 0.1, 0.9);
+      await addDrop(am1);
+
+      const am2 = isubset(am1, [0, 1, 2, 20, 30, 400]);
+      await addDrop(am2);
+
+      const am3 = isubset(annoMatrix, [10, 0, 7, 3]);
+      await addDrop(am3);
+
+      const am4 = clip(am3, 0, 1);
+      await addDrop(am4);
+
+      fetch.mockResponse(serverMocks.responder);
+
+      await am1.fetch("obs", am1.getMatrixColumns("obs"));
+      await am2.fetch("obs", am2.getMatrixColumns("obs"));
+      await am3.fetch("obs", am3.getMatrixColumns("obs"));
+      await am4.fetch("obs", am4.getMatrixColumns("obs"));
+
+      fetch.resetMocks();
+
+      await addDrop(am1);
+      await addDrop(am2);
+      await addDrop(am3);
+      await addDrop(am4);
+    });
+  });
+
+  describe("setObsColumnValues", () => {
+    async function addSetDrop(base) {
+      /* add column */
+      let am = base.addObsColumn(
+        {
+          name: "test",
+          type: "categorical",
+          categories: ["unassigned", "red", "green"],
+          writable: true,
+        },
+        Array,
+        "unassigned"
+      );
+
+      const testVal = await am.fetch("obs", "test");
+      expect(testVal.col("test").asArray()).toEqual(
+        new Array(am.nObs).fill("unassigned")
+      );
+
+      /* set values in column */
+      const whichRows = [1, 2, 10];
+      const am1 = await am.setObsColumnValues("test", whichRows, "yo");
+      const testVal1 = await am1.fetch("obs", "test");
+      const expt = new Array(am1.nObs).fill("unassigned");
+      for (let i = 0; i < whichRows.length; i += 1) {
+        const offset = am1.rowIndex.getOffset(whichRows[i]);
+        expt[offset] = "yo";
+      }
+      expect(testVal1).not.toBe(testVal);
+      expect(testVal1.col("test").asArray()).toEqual(expt);
+      expect(am1.getColumnSchema("obs", "test").type).toBe("categorical");
+      expect(am1.getColumnSchema("obs", "test").categories).toEqual(
+        expect.arrayContaining(["unassigned", "red", "green", "yo"])
+      );
+
+      /* drop column */
+      fetch.mockRejectOnce(new Error("unknown column name"));
+      am = am1.dropObsColumn("test");
+      await expect(am.fetch("obs", "test")).rejects.toThrow(
+        "unknown column name"
+      );
+    }
+
+    test("set, without a view", async () => {
+      await addSetDrop(annoMatrix);
+    });
+
+    test("set, with a view", async () => {
+      const am1 = clip(annoMatrix, 0.1, 0.9);
+      await addSetDrop(am1);
+
+      const am2 = isubset(am1, [0, 1, 2, 10, 20, 30, 400]);
+      await addSetDrop(am2);
+
+      const am3 = isubset(annoMatrix, [10, 1, 0, 30, 2]);
+      await addSetDrop(am3);
+
+      fetch.mockResponse(serverMocks.responder);
+
+      await am1.fetch("obs", am1.getMatrixColumns("obs"));
+      await am2.fetch("obs", am2.getMatrixColumns("obs"));
+      await am3.fetch("obs", am3.getMatrixColumns("obs"));
+
+      await addSetDrop(am1);
+      await addSetDrop(am2);
+      await addSetDrop(am3);
+    });
+  });
+});

client/__tests__/util/annoMatrix/crossfilter.test.js

@@ -0,0 +1,692 @@
+// these TWO statements MUST be first in the file, before any other imports
+import { enableFetchMocks } from "jest-fetch-mock";
+import * as serverMocks from "./serverMocks";
+// OK, continue on!
+
+import obsLouvain from "./louvain.json";
+import obsNGenes from "./n_genes.json";
+import embUmap from "./umap.json";
+
+import {
+  AnnoMatrixLoader,
+  AnnoMatrixObsCrossfilter,
+  isubsetMask,
+} from "../../../src/annoMatrix";
+import { rangeFill } from "../../../src/util/range";
+
+enableFetchMocks();
+
+describe("AnnoMatrixCrossfilter", () => {
+  let annoMatrix;
+  let crossfilter;
+
+  beforeEach(async () => {
+    fetch.resetMocks(); // reset all fetch mocking state
+    annoMatrix = new AnnoMatrixLoader(
+      serverMocks.baseDataURL,
+      serverMocks.schema.schema
+    );
+    crossfilter = new AnnoMatrixObsCrossfilter(annoMatrix);
+  });
+
+  test("initial state of crossfilter", () => {
+    const { nObs } = annoMatrix;
+
+    expect(crossfilter).toBeDefined();
+    expect(crossfilter.size()).toEqual(nObs);
+    expect(crossfilter.annoMatrix).toBe(annoMatrix);
+
+    /* by default, everything should be selected, even if no data in cache */
+    expect(crossfilter.countSelected()).toEqual(nObs);
+    expect(crossfilter.allSelectedLabels()).toEqual(
+      rangeFill(new Int32Array(nObs))
+    );
+    expect(crossfilter.allSelectedMask()).toEqual(new Uint8Array(nObs).fill(1));
+    expect(crossfilter.fillByIsSelected(new Uint8Array(nObs), 2, 1)).toEqual(
+      new Uint8Array(nObs).fill(2)
+    );
+  });
+
+  describe("select", () => {
+    /*
+    test the selection state via crossfilter proxy
+    */
+
+    test("select loads index", async () => {
+      /*
+      Select should transparently load/create dimension index.
+
+      Internal dimension names are field/col:col:col..., eg,
+
+        obs:louvain
+        emb:umap_0:umap_1
+
+      */
+      expect(crossfilter.obsCrossfilter.dimensionNames()).toEqual([]);
+      expect(
+        crossfilter.obsCrossfilter.hasDimension("obs/louvain")
+      ).toBeFalsy();
+
+      fetch.once(serverMocks.dataframeResponse(["louvain"], [obsLouvain]));
+      let newCrossfilter = await crossfilter.select("obs", "louvain", {
+        mode: "none",
+      });
+
+      expect(newCrossfilter.countSelected()).toEqual(0);
+      expect(
+        newCrossfilter.obsCrossfilter.hasDimension("obs/louvain")
+      ).toBeTruthy();
+      expect(fetch.mock.calls).toHaveLength(1);
+
+      newCrossfilter = await crossfilter.select("obs", "louvain", {
+        mode: "all",
+      });
+      expect(newCrossfilter.countSelected()).toEqual(annoMatrix.nObs);
+    });
+
+    test("simple column select", async () => {
+      let xfltr;
+
+      fetch.once(serverMocks.dataframeResponse(["louvain"], [obsLouvain]));
+      xfltr = await crossfilter.select("obs", "louvain", {
+        mode: "exact",
+        values: ["NK cells", "B cells"],
+      });
+
+      expect(xfltr).toBeDefined();
+      expect(xfltr.countSelected()).toEqual(496);
+      expect(xfltr.allSelectedMask()).toEqual(
+        Uint8Array.from(
+          obsLouvain.map((val) =>
+            val === "NK cells" || val === "B cells" ? 1 : 0
+          )
+        )
+      );
+      expect(xfltr.allSelectedLabels()).toEqual(
+        Int32Array.from(
+          obsLouvain.reduce((acc, val, idx) => {
+            if (val === "NK cells" || val === "B cells") acc.push(idx);
+            return acc;
+          }, [])
+        )
+      );
+      expect(
+        xfltr.fillByIsSelected(new Uint8Array(annoMatrix.nObs), 3, 1)
+      ).toEqual(
+        Uint8Array.from(
+          obsLouvain.map((val) =>
+            val === "NK cells" || val === "B cells" ? 3 : 1
+          )
+        )
+      );
+
+      const df = await annoMatrix.fetch("obs", "louvain");
+      const values = df.col("louvain").asArray();
+      const selected = xfltr.allSelectedMask();
+      values.every(
+        (val, idx) => !["NK cells", "B cells"].includes(val) !== !selected[idx]
+      );
+
+      fetch.once(
+        serverMocks.dataframeResponse(["n_genes"], [new Int32Array(obsNGenes)])
+      );
+      xfltr = await xfltr.select("obs", "n_genes", {
+        mode: "range",
+        lo: 0,
+        hi: 500,
+        inclusive: false,
+      });
+      expect(xfltr.countSelected()).toEqual(33);
+      expect(xfltr.allSelectedLabels()).toEqual(
+        Int32Array.from(
+          obsNGenes.reduce((acc, val, idx) => {
+            const louvain = obsLouvain[idx];
+            if (
+              val >= 0 &&
+              val < 500 &&
+              (louvain === "NK cells" || louvain === "B cells")
+            )
+              acc.push(idx);
+            return acc;
+          }, [])
+        )
+      );
+
+      xfltr = await xfltr.selectAll();
+      expect(xfltr.countSelected()).toEqual(annoMatrix.nObs);
+    });
+
+    test("join column select", async () => {
+      const varIndex = annoMatrix.schema.annotations.var.index;
+
+      const { nObs } = annoMatrix.schema.dataframe;
+      fetch.once(
+        serverMocks.dataframeResponse(
+          ["TEST"],
+          [rangeFill(new Float32Array(nObs), 0, 0.1)]
+        )
+      );
+
+      const xfltr = await crossfilter.select(
+        "X",
+        {
+          where: {
+            field: "var",
+            column: varIndex,
+            value: "TYMP",
+          },
+        },
+        {
+          mode: "range",
+          lo: 0,
+          hi: 50,
+          inclusive: true,
+        }
+      );
+
+      expect(xfltr).toBeDefined();
+      expect(xfltr.countSelected()).toEqual(501);
+
+      const df = await annoMatrix.fetch("X", {
+        where: {
+          field: "var",
+          column: varIndex,
+          value: "TYMP",
+        },
+      });
+      const values = df.icol(0).asArray();
+      const selected = xfltr.allSelectedMask();
+      values.every((val, idx) => !(val >= 0 && val <= 50) !== !selected[idx]);
+      expect(selected.reduce((acc, val) => (val ? acc + 1 : acc), 0)).toEqual(
+        xfltr.countSelected()
+      );
+    });
+
+    test("spatial column select", async () => {
+      fetch.once(
+        serverMocks.dataframeResponse(
+          ["umap_0", "umap_1"],
+          [Float32Array.from(embUmap[0]), Float32Array.from(embUmap[1])]
+        )
+      );
+      const xfltr = await crossfilter.select("emb", "umap", {
+        mode: "within-rect",
+        minX: 0,
+        minY: 0,
+        maxX: 0.5,
+        maxY: 0.5,
+      });
+      expect(xfltr.countSelected()).toEqual(16);
+    });
+
+    test("select on subset", async () => {
+      const mask = new Uint8Array(annoMatrix.nObs).fill(0);
+      for (let i = 0; i < mask.length; i += 2) {
+        mask[i] = true;
+      }
+      const annoMatrixSubset = isubsetMask(annoMatrix, mask);
+      expect(annoMatrixSubset.nObs).toEqual(Math.floor(annoMatrix.nObs / 2));
+
+      let xfltr = new AnnoMatrixObsCrossfilter(annoMatrixSubset);
+      expect(xfltr.countSelected()).toEqual(annoMatrixSubset.nObs);
+
+      fetch.once(serverMocks.dataframeResponse(["louvain"], [obsLouvain]));
+      xfltr = await xfltr.select("obs", "louvain", {
+        mode: "exact",
+        values: ["NK cells", "B cells"],
+      });
+
+      expect(xfltr).toBeDefined();
+      expect(xfltr.countSelected()).toEqual(240);
+
+      const df = await annoMatrixSubset.fetch("obs", "louvain");
+      const values = df.col("louvain").asArray();
+      const selected = xfltr.allSelectedMask();
+      values.every(
+        (val, idx) => !["NK cells", "B cells"].includes(val) !== !selected[idx]
+      );
+    });
+
+    test("select catches errors", async () => {
+      await expect(crossfilter.select("NADA", "foo")).rejects.toThrow(
+        "Unknown field name"
+      );
+
+      await expect(crossfilter.select("var", "foo")).rejects.toThrow(
+        "unable to obsSelect upon the var dimension"
+      );
+
+      fetch.mockRejectOnce(new Error("unknown column name"));
+      await expect(crossfilter.select("obs", "foo")).rejects.toThrow(
+        "unknown column name"
+      );
+    });
+  });
+
+  describe("mutate matrix", () => {
+    /*
+    test the matrix mutators via crossfilter proxy
+    */
+    async function helperAddTestCol(cf, colName, colSchema = null) {
+      expect(
+        cf.annoMatrix.getMatrixColumns("obs").includes(colName)
+      ).toBeFalsy();
+
+      if (colSchema === null) {
+        colSchema = {
+          name: colName,
+          type: "categorical",
+          categories: ["toasty"],
+        };
+      }
+      colSchema.name = colName;
+      const initValue = colSchema.categories[0];
+      const xfltr = cf.addObsColumn(colSchema, Array, initValue);
+      expect(
+        xfltr.annoMatrix.schema.annotations.obs.columns.filter(
+          (v) => v.name === colName
+        )
+      ).toHaveLength(1);
+      const df = await xfltr.annoMatrix.fetch("obs", colName);
+      expect(df.hasCol(colName)).toBeTruthy();
+      return xfltr;
+    }
+
+    test("addObsColumn", async () => {
+      expect(crossfilter.countSelected()).toBe(annoMatrix.nObs);
+      expect(
+        crossfilter.annoMatrix.getMatrixColumns("obs").includes("foo")
+      ).toBeFalsy();
+      const xfltr = crossfilter.addObsColumn(
+        { name: "foo", type: "categorical", categories: ["A"] },
+        Array,
+        "A"
+      );
+
+      // check schema updates correctly.
+      expect(xfltr.countSelected()).toBe(annoMatrix.nObs);
+      expect(
+        xfltr.annoMatrix.getMatrixColumns("obs").includes("foo")
+      ).toBeTruthy();
+      expect(xfltr.annoMatrix.schema.annotations.obsByName.foo).toMatchObject({
+        name: "foo",
+        type: "categorical",
+      });
+      expect(
+        xfltr.annoMatrix.schema.annotations.obs.columns.filter(
+          (v) => v.name === "foo"
+        )
+      ).toHaveLength(1);
+
+      // check data update.
+      const df = await xfltr.annoMatrix.fetch("obs", "foo");
+      expect(
+        df
+          .col("foo")
+          .asArray()
+          .every((v) => v === "A")
+      ).toBeTruthy();
+
+      // check that we catch dups
+      expect(() =>
+        xfltr.addObsColumn(
+          { name: "foo", type: "categorical" },
+          Array,
+          "toasty"
+        )
+      ).toThrow("column already exists");
+      expect(() =>
+        xfltr.addObsColumn(
+          { name: "louvain", type: "categorical" },
+          Array,
+          "toasty"
+        )
+      ).toThrow("column already exists");
+    });
+
+    test("dropObsColumn", async () => {
+      let xfltr;
+
+      /* check that we catch attempt to drop readonly dimension */
+      expect(() => crossfilter.dropObsColumn("louvain")).toThrow(
+        "Unknown or readonly obs column"
+      );
+      /* non-existent column */
+      expect(() => crossfilter.dropObsColumn("does-not-exist")).toThrow(
+        "Unknown or readonly obs column"
+      );
+
+      // add a column, then drop it.
+      xfltr = await helperAddTestCol(crossfilter, "foo");
+      xfltr = xfltr.dropObsColumn("foo");
+      expect(
+        xfltr.annoMatrix.schema.annotations.obs.columns.filter(
+          (v) => v.name === "foo"
+        )
+      ).toHaveLength(0);
+      expect(xfltr.annoMatrix.schema.annotations.obsByName.foo).toBeUndefined();
+      fetch.mockRejectOnce(new Error("unknown column name"));
+      await expect(xfltr.annoMatrix.fetch("obs", "foo")).rejects.toThrow(
+        "unknown column name"
+      );
+
+      // now same, but ensure we have built an index before doing the drop
+      xfltr = await helperAddTestCol(crossfilter, "bar");
+      xfltr = await xfltr.select("obs", "bar", {
+        mode: "exact",
+        values: "whatever",
+      });
+      xfltr = xfltr.dropObsColumn("bar");
+
+      fetch.mockRejectOnce(new Error("unknown column name"));
+      await expect(xfltr.select("obs", "bar", { mode: "all" })).rejects.toThrow(
+        "unknown column name"
+      );
+    });
+
+    test("renameObsColumn", async () => {
+      let xfltr;
+
+      /* catch attempts to rename non-existent or readonly columns */
+      expect(() =>
+        crossfilter.renameObsColumn("does-not-exist", "foo")
+      ).toThrow("Unknown or readonly obs column");
+      expect(() => crossfilter.renameObsColumn("louvain", "foo")).toThrow(
+        "Unknown or readonly obs column"
+      );
+
+      // add a column, then rename it.
+      xfltr = await helperAddTestCol(crossfilter, "foo");
+      xfltr = xfltr.renameObsColumn("foo", "bar");
+      expect(xfltr.annoMatrix.getColumnSchema("obs", "foo")).toBeUndefined();
+      expect(xfltr.annoMatrix.getColumnSchema("obs", "bar")).toMatchObject({
+        name: "bar",
+        type: "categorical",
+      });
+
+      fetch.mockRejectOnce(new Error("unknown column name"));
+      await expect(xfltr.annoMatrix.fetch("obs", "foo")).rejects.toThrow(
+        "unknown column name"
+      );
+      const df = await xfltr.annoMatrix.fetch("obs", "bar");
+      expect(df.hasCol("bar")).toBeTruthy();
+
+      // now same, but ensure we have built an index before doing the rename
+      xfltr = await helperAddTestCol(crossfilter, "bar");
+      xfltr = await xfltr.select("obs", "bar", {
+        mode: "exact",
+        values: "whatever",
+      });
+      xfltr = xfltr.renameObsColumn("bar", "xyz");
+
+      fetch.mockRejectOnce(new Error("unknown column name"));
+      await expect(xfltr.select("obs", "bar", { mode: "all" })).rejects.toThrow(
+        "unknown column name"
+      );
+      await expect(
+        xfltr.select("obs", "xyz", { mode: "none" })
+      ).resolves.toBeInstanceOf(AnnoMatrixObsCrossfilter);
+    });
+
+    test("addObsAnnoCategory", async () => {
+      let xfltr;
+
+      // catch unknown or readonly columns
+      expect(() => crossfilter.addObsAnnoCategory("louvain", "mumble")).toThrow(
+        "Unknown or readonly obs column"
+      );
+      expect(() =>
+        crossfilter.addObsAnnoCategory("undefined-name", "mumble")
+      ).toThrow("Unknown or readonly obs column");
+
+      // add a column and then add category to it
+      xfltr = await helperAddTestCol(crossfilter, "foo", {
+        name: "foo",
+        type: "categorical",
+        categories: ["unassigned"],
+      });
+      xfltr = xfltr.addObsAnnoCategory("foo", "a-new-label");
+      expect(xfltr.annoMatrix.getColumnSchema("obs", "foo")).toMatchObject({
+        name: "foo",
+        type: "categorical",
+        categories: expect.arrayContaining(["a-new-label", "unassigned"]),
+      });
+
+      // do it again, dup; should throw
+      expect(() => xfltr.addObsAnnoCategory("foo", "a-new-label")).toThrow(
+        "category already exists"
+      );
+
+      // now same, but ensure we have built an index before doing the operation
+      xfltr = await helperAddTestCol(crossfilter, "bar", {
+        name: "bar",
+        type: "categorical",
+        categories: ["unassigned"],
+      });
+      xfltr = await xfltr.select("obs", "bar", {
+        mode: "exact",
+        values: "something",
+      });
+      xfltr = xfltr.addObsAnnoCategory("bar", "a-new-label");
+      expect(xfltr.annoMatrix.getColumnSchema("obs", "bar")).toMatchObject({
+        name: "bar",
+        type: "categorical",
+        categories: expect.arrayContaining(["a-new-label", "unassigned"]),
+      });
+    });
+
+    test("removeObsAnnoCategory", async () => {
+      let xfltr;
+
+      // catch unknown or readonly categories
+      await expect(() =>
+        crossfilter.removeObsAnnoCategory("louvain", "mumble", "unassigned")
+      ).rejects.toThrow("Unknown or readonly obs column");
+      await expect(() =>
+        crossfilter.removeObsAnnoCategory("undefined-name", "mumble")
+      ).rejects.toThrow("Unknown or readonly obs column");
+
+      xfltr = await helperAddTestCol(crossfilter, "foo", {
+        name: "foo",
+        type: "categorical",
+        categories: ["unassigned", "red", "green", "blue"],
+      });
+      xfltr = await xfltr.select("obs", "foo", { mode: "all" });
+      expect(
+        (await xfltr.annoMatrix.fetch("obs", "foo"))
+          .col("foo")
+          .asArray()
+          .every((v) => v === "unassigned")
+      ).toBeTruthy();
+      expect(xfltr.annoMatrix.getColumnSchema("obs", "foo")).toMatchObject({
+        name: "foo",
+        type: "categorical",
+        categories: expect.arrayContaining([
+          "unassigned",
+          "red",
+          "green",
+          "blue",
+        ]),
+      });
+
+      // remove an unused category
+      const xfltr1 = await xfltr.removeObsAnnoCategory("foo", "red", "mumble");
+      expect(
+        (await xfltr1.annoMatrix.fetch("obs", "foo"))
+          .col("foo")
+          .asArray()
+          .every((v) => v === "unassigned")
+      ).toBeTruthy();
+      expect(xfltr1.annoMatrix.getColumnSchema("obs", "foo")).toMatchObject({
+        name: "foo",
+        type: "categorical",
+        categories: expect.arrayContaining([
+          "unassigned",
+          "green",
+          "blue",
+          "mumble",
+        ]),
+      });
+
+      // remove a used category
+      const xfltr2 = await xfltr.removeObsAnnoCategory(
+        "foo",
+        "unassigned",
+        "red"
+      );
+      expect(
+        (await xfltr2.annoMatrix.fetch("obs", "foo"))
+          .col("foo")
+          .asArray()
+          .every((v) => v === "red")
+      ).toBeTruthy();
+      expect(xfltr2.annoMatrix.getColumnSchema("obs", "foo")).toMatchObject({
+        name: "foo",
+        type: "categorical",
+        categories: expect.arrayContaining(["green", "blue", "red"]),
+      });
+    });
+
+    test("setObsColumnValues", async () => {
+      // catch unknown or readonly categories
+      await expect(() =>
+        crossfilter.setObsColumnValues("louvain", [0, 1], "unassigned")
+      ).rejects.toThrow("Unknown or readonly obs column");
+      await expect(() =>
+        crossfilter.setObsColumnValues("undefined-name", [0], "mumble")
+      ).rejects.toThrow("Unknown or readonly obs column");
+
+      let xfltr = await helperAddTestCol(crossfilter, "foo", {
+        name: "foo",
+        type: "categorical",
+        categories: ["unassigned", "red", "green", "blue"],
+      });
+      xfltr = await xfltr.select("obs", "foo", { mode: "all" });
+
+      // catch unknown row label
+      await expect(() =>
+        xfltr.setObsColumnValues("foo", [-1], "red")
+      ).rejects.toThrow("Unknown row label");
+
+      // set a few rows
+      expect(
+        (await xfltr.annoMatrix.fetch("obs", "foo"))
+          .col("foo")
+          .asArray()
+          .every((v) => v === "unassigned")
+      ).toBeTruthy();
+      const xfltr1 = await xfltr.setObsColumnValues("foo", [0, 10], "purple");
+      expect(
+        (await xfltr1.annoMatrix.fetch("obs", "foo"))
+          .col("foo")
+          .asArray()
+          .every(
+            (v, i) =>
+              v === "unassigned" || (v === "purple" && (i === 0 || i === 10))
+          )
+      ).toBeTruthy();
+      expect(xfltr1.annoMatrix.getColumnSchema("obs", "foo")).toMatchObject({
+        name: "foo",
+        type: "categorical",
+        categories: expect.arrayContaining([
+          "unassigned",
+          "red",
+          "green",
+          "blue",
+          "purple",
+        ]),
+      });
+
+      expect(xfltr1.countSelected()).toEqual(xfltr1.annoMatrix.nObs);
+      const xfltr2 = await xfltr1.select("obs", "foo", {
+        mode: "exact",
+        values: ["purple"],
+      });
+      expect(xfltr2.countSelected()).toEqual(2);
+      expect(xfltr2.allSelectedLabels()).toEqual(Int32Array.from([0, 10]));
+    });
+
+    test("resetObsColumnValues", async () => {
+      // catch unknown or readonly categories
+      await expect(() =>
+        crossfilter.resetObsColumnValues("louvain", "red", "blue")
+      ).rejects.toThrow("Unknown or readonly obs column");
+      await expect(() =>
+        crossfilter.resetObsColumnValues("undefined-name", "red", "blue")
+      ).rejects.toThrow("Unknown or readonly obs column");
+
+      let xfltr = await helperAddTestCol(crossfilter, "foo", {
+        name: "foo",
+        type: "categorical",
+        categories: ["unassigned", "red", "green", "blue"],
+      });
+      xfltr = await xfltr.select("obs", "foo", {
+        mode: "exact",
+        values: "red",
+      });
+
+      // catch unknown category name label
+      await expect(() =>
+        xfltr.resetObsColumnValues("foo", "unknown-label", "red")
+      ).rejects.toThrow("unknown category");
+
+      let xfltr1 = await xfltr.setObsColumnValues("foo", [0, 10], "purple");
+      xfltr1 = await xfltr1.select("obs", "foo", {
+        mode: "exact",
+        values: "purple",
+      });
+      expect(
+        (await xfltr1.annoMatrix.fetch("obs", "foo"))
+          .col("foo")
+          .asArray()
+          .filter((v) => v === "purple")
+      ).toHaveLength(2);
+
+      xfltr1 = await xfltr1.resetObsColumnValues("foo", "purple", "magenta");
+      expect(
+        (await xfltr1.annoMatrix.fetch("obs", "foo"))
+          .col("foo")
+          .asArray()
+          .filter((v) => v === "magenta")
+      ).toHaveLength(2);
+      expect(
+        (await xfltr1.annoMatrix.fetch("obs", "foo"))
+          .col("foo")
+          .asArray()
+          .filter((v) => v === "purple")
+      ).toHaveLength(0);
+      expect(xfltr1.annoMatrix.getColumnSchema("obs", "foo")).toMatchObject({
+        name: "foo",
+        type: "categorical",
+        categories: expect.arrayContaining([
+          "unassigned",
+          "red",
+          "green",
+          "blue",
+          "purple",
+          "magenta",
+        ]),
+      });
+    });
+  });
+
+  describe("edge cases", () => {
+    test("transition from empty annoMatrix", async () => {
+      // select before fetch needs to work
+      fetch.once(serverMocks.dataframeResponse(["louvain"], [obsLouvain]));
+      const xfltr = await crossfilter.select("obs", "louvain", {
+        mode: "exact",
+        values: "B cells",
+      });
+      expect(fetch.mock.calls).toHaveLength(1);
+      expect(xfltr.obsCrossfilter.hasDimension("obs/louvain")).toBeTruthy();
+      expect(xfltr.obsCrossfilter.all()).toBe(xfltr.annoMatrix._cache.obs);
+      expect(xfltr.countSelected()).toEqual(
+        obsLouvain.reduce(
+          (count, v) => (v === "B cells" ? count + 1 : count),
+          0
+        )
+      );
+    });
+  });
+});

client/__tests__/util/annoMatrix/n_genes.json

@@ -0,0 +1,180 @@
+[
+  781, 1352, 1131, 960, 522, 782, 783, 790, 533, 550, 1116, 751, 866, 1059, 458,
+  335, 1424, 1014, 1446, 446, 1020, 417, 878, 789, 510, 824, 1545, 996, 937,
+  1368, 428, 406, 1020, 786, 1019, 750, 822, 982, 876, 930, 838, 1014, 732, 877,
+  782, 787, 791, 880, 801, 1215, 343, 1460, 1250, 756, 836, 824, 827, 1238,
+  1243, 1652, 843, 825, 656, 776, 766, 1465, 790, 871, 803, 965, 800, 876, 690,
+  988, 906, 741, 620, 867, 916, 969, 803, 732, 555, 790, 862, 900, 674, 397,
+  663, 563, 786, 859, 568, 412, 1043, 1206, 702, 1263, 929, 1079, 938, 316, 900,
+  919, 862, 903, 390, 1717, 819, 1877, 660, 791, 478, 769, 481, 819, 866, 600,
+  1185, 650, 775, 699, 642, 857, 832, 388, 710, 341, 894, 935, 604, 1008, 985,
+  679, 603, 864, 1031, 887, 603, 610, 1119, 669, 794, 963, 756, 637, 1032, 776,
+  860, 825, 852, 745, 858, 925, 1228, 806, 715, 668, 779, 1197, 888, 1273, 873,
+  847, 781, 959, 805, 554, 604, 785, 978, 910, 936, 997, 961, 1314, 799, 1112,
+  677, 775, 1298, 657, 626, 1313, 467, 936, 977, 780, 1311, 432, 579, 850, 736,
+  800, 892, 860, 720, 822, 681, 954, 889, 1265, 919, 799, 833, 496, 1476, 848,
+  869, 1059, 490, 897, 832, 864, 419, 856, 907, 791, 756, 771, 957, 1190, 680,
+  524, 908, 506, 851, 775, 793, 748, 951, 643, 1277, 828, 480, 969, 1112, 648,
+  805, 1223, 1023, 669, 489, 390, 350, 1113, 837, 1547, 840, 581, 748, 1861,
+  735, 488, 1016, 585, 797, 769, 490, 1307, 895, 686, 602, 772, 704, 892, 1169,
+  1375, 1189, 892, 2455, 355, 1856, 1317, 703, 825, 736, 1997, 892, 1034, 545,
+  1188, 659, 1056, 819, 979, 632, 598, 690, 310, 803, 743, 560, 1073, 844, 882,
+  841, 815, 771, 976, 986, 820, 957, 640, 1012, 927, 794, 753, 791, 366, 539,
+  752, 769, 650, 947, 771, 824, 744, 837, 723, 640, 923, 1174, 1597, 699, 618,
+  1418, 820, 1047, 981, 866, 527, 762, 717, 860, 603, 828, 476, 1071, 775, 614,
+  913, 836, 669, 942, 792, 871, 1046, 859, 793, 822, 751, 435, 1142, 781, 718,
+  471, 1750, 892, 841, 1156, 1031, 912, 873, 824, 1233, 1312, 575, 605, 717,
+  1019, 1215, 928, 1780, 657, 718, 646, 808, 1120, 750, 390, 1100, 456, 755,
+  1118, 571, 867, 728, 916, 491, 960, 625, 1090, 772, 968, 480, 810, 725, 1016,
+  1011, 1075, 808, 672, 950, 862, 766, 963, 507, 570, 678, 768, 1037, 885, 1426,
+  1496, 587, 879, 924, 783, 696, 1057, 783, 867, 919, 1251, 1023, 727, 645,
+  1217, 929, 792, 994, 1025, 946, 600, 881, 975, 1609, 758, 772, 682, 998, 979,
+  1045, 706, 808, 855, 819, 1147, 742, 914, 969, 704, 1398, 581, 809, 921, 805,
+  542, 888, 519, 1092, 762, 698, 752, 771, 899, 1101, 760, 881, 1124, 809, 445,
+  1703, 789, 641, 819, 890, 767, 806, 1323, 942, 807, 981, 888, 726, 1190, 826,
+  661, 713, 816, 822, 806, 864, 464, 664, 931, 860, 674, 803, 464, 788, 1068,
+  781, 843, 779, 873, 707, 492, 669, 982, 749, 789, 780, 561, 655, 432, 801,
+  945, 770, 503, 766, 776, 970, 989, 654, 762, 882, 1236, 1499, 626, 1380, 1170,
+  491, 833, 924, 581, 842, 596, 811, 542, 1572, 758, 854, 274, 1413, 872, 559,
+  907, 951, 1175, 946, 905, 737, 604, 843, 606, 1079, 668, 785, 726, 978, 941,
+  994, 655, 1013, 987, 591, 1041, 625, 582, 814, 570, 775, 822, 715, 738, 956,
+  1178, 743, 1861, 841, 944, 783, 643, 924, 936, 431, 490, 532, 524, 620, 749,
+  1334, 834, 790, 702, 892, 693, 784, 944, 471, 839, 529, 729, 1084, 802, 886,
+  815, 856, 746, 1318, 545, 696, 872, 1154, 467, 725, 1027, 479, 728, 926, 1282,
+  907, 833, 1024, 838, 900, 737, 367, 459, 1030, 1279, 756, 662, 1323, 1003,
+  359, 770, 813, 634, 924, 1184, 901, 816, 1421, 771, 706, 953, 348, 716, 870,
+  715, 550, 689, 947, 1157, 690, 383, 374, 882, 697, 246, 833, 1006, 1181, 974,
+  856, 978, 1551, 965, 907, 565, 417, 907, 927, 966, 658, 727, 743, 381, 385,
+  905, 645, 1167, 936, 724, 618, 1038, 853, 808, 1403, 762, 741, 687, 932, 1096,
+  601, 652, 895, 682, 1553, 604, 823, 803, 646, 917, 942, 850, 795, 771, 949,
+  516, 664, 1001, 637, 619, 907, 961, 812, 793, 1043, 1343, 1326, 981, 675, 937,
+  631, 1026, 1135, 499, 948, 801, 848, 741, 604, 864, 1076, 1106, 1111, 624,
+  1008, 908, 815, 346, 1062, 803, 749, 779, 1027, 1032, 1040, 654, 631, 755,
+  854, 850, 798, 864, 1078, 690, 864, 1523, 838, 966, 389, 1654, 808, 885, 1665,
+  920, 855, 807, 859, 1276, 987, 1079, 678, 626, 831, 829, 1009, 547, 893, 722,
+  656, 415, 773, 1262, 1218, 365, 661, 805, 1409, 1094, 779, 898, 830, 1242,
+  864, 576, 901, 1274, 852, 1006, 719, 763, 683, 957, 831, 724, 354, 763, 910,
+  749, 626, 870, 795, 1078, 736, 835, 402, 1203, 699, 755, 697, 1229, 637, 666,
+  846, 1036, 1027, 831, 999, 942, 891, 1007, 712, 990, 725, 745, 921, 333, 1042,
+  895, 873, 1612, 724, 929, 601, 862, 908, 658, 775, 724, 753, 741, 690, 379,
+  608, 927, 777, 969, 827, 709, 385, 690, 769, 554, 892, 761, 367, 731, 1103,
+  944, 832, 675, 652, 418, 727, 745, 872, 1336, 863, 934, 844, 721, 432, 782,
+  1006, 834, 840, 840, 819, 699, 489, 665, 576, 1291, 1102, 826, 880, 738, 904,
+  686, 874, 887, 873, 560, 766, 710, 1135, 1054, 805, 724, 973, 1201, 575, 838,
+  865, 546, 811, 884, 886, 791, 1026, 2000, 644, 763, 969, 800, 359, 624, 993,
+  800, 1167, 833, 1871, 616, 822, 647, 1000, 618, 734, 618, 1938, 861, 945,
+  1032, 723, 984, 994, 771, 738, 1583, 1113, 614, 1146, 615, 848, 983, 677, 972,
+  791, 827, 804, 395, 843, 493, 741, 941, 1659, 742, 1517, 559, 937, 740, 781,
+  819, 813, 578, 1022, 1191, 824, 1146, 757, 638, 830, 713, 609, 1271, 680, 769,
+  1119, 731, 804, 781, 916, 735, 835, 1257, 472, 879, 851, 1023, 661, 1008, 748,
+  845, 393, 675, 843, 876, 939, 932, 760, 735, 1561, 752, 940, 705, 405, 690,
+  1071, 544, 927, 817, 388, 560, 1322, 640, 886, 1075, 689, 524, 606, 802, 868,
+  939, 753, 770, 1105, 841, 786, 445, 703, 593, 875, 901, 927, 798, 1221, 415,
+  1381, 949, 1322, 1169, 745, 727, 799, 490, 767, 943, 808, 926, 664, 569, 843,
+  727, 1222, 457, 1515, 1138, 1174, 525, 878, 525, 999, 778, 772, 819, 1015,
+  939, 856, 715, 793, 837, 1193, 764, 834, 677, 625, 420, 837, 874, 679, 811,
+  546, 787, 587, 821, 669, 813, 780, 649, 924, 1322, 701, 792, 817, 667, 804,
+  593, 740, 1243, 859, 685, 596, 1193, 859, 775, 947, 689, 907, 734, 621, 336,
+  922, 802, 812, 941, 943, 868, 947, 767, 701, 692, 423, 775, 500, 1100, 812,
+  728, 616, 928, 835, 454, 812, 769, 1058, 914, 628, 649, 452, 754, 1327, 776,
+  670, 1017, 599, 431, 967, 1077, 2033, 731, 533, 926, 725, 559, 870, 964, 1341,
+  1981, 1103, 326, 428, 808, 821, 554, 596, 680, 1034, 849, 566, 879, 1091, 823,
+  447, 1688, 868, 1254, 942, 462, 1055, 852, 738, 804, 775, 726, 993, 1462,
+  1007, 798, 1036, 786, 948, 743, 761, 838, 1040, 859, 867, 1188, 846, 687, 672,
+  629, 725, 660, 809, 469, 600, 812, 856, 397, 786, 895, 882, 449, 890, 823,
+  1051, 823, 1055, 741, 999, 1241, 790, 878, 778, 1066, 815, 465, 1079, 743,
+  1098, 807, 1120, 1025, 805, 676, 828, 763, 997, 852, 866, 1118, 508, 928, 958,
+  932, 892, 905, 494, 710, 1068, 795, 787, 951, 720, 842, 890, 1355, 1005, 872,
+  1185, 912, 869, 894, 997, 770, 554, 806, 1426, 1012, 452, 896, 426, 239, 829,
+  895, 787, 1139, 925, 1015, 1360, 1097, 650, 853, 549, 1052, 307, 1152, 907,
+  1628, 731, 897, 1749, 762, 712, 1195, 851, 864, 968, 845, 331, 840, 734, 948,
+  842, 1543, 661, 981, 912, 912, 1063, 683, 823, 996, 695, 1483, 927, 574, 1052,
+  571, 1028, 1263, 671, 958, 747, 866, 896, 489, 643, 923, 820, 1466, 550, 1112,
+  1006, 1448, 727, 899, 998, 563, 870, 903, 516, 754, 879, 588, 740, 798, 798,
+  653, 902, 990, 724, 953, 891, 1437, 653, 714, 956, 877, 1012, 824, 1077, 740,
+  692, 1063, 771, 808, 1389, 1264, 952, 816, 795, 795, 760, 886, 349, 868, 842,
+  819, 626, 418, 903, 838, 723, 436, 1112, 724, 1299, 719, 843, 1090, 696, 885,
+  627, 809, 423, 729, 853, 855, 608, 627, 823, 1063, 575, 743, 1528, 681, 544,
+  422, 731, 920, 761, 884, 982, 784, 496, 573, 521, 663, 794, 975, 856, 978,
+  590, 905, 695, 816, 976, 816, 753, 791, 858, 813, 841, 1085, 1692, 716, 955,
+  1467, 741, 296, 738, 1573, 1119, 918, 283, 703, 842, 1253, 676, 1636, 1273,
+  380, 799, 1491, 878, 939, 725, 1365, 818, 719, 1343, 905, 837, 803, 990, 1084,
+  976, 1630, 795, 1408, 771, 650, 779, 648, 817, 1127, 882, 954, 830, 732, 783,
+  756, 708, 976, 718, 887, 809, 795, 662, 912, 1550, 1509, 1021, 1751, 776, 910,
+  714, 530, 846, 631, 1152, 1118, 755, 573, 1176, 267, 918, 1132, 849, 938,
+  1140, 909, 840, 806, 904, 788, 778, 715, 869, 714, 883, 767, 858, 788, 553,
+  634, 1230, 1131, 849, 811, 827, 1753, 713, 484, 783, 722, 596, 514, 372, 925,
+  747, 840, 673, 955, 796, 719, 718, 857, 578, 1063, 594, 997, 1268, 341, 388,
+  753, 834, 593, 1189, 911, 738, 476, 816, 641, 746, 635, 953, 801, 326, 849,
+  499, 865, 1420, 487, 876, 797, 981, 756, 850, 1097, 998, 829, 1159, 955, 1061,
+  696, 786, 743, 1211, 893, 491, 744, 1447, 767, 1050, 853, 1118, 1428, 555,
+  612, 854, 789, 889, 784, 712, 1323, 921, 682, 794, 846, 1012, 839, 807, 587,
+  881, 781, 1010, 1182, 1149, 812, 607, 1155, 714, 642, 1088, 1291, 1196, 325,
+  804, 370, 359, 855, 1165, 836, 923, 863, 1284, 1011, 889, 984, 512, 939, 883,
+  697, 1211, 362, 969, 1135, 1239, 580, 1103, 975, 825, 1170, 921, 640, 1180,
+  378, 982, 916, 1122, 792, 619, 750, 913, 775, 661, 766, 768, 675, 944, 940,
+  761, 727, 767, 873, 1043, 850, 995, 680, 595, 700, 753, 736, 891, 685, 780,
+  986, 989, 830, 810, 685, 784, 642, 742, 961, 906, 829, 621, 822, 717, 1210,
+  800, 1963, 749, 757, 570, 831, 721, 336, 802, 1001, 886, 631, 759, 631, 550,
+  984, 767, 835, 777, 639, 860, 1413, 747, 779, 540, 367, 1629, 1380, 689, 1001,
+  809, 337, 1103, 796, 966, 782, 1018, 642, 967, 436, 826, 779, 1000, 601, 796,
+  945, 1679, 1123, 596, 995, 720, 588, 759, 452, 780, 836, 515, 846, 392, 283,
+  710, 1158, 796, 895, 585, 559, 859, 879, 858, 842, 643, 1308, 595, 1181, 909,
+  710, 821, 817, 841, 1197, 640, 1425, 947, 900, 852, 460, 1100, 824, 780, 932,
+  542, 1137, 1225, 997, 572, 780, 765, 906, 793, 753, 772, 854, 936, 1048, 819,
+  645, 619, 314, 726, 737, 1162, 1081, 868, 1032, 913, 476, 490, 799, 1201, 997,
+  898, 212, 1586, 427, 947, 937, 724, 380, 715, 739, 931, 973, 773, 1497, 906,
+  798, 953, 471, 821, 806, 714, 828, 727, 773, 976, 856, 727, 761, 1084, 1557,
+  693, 559, 627, 795, 750, 838, 803, 453, 734, 607, 1029, 805, 669, 505, 858,
+  832, 1019, 585, 1225, 1287, 903, 752, 2020, 774, 666, 843, 857, 887, 1082,
+  656, 674, 911, 734, 910, 672, 802, 539, 699, 941, 828, 800, 642, 733, 607,
+  992, 379, 562, 847, 787, 1461, 732, 941, 785, 696, 795, 809, 828, 651, 882,
+  972, 1323, 679, 774, 784, 766, 520, 671, 796, 644, 1549, 756, 723, 788, 643,
+  856, 825, 730, 831, 653, 429, 641, 637, 812, 1527, 859, 972, 744, 869, 508,
+  624, 923, 976, 801, 1014, 1429, 586, 692, 704, 1176, 806, 883, 1249, 765, 743,
+  907, 666, 669, 364, 794, 959, 766, 937, 1398, 942, 1469, 905, 812, 572, 1378,
+  1058, 1215, 697, 531, 676, 1819, 503, 801, 943, 874, 813, 694, 494, 660, 1467,
+  976, 833, 689, 921, 625, 1428, 817, 909, 956, 765, 1207, 829, 1648, 554, 1500,
+  953, 647, 537, 786, 814, 761, 862, 838, 1102, 1392, 1042, 372, 971, 1364,
+  1137, 847, 935, 710, 1070, 914, 855, 759, 654, 981, 1193, 397, 1123, 616, 747,
+  876, 949, 965, 789, 752, 717, 1198, 952, 625, 784, 914, 659, 913, 863, 959,
+  1637, 796, 1508, 906, 714, 1195, 867, 819, 375, 656, 1047, 745, 866, 1186,
+  1669, 539, 942, 839, 927, 796, 734, 831, 967, 620, 814, 605, 1391, 655, 1512,
+  625, 719, 547, 864, 902, 853, 1143, 990, 858, 604, 1291, 701, 859, 768, 1621,
+  715, 594, 783, 1608, 927, 740, 805, 705, 491, 514, 768, 880, 993, 356, 748,
+  993, 801, 843, 1194, 794, 606, 810, 882, 682, 1126, 792, 829, 1657, 786, 610,
+  664, 790, 852, 678, 589, 930, 690, 840, 647, 622, 427, 856, 522, 1262, 722,
+  641, 997, 983, 1099, 607, 704, 947, 886, 1062, 659, 995, 492, 754, 676, 837,
+  532, 598, 1274, 921, 700, 828, 828, 543, 815, 709, 842, 1200, 1126, 1327, 849,
+  1160, 707, 675, 709, 990, 270, 445, 892, 1040, 832, 815, 437, 1401, 1341, 480,
+  1517, 704, 622, 823, 731, 781, 849, 1209, 472, 838, 643, 1101, 879, 895, 408,
+  832, 845, 928, 920, 1269, 652, 1463, 840, 706, 956, 1219, 612, 970, 789, 382,
+  533, 627, 388, 655, 1103, 782, 1231, 617, 589, 758, 882, 909, 661, 558, 947,
+  491, 1386, 1481, 823, 1064, 681, 652, 363, 824, 933, 611, 886, 806, 531, 864,
+  648, 1142, 736, 955, 826, 725, 1174, 841, 1006, 1418, 955, 829, 1038, 632,
+  750, 846, 1033, 859, 890, 893, 1030, 913, 1385, 750, 416, 703, 522, 856, 914,
+  742, 381, 1079, 671, 1101, 858, 663, 581, 845, 881, 823, 835, 1290, 598, 1130,
+  793, 838, 765, 768, 815, 810, 912, 818, 790, 665, 703, 875, 657, 1567, 688,
+  749, 1076, 782, 543, 1100, 841, 809, 789, 1263, 758, 368, 1022, 730, 1101,
+  478, 488, 618, 940, 771, 784, 847, 1303, 821, 1111, 752, 1128, 958, 742, 782,
+  824, 751, 872, 939, 606, 698, 705, 963, 607, 621, 1650, 1093, 545, 670, 617,
+  723, 1194, 1019, 1291, 758, 855, 1549, 743, 1372, 802, 337, 1121, 1028, 1524,
+  645, 847, 866, 941, 751, 583, 796, 793, 975, 936, 524, 659, 607, 1433, 562,
+  696, 927, 517, 719, 599, 639, 977, 1019, 816, 672, 1903, 1162, 964, 936, 947,
+  989, 790, 902, 982, 673, 856, 629, 692, 843, 940, 795, 780, 821, 471, 702,
+  631, 1557, 868, 901, 798, 1020, 885, 881, 719, 1043, 1238, 565, 776, 696, 725,
+  365, 811, 1212, 1178, 1132, 661, 831, 786, 471, 835, 564, 929, 958, 706, 388,
+  842, 965, 1088, 511, 794, 900, 865, 789, 504, 701, 817, 796, 972, 906, 871,
+  922, 724, 628, 1479, 533, 1101, 1913, 855, 1266, 884, 817, 619, 591, 685, 887,
+  1336, 656, 1227, 980, 817, 582, 1370, 460, 638, 471, 650, 414, 907, 1147, 732,
+  992, 801, 822, 529, 737, 806, 816, 889, 1305, 588, 657, 1154, 713, 326, 1129,
+  1603, 879, 1156, 642, 285, 825, 823, 719, 1253, 971, 853, 916, 1053, 515,
+  1017, 953, 832, 645, 667, 1326, 547, 636, 1783, 1211, 788, 807, 1104, 884,
+  848, 788, 1013, 1003, 916, 818, 828, 882, 959, 395, 368, 787, 929, 1379, 711,
+  733, 752, 464, 626, 735, 946, 876, 647, 536, 954, 486, 712, 786, 438, 807,
+  1016, 551, 841, 929, 757, 971, 708, 567, 881, 801, 873, 805, 1359, 866, 945,
+  1068, 819, 815, 1058, 845, 881, 1051, 1179, 718, 657, 882, 709, 754, 735, 603,
+  944, 1794, 712, 721, 1097, 813, 788, 917, 656, 1104, 1268, 1239, 862, 739,
+  858, 1066, 752, 615, 721, 571, 861, 933, 807, 1082, 820, 887, 850, 1567, 803,
+  1156, 721, 692, 700, 458, 637, 873, 1544, 1155, 1227, 622, 454, 724
+]

client/__tests__/util/annoMatrix/serverMocks/index.js

@@ -0,0 +1,4 @@
+export const baseDataURL = "https://a.fake.url/api/v0.2";
+
+export { schema } from "./schema";
+export * from "./routes";

client/__tests__/util/annoMatrix/serverMocks/routes.js

@@ -0,0 +1,211 @@
+import { schema } from "./schema";
+import { Dataframe, KeyIndex } from "../../../../src/util/dataframe";
+import { encodeMatrixFBS } from "../../../../src/util/stateManager/matrix";
+
+const indexedSchema = {
+  obsByName: Object.fromEntries(
+    schema.schema.annotations.obs.columns.map((v) => [v.name, v]) ?? []
+  ),
+  varByName: Object.fromEntries(
+    schema.schema.annotations.var.columns.map((v) => [v.name, v]) ?? []
+  ),
+  embByName: Object.fromEntries(
+    schema.schema.layout.obs.map((v) => [v.name, v]) ?? []
+  ),
+};
+
+function makeMockColumn(s, length) {
+  const { type } = s;
+  switch (type) {
+    case "int32":
+      return new Int32Array(length).fill(Math.floor(99 * Math.random()));
+
+    case "string":
+      return new Array(length).fill("test");
+
+    case "float32":
+      return new Float32Array(length).fill(99 * Math.random());
+
+    case "boolean":
+      return new Array(length).fill(false);
+
+    case "categorical":
+      return new Array(length).fill(s.categories[0]);
+
+    default:
+      throw new Error("unkonwn type");
+  }
+}
+
+function getEncodedDataframe(colNames, length, colSchemas) {
+  const colIndex = new KeyIndex(colNames);
+  const columns = colSchemas.map((s) => makeMockColumn(s, length));
+  const df = new Dataframe([length, colNames.length], columns, null, colIndex);
+  const body = encodeMatrixFBS(df);
+  return body;
+}
+
+export function dataframeResponse(colNames, columns) {
+  const colIndex = new KeyIndex(colNames);
+  const df = new Dataframe(
+    [columns[0].length, colNames.length],
+    columns,
+    null,
+    colIndex
+  );
+  const body = encodeMatrixFBS(df);
+  const headers = new Headers({
+    "Content-Type": "application/octet-stream",
+  });
+  return () => Promise.resolve({ body, init: { status: 200, headers } });
+}
+
+function annotationObsResponse(request) {
+  const url = new URL(request.url);
+  const params = Array.from(url.searchParams.entries());
+  const names = params
+    .filter(([k]) => k === "annotation-name")
+    .map(([, v]) => v);
+  if (!names.every((n) => indexedSchema.obsByName[n])) {
+    return Promise.reject(new Error("bad obs annotation name in URL"));
+  }
+  const colSchemas = names.map((n) => indexedSchema.obsByName[n]);
+  const body = getEncodedDataframe(
+    names,
+    schema.schema.dataframe.nObs,
+    colSchemas
+  );
+
+  const headers = new Headers({
+    "Content-Type": "application/octet-stream",
+  });
+  return Promise.resolve({
+    body,
+    init: { status: 200, headers },
+  });
+}
+
+function annotationVarResponse(request) {
+  const url = new URL(request.url);
+  const params = Array.from(url.searchParams.entries());
+  const names = params
+    .filter(([k]) => k === "annotation-name")
+    .map(([, v]) => v);
+  if (!names.every((n) => indexedSchema.varByName[n])) {
+    return Promise.reject(new Error("bad var annotation name in URL"));
+  }
+  const colSchemas = names.map((n) => indexedSchema.varByName[n]);
+  const body = getEncodedDataframe(
+    names,
+    schema.schema.dataframe.nVar,
+    colSchemas
+  );
+
+  const headers = new Headers({
+    "Content-Type": "application/octet-stream",
+  });
+  return Promise.resolve({
+    body,
+    init: { status: 200, headers },
+  });
+}
+
+function layoutObsResponse(request) {
+  const url = new URL(request.url);
+  const params = Array.from(url.searchParams.entries());
+  const names = params.filter(([k]) => k === "layout-name").map(([, v]) => v);
+  if (!names.every((n) => indexedSchema.embByName[n])) {
+    return Promise.reject(new Error("bad layout name in URL"));
+  }
+  const dims = names.map((n) => indexedSchema.embByName[n].dims).flat();
+  const colSchemas = names
+    .map((n) => [indexedSchema.embByName[n], indexedSchema.embByName[n]])
+    .flat();
+  const body = getEncodedDataframe(
+    dims,
+    schema.schema.dataframe.nObs,
+    colSchemas
+  );
+
+  const headers = new Headers({
+    "Content-Type": "application/octet-stream",
+  });
+  return Promise.resolve({
+    body,
+    init: { status: 200, headers },
+  });
+}
+
+function dataVarResponse(request) {
+  const url = new URL(request.url);
+  const params = Array.from(url.searchParams.entries());
+
+  const colNames = params.map((v) => `${v[0]}/${v[1]}`);
+  const colSchemas = colNames.map(() => schema.schema.dataframe);
+  const body = getEncodedDataframe(
+    colNames,
+    schema.schema.dataframe.nObs,
+    colSchemas
+  );
+
+  const headers = new Headers({
+    "Content-Type": "application/octet-stream",
+  });
+  return Promise.resolve({
+    body,
+    init: { status: 200, headers },
+  });
+}
+
+export function responder(request) {
+  const url = new URL(request.url);
+  const { pathname } = url;
+  if (pathname.endsWith("/annotations/obs")) {
+    return annotationObsResponse(request);
+  }
+  if (pathname.endsWith("/annotations/var")) {
+    return annotationVarResponse(request);
+  }
+  if (pathname.endsWith("/layout/obs")) {
+    return layoutObsResponse(request);
+  }
+  if (pathname.endsWith("/data/var")) {
+    return dataVarResponse(request);
+  }
+  return Promise.reject(new Error("bad URL"));
+}
+
+export function withExpected(expectedURL, expectedParams) {
+  /*
+  Do some additional error checking
+  */
+  return (request) => {
+    // if URL is bogus, reject the promise
+    const url = new URL(request.url);
+    if (!url.pathname.endsWith(expectedURL)) {
+      return Promise.reject(new Error("Unexpected URL!"));
+    }
+    const params = Array.from(url.searchParams.entries()).sort(
+      (a, b) => a[0] < b[0]
+    );
+    expectedParams = expectedParams.slice().sort((a, b) => a[0] < b[0]);
+
+    if (
+      params.length !== expectedParams.length ||
+      !params.every(
+        (p, i) => p[0] === expectedParams[i][0] && p[1] === expectedParams[i][1]
+      )
+    ) {
+      return Promise.reject(new Error("unexpected name requested in URL"));
+    }
+
+    return responder(request);
+  };
+}
+
+export function annotationsObs(names) {
+  return withExpected(
+    "/annotations/obs",
+    names.map((name) => ["annotation-name", name])
+  );
+}

client/__tests__/util/annoMatrix/serverMocks/schema.js

@@ -0,0 +1,80 @@
+export const schema = {
+  schema: {
+    annotations: {
+      obs: {
+        columns: [
+          {
+            name: "name_0",
+            type: "string",
+            writable: false,
+          },
+          {
+            name: "n_genes",
+            type: "int32",
+            writable: false,
+          },
+          {
+            name: "percent_mito",
+            type: "float32",
+            writable: false,
+          },
+          {
+            name: "n_counts",
+            type: "float32",
+            writable: false,
+          },
+          {
+            name: "louvain",
+            type: "string",
+            writable: false,
+          },
+        ],
+        index: "name_0",
+      },
+      var: {
+        columns: [
+          {
+            name: "name_0",
+            type: "string",
+            writable: false,
+          },
+          {
+            name: "n_cells",
+            type: "int32",
+            writable: false,
+          },
+        ],
+        index: "name_0",
+      },
+    },
+    dataframe: {
+      nObs: 2638,
+      nVar: 1838,
+      type: "float32",
+    },
+    layout: {
+      obs: [
+        {
+          dims: ["draw_graph_fr_0", "draw_graph_fr_1"],
+          name: "draw_graph_fr",
+          type: "float32",
+        },
+        {
+          dims: ["pca_0", "pca_1"],
+          name: "pca",
+          type: "float32",
+        },
+        {
+          dims: ["tsne_0", "tsne_1"],
+          name: "tsne",
+          type: "float32",
+        },
+        {
+          dims: ["umap_0", "umap_1"],
+          name: "umap",
+          type: "float32",
+        },
+      ],
+    },
+  },
+};

client/__tests__/util/annoMatrix/whereCache.test.js

@@ -0,0 +1,382 @@
+import sha1 from "sha1";
+import {
+  _whereCacheGet,
+  _whereCacheCreate,
+  _whereCacheMerge,
+} from "../../../src/annoMatrix/whereCache";
+
+const schema = {};
+
+describe("whereCache", () => {
+  test("whereCacheGet - where query, missing cache values", () => {
+    expect(
+      _whereCacheGet({}, schema, "X", {
+        where: {
+          field: "var",
+          column: "foo",
+          value: "bar",
+        },
+      })
+    ).toEqual([undefined]);
+    expect(
+      _whereCacheGet({}, schema, "X", {
+        summarize: {
+          field: "var",
+          column: "foo",
+          values: ["bar"],
+        },
+      })
+    ).toEqual([undefined]);
+    expect(
+      _whereCacheGet({ where: { X: {} } }, schema, "X", {
+        where: {
+          field: "var",
+          column: "foo",
+          value: "bar",
+        },
+      })
+    ).toEqual([undefined]);
+    expect(
+      _whereCacheGet({ where: { X: { var: new Map() } } }, schema, "X", {
+        where: {
+          field: "var",
+          column: "foo",
+          value: "bar",
+        },
+      })
+    ).toEqual([undefined]);
+    expect(
+      _whereCacheGet(
+        { where: { X: { var: new Map([["foo", new Map()]]) } } },
+        schema,
+        "X",
+        {
+          where: {
+            field: "var",
+            column: "foo",
+            value: "bar",
+          },
+        }
+      )
+    ).toEqual([undefined]);
+  });
+
+  test("whereCacheGet - summarize query, missing cache values", () => {
+    expect(
+      _whereCacheGet({}, schema, "X", {
+        summarize: {
+          method: "mean",
+          field: "var",
+          column: "foo",
+          values: ["bar"],
+        },
+      })
+    ).toEqual([undefined]);
+    expect(
+      _whereCacheGet(
+        { summarize: { X: { mean: { var: new Map() } } } },
+        schema,
+        "X",
+        {
+          summarize: {
+            method: "mean",
+            field: "var",
+            column: "foo",
+            values: ["bar"],
+          },
+        }
+      )
+    ).toEqual([undefined]);
+  });
+
+  test("whereCacheGet - varied lookups", () => {
+    const whereCache = {
+      where: {
+        X: {
+          var: new Map([
+            [
+              "foo",
+              new Map([
+                ["bar", [0]],
+                ["baz", [1, 2]],
+              ]),
+            ],
+          ]),
+        },
+      },
+      summarize: {
+        X: {
+          mean: {
+            var: new Map([
+              [
+                "foo",
+                new Map([
+                  [sha1("bar"), [0]],
+                  [sha1("baz"), [1, 2]],
+                ]),
+              ],
+            ]),
+          },
+        },
+      },
+    };
+
+    expect(
+      _whereCacheGet(whereCache, schema, "X", {
+        where: {
+          field: "var",
+          column: "foo",
+          value: "bar",
+        },
+      })
+    ).toEqual([0]);
+    expect(
+      _whereCacheGet(whereCache, schema, "X", {
+        summarize: {
+          method: "mean",
+          field: "var",
+          column: "foo",
+          values: ["bar"],
+        },
+      })
+    ).toEqual([0]);
+    expect(
+      _whereCacheGet(whereCache, schema, "X", {
+        where: {
+          field: "var",
+          column: "foo",
+          value: "baz",
+        },
+      })
+    ).toEqual([1, 2]);
+    expect(
+      _whereCacheGet(whereCache, schema, "X", {
+        summarize: {
+          method: "mean",
+          field: "var",
+          column: "foo",
+          values: ["baz"],
+        },
+      })
+    ).toEqual([1, 2]);
+    expect(_whereCacheGet(whereCache, schema, "Y", {})).toEqual([undefined]);
+    expect(
+      _whereCacheGet(whereCache, schema, "X", {
+        where: {
+          field: "whoknows",
+          column: "whatever",
+          value: "snork",
+        },
+      })
+    ).toEqual([undefined]);
+    expect(
+      _whereCacheGet(whereCache, schema, "X", {
+        where: {
+          field: "var",
+          column: "whatever",
+          value: "snork",
+        },
+      })
+    ).toEqual([undefined]);
+    expect(
+      _whereCacheGet(whereCache, schema, "X", {
+        where: {
+          field: "var",
+          column: "foo",
+          value: "snork",
+        },
+      })
+    ).toEqual([undefined]);
+  });
+
+  test("whereCacheCreate, where query", () => {
+    const query = {
+      where: {
+        field: "queryField",
+        column: "queryColumn",
+        value: "queryValue",
+      },
+    };
+    const wc = _whereCacheCreate(
+      "field",
+      {
+        where: {
+          field: "queryField",
+          column: "queryColumn",
+          value: "queryValue",
+        },
+      },
+      [0, 1, 2]
+    );
+    expect(wc).toBeDefined();
+    expect(wc).toEqual(
+      expect.objectContaining({
+        where: {
+          field: {
+            queryField: expect.any(Map),
+          },
+        },
+      })
+    );
+    expect(wc.where.field.queryField.has("queryColumn")).toEqual(true);
+    expect(wc.where.field.queryField.get("queryColumn")).toBeInstanceOf(Map);
+    expect(
+      wc.where.field.queryField.get("queryColumn").has("queryValue")
+    ).toEqual(true);
+    expect(_whereCacheGet(wc, schema, "field", query)).toEqual([0, 1, 2]);
+  });
+
+  test("whereCacheCreate, summarize query", () => {
+    const query = {
+      summarize: {
+        method: "method",
+        field: "queryField",
+        column: "queryColumn",
+        values: ["queryValue"],
+      },
+    };
+    const wc = _whereCacheCreate("field", query, [0, 1, 2]);
+    expect(_whereCacheGet(wc, schema, "field", query)).toEqual([0, 1, 2]);
+  });
+
+  test("whereCacheCreate, unknown query type", () => {
+    expect(_whereCacheCreate("field", { foobar: true }, [1])).toEqual({});
+  });
+
+  test("whereCacheMerge, where queries", () => {
+    let wc;
+
+    // remember, will mutate dst
+    const src = _whereCacheCreate(
+      "field",
+      { where: { field: "queryField", column: "queryColumn", value: "foo" } },
+      ["foo"]
+    );
+
+    const dst1 = _whereCacheCreate(
+      "field",
+      { where: { field: "queryField", column: "queryColumn", value: "bar" } },
+      ["dst1"]
+    );
+    wc = _whereCacheMerge(dst1, src);
+    expect(
+      _whereCacheGet(wc, schema, "field", {
+        where: {
+          field: "queryField",
+          column: "queryColumn",
+          value: "foo",
+        },
+      })
+    ).toEqual(["foo"]);
+    expect(
+      _whereCacheGet(wc, schema, "field", {
+        where: {
+          field: "queryField",
+          column: "queryColumn",
+          value: "bar",
+        },
+      })
+    ).toEqual(["dst1"]);
+
+    const dst2 = _whereCacheCreate(
+      "field",
+      { where: { field: "queryField", column: "queryColumn", value: "bar" } },
+      ["dst2"]
+    );
+    wc = _whereCacheMerge(dst2, dst1, src);
+    expect(
+      _whereCacheGet(wc, schema, "field", {
+        where: {
+          field: "queryField",
+          column: "queryColumn",
+          value: "foo",
+        },
+      })
+    ).toEqual(["foo"]);
+    expect(
+      _whereCacheGet(wc, schema, "field", {
+        where: {
+          field: "queryField",
+          column: "queryColumn",
+          value: "bar",
+        },
+      })
+    ).toEqual(["dst1"]);
+
+    wc = _whereCacheMerge({}, src);
+    expect(wc).toEqual(src);
+
+    wc = _whereCacheMerge({ where: { field: { queryField: new Map() } } }, src);
+    expect(wc).toEqual(src);
+  });
+
+  test("whereCacheMerge, mixed queries", () => {
+    const wc = _whereCacheMerge(
+      _whereCacheCreate(
+        "field",
+        {
+          where: {
+            field: "queryField",
+            column: "queryColumn",
+            value: "foo",
+          },
+        },
+        ["a"]
+      ),
+      _whereCacheCreate(
+        "field",
+        {
+          summarize: {
+            method: "mean",
+            field: "queryField",
+            column: "queryColumn",
+            values: ["foo", "bar", "baz"],
+          },
+        },
+        ["b"]
+      )
+    );
+
+    expect(
+      _whereCacheGet(wc, schema, "field", {
+        where: {
+          field: "queryField",
+          column: "queryColumn",
+          value: "foo",
+        },
+      })
+    ).toEqual(["a"]);
+
+    expect(
+      _whereCacheGet(wc, schema, "field", {
+        summarize: {
+          method: "mean",
+          field: "queryField",
+          column: "queryColumn",
+          values: ["foo", "bar", "baz"],
+        },
+      })
+    ).toEqual(["b"]);
+
+    expect(
+      _whereCacheGet(wc, schema, "field", {
+        where: {
+          field: "queryField",
+          column: "queryColumn",
+          value: "does-not-exist",
+        },
+      })
+    ).toEqual([undefined]);
+
+    expect(
+      _whereCacheGet(wc, schema, "field", {
+        summarize: {
+          method: "no-such-method",
+          field: "queryField",
+          column: "queryColumn",
+          values: ["does-not-exist"],
+        },
+      })
+    ).toEqual([undefined]);
+  });
+});

client/__tests__/util/centroid.test.js

@@ -1,55 +1,35 @@
-import _ from "lodash";
+import cloneDeep from "lodash.clonedeep";
 
 import calcCentroid from "../../src/util/centroid";
-
 import quantile from "../../src/util/quantile";
-import * as Universe from "../../src/util/stateManager/universe";
-import * as World from "../../src/util/stateManager/world";
+import { matrixFBSToDataframe } from "../../src/util/stateManager/matrix";
 import * as REST from "./stateManager/sampleResponses";
-import { ControlsHelpers as CH } from "../../src/util/stateManager";
+import { indexEntireSchema } from "../../src/util/stateManager/schemaHelpers";
+import { normalizeWritableCategoricalSchema } from "../../src/annoMatrix/normalize";
 
 describe("centroid", () => {
-  let world;
-  let categoricalSelection;
+  let schema;
+  let obsAnnotations;
+  let obsLayout;
+
   beforeAll(() => {
-    // Create world + universe
-    let universe = Universe.createUniverseFromResponse(
-      _.cloneDeep(REST.config),
-      _.cloneDeep(REST.schema)
-    );
+    schema = indexEntireSchema(cloneDeep(REST.schema.schema));
+    obsAnnotations = matrixFBSToDataframe(REST.annotationsObs);
+    obsLayout = matrixFBSToDataframe(REST.layoutObs);
 
-    universe = {
-      ...universe,
-      ...Universe.addObsAnnotations(
-        universe,
-        Universe.matrixFBSToDataframe(REST.annotationsObs)
-      ),
-      ...Universe.addVarAnnotations(
-        universe,
-        Universe.matrixFBSToDataframe(REST.annotationsVar)
-      ),
-      ...Universe.addObsLayout(
-        universe,
-        Universe.matrixFBSToDataframe(REST.layoutObs)
-      )
-    };
-    world = World.createWorldFromEntireUniverse(universe);
-
-    // Create categorical selection from world
-    categoricalSelection = CH.createCategoricalSelection(
-      world,
-      CH.selectableCategoryNames(world.schema, CH.maxCategoryItems(REST.config))
+    normalizeWritableCategoricalSchema(
+      schema.annotations.obsByName.field3,
+      obsAnnotations.col("field3")
     );
   });
 
   test("field4 (categorical obsAnnotation)", () => {
     const centroidResult = calcCentroid(
-      world.obsAnnotations,
-      world.obsLayout,
+      schema,
       "field4",
-      ["umap_0", "umap_1"],
-      categoricalSelection,
-      world.schema.annotations.obsByName
+      obsAnnotations,
+      { current: "umap", currentDimNames: ["umap_0", "umap_1"] },
+      obsLayout
     );
 
     // Check to see that a centroid has been calculated for every categorical value
@@ -60,23 +40,22 @@ describe("centroid", () => {
 
     // This expected result assumes that all cells belong in all categorical values inside of sample response
     const expectedResult = [
-      quantile([0.5], world.obsLayout.col("umap_0").asArray())[0],
-      quantile([0.5], world.obsLayout.col("umap_1").asArray())[0]
+      quantile([0.5], obsLayout.col("umap_0").asArray())[0],
+      quantile([0.5], obsLayout.col("umap_1").asArray())[0],
     ];
 
-    centroidResult.forEach(coordinate => {
+    centroidResult.forEach((coordinate) => {
       expect(coordinate).toEqual(expectedResult);
     });
   });
 
-  test("field3 (boolean obsAnnotation", () => {
+  test("field3 (boolean obsAnnotation)", () => {
     const centroidResult = calcCentroid(
-      world.obsAnnotations,
-      world.obsLayout,
+      schema,
       "field3",
-      ["umap_0", "umap_1"],
-      categoricalSelection,
-      world.schema.annotations.obsByName
+      obsAnnotations,
+      { current: "umap", currentDimNames: ["umap_0", "umap_1"] },
+      obsLayout
     );
 
     // Check to see that a centroid has been calculated for every categorical value
@@ -85,11 +64,11 @@ describe("centroid", () => {
 
     // This expected result assumes that all cells belong in all categorical values inside of sample response
     const expectedResult = [
-      quantile([0.5], world.obsLayout.col("umap_0").asArray())[0],
-      quantile([0.5], world.obsLayout.col("umap_1").asArray())[0]
+      quantile([0.5], obsLayout.col("umap_0").asArray())[0],
+      quantile([0.5], obsLayout.col("umap_1").asArray())[0],
     ];
 
-    centroidResult.forEach(coordinate => {
+    centroidResult.forEach((coordinate) => {
       expect(coordinate).toEqual(expectedResult);
     });
   });

client/__tests__/util/dataframe/histogram.test.js

@@ -14,7 +14,7 @@ describe("Dataframe column histogram", () => {
       new Map([
         ["n1", new Map([["c1", 1]])],
         ["n2", new Map([["c2", 1]])],
-        ["n3", new Map([["c3", 1]])]
+        ["n3", new Map([["c3", 1]])],
       ])
     );
     // memoized?
@@ -31,7 +31,11 @@ describe("Dataframe column histogram", () => {
 
     const h1 = df.col("value").histogram(3, [0, 2], df.col("name"));
     expect(h1).toMatchObject(
-      new Map([["n1", [1, 0, 0]], ["n2", [0, 1, 0]], ["n3", [0, 0, 1]]])
+      new Map([
+        ["n1", [1, 0, 0]],
+        ["n2", [0, 1, 0]],
+        ["n3", [0, 0, 1]],
+      ])
     );
     // memoized?
     expect(df.col("value").histogram(3, [0, 2], df.col("name"))).toMatchObject(
@@ -48,7 +52,13 @@ describe("Dataframe column histogram", () => {
     );
 
     const h1 = df.col("cat").histogram();
-    expect(h1).toMatchObject(new Map([["c1", 1], ["c2", 1], ["c3", 1]]));
+    expect(h1).toMatchObject(
+      new Map([
+        ["c1", 1],
+        ["c2", 1],
+        ["c3", 1],
+      ])
+    );
     // memoized?
     expect(df.col("value").histogram(3, [0, 2])).toMatchObject(h1);
   });
@@ -78,16 +88,7 @@ describe("Dataframe column histogram", () => {
     expect(df.col(1).histogram(5, [0, 100])).toEqual([5, 1, 0, 0, 2]);
     expect(df.col(0).histogram(2, [0, 10])).toEqual([2, 2]);
     expect(df.col(0).histogram(10, [0, 100])).toEqual([
-      3,
-      2,
-      1,
-      0,
-      0,
-      0,
-      0,
-      0,
-      0,
-      2
+      3, 2, 1, 0, 0, 0, 0, 0, 0, 2,
     ]);
   });
 });

client/__tests__/util/dataframe/summarize.test.js

@@ -13,7 +13,7 @@ describe("Dataframe column summary", () => {
         categorical: true,
         categories: [],
         categoryCounts: new Map(),
-        numCategories: 0
+        numCategories: 0,
       })
     );
   });
@@ -27,7 +27,7 @@ describe("Dataframe column summary", () => {
         [true],
         new Float32Array([39.3]),
         new Int32Array([99]),
-        [1]
+        [1],
       ],
       null,
       new Dataframe.KeyIndex([
@@ -36,7 +36,7 @@ describe("Dataframe column summary", () => {
         "nameBoolean",
         "nameFloat32",
         "nameInt32",
-        "nameCategorical"
+        "nameCategorical",
       ])
     );
 
@@ -45,7 +45,7 @@ describe("Dataframe column summary", () => {
         categorical: true,
         categories: ["n1"],
         categoryCounts: new Map([["n1", 1]]),
-        numCategories: 1
+        numCategories: 1,
       })
     );
     expect(df.icol(1).summarize()).toEqual(
@@ -53,7 +53,7 @@ describe("Dataframe column summary", () => {
         categorical: true,
         categories: ["hi"],
         categoryCounts: new Map([["hi", 1]]),
-        numCategories: 1
+        numCategories: 1,
       })
     );
     expect(df.icol(2).summarize()).toEqual(
@@ -61,7 +61,7 @@ describe("Dataframe column summary", () => {
         categorical: true,
         categories: [true],
         categoryCounts: new Map([[true, 1]]),
-        numCategories: 1
+        numCategories: 1,
       })
     );
     expect(df.icol(3).summarize()).toEqual(
@@ -71,7 +71,7 @@ describe("Dataframe column summary", () => {
         max: float32Conversion(39.3),
         nan: 0,
         ninf: 0,
-        pinf: 0
+        pinf: 0,
       })
     );
     expect(df.icol(4).summarize()).toEqual(
@@ -81,7 +81,7 @@ describe("Dataframe column summary", () => {
         max: 99,
         nan: 0,
         ninf: 0,
-        pinf: 0
+        pinf: 0,
       })
     );
     expect(df.icol(5).summarize()).toEqual(
@@ -89,7 +89,7 @@ describe("Dataframe column summary", () => {
         categorical: true,
         categories: [1],
         categoryCounts: new Map([[1, 1]]),
-        numCategories: 1
+        numCategories: 1,
       })
     );
   });
@@ -103,7 +103,7 @@ describe("Dataframe column summary", () => {
         [false, true, true],
         new Float32Array([39.3, 39.3, 0]),
         new Int32Array([99, 99, 99]),
-        [1, false, "0"]
+        [1, false, "0"],
       ],
       null,
       new Dataframe.KeyIndex([
@@ -112,7 +112,7 @@ describe("Dataframe column summary", () => {
         "nameBoolean",
         "nameFloat32",
         "nameInt32",
-        "nameCategorical"
+        "nameCategorical",
       ])
     );
 
@@ -120,24 +120,34 @@ describe("Dataframe column summary", () => {
       expect.objectContaining({
         categorical: true,
         categories: expect.arrayContaining(["n0", "n1", "n2"]),
-        categoryCounts: new Map([["n0", 1], ["n1", 1], ["n2", 1]]),
-        numCategories: 3
+        categoryCounts: new Map([
+          ["n0", 1],
+          ["n1", 1],
+          ["n2", 1],
+        ]),
+        numCategories: 3,
       })
     );
     expect(df.icol(1).summarize()).toEqual(
       expect.objectContaining({
         categorical: true,
         categories: expect.arrayContaining(["hi", "bye"]),
-        categoryCounts: new Map([["hi", 2], ["bye", 1]]),
-        numCategories: 2
+        categoryCounts: new Map([
+          ["hi", 2],
+          ["bye", 1],
+        ]),
+        numCategories: 2,
       })
     );
     expect(df.icol(2).summarize()).toEqual(
       expect.objectContaining({
         categorical: true,
         categories: expect.arrayContaining([true, false]),
-        categoryCounts: new Map([[true, 2], [false, 1]]),
-        numCategories: 2
+        categoryCounts: new Map([
+          [true, 2],
+          [false, 1],
+        ]),
+        numCategories: 2,
       })
     );
     expect(df.icol(3).summarize()).toEqual(
@@ -147,7 +157,7 @@ describe("Dataframe column summary", () => {
         max: float32Conversion(39.3),
         nan: 0,
         ninf: 0,
-        pinf: 0
+        pinf: 0,
       })
     );
     expect(df.icol(4).summarize()).toEqual(
@@ -157,15 +167,19 @@ describe("Dataframe column summary", () => {
         max: 99,
         nan: 0,
         ninf: 0,
-        pinf: 0
+        pinf: 0,
       })
     );
     expect(df.icol(5).summarize()).toEqual(
       expect.objectContaining({
         categorical: true,
         categories: expect.arrayContaining([1, false, "0"]),
-        categoryCounts: new Map([[1, 1], [false, 1], ["0", 1]]),
-        numCategories: 3
+        categoryCounts: new Map([
+          [1, 1],
+          [false, 1],
+          ["0", 1],
+        ]),
+        numCategories: 3,
       })
     );
   });
@@ -181,10 +195,10 @@ describe("Dataframe column summary", () => {
           39.3,
           Number.NEGATIVE_INFINITY,
           Number.NaN,
-          Number.POSITIVE_INFINITY
+          Number.POSITIVE_INFINITY,
         ]),
         new Int32Array([99, 99, 99, 99]),
-        [1, false, "0", "0"]
+        [1, false, "0", "0"],
       ],
       null,
       new Dataframe.KeyIndex([
@@ -193,7 +207,7 @@ describe("Dataframe column summary", () => {
         "nameBoolean",
         "nameFloat32",
         "nameInt32",
-        "nameCategorical"
+        "nameCategorical",
       ])
     );
 
@@ -201,24 +215,34 @@ describe("Dataframe column summary", () => {
       expect.objectContaining({
         categorical: true,
         categories: expect.arrayContaining(["n0", "n1", "n2"]),
-        categoryCounts: new Map([["n0", 1], ["n1", 1], ["n2", 2]]),
-        numCategories: 3
+        categoryCounts: new Map([
+          ["n0", 1],
+          ["n1", 1],
+          ["n2", 2],
+        ]),
+        numCategories: 3,
       })
     );
     expect(df.icol(1).summarize()).toEqual(
       expect.objectContaining({
         categorical: true,
         categories: expect.arrayContaining(["hi", "bye"]),
-        categoryCounts: new Map([["hi", 2], ["bye", 1]]),
-        numCategories: 2
+        categoryCounts: new Map([
+          ["hi", 2],
+          ["bye", 1],
+        ]),
+        numCategories: 2,
       })
     );
     expect(df.icol(2).summarize()).toEqual(
       expect.objectContaining({
         categorical: true,
         categories: expect.arrayContaining([true, false]),
-        categoryCounts: new Map([[true, 2], [false, 1]]),
-        numCategories: 2
+        categoryCounts: new Map([
+          [true, 2],
+          [false, 1],
+        ]),
+        numCategories: 2,
       })
     );
     expect(df.icol(3).summarize()).toEqual(
@@ -228,7 +252,7 @@ describe("Dataframe column summary", () => {
         max: float32Conversion(39.3),
         nan: 1,
         ninf: 1,
-        pinf: 1
+        pinf: 1,
       })
     );
     expect(df.icol(4).summarize()).toEqual(
@@ -238,15 +262,19 @@ describe("Dataframe column summary", () => {
         max: 99,
         nan: 0,
         ninf: 0,
-        pinf: 0
+        pinf: 0,
       })
     );
     expect(df.icol(5).summarize()).toEqual(
       expect.objectContaining({
         categorical: true,
         categories: expect.arrayContaining([1, false, "0"]),
-        categoryCounts: new Map([[1, 1], [false, 1], ["0", 1]]),
-        numCategories: 3
+        categoryCounts: new Map([
+          [1, 1],
+          [false, 1],
+          ["0", 1],
+        ]),
+        numCategories: 3,
       })
     );
   });

client/__tests__/util/nameCreators.test.js

@@ -6,7 +6,7 @@ import {
   obsAnnoDimensionName,
   diffexpDimensionName,
   userDefinedDimensionName,
-  makeContinuousDimensionName
+  makeContinuousDimensionName,
 } from "../../src/util/nameCreators";
 
 describe("nameCreators", () => {
@@ -14,25 +14,25 @@ describe("nameCreators", () => {
     layoutDimensionName,
     obsAnnoDimensionName,
     diffexpDimensionName,
-    userDefinedDimensionName
+    userDefinedDimensionName,
   ];
 
   test("check for namespace isolation", () => {
     const foo = "foo";
-    nameCreators.forEach(fn => expect(fn(foo)).not.toBe(foo));
+    nameCreators.forEach((fn) => expect(fn(foo)).not.toBe(foo));
 
     const bar = "bar";
-    nameCreators.forEach(fn => expect(fn(foo)).not.toBe(fn(bar)));
+    nameCreators.forEach((fn) => expect(fn(foo)).not.toBe(fn(bar)));
 
-    nameCreators.forEach(fn => {
-      const allOtherFn = nameCreators.filter(elmnt => elmnt !== fn);
-      allOtherFn.forEach(otherFn => expect(fn(foo)).not.toBe(otherFn(foo)));
+    nameCreators.forEach((fn) => {
+      const allOtherFn = nameCreators.filter((elmnt) => elmnt !== fn);
+      allOtherFn.forEach((otherFn) => expect(fn(foo)).not.toBe(otherFn(foo)));
     });
   });
 
   test("check for legal keys", () => {
     /* need namespace creators to return strings only */
-    nameCreators.forEach(fn => expect(fn("X")).toMatch(/X/));
+    nameCreators.forEach((fn) => expect(fn("X")).toMatch(/X/));
   });
 });
 

client/__tests__/util/promiseLimit.test.js

@@ -0,0 +1,96 @@
+import PromiseLimit from "../../src/util/promiseLimit";
+import { range } from "../../src/util/range";
+
+const delay = (t) => new Promise((resolve) => setTimeout(resolve, t));
+
+describe("PromiseLimit", () => {
+  test("simple evaluation, concurrency 1", async () => {
+    const plimit = new PromiseLimit(1);
+    const result = await Promise.all([
+      plimit.add(() => Promise.resolve(1)),
+      plimit.add(() => Promise.resolve(2)),
+      plimit.add(() => Promise.resolve(3)),
+      plimit.add(() => Promise.resolve(4)),
+    ]);
+    expect(result).toEqual([1, 2, 3, 4]);
+  });
+
+  test("simple evaluation, concurrency > 1", async () => {
+    const plimit = new PromiseLimit(100);
+    const result = await Promise.all([
+      plimit.add(() => Promise.resolve(1)),
+      plimit.add(() => Promise.resolve(2)),
+      plimit.add(() => Promise.resolve(3)),
+      plimit.add(() => Promise.resolve(4)),
+    ]);
+    expect(result).toEqual([1, 2, 3, 4]);
+  });
+
+  test("eval in order of insertion", async () => {
+    const plimit = new PromiseLimit(100);
+
+    const result = await Promise.all([
+      plimit.add(() => Promise.resolve(1)),
+      plimit.add(() => Promise.resolve(2)),
+      plimit.add(() => Promise.resolve(3)),
+      plimit.add(() => Promise.resolve(4)),
+    ]);
+
+    expect(result).toEqual([1, 2, 3, 4]);
+  });
+
+  test("obeys concurrency limit", async () => {
+    const plimit = new PromiseLimit(2);
+    let running = 0;
+    let maxRunning = 0;
+
+    const callback = async () => {
+      running += 1;
+      maxRunning = running > maxRunning ? running : maxRunning;
+      await delay(100);
+      running -= 1;
+    };
+
+    await Promise.all(range(10).map((i) => plimit.add(() => callback(i))));
+
+    expect(maxRunning).toEqual(2);
+  });
+
+  test("rejection", async () => {
+    const plimit = new PromiseLimit(2);
+    const result = await Promise.all([
+      plimit.add(() => Promise.resolve("OK")),
+      // eslint-disable-next-line prefer-promise-reject-errors -- unit test
+      plimit.add(() => Promise.reject("not OK")).catch((e) => e),
+      plimit.add(() => Promise.resolve("OK")),
+      plimit
+        .add(() => {
+          throw new Error("not OK");
+        })
+        .catch((e) => e.message),
+    ]);
+    expect(result).toEqual(["OK", "not OK", "OK", "not OK"]);
+  });
+
+  test("priority queue", async () => {
+    const plimit = new PromiseLimit(1);
+
+    let finishOrder = 0;
+    const callback = () => async () => {
+      await delay(100);
+      const result = finishOrder;
+      finishOrder += 1;
+      return result;
+    };
+
+    const result = await Promise.all([
+      plimit.add(callback()),
+      plimit.priorityAdd(4, callback()),
+      plimit.priorityAdd(0, callback()),
+      plimit.priorityAdd(1, callback()),
+      plimit.priorityAdd(-1, callback()),
+    ]);
+
+    expect(result).toEqual([0, 4, 2, 3, 1]);
+  });
+});

client/__tests__/util/quantile.test.js

@@ -1,29 +1,25 @@
 import quantile from "../../src/util/quantile";
 
 describe("quantile", () => {
-	test("single q", () => {
-		const arr = new Float32Array([9, 3, 5, 6, 0]);
-		expect(quantile([1.0], arr)).toMatchObject([9]);
-		expect(quantile([0.9], arr)).toMatchObject([9]);
-		expect(quantile([0.8], arr)).toMatchObject([9]);
-		expect(quantile([0.7], arr)).toMatchObject([6]);
-		expect(quantile([0.6], arr)).toMatchObject([6]);
-		expect(quantile([0.5], arr)).toMatchObject([5]);
-		expect(quantile([0.4], arr)).toMatchObject([5]);
-		expect(quantile([0.3], arr)).toMatchObject([3]);
-		expect(quantile([0.2], arr)).toMatchObject([3]);
-		expect(quantile([0.1], arr)).toMatchObject([0]);
-		expect(quantile([0], arr)).toMatchObject([0]);
-	});
+  test("single q", () => {
+    const arr = new Float32Array([9, 3, 5, 6, 0]);
+    expect(quantile([1.0], arr)).toMatchObject([9]);
+    expect(quantile([0.9], arr)).toMatchObject([9]);
+    expect(quantile([0.8], arr)).toMatchObject([9]);
+    expect(quantile([0.7], arr)).toMatchObject([6]);
+    expect(quantile([0.6], arr)).toMatchObject([6]);
+    expect(quantile([0.5], arr)).toMatchObject([5]);
+    expect(quantile([0.4], arr)).toMatchObject([5]);
+    expect(quantile([0.3], arr)).toMatchObject([3]);
+    expect(quantile([0.2], arr)).toMatchObject([3]);
+    expect(quantile([0.1], arr)).toMatchObject([0]);
+    expect(quantile([0], arr)).toMatchObject([0]);
+  });
 
-	test("multi q", () => {
-		const arr = new Float32Array([9, 3, 5, 6, 0]);
-		expect(quantile([0, 0.25, 0.5, 0.75, 1.0], arr)).toMatchObject([
-			0,
-			3,
-			5,
-			6,
-			9
-		]);
-	});
+  test("multi q", () => {
+    const arr = new Float32Array([9, 3, 5, 6, 0]);
+    expect(quantile([0, 0.25, 0.5, 0.75, 1.0], arr)).toMatchObject([
+      0, 3, 5, 6, 9,
+    ]);
+  });
 });

client/__tests__/util/range.test.js

@@ -1,50 +1,48 @@
 import { range, rangeFill, linspace } from "../../src/util/range";
 
 describe("range", () => {
-	test("no defaults", () => {
-		expect(range(0, 3, 1)).toMatchObject([0, 1, 2]);
-	});
+  test("no defaults", () => {
+    expect(range(0, 3, 1)).toMatchObject([0, 1, 2]);
+  });
 
-	test("range(stop)", () => {
-		expect(range(3)).toMatchObject([0, 1, 2]);
-		expect(range(0)).toMatchObject([]);
-		expect(range(1)).toMatchObject([0]);
-	});
+  test("range(stop)", () => {
+    expect(range(3)).toMatchObject([0, 1, 2]);
+    expect(range(0)).toMatchObject([]);
+    expect(range(1)).toMatchObject([0]);
+  });
 
-	test("range(start,stop)", () => {
-		expect(range(0, 0)).toMatchObject([]);
-		expect(range(0, 2)).toMatchObject([0, 1]);
-		expect(range(4, 8)).toMatchObject([4, 5, 6, 7]);
-	});
+  test("range(start,stop)", () => {
+    expect(range(0, 0)).toMatchObject([]);
+    expect(range(0, 2)).toMatchObject([0, 1]);
+    expect(range(4, 8)).toMatchObject([4, 5, 6, 7]);
+  });
 
-	test("range(start, stop, step", () => {
-		expect(range(4, 0, -1)).toMatchObject([4, 3, 2, 1]);
-		expect(range(0, 4, 2)).toMatchObject([0, 2]);
-	});
+  test("range(start, stop, step", () => {
+    expect(range(4, 0, -1)).toMatchObject([4, 3, 2, 1]);
+    expect(range(0, 4, 2)).toMatchObject([0, 2]);
+  });
 });
 
 describe("rangefill", () => {
-	test("rangeFill(arr)", () => {
-		expect(rangeFill(new Int32Array(3))).toMatchObject(
-			new Int32Array([0, 1, 2])
-		);
-	});
-	test("rangeFill(arr, start)", () => {
-		expect(rangeFill(new Int32Array(2), 1)).toMatchObject(
-			new Int32Array([1, 2])
-		);
-	});
-	test("rangeFill(arr, start, step)", () => {
-		expect(rangeFill(new Int32Array(3), 2, -1)).toMatchObject(
-			new Int32Array([2, 1, 0])
-		);
-	});
+  test("rangeFill(arr)", () => {
+    expect(rangeFill(new Int32Array(3))).toMatchObject(
+      new Int32Array([0, 1, 2])
+    );
+  });
+  test("rangeFill(arr, start)", () => {
+    expect(rangeFill(new Int32Array(2), 1)).toMatchObject(
+      new Int32Array([1, 2])
+    );
+  });
+  test("rangeFill(arr, start, step)", () => {
+    expect(rangeFill(new Int32Array(3), 2, -1)).toMatchObject(
+      new Int32Array([2, 1, 0])
+    );
+  });
 });
 
 describe("linspace", () => {
-	test("linspace(arr, start, step)", () => {
-		expect(linspace(0.0, 2.0, 5)).toMatchObject(
-		    [0.0, 0.5, 1.0, 1.5, 2.0]
-		);
-	});
+  test("linspace(arr, start, step)", () => {
+    expect(linspace(0.0, 2.0, 5)).toMatchObject([0.0, 0.5, 1.0, 1.5, 2.0]);
+  });
 });

client/__tests__/util/stateManager/colorHelpers.test.js

@@ -0,0 +1,198 @@
+/* eslint-disable no-bitwise -- unsigned right shift better than Math.round */
+
+/*
+test color helpers
+*/
+import {
+  createColorTable,
+  loadUserColorConfig,
+} from "../../../src/util/stateManager/colorHelpers";
+import * as Dataframe from "../../../src/util/dataframe";
+
+describe("categorical color helpers", () => {
+  /*
+  Primary test constraint for categorical colors is that they are ordered/identified
+  by schema order, NOT by value.  Ie,
+
+    scale(schemaIndex) should match rgb[obsOffset]
+  */
+
+  const schema = indexSchema({
+    annotations: {
+      obs: {
+        columns: [
+          {
+            name: "name_0",
+            type: "string",
+            writable: false,
+          },
+          {
+            name: "continuousColumn",
+            type: "float32",
+            writable: false,
+          },
+          {
+            categories: [
+              "CD4 T cells",
+              "CD14+ Monocytes",
+              "B cells",
+              "CD8 T cells",
+              "NK cells",
+              "FCGR3A+ Monocytes",
+              "Dendritic cells",
+              "Megakaryocytes",
+            ],
+            name: "categoricalColumn",
+            type: "categorical",
+            writable: false,
+          },
+        ],
+        index: "name_0",
+      },
+      var: {
+        columns: [
+          {
+            name: "name_0",
+            type: "string",
+            writable: false,
+          },
+        ],
+        index: "name_0",
+      },
+    },
+    dataframe: {
+      nObs: 2638,
+      nVar: 1838,
+      type: "float32",
+    },
+    layout: {},
+  });
+
+  const catColCategories = schema.annotations.obs.columns[2].categories;
+  const obsDataframe = new Dataframe.Dataframe(
+    [schema.dataframe.nObs, 2],
+    [
+      new Float32Array(schema.dataframe.nObs).map(() => Math.random()),
+      new Array(schema.dataframe.nObs)
+        .fill("")
+        .map(
+          () =>
+            catColCategories[(Math.random() * catColCategories.length) >>> 0]
+        ),
+    ],
+    null,
+    new Dataframe.KeyIndex(["continuousColumn", "categoricalColumn"])
+  );
+
+  test("default category order", () => {
+    const ct = createColorTable(
+      "color by categorical metadata",
+      "categoricalColumn",
+      obsDataframe,
+      schema
+    );
+    expect(ct).toBeDefined();
+    const data = obsDataframe.col("categoricalColumn").asArray();
+    const cats = schema.annotations.obsByName.categoricalColumn.categories;
+    for (let i = 0; i < schema.dataframe.nObs; i += 1) {
+      expect(makeScale(ct.rgb[i])).toEqual(ct.scale(cats.indexOf(data[i])));
+    }
+  });
+
+  test("shuffle category order", () => {
+    const schemaClone = indexSchema(JSON.parse(JSON.stringify(schema)));
+    shuffle(schemaClone.annotations.obsByName.categoricalColumn.categories);
+    const ct = createColorTable(
+      "color by categorical metadata",
+      "categoricalColumn",
+      obsDataframe,
+      schemaClone
+    );
+    expect(ct).toBeDefined();
+    const data = obsDataframe.col("categoricalColumn").asArray();
+    const cats = schemaClone.annotations.obsByName.categoricalColumn.categories;
+    for (let i = 0; i < schemaClone.dataframe.nObs; i += 1) {
+      expect(makeScale(ct.rgb[i])).toEqual(ct.scale(cats.indexOf(data[i])));
+    }
+  });
+
+  test("user defined color order", () => {
+    const cats = schema.annotations.obsByName.categoricalColumn.categories;
+    const shuffleCats = shuffle(
+      Array.from(schema.annotations.obsByName.categoricalColumn.categories)
+    );
+    const userDefinedColorTable = {
+      categoricalColumn: shuffleCats.reduce((acc, label) => {
+        acc[label] = randRGBColor();
+        return acc;
+      }, {}),
+    };
+
+    const userColors = loadUserColorConfig(userDefinedColorTable);
+    expect(userColors).toBeDefined();
+
+    const ct = createColorTable(
+      "color by categorical metadata",
+      "categoricalColumn",
+      obsDataframe,
+      schema,
+      userColors
+    );
+    expect(ct).toBeDefined();
+    const data = obsDataframe.col("categoricalColumn").asArray();
+    for (let i = 0; i < schema.dataframe.nObs; i += 1) {
+      expect(makeScale(ct.rgb[i])).toEqual(
+        ct.scale(cats.indexOf(data[i])).toString()
+      );
+    }
+  });
+});
+
+/*
+TODO:
+1. mix up category order in schema to make sure it works with varied order
+2. user defined colors
+*/
+
+function indexSchema(schema) {
+  schema.annotations.obsByName = Object.fromEntries(
+    schema.annotations?.obs?.columns?.map((v) => [v.name, v]) ?? []
+  );
+  schema.annotations.varByName = Object.fromEntries(
+    schema.annotations?.var?.columns?.map((v) => [v.name, v]) ?? []
+  );
+  schema.layout.obsByName = Object.fromEntries(
+    schema.layout?.obs?.map((v) => [v.name, v]) ?? []
+  );
+  schema.layout.varByName = Object.fromEntries(
+    schema.layout?.var?.map((v) => [v.name, v]) ?? []
+  );
+
+  return schema;
+}
+
+function makeScale(rgb) {
+  // make a scale string from a rgb float triple
+  return `rgb(${(rgb[0] * 255) >>> 0}, ${(rgb[1] * 255) >>> 0}, ${
+    (rgb[2] * 256) >>> 0
+  })`;
+}
+
+function shuffle(array) {
+  for (let i = array.length - 1; i > 0; i -= 1) {
+    const j = (Math.random() * (i + 1)) >>> 0;
+    [array[i], array[j]] = [array[j], array[i]];
+  }
+  return array;
+}
+
+function randHexColor() {
+  const hex = ((Math.random() * 255) >>> 0).toString(16);
+  return `0${hex}`.slice(-2);
+}
+
+function randRGBColor() {
+  return `#${randHexColor()}${randHexColor()}${randHexColor()}`;
+}
+
+/* eslint-enable no-bitwise -- unsigned right shift better than Math.round */

client/__tests__/util/stateManager/controlsHelpers.test.js

@@ -1,24 +1,9 @@
 /*
 test controls helpers
 */
-import { subsetAndResetGeneLists } from "../../../src/util/stateManager/controlsHelpers";
-import * as globals from "../../../src/globals";
+// TODO #2227 test: improve test coverage on control helper functions
+// (`isSelectableCategoryName()`, `selectableCategoryNames()`, `createCategorySummaryFromDfCol()`, `createCategoricalSelection()`, )
 
 describe("controls helpers", () => {
-
-  test("subsetAndResetGeneLists", () => {
-    const geneList = [...Array(150).keys()].map(() =>
-      Math.random().toString(36).substring(2, 6) // random string of 4 characters
-    );
-    const state = {
-      userDefinedGenes: geneList.slice(0, 20),
-      diffexpGenes: geneList.slice(20),
-    };
-    const [newUserDefinedGenes, newDiffExpGenes] = subsetAndResetGeneLists(state);
-    expect(globals.maxUserDefinedGenes).toBeLessThan(globals.maxGenes);
-    expect(geneList.length).toBeGreaterThan(globals.maxGenes);
-    expect(newUserDefinedGenes).toHaveLength(globals.maxGenes);
-    expect(newUserDefinedGenes).toStrictEqual(geneList.slice(0, globals.maxGenes));
-    expect(newDiffExpGenes).toStrictEqual([]);
-  });
+  test("placeholder", () => {});
 });

client/__tests__/util/stateManager/fbs.test.js

@@ -3,32 +3,32 @@ test FBS encode/decode API
 */
 import { Dataframe, KeyIndex } from "../../../src/util/dataframe";
 import {
-	decodeMatrixFBS,
-	encodeMatrixFBS
+  decodeMatrixFBS,
+  encodeMatrixFBS,
 } from "../../../src/util/stateManager/matrix";
 
 describe("encode/decode", () => {
-	test("round trip", () => {
-		const columns = [
-			["red", "green", "blue"],
-			new Int32Array(3).fill(0),
-			new Uint32Array(3).fill(1),
-			new Float32Array(3).fill(2)
-		];
+  test("round trip", () => {
+    const columns = [
+      ["red", "green", "blue"],
+      new Int32Array(3).fill(0),
+      new Uint32Array(3).fill(1),
+      new Float32Array(3).fill(2),
+    ];
 
-		const dfNoColIdx = new Dataframe([3, 4], columns);
-		const dfA = decodeMatrixFBS(encodeMatrixFBS(dfNoColIdx));
-		expect([dfA.nRows, dfA.nCols]).toEqual(dfNoColIdx.dims);
-		expect(dfA.colIdx).toBeNull();
-		expect(dfA.rowIdx).toBeNull();
-		expect(dfA.columns).toEqual(columns);
+    const dfNoColIdx = new Dataframe([3, 4], columns);
+    const dfA = decodeMatrixFBS(encodeMatrixFBS(dfNoColIdx));
+    expect([dfA.nRows, dfA.nCols]).toEqual(dfNoColIdx.dims);
+    expect(dfA.colIdx).toBeNull();
+    expect(dfA.rowIdx).toBeNull();
+    expect(dfA.columns).toEqual(columns);
 
-		const colIndex = new KeyIndex(["a", "b", "c", "d"]);
-		const dfWithColIdx = new Dataframe([3, 4], columns, null, colIndex);
-		const dfB = decodeMatrixFBS(encodeMatrixFBS(dfWithColIdx));
-		expect([dfB.nRows, dfB.nCols]).toEqual(dfWithColIdx.dims);
-		expect(dfB.colIdx).toEqual(colIndex.keys());
-		expect(dfB.rowIdx).toBeNull();
-		expect(dfB.columns).toEqual(columns);
-	});
+    const colIndex = new KeyIndex(["a", "b", "c", "d"]);
+    const dfWithColIdx = new Dataframe([3, 4], columns, null, colIndex);
+    const dfB = decodeMatrixFBS(encodeMatrixFBS(dfWithColIdx));
+    expect([dfB.nRows, dfB.nCols]).toEqual(dfWithColIdx.dims);
+    expect(dfB.colIdx).toEqual(colIndex.labels());
+    expect(dfB.rowIdx).toBeNull();
+    expect(dfB.columns).toEqual(columns);
+  });
 });

client/__tests__/util/stateManager/sampleResponses.js

@@ -1,4 +1,7 @@
-/* eslint no-bitwise: "off" */
+import every from "lodash.every";
+import map from "lodash.map";
+import isNumber from "lodash.isnumber";
+import zip from "lodash.zip";
 import _ from "lodash";
 import { flatbuffers } from "flatbuffers";
 import { NetEncoding } from "../../../src/util/stateManager/matrix_generated";
@@ -17,13 +20,13 @@ const aConfigResponse = {
       { method: "POST", path: "/cluster/", available: false },
       { method: "POST", path: "/layout/", available: false },
       { method: "POST", path: "/diffexp/", available: false },
-      { method: "POST", path: "/saveLocal/", available: false }
+      { method: "POST", path: "/saveLocal/", available: false },
     ],
     displayNames: {
       engine: "the little engine that could",
-      dataset: "all your zeros are mine"
-    }
-  }
+      dataset: "all your zeros are mine",
+    },
+  },
 };
 
 const aSchemaResponse = {
@@ -31,7 +34,7 @@ const aSchemaResponse = {
     dataframe: {
       nObs,
       nVar,
-      type: "float32"
+      type: "float32",
     },
     annotations: {
       obs: {
@@ -44,9 +47,9 @@ const aSchemaResponse = {
           {
             name: "field4",
             type: "categorical",
-            categories: field4Categories
-          }
-        ]
+            categories: field4Categories,
+          },
+        ],
       },
       var: {
         index: "name",
@@ -58,46 +61,48 @@ const aSchemaResponse = {
           {
             name: "fieldD",
             type: "categorical",
-            categories: fieldDCategories
-          }
-        ]
-      }
+            categories: fieldDCategories,
+          },
+        ],
+      },
     },
     layout: {
       obs: [{ name: "umap", type: "float32", dims: ["umap_0", "umap_1"] }],
-      var: []
-    }
-  }
+      var: [],
+    },
+  },
 };
 
 const anAnnotationsObsJSONResponse = {
   names: ["name", "field1", "field2", "field3", "field4"],
   data: _()
     .range(nObs)
-    .map(idx => [
+    .map((idx) => [
       idx,
       `obs${idx}`,
       2 * idx,
       idx + 0.0133,
+      // eslint-disable-next-line no-bitwise -- idx & 1 to check for odd numbers
       !!(idx & 1),
-      field4Categories[idx % field4Categories.length]
+      field4Categories[idx % field4Categories.length],
     ])
-    .value()
+    .value(),
 };
 
 const anAnnotationsVarJSONResponse = {
   names: ["fieldA", "fieldB", "fieldC", "fieldD", "name"],
   data: _()
     .range(nVar)
-    .map(idx => [
+    .map((idx) => [
       idx,
       10 * idx,
       idx + 2.90143,
+      // eslint-disable-next-line no-bitwise -- idx & 1 to check for odd numbers
       !!(idx & 1),
       fieldDCategories[idx % fieldDCategories.length],
-      `var${idx}`
+      `var${idx}`,
     ])
-    .value()
+    .value(),
 };
 
 function encodeTypedArray(builder, uType, uData) {
@@ -112,7 +117,7 @@ function encodeTypedArray(builder, uType, uData) {
 function encodeMatrix(columns, colIndex = undefined) {
   /*
   IMPORTANT: this is not a general purpose encoder.  in particular,
-  it doesn't correctly handle all column index types, nor does it 
+  it doesn't correctly handle all column index types, nor does it
   handle all column typedarray types.
 
   encodeMatrixFBS in matrix.py is more general.  This is used only
@@ -120,10 +125,10 @@ function encodeMatrix(columns, colIndex = undefined) {
   */
   const utf8Encoder = new TextEncoder("utf-8");
   const builder = new flatbuffers.Builder(1024);
-  const cols = _.map(columns, carr => {
+  const cols = map(columns, (carr) => {
     let uType;
     let tarr;
-    if (_.every(carr, _.isNumber)) {
+    if (every(carr, isNumber)) {
       uType = NetEncoding.TypedArray.Float32Array;
       tarr = encodeTypedArray(builder, uType, new Float32Array(carr));
     } else {
@@ -166,19 +171,19 @@ function encodeMatrix(columns, colIndex = undefined) {
 }
 
 const anAnnotationsObsFBSResponse = (() => {
-  const columns = _.zip(...anAnnotationsObsJSONResponse.data).slice(1);
+  const columns = zip(...anAnnotationsObsJSONResponse.data).slice(1);
   return encodeMatrix(columns, anAnnotationsObsJSONResponse.names);
 })();
 
 const anAnnotationsVarFBSResponse = (() => {
-  const columns = _.zip(...anAnnotationsVarJSONResponse.data).slice(1);
+  const columns = zip(...anAnnotationsVarJSONResponse.data).slice(1);
   return encodeMatrix(columns, anAnnotationsVarJSONResponse.names);
 })();
 
 const aLayoutFBSResponse = (() => {
   const coords = [
     new Float32Array(nObs).fill(Math.random()),
-    new Float32Array(nObs).fill(Math.random())
+    new Float32Array(nObs).fill(Math.random()),
   ];
   return encodeMatrix(coords, ["umap_0", "umap_1"]);
 })();
@@ -187,8 +192,8 @@ const aDataObsResponse = {
   var: [2, 4, 29],
   obs: _()
     .range(nObs)
-    .map(idx => [idx, Math.random(), Math.random(), Math.random()])
-    .value()
+    .map((idx) => [idx, Math.random(), Math.random(), Math.random()])
+    .value(),
 };
 
 export {
@@ -197,5 +202,5 @@ export {
   anAnnotationsVarFBSResponse as annotationsVar,
   anAnnotationsObsFBSResponse as annotationsObs,
   aSchemaResponse as schema,
-  aConfigResponse as config
+  aConfigResponse as config,
 };

client/__tests__/util/stateManager/universe.test.js

@@ -1,92 +0,0 @@
-import * as Universe from "../../../src/util/stateManager/universe";
-import * as Dataframe from "../../../src/util/dataframe";
-import * as REST from "./sampleResponses";
-
-describe("createUniverseFromResponse", () => {
-  /*
-  test createUniverseFromResponse - this function converts
-  a set of REST 0.2 responses into a "new" Universe.
-
-  createUniverseFromResponse(
-    configResponse,
-    schemaResponse,
-    annotationsObsResponse,
-    annotationsVarResponse,
-    layoutObsResponse
-  ) --> Universe
-
-  where:
-  configResponse: GET /.../config
-  schemaResponse: GET /.../schema
-  annotationsObsResponse: GET /.../annotations/obs
-  annotationsVarResponse: GET /.../annotations/var
-  layoutObsResponse: GET /.../layout/obs
-
-  See spec in docs/REST_API.md.
-  */
-
-  test("create from test data", () => {
-    /*
-    create a universe from sample data nad validate its shape & contents
-    */
-    const { nObs, nVar } = REST.schema.schema.dataframe;
-    let universe = Universe.createUniverseFromResponse(
-      REST.config,
-      REST.schema
-    );
-    expect(universe).toBeDefined();
-    expect(universe).toMatchObject(
-      expect.objectContaining({
-        nObs,
-        nVar,
-        schema: REST.schema.schema,
-        obsAnnotations: expect.any(Dataframe.Dataframe),
-        varAnnotations: expect.any(Dataframe.Dataframe),
-        obsLayout: expect.any(Dataframe.Dataframe),
-        varData: expect.any(Dataframe.Dataframe)
-      })
-    );
-
-    universe = {
-      ...universe,
-      ...Universe.addObsAnnotations(
-        universe,
-        Universe.matrixFBSToDataframe(REST.annotationsObs)
-      ),
-      ...Universe.addVarAnnotations(
-        universe,
-        Universe.matrixFBSToDataframe(REST.annotationsVar)
-      ),
-      ...Universe.addObsLayout(
-        universe,
-        Universe.matrixFBSToDataframe(REST.layoutObs)
-      )
-    };
-
-    expect(universe).toMatchObject(
-      expect.objectContaining({
-        nObs,
-        nVar,
-        schema: REST.schema.schema,
-        obsAnnotations: expect.any(Dataframe.Dataframe),
-        varAnnotations: expect.any(Dataframe.Dataframe),
-        obsLayout: expect.any(Dataframe.Dataframe),
-        varData: expect.any(Dataframe.Dataframe)
-      })
-    );
-
-    expect(universe.obsAnnotations.dims).toEqual([
-      nObs,
-      REST.schema.schema.annotations.obs.columns.length
-    ]);
-    expect(universe.obsLayout.dims).toEqual([nObs, 2]);
-    expect(universe.obsLayout.colIndex.keys()).toEqual(
-      universe.schema.layout.obs[0].dims
-    );
-    expect(universe.varAnnotations.dims).toEqual([
-      nVar,
-      REST.schema.schema.annotations.var.columns.length
-    ]);
-    expect(universe.varData.isEmpty()).toBeTruthy();
-  });
-});

client/__tests__/util/stateManager/world.test.js

@@ -1,204 +0,0 @@
-import _ from "lodash";
-import * as Universe from "../../../src/util/stateManager/universe";
-import * as World from "../../../src/util/stateManager/world";
-import * as Dataframe from "../../../src/util/dataframe";
-import Crossfilter from "../../../src/util/typedCrossfilter";
-import { DimTypes } from "../../../src/util/typedCrossfilter/crossfilter";
-import * as REST from "./sampleResponses";
-import {
-  obsAnnoDimensionName,
-  layoutDimensionName
-} from "../../../src/util/nameCreators";
-
-/*
-Helper - creates universe, world, corssfilter and dimensionMap from
-the default REST test response.
-*/
-const defaultBigBang = () => {
-  /* create unverse, world, crossfilter and dimensionMap */
-  /* create universe */
-  let universe = Universe.createUniverseFromResponse(
-    _.cloneDeep(REST.config),
-    _.cloneDeep(REST.schema)
-  );
-
-  universe = {
-    ...universe,
-    ...Universe.addObsAnnotations(
-      universe,
-      Universe.matrixFBSToDataframe(REST.annotationsObs)
-    ),
-    ...Universe.addVarAnnotations(
-      universe,
-      Universe.matrixFBSToDataframe(REST.annotationsVar)
-    ),
-    ...Universe.addObsLayout(
-      universe,
-      Universe.matrixFBSToDataframe(REST.layoutObs)
-    )
-  };
-
-  /* create world */
-  const world = World.createWorldFromEntireUniverse(universe);
-  /* create crossfilter */
-  const crossfilter = World.createObsDimensions(
-    new Crossfilter(world.obsAnnotations),
-    world,
-    REST.schema.schema.layout.obs[0].dims
-  );
-
-  return {
-    universe,
-    world,
-    crossfilter
-  };
-};
-
-describe("createWorldFromEntireUniverse", () => {
-  test("create from REST sample", () => {
-    const universe = Universe.createUniverseFromResponse(
-      _.cloneDeep(REST.config),
-      _.cloneDeep(REST.schema),
-      Universe.matrixFBSToDataframe(_.cloneDeep(REST.annotationsObs)),
-      Universe.matrixFBSToDataframe(_.cloneDeep(REST.annotationsVar)),
-      Universe.matrixFBSToDataframe(_.cloneDeep(REST.layoutObs))
-    );
-    expect(universe).toBeDefined();
-
-    const world = World.createWorldFromEntireUniverse(universe);
-    expect(world).toBeDefined();
-
-    expect(world).toMatchObject(
-      expect.objectContaining({
-        nObs: universe.nObs,
-        nVar: universe.nVar,
-        schema: universe.schema,
-        obsAnnotations: expect.any(Dataframe.Dataframe),
-        varAnnotations: expect.any(Dataframe.Dataframe),
-        obsLayout: expect.any(Dataframe.Dataframe),
-        varData: expect.any(Dataframe.Dataframe),
-        clipQuantiles: { min: 0, max: 1 },
-        unclipped: {
-          obsAnnotations: expect.any(Dataframe.Dataframe),
-          varData: expect.any(Dataframe.Dataframe)
-        }
-      })
-    );
-  });
-});
-
-describe("createWorldFromCurrentSelection", () => {
-  test("create from REST sample", () => {
-    const {
-      universe,
-      world: originalWorld,
-      crossfilter: originalCrossfilter
-    } = defaultBigBang();
-
-    /* mock a selection */
-    const crossfilter = originalCrossfilter
-      .select(obsAnnoDimensionName("field1"), { mode: "range", lo: 0, hi: 5 })
-      .select(obsAnnoDimensionName("field3"), {
-        mode: "exact",
-        values: [false]
-      });
-
-    /* create the world from the selection */
-    const world = World.createWorldBySelection(
-      universe,
-      originalWorld,
-      crossfilter
-    );
-    expect(world).toBeDefined();
-    expect(world.nObs).toEqual(crossfilter.countSelected());
-
-    /*
-    calculate expected values and match against result
-    */
-
-    /* matchFilter must match the dimension filters above */
-    const matchFilter = (df, row) => {
-      const field1 = df.at(row, "field1");
-      const field3 = df.at(row, "field3");
-      return field1 >= 0 && field1 < 5 && !field3;
-    };
-    const matchingIndices = _()
-      .range(universe.nObs)
-      .filter(idx => matchFilter(universe.obsAnnotations, idx))
-      .value();
-
-    expect(world).toMatchObject(
-      expect.objectContaining({
-        nObs: matchingIndices.length,
-        nVar: universe.nVar,
-        schema: universe.schema,
-        clipQuantiles: { min: 0, max: 1 },
-        obsAnnotations: expect.any(Dataframe.Dataframe),
-        varAnnotations: expect.any(Dataframe.Dataframe),
-        obsLayout: expect.any(Dataframe.Dataframe),
-        varData: expect.any(Dataframe.Dataframe),
-        unclipped: {
-          obsAnnotations: expect.any(Dataframe.Dataframe),
-          varData: expect.any(Dataframe.Dataframe)
-        }
-      })
-    );
-
-    expect(world.obsAnnotations.rowIndex.keys()).toEqual(
-      new Int32Array(matchingIndices)
-    );
-    expect(world.obsAnnotations.colIndex.keys()).toEqual(
-      universe.obsAnnotations.colIndex.keys()
-    );
-    expect(world.obsLayout.rowIndex.keys()).toEqual(
-      new Int32Array(matchingIndices)
-    );
-    expect(world.obsLayout.colIndex.keys()).toEqual(
-      world.schema.layout.obs[0].dims
-    );
-  });
-});
-
-describe("createObsDimensionMap", () => {
-  test("when universe eq world", () => {
-    /*
-    check for:
-    - creates a dimension for all obsAnnotations, PLUS X/Y layout
-    - check that dimension typing is sane
-    */
-
-    const { crossfilter } = defaultBigBang();
-    const annotationNames = _.map(
-      REST.schema.schema.annotations.obs.columns,
-      c => c.name
-    );
-    const obsIndexColName = REST.schema.schema.annotations.obs.index;
-    const schemaByObsName = _.keyBy(
-      REST.schema.schema.annotations.obs.columns,
-      "name"
-    );
-    expect(crossfilter).toBeDefined();
-    annotationNames.forEach(name => {
-      const dim = crossfilter.dimensions[obsAnnoDimensionName(name)];
-      if (name === obsIndexColName) {
-        expect(dim).toBeUndefined();
-      } else {
-        const { type } = schemaByObsName[name];
-        if (type === "string" || type === "boolean" || type === "categorical") {
-          expect(dim.dim).toBeInstanceOf(DimTypes.enum);
-        } else {
-          expect(dim.dim).toBeInstanceOf(DimTypes.scalar);
-        }
-      }
-    });
-    expect(
-      crossfilter.dimensions[layoutDimensionName("XY")].dim
-    ).toBeInstanceOf(DimTypes.spatial);
-  });
-});
-
-describe("worldEqUniverse", () => {
-  const { universe, world } = defaultBigBang();
-  const result = World.worldEqUniverse(world, universe);
-  expect(result).toBe(true);
-});

client/__tests__/util/typedCrossfilter/bitArray.test.js

@@ -1,5 +1,3 @@
-// jshint esversion: 6
-
 import BitArray from "../../../src/util/typedCrossfilter/bitArray";
 
 const defaultTestLength = 8;
@@ -187,7 +185,7 @@ describe("fillBySelection", () => {
 describe("wide bitarray", () => {
   test.each([9, 30, 31, 32, 33, 54, 63, 64, 65, 127, 128, 129])(
     "more than %d dimensions",
-    ndim => {
+    (ndim) => {
       /* ensure we move across the uint boundary correctly */
 
       const ba = new BitArray(defaultTestLength);

client/__tests__/util/typedCrossfilter/crossfilter.test.js

@@ -1,4 +1,5 @@
-import _ from "lodash";
+import filter from "lodash.filter";
+import zip from "lodash.zip";
 
 import Crossfilter from "../../../src/util/typedCrossfilter";
 
@@ -11,7 +12,7 @@ const someData = [
     type: "tab",
     productIDs: ["001"],
     coords: [0, 0],
-    nonFinite: 0.0
+    nonFinite: 0.0,
   },
   {
     date: "2011-11-14T16:20:19Z",
@@ -21,7 +22,7 @@ const someData = [
     type: "tab",
     productIDs: ["001", "005"],
     coords: [0.4, 0.4],
-    nonFinite: Number.NaN
+    nonFinite: Number.NaN,
   },
   {
     date: "2011-11-14T16:28:54Z",
@@ -31,7 +32,7 @@ const someData = [
     type: "visa",
     productIDs: ["004", "005"],
     coords: [0.3, 0.1],
-    nonFinite: Number.POSITIVE_INFINITY
+    nonFinite: Number.POSITIVE_INFINITY,
   },
   {
     date: "2011-11-14T16:30:43Z",
@@ -41,7 +42,7 @@ const someData = [
     type: "tab",
     productIDs: ["001", "002"],
     coords: [0.392, 0.1],
-    nonFinite: Number.NEGATIVE_INFINITY
+    nonFinite: Number.NEGATIVE_INFINITY,
   },
   {
     date: "2011-11-14T16:48:46Z",
@@ -51,7 +52,7 @@ const someData = [
     type: "tab",
     productIDs: ["005"],
     coords: [0.7, 0.0482],
-    nonFinite: 1.0
+    nonFinite: 1.0,
   },
   {
     date: "2011-11-14T16:53:41Z",
@@ -61,7 +62,7 @@ const someData = [
     type: "tab",
     productIDs: ["001", "004", "005"],
     coords: [0.9999, 1.0],
-    nonFinite: Number.NaN
+    nonFinite: Number.NaN,
   },
   {
     date: "2011-11-14T16:54:06Z",
@@ -71,7 +72,7 @@ const someData = [
     type: "cash",
     productIDs: ["001", "002", "003", "004", "005"],
     coords: [0.384, 0.6938],
-    nonFinite: 99.0
+    nonFinite: 99.0,
   },
   {
     date: "2011-11-14T16:58:03Z",
@@ -81,7 +82,7 @@ const someData = [
     type: "tab",
     productIDs: ["001"],
     coords: [0.4822, 0.482],
-    nonFinite: Number.NaN
+    nonFinite: Number.NaN,
   },
   {
     date: "2011-11-14T17:07:21Z",
@@ -91,7 +92,7 @@ const someData = [
     type: "tab",
     productIDs: ["004", "005"],
     coords: [0.2234, 0],
-    nonFinite: Number.NaN
+    nonFinite: Number.NaN,
   },
   {
     date: "2011-11-14T17:22:59Z",
@@ -101,7 +102,7 @@ const someData = [
     type: "tab",
     productIDs: ["001", "002", "004", "005"],
     coords: [0.382, 0.38485],
-    nonFinite: -1
+    nonFinite: -1,
   },
   {
     date: "2011-11-14T17:25:45Z",
@@ -111,7 +112,7 @@ const someData = [
     type: "cash",
     productIDs: ["002"],
     coords: [0.998, 0.8472],
-    nonFinite: 0.0
+    nonFinite: 0.0,
   },
   {
     date: "2011-11-14T17:29:52Z",
@@ -121,8 +122,8 @@ const someData = [
     type: "visa",
     productIDs: ["004"],
     coords: [0.8273, 0.3384],
-    nonFinite: 0.0
-  }
+    nonFinite: 0.0,
+  },
 ];
 
 let payments = null;
@@ -248,16 +249,26 @@ describe("ImmutableTypedCrossfilter", () => {
     test("none", () => {
       expect(p.select("quantity", { mode: "none" }).countSelected()).toEqual(0);
     });
-    test.each([[[]], [[2]], [[2, 1]], [[9, 82]], [[0, 1]]])("exact: %p", v =>
+    test.each([[[]], [[2]], [[2, 1]], [[9, 82]], [[0, 1]]])("exact: %p", (v) =>
       expect(
         p.select("quantity", { mode: "exact", values: v }).countSelected()
-      ).toEqual(_.filter(someData, d => v.includes(d.quantity)).length)
+      ).toEqual(filter(someData, (d) => v.includes(d.quantity)).length)
     );
-    test.each([[0, 1], [1, 2], [0, 99], [99, 100000]])("range %p", (lo, hi) =>
+    test("single value exact", () => {
+      expect(
+        p.select("quantity", { mode: "exact", values: 2 }).countSelected()
+      ).toEqual(filter(someData, (d) => d.quantity === 2).length);
+    });
+    test.each([
+      [0, 1],
+      [1, 2],
+      [0, 99],
+      [99, 100000],
+    ])("range %p", (lo, hi) =>
       expect(
         p.select("quantity", { mode: "range", lo, hi }).countSelected()
       ).toEqual(
-        _.filter(someData, d => d.quantity >= lo && d.quantity < hi).length
+        filter(someData, (d) => d.quantity >= lo && d.quantity < hi).length
       )
     );
     test("bad mode", () => {
@@ -284,12 +295,17 @@ describe("ImmutableTypedCrossfilter", () => {
       [["tab"]],
       [["visa"]],
       [["visa", "tab"]],
-      [["cash", "tab", "visa"]]
-    ])("exact: %p", v =>
+      [["cash", "tab", "visa"]],
+    ])("exact: %p", (v) =>
       expect(
         p.select("type", { mode: "exact", values: v }).countSelected()
-      ).toEqual(_.filter(someData, d => v.includes(d.type)).length)
+      ).toEqual(filter(someData, (d) => v.includes(d.type)).length)
     );
+    test("single value exact", () => {
+      expect(
+        p.select("type", { mode: "exact", values: "tab" }).countSelected()
+      ).toEqual(filter(someData, (d) => d.type === "tab").length);
+    });
     test("range", () => {
       expect(() => p.select("type", { mode: "range", lo: 0, hi: 9 })).toThrow(
         Error
@@ -303,8 +319,8 @@ describe("ImmutableTypedCrossfilter", () => {
   describe("spatial dimension", () => {
     let p;
     beforeEach(() => {
-      const X = someData.map(r => r.coords[0]);
-      const Y = someData.map(r => r.coords[1]);
+      const X = someData.map((r) => r.coords[0]);
+      const Y = someData.map((r) => r.coords[1]);
       p = payments.addDimension("coords", "spatial", X, Y);
     });
 
@@ -316,34 +332,76 @@ describe("ImmutableTypedCrossfilter", () => {
     test("none", () => {
       expect(p.select("coords", { mode: "none" }).countSelected()).toEqual(0);
     });
-    test.each([[0, 0, 1, 1], [0, 0, 0.5, 0.5], [0.5, 0.5, 1, 1]])(
-      "within-rect %d %d %d %d",
-      (minX, minY, maxX, maxY) => {
-        expect(
-          p
-            .select("coords", { mode: "within-rect", minX, minY, maxX, maxY })
-            .allSelected()
-        ).toEqual(
-          _.filter(someData, d => {
-            const [x, y] = d.coords;
-            return minX <= x && x < maxX && minY <= y && y < maxY;
-          })
-        );
-      }
-    );
+    test.each([
+      [0, 0, 1, 1],
+      [0, 0, 0.5, 0.5],
+      [0.5, 0.5, 1, 1],
+    ])("within-rect %d %d %d %d", (minX, minY, maxX, maxY) => {
+      expect(
+        p
+          .select("coords", { mode: "within-rect", minX, minY, maxX, maxY })
+          .allSelected()
+      ).toEqual(
+        filter(someData, (d) => {
+          const [x, y] = d.coords;
+          return minX <= x && x < maxX && minY <= y && y < maxY;
+        })
+      );
+    });
 
     test.each([
       [
-          [[0, 0], [0, 1], [1, 1], [1, 0]],
-          [true, true, true, true, true, false, true, true, true, true, true, true]
+        [
+          [0, 0],
+          [0, 1],
+          [1, 1],
+          [1, 0],
+        ],
+        [
+          true,
+          true,
+          true,
+          true,
+          true,
+          false,
+          true,
+          true,
+          true,
+          true,
+          true,
+          true,
+        ],
       ],
       [
-          [[0, 0], [0, 0.5], [0.5, 0.5], [0.5, 0]],
-          [true, true, true, true, false, false, false, true, true, true, false, false]
-      ]
+        [
+          [0, 0],
+          [0, 0.5],
+          [0.5, 0.5],
+          [0.5, 0],
+        ],
+        [
+          true,
+          true,
+          true,
+          true,
+          false,
+          false,
+          false,
+          true,
+          true,
+          true,
+          false,
+          false,
+        ],
+      ],
     ])("within-polygon %p", (polygon, expected) => {
-      expect(p.select("coords", { mode: "within-polygon", polygon }).allSelected())
-          .toEqual(_.zip(someData, expected).filter(x => x[1]).map(x => x[0]));
+      expect(
+        p.select("coords", { mode: "within-polygon", polygon }).allSelected()
+      ).toEqual(
+        zip(someData, expected)
+          .filter((x) => x[1])
+          .map((x) => x[0])
+      );
     });
   });
 
@@ -381,7 +439,7 @@ describe("ImmutableTypedCrossfilter", () => {
         p
           .select("nonFinite", {
             mode: "exact",
-            values: [Number.POSITIVE_INFINITY]
+            values: [Number.POSITIVE_INFINITY],
           })
           .countSelected()
       ).toEqual(1);
@@ -389,7 +447,7 @@ describe("ImmutableTypedCrossfilter", () => {
         p
           .select("nonFinite", {
             mode: "exact",
-            values: [Number.NEGATIVE_INFINITY]
+            values: [Number.NEGATIVE_INFINITY],
           })
           .countSelected()
       ).toEqual(1);
@@ -402,7 +460,7 @@ describe("ImmutableTypedCrossfilter", () => {
         p
           .select("nonFinite", {
             mode: "exact",
-            values: [Number.POSITIVE_INFINITY, 0, 1, 99]
+            values: [Number.POSITIVE_INFINITY, 0, 1, 99],
           })
           .countSelected()
       ).toEqual(6);
@@ -414,7 +472,7 @@ describe("ImmutableTypedCrossfilter", () => {
           .select("nonFinite", {
             mode: "range",
             lo: 0,
-            hi: Number.POSITIVE_INFINITY
+            hi: Number.POSITIVE_INFINITY,
           })
           .countSelected()
       ).toEqual(5);
@@ -423,7 +481,7 @@ describe("ImmutableTypedCrossfilter", () => {
           .select("nonFinite", {
             mode: "range",
             lo: 0,
-            hi: Number.NaN
+            hi: Number.NaN,
           })
           .countSelected()
       ).toEqual(6);
@@ -432,7 +490,7 @@ describe("ImmutableTypedCrossfilter", () => {
           .select("nonFinite", {
             mode: "range",
             lo: Number.NEGATIVE_INFINITY,
-            hi: Number.POSITIVE_INFINITY
+            hi: Number.POSITIVE_INFINITY,
           })
           .countSelected()
       ).toEqual(7);

client/__tests__/util/typedCrossfilter/positiveInterval.test.js

@@ -1,5 +1,3 @@
-// jshint esversion: 6
-
 // const PositiveIntervals = require("../../src/util/typedCrossfilter/positiveIntervals");
 import PositiveIntervals from "../../../src/util/typedCrossfilter/positiveIntervals";
 
@@ -10,15 +8,30 @@ describe("canonicalize", () => {
 
   test("simple, already correct", () => {
     expect(PositiveIntervals.canonicalize([[0, 1]])).toEqual([[0, 1]]);
-    expect(PositiveIntervals.canonicalize([[0, 1], [2, 3]])).toEqual([
+    expect(
+      PositiveIntervals.canonicalize([
+        [0, 1],
+        [2, 3],
+      ])
+    ).toEqual([
       [0, 1],
-      [2, 3]
+      [2, 3],
     ]);
   });
 
   test("non-canonical, need to be canonicalized", () => {
-    expect(PositiveIntervals.canonicalize([[0, 1], [1, 2]])).toEqual([[0, 2]]);
-    expect(PositiveIntervals.canonicalize([[1, 2], [2, 3]])).toEqual([[1, 3]]);
+    expect(
+      PositiveIntervals.canonicalize([
+        [0, 1],
+        [1, 2],
+      ])
+    ).toEqual([[0, 2]]);
+    expect(
+      PositiveIntervals.canonicalize([
+        [1, 2],
+        [2, 3],
+      ])
+    ).toEqual([[1, 3]]);
   });
 });
 
@@ -26,14 +39,30 @@ describe("union", () => {
   test("empty range", () => {
     expect(PositiveIntervals.union([], [])).toEqual([]);
     expect(PositiveIntervals.union([], [[1, 2]])).toEqual([[1, 2]]);
-    expect(PositiveIntervals.union([], [[1, 2], [3, 4]])).toEqual([
+    expect(
+      PositiveIntervals.union(
+        [],
+        [
+          [1, 2],
+          [3, 4],
+        ]
+      )
+    ).toEqual([
       [1, 2],
-      [3, 4]
+      [3, 4],
     ]);
     expect(PositiveIntervals.union([[3, 4]], [])).toEqual([[3, 4]]);
-    expect(PositiveIntervals.union([[1, 2], [3, 4]], [])).toEqual([
+    expect(
+      PositiveIntervals.union(
+        [
+          [1, 2],
+          [3, 4],
+        ],
+        []
+      )
+    ).toEqual([
       [1, 2],
-      [3, 4]
+      [3, 4],
     ]);
     expect(PositiveIntervals.union([[3, 3]], [])).toEqual([[3, 3]]);
     expect(PositiveIntervals.union([], [[3, 3]])).toEqual([[3, 3]]);
@@ -44,17 +73,37 @@ describe("union", () => {
     expect(PositiveIntervals.union([[2, 3]], [[1, 2]])).toEqual([[1, 3]]);
     expect(PositiveIntervals.union([[1, 2]], [[3, 4]])).toEqual([
       [1, 2],
-      [3, 4]
+      [3, 4],
     ]);
     expect(
-      PositiveIntervals.union([[1, 2], [3, 4]], [[6, 7], [19, 40]])
-    ).toEqual([[1, 2], [3, 4], [6, 7], [19, 40]]);
-    expect(PositiveIntervals.union([[1, 4]], [[1, 1], [3, 4]])).toEqual([
-      [1, 4]
+      PositiveIntervals.union(
+        [
+          [1, 2],
+          [3, 4],
+        ],
+        [
+          [6, 7],
+          [19, 40],
+        ]
+      )
+    ).toEqual([
+      [1, 2],
+      [3, 4],
+      [6, 7],
+      [19, 40],
     ]);
+    expect(
+      PositiveIntervals.union(
+        [[1, 4]],
+        [
+          [1, 1],
+          [3, 4],
+        ]
+      )
+    ).toEqual([[1, 4]]);
     expect(PositiveIntervals.union([[3, 3]], [[4, 4]])).toEqual([
       [3, 3],
-      [4, 4]
+      [4, 4],
     ]);
   });
 });
@@ -70,34 +119,43 @@ describe("intersection", () => {
     expect(PositiveIntervals.intersection([[1, 2]], [[2, 3]])).toEqual([]);
     expect(PositiveIntervals.intersection([[2, 3]], [[1, 2]])).toEqual([]);
     expect(PositiveIntervals.intersection([[1, 10]], [[1, 10]])).toEqual([
-      [1, 10]
+      [1, 10],
     ]);
     expect(PositiveIntervals.intersection([[1, 10]], [[2, 8]])).toEqual([
-      [2, 8]
+      [2, 8],
     ]);
     expect(PositiveIntervals.intersection([[2, 8]], [[1, 10]])).toEqual([
-      [2, 8]
+      [2, 8],
     ]);
     expect(PositiveIntervals.intersection([[1, 10]], [[2, 12]])).toEqual([
-      [2, 10]
+      [2, 10],
     ]);
     expect(PositiveIntervals.intersection([[2, 12]], [[1, 10]])).toEqual([
-      [2, 10]
+      [2, 10],
     ]);
     expect(PositiveIntervals.intersection([[1, 10]], [[1, 8]])).toEqual([
-      [1, 8]
+      [1, 8],
     ]);
     expect(PositiveIntervals.intersection([[1, 8]], [[1, 10]])).toEqual([
-      [1, 8]
+      [1, 8],
+    ]);
+    expect(
+      PositiveIntervals.intersection(
+        [[1, 10]],
+        [
+          [1, 2],
+          [6, 9],
+        ]
+      )
+    ).toEqual([
+      [1, 2],
+      [6, 9],
     ]);
-    expect(PositiveIntervals.intersection([[1, 10]], [[1, 2], [6, 9]])).toEqual(
-      [[1, 2], [6, 9]]
-    );
     expect(PositiveIntervals.intersection([[0, 2638]], [[1363, 2638]])).toEqual(
       [[1363, 2638]]
     );
     expect(PositiveIntervals.intersection([[1, 2]], [[1, 2]])).toEqual([
-      [1, 2]
+      [1, 2],
     ]);
   });
 });
@@ -110,32 +168,66 @@ describe("difference", () => {
   });
 
   test("simple", () => {
-    expect(PositiveIntervals.difference([[1, 2], [3, 4]], [])).toEqual([
+    expect(
+      PositiveIntervals.difference(
+        [
+          [1, 2],
+          [3, 4],
+        ],
+        []
+      )
+    ).toEqual([
       [1, 2],
-      [3, 4]
-    ]);
-    expect(PositiveIntervals.difference([[1, 2], [3, 10]], [[5, 10]])).toEqual([
-      [1, 2],
-      [3, 5]
-    ]);
-    expect(PositiveIntervals.difference([[1, 2], [3, 10]], [[0, 5]])).toEqual([
-      [5, 10]
+      [3, 4],
     ]);
     expect(
-      PositiveIntervals.difference([[0, 2638]], [[0, 1363], [2055, 2638]])
+      PositiveIntervals.difference(
+        [
+          [1, 2],
+          [3, 10],
+        ],
+        [[5, 10]]
+      )
+    ).toEqual([
+      [1, 2],
+      [3, 5],
+    ]);
+    expect(
+      PositiveIntervals.difference(
+        [
+          [1, 2],
+          [3, 10],
+        ],
+        [[0, 5]]
+      )
+    ).toEqual([[5, 10]]);
+    expect(
+      PositiveIntervals.difference(
+        [[0, 2638]],
+        [
+          [0, 1363],
+          [2055, 2638],
+        ]
+      )
     ).toEqual([[1363, 2055]]);
     expect(
-      PositiveIntervals.difference([[0, 1363], [2055, 2638]], [[0, 2638]])
+      PositiveIntervals.difference(
+        [
+          [0, 1363],
+          [2055, 2638],
+        ],
+        [[0, 2638]]
+      )
     ).toEqual([]);
     expect(PositiveIntervals.difference([[0, 10]], [[0, 1]])).toEqual([
-      [1, 10]
+      [1, 10],
     ]);
     expect(PositiveIntervals.difference([[0, 10]], [[1, 2]])).toEqual([
       [0, 1],
-      [2, 10]
+      [2, 10],
     ]);
     expect(PositiveIntervals.difference([[0, 10]], [[9, 10]])).toEqual([
-      [0, 9]
+      [0, 9],
     ]);
   });
 });

client/__tests__/util/typedCrossfilter/sort.test.js

@@ -2,9 +2,6 @@ import {
   sortArray,
   sortIndex,
   lowerBound,
-  upperBound,
-  lowerBoundIndirect,
-  upperBoundIndirect
 } from "../../../src/util/typedCrossfilter/sort";
 
 /*
@@ -40,7 +37,7 @@ describe("sortArray", () => {
       ["a", "b", "0", "1"],
       [0, "a", true, null, undefined, 3.1415],
       fillRand(new Array(1000)),
-      ["a", NaN, null, pInf]
+      ["a", NaN, null, pInf],
     ].map((val, idx) =>
       test(`JS vals ${idx}`, () => {
         expect(sortArray(val)).toMatchObject(val.sort());
@@ -49,7 +46,7 @@ describe("sortArray", () => {
   });
 
   describe("finite numbers", () => {
-    [Array, Float32Array, Uint32Array, Int32Array, Float64Array].map(Type =>
+    [Array, Float32Array, Uint32Array, Int32Array, Float64Array].map((Type) =>
       test(Type.name, () => {
         expect(sortArray(Type.from([6, 5, 4, 3, 2, 1, 0]))).toMatchObject(
           Type.from([0, 1, 2, 3, 4, 5, 6])
@@ -68,7 +65,7 @@ describe("sortArray", () => {
   });
 
   describe("non-finite numbers", () => {
-    test("inifinity", () => {
+    test("infinity", () => {
       expect(sortArray(new Float32Array([pInf, nInf, 0, 1, 2]))).toMatchObject(
         new Float32Array([nInf, 0, 1, 2, pInf])
       );
@@ -131,7 +128,7 @@ describe("sortArray", () => {
 
 describe("sortIndex", () => {
   describe("finite numbers", () => {
-    [Array, Float32Array, Uint32Array, Int32Array, Float64Array].map(Type =>
+    [Array, Float32Array, Uint32Array, Int32Array, Float64Array].map((Type) =>
       test(Type.name, () => {
         const source1 = Type.from([6, 5, 4, 3, 2, 1, 0]);
         const index1 = fillRange(new Uint32Array(source1.length));

client/__tests__/util/typedCrossfilter/util.test.js

@@ -1,6 +1,6 @@
 import {
   sliceByIndex,
-  makeSortIndex
+  makeSortIndex,
 } from "../../../src/util/typedCrossfilter/util";
 import { rangeFill as fillRange } from "../../../src/util/range";
 

client/configuration/babel/babel.dev.js

@@ -2,15 +2,24 @@ module.exports = {
   babelrc: false,
   cacheDirectory: true,
   presets: [
-    ["modern-browsers", { loose: true, modules: false }],
-    "@babel/preset-react"
+    [
+      "@babel/preset-env",
+      {
+        useBuiltIns: "entry",
+        corejs: 3,
+        modules: false,
+      },
+    ],
+    "@babel/preset-react",
   ],
   plugins: [
     "@babel/plugin-proposal-function-bind",
     ["@babel/plugin-proposal-decorators", { legacy: true }],
     ["@babel/plugin-proposal-class-properties", { loose: true }],
+    ["@babel/plugin-proposal-private-methods", { loose: true }],
+    ["@babel/plugin-proposal-private-property-in-object", { loose: true }],
     "@babel/plugin-proposal-export-namespace-from",
     "@babel/plugin-proposal-optional-chaining",
-    "@babel/plugin-proposal-nullish-coalescing-operator"
-  ]
+    "@babel/plugin-proposal-nullish-coalescing-operator",
+  ],
 };

client/configuration/babel/babel.prod.js

@@ -1,17 +1,26 @@
 module.exports = {
   babelrc: false,
   presets: [
-    ["modern-browsers", { loose: true, modules: false }],
-    "@babel/preset-react"
+    [
+      "@babel/preset-env",
+      {
+        useBuiltIns: "entry",
+        corejs: 3,
+        modules: false,
+      },
+    ],
+    "@babel/preset-react",
   ],
   plugins: [
     "@babel/plugin-proposal-function-bind",
     ["@babel/plugin-proposal-decorators", { legacy: true }],
     ["@babel/plugin-proposal-class-properties", { loose: true }],
+    ["@babel/plugin-proposal-private-methods", { loose: true }],
+    ["@babel/plugin-proposal-private-property-in-object", { loose: true }],
     "@babel/plugin-proposal-export-namespace-from",
     "@babel/plugin-transform-react-constant-elements",
     "@babel/plugin-transform-runtime",
     "@babel/plugin-proposal-optional-chaining",
-    "@babel/plugin-proposal-nullish-coalescing-operator"
-  ]
+    "@babel/plugin-proposal-nullish-coalescing-operator",
+  ],
 };

client/configuration/eslint/eslint.js

@@ -1,18 +1,48 @@
 module.exports = {
   root: true,
-  parser: "babel-eslint",
-  extends: ["airbnb", "prettier", "prettier/react"],
+
+  extends: [
+    "airbnb",
+    "plugin:eslint-comments/recommended",
+    "plugin:@blueprintjs/recommended",
+    "plugin:compat/recommended",
+    "plugin:jsx-a11y/recommended",
+    // (thuang) disable eslint formatting rules, so prettier can do its job
+    // Do not use `plugin:prettier/recommended` per doc below:
+    // https://prettier.io/docs/en/integrating-with-linters.html
+    "prettier",
+  ],
+  settings: {
+    // AbortController is not supported in iOS Safari 10.3, Chrome 61
+    // Headers is not supported in iOS Safari 10.3
+    polyfills: ["Headers", "AbortController"],
+  },
   env: { browser: true, commonjs: true, es6: true },
-  globals: { expect: true },
+  globals: {
+    expect: true,
+    jest: true,
+    jestPuppeteer: true,
+    it: true,
+    page: true,
+    browser: true,
+    context: true,
+    beforeEach: true,
+  },
+  parser: "@babel/eslint-parser",
   parserOptions: {
     ecmaVersion: 2017,
     sourceType: "module",
     ecmaFeatures: {
       jsx: true,
-      generators: true
-    }
+      generators: true,
+    },
+    babelOptions: {
+      configFile: "./configuration/babel/babel.prod.js",
+    },
   },
   rules: {
+    "react/jsx-no-target-blank": "off",
+    "eslint-comments/require-description": ["error"],
     "no-magic-numbers": "off",
     "no-nested-ternary": "off",
     "func-style": "off",
@@ -21,13 +51,7 @@ module.exports = {
     "react/jsx-filename-extension": "off",
     "comma-dangle": "off",
     "no-underscore-dangle": "off",
-    quotes: ["error", "double"],
     "implicit-arrow-linebreak": "off",
-    "operator-linebreak": [
-      "error",
-      "after",
-      { overrides: { "?": "before", ":": "before" } }
-    ],
     "no-console": "off",
     "spaced-comment": ["error", "always", { exceptions: ["*"] }],
     "no-param-reassign": "off",
@@ -35,13 +59,26 @@ module.exports = {
     "react/prop-types": [0],
     "space-before-function-paren": "off",
     "function-paren-newline": "off",
-    "prefer-destructuring": ["error", { object: true, array: false }]
+    "prefer-destructuring": ["error", { object: true, array: false }],
+    "import/prefer-default-export": "off",
+    "no-restricted-syntax": [
+      "error",
+      "ForInStatement",
+      "LabeledStatement",
+      "WithStatement",
+    ],
+    "import/no-extraneous-dependencies": [
+      "error",
+      {
+        devDependencies: true,
+      },
+    ],
   },
   overrides: [
     {
       files: ["**/*.test.js"],
       env: {
-        jest: true // now **/*.test.js files' env has both es6 *and* jest
+        jest: true, // now **/*.test.js files' env has both es6 *and* jest
       },
       // Can't extend in overrides: https://github.com/eslint/eslint/issues/8813
       // "extends": ["plugin:jest/recommended"]
@@ -51,8 +88,8 @@ module.exports = {
         "jest/no-focused-tests": "error",
         "jest/no-identical-title": "error",
         "jest/prefer-to-have-length": "warn",
-        "jest/valid-expect": "error"
-      }
-    }
-  ]
+        "jest/valid-expect": "error",
+      },
+    },
+  ],
 };

client/configuration/lint-staged/lint-staged.config.js

@@ -0,0 +1,4 @@
+module.exports = {
+  "*.js": "eslint --fix",
+  "**/*": "prettier --write --ignore-unknown",
+};

client/configuration/webpack/cspHashPlugin.js

@@ -0,0 +1,53 @@
+/* eslint-disable import/no-extraneous-dependencies -- this file is a devDependency*/
+const cheerio = require("cheerio");
+const crypto = require("crypto");
+const HtmlWebpackPlugin = require("html-webpack-plugin");
+
+const digest = (str) => {
+  const hash = crypto.createHash("sha256").update(str, "utf8").digest("base64");
+  return `sha256-${hash}`;
+};
+class CspHashPlugin {
+  constructor(opts) {
+    this.opts = { ...opts };
+  }
+
+  apply(compiler) {
+    compiler.hooks.compilation.tap("CspHashPlugin", (compilation) => {
+      HtmlWebpackPlugin.getHooks(compilation).beforeEmit.tapAsync(
+        "CspHashPlugin",
+        (data, cb) => {
+          const { filename } = this.opts;
+
+          const $ = cheerio.load(data.html, { decodeEntities: false });
+
+          if (filename) {
+            const results = {};
+            results["script-hashes"] = $("script:not([src]):not([no-csp-hash])")
+              .map((i, elmt) => digest($(elmt).html()))
+              .get();
+
+            const json = JSON.stringify(results);
+            compilation.assets[filename] = {
+              source: () => json,
+              size: () => json.length,
+            };
+          }
+
+          // Remove no-csp-hash attributes. Cheerio does not parse Jinja templates
+          // correctly, so we brute force this with a regular expression.
+          data.html = data.html.replace(
+            /(<script .*)no-csp-hash(.*>)/,
+            (match, p1, p2) => [p1, p2].join("")
+          );
+
+          // Tell webpack to move on
+          cb(null, data);
+        }
+      );
+    });
+  }
+}
+
+module.exports = CspHashPlugin;
+/* eslint-enable import/no-extraneous-dependencies -- enable*/

client/configuration/webpack/obsoleteHTMLTemplate.html

@@ -0,0 +1,75 @@
+<script>
+  /** Reflect ANY changes to this script in the script hash in `server/eb/app.py` **/
+  var root = document.getElementById("root");
+  root.remove();
+  var portals = document.getElementsByClassName("bp3-portal");
+  for (var i = 0; i < portals.length; i += 1) {
+    portals[i].remove();
+  }
+</script>
+<div
+  style="
+    display: flex;
+    flex-direction: column;
+    width: 100vw;
+    height: 100vh;
+    text-align: center;
+    justify-content: center;
+    align-items: center;
+    background: #8080801a;
+    font-family: 'Roboto Condensed, sans serif';
+  "
+>
+  <img
+    src="https://cellxgene.cziscience.com/s3/cellxgene/static/images/cellxgene-logo.png"
+    style="width: 320px;"
+  />
+  <div
+    style="
+      margin-top: 16px;
+      background: white;
+      width: 40vw;
+      border-radius: 4px;
+      padding: 24px 64px;
+      -webkit-box-shadow: 0px 0px 3px 2px rgba(0, 0, 0, 0.38);
+      -moz-box-shadow: 0px 0px 3px 2px rgba(0, 0, 0, 0.38);
+      box-shadow: 0px 0px 3px 2px rgba(0, 0, 0, 0.38);
+      max-width: 550px;
+    "
+  >
+    <div style="margin-bottom: 0; font-weight: bolder; font-size: 1.2em;">
+      Unsupported Browser
+    </div>
+    <div style="margin-top: 0;">
+      cellxgene is currently supported on the following browsers
+    </div>
+    <div
+      style="display: flex; justify-content: space-around; margin-top: 16px;"
+    >
+      <a
+        href="https://www.google.com/chrome/?hl=en%22"
+        aria-label="Download Google Chrome"
+      >
+        <img
+          src="https://cellxgene.cziscience.com/s3/cellxgene/static/images/chrome.png"
+          style="width: 80px; height: 80px;"
+        />
+        <div>Chrome &gt; 60</div>
+      </a>
+      <a href="https://www.mozilla.com/firefox/" aria-label="Download Firefox">
+        <img
+          src="https://cellxgene.cziscience.com/s3/cellxgene/static/images/firefox.png"
+          style="width: 80px; height: 80px;"
+        />
+        <div>Firefox ≥ 60</div>
+      </a>
+      <a href="//www.microsoft.com/edge" aria-label="Download Edge">
+        <img
+          src="https://cellxgene.cziscience.com/s3/cellxgene/static/images/edge.png"
+          style="width: 80px; height: 80px;"
+        />
+        <div>Edge ≥ 79</div>
+      </a>
+    </div>
+  </div>
+</div>

client/configuration/webpack/webpack.config.dev.js

@@ -1,96 +1,81 @@
-// jshint esversion: 6
 const path = require("path");
 const webpack = require("webpack");
 const HtmlWebpackPlugin = require("html-webpack-plugin");
+const FaviconsWebpackPlugin = require("favicons-webpack-plugin");
+const MiniCssExtractPlugin = require("mini-css-extract-plugin");
+
+const { merge } = require("webpack-merge");
+
+const sharedConfig = require("./webpack.config.shared");
+const babelOptions = require("../babel/babel.dev");
 
-const src = path.resolve("src");
 const fonts = path.resolve("src/fonts");
 const nodeModules = path.resolve("node_modules");
 
-const babelOptions = require("../babel/babel.dev");
-
-module.exports = {
+const devConfig = {
   mode: "development",
   devtool: "eval",
-  entry: ["./src/index"],
   output: {
-    path: path.resolve("build"),
     pathinfo: true,
     filename: "static/js/bundle.js",
-    publicPath: "/"
   },
   module: {
     rules: [
       {
-        test: /\.js$/,
-        include: src,
+        test: /\.jsx?$/,
         loader: "babel-loader",
-        options: babelOptions
+        options: babelOptions,
       },
       {
-        test: /\.css$/,
-        include: src,
-        exclude: [path.resolve(src, "index.css")],
-        loader: [
-          {
-            loader: "style-loader"
-          },
-          {
-            loader: "css-loader",
-            options: {
-              modules: {
-                localIdentName: "[name]__[local]___[hash:base64:5]"
-              }
-            }
-          }
-        ]
-      },
-      {
-        test: /index\.css$/,
-        include: [path.resolve(src, "index.css")],
-        loader: [
-          {
-            loader: "style-loader"
-          },
-          {
-            loader: "css-loader"
-          }
-        ]
-      },
-
-      { test: /\.json$/, include: [src, nodeModules], loader: "json-loader" },
-      {
-        test: /\.(jpg|png|gif|eot|svg|ttf|woff|woff2)(\?.*)?$/,
-        include: nodeModules,
+        test: /\.(jpg|png|gif|eot|svg|ttf|woff|woff2|otf)$/i,
         loader: "file-loader",
-        query: { name: "static/media/[name].[ext]" }
+        include: [nodeModules, fonts],
+        options: {
+          name: "static/assets/[name].[ext]",
+          // (thuang): This is needed to make sure @font url path is '/static/assets/'
+          publicPath: "/",
+        },
       },
-      {
-        test: /\.(woff|woff2|eot|ttf|otf)$/,
-        include: fonts,
-        loader: "file-loader",
-        query: { name: "static/fonts/[name].[ext]" }
-      },
-      {
-        test: /\.(mp4|webm)(\?.*)?$/,
-        include: [src, nodeModules],
-        loader: "url-loader",
-        query: { limit: 10000, name: "static/media/[name].[ext]" }
-      }
-    ]
+    ],
   },
   plugins: [
     new HtmlWebpackPlugin({
       inject: true,
       template: path.resolve("index.html"),
-      favicon: path.resolve("favicon.png")
+    }),
+    new FaviconsWebpackPlugin({
+      logo: "./favicon.png",
+      prefix: "static/img/",
+      favicons: {
+        icons: {
+          android: false,
+          appleIcon: false,
+          appleStartup: false,
+          coast: false,
+          firefox: false,
+          windows: false,
+          yandex: false,
+        },
+      },
+    }),
+    new MiniCssExtractPlugin({
+      filename: "static/[name].css",
     }),
     new webpack.NoEmitOnErrorsPlugin(),
     new webpack.DefinePlugin({
-      __REACT_DEVTOOLS_GLOBAL_HOOK__: "({ isDisabled: true })"
+      __REACT_DEVTOOLS_GLOBAL_HOOK__: "({ isDisabled: true })",
     }),
     new webpack.DefinePlugin({
-      "process.env.CXG_SERVER_PORT": JSON.stringify(process.env.CXG_SERVER_PORT),
-    })
-  ]
+      // webpack 5 no longer polyfills NodeJS modules, so fake the one we need
+      "process.env": JSON.stringify({
+        NODE_ENV: process.env.NODE_ENV || "development",
+        CXG_SERVER_PORT: process.env.CXG_SERVER_PORT || "5005",
+      }),
+    }),
+  ],
+  infrastructureLogging: {
+    level: "warn",
+  },
 };
+
+module.exports = merge(sharedConfig, devConfig);

client/configuration/webpack/webpack.config.prod.js

@@ -1,124 +1,96 @@
-// jshint esversion: 6
 const path = require("path");
+const webpack = require("webpack");
 const HtmlWebpackPlugin = require("html-webpack-plugin");
+const { CleanWebpackPlugin } = require("clean-webpack-plugin");
+const TerserJSPlugin = require("terser-webpack-plugin");
+const FaviconsWebpackPlugin = require("favicons-webpack-plugin");
 const MiniCssExtractPlugin = require("mini-css-extract-plugin");
-const SWPrecacheWebpackPlugin = require("sw-precache-webpack-plugin");
-const HtmlWebpackInlineSourcePlugin = require("html-webpack-inline-source-plugin");
 
-const src = path.resolve("src");
-const fonts = path.resolve("src/fonts");
-const nodeModules = path.resolve("node_modules");
+const { merge } = require("webpack-merge");
 
 const babelOptions = require("../babel/babel.prod");
 
-const publicPath = "";
+const CspHashPlugin = require("./cspHashPlugin");
+const sharedConfig = require("./webpack.config.shared");
 
-module.exports = {
+const fonts = path.resolve("src/fonts");
+const nodeModules = path.resolve("node_modules");
+
+const prodConfig = {
   mode: "production",
   bail: true,
   cache: false,
-  devtool: "cheap-source-map",
-  entry: ["./src/index.js"],
   output: {
-    path: path.resolve("build"),
-    filename: "static/js/[name].[chunkhash:8].js",
-    publicPath
+    filename: "static/[name]-[contenthash].js",
   },
+  optimization: {
+    minimize: true,
+    minimizer: [new TerserJSPlugin({})],
+  },
+  devtool: "source-map",
   module: {
     rules: [
       {
-        test: /\.js$/,
-        include: src,
+        test: /\.jsx?$/,
         loader: "babel-loader",
-        options: babelOptions
+        options: babelOptions,
       },
       {
-        test: /\.css$/,
-        include: src,
-        exclude: [path.resolve(src, "index.css")],
-        use: [
-          MiniCssExtractPlugin.loader,
-          {
-            loader: "css-loader",
-            options: {
-              modules: {
-                localIdentName: "[name]__[local]___[hash:base64:5]"
-              },
-              importLoaders: 1
-            }
-          }
-        ]
-      },
-      {
-        test: /index\.css$/,
-        include: [path.resolve(src, "index.css")],
-        use: [
-          MiniCssExtractPlugin.loader,
-          {
-            loader: "css-loader",
-            options: {
-              importLoaders: 1
-            }
-          }
-        ]
-      },
-      {
-        test: /\.json$/,
-        include: [src, nodeModules],
-        loader: "json-loader",
-        exclude: /manifest.json$/
-      },
-      {
-        test: /\.(jpg|png|gif|eot|svg|ttf|woff|woff2)(\?.*)?$/,
-        include: nodeModules,
+        test: /\.(jpg|png|gif|eot|svg|ttf|woff|woff2|otf)$/i,
         loader: "file-loader",
-        query: { name: "static/media/[name].[ext]" }
+        include: [nodeModules, fonts],
+        options: {
+          name: "static/assets/[name]-[contenthash].[ext]",
+          // (thuang): This is needed to make sure @font url path is '../static/assets/'
+          publicPath: "static/",
+        },
       },
-      {
-        test: /\.(woff|woff2|eot|ttf|otf)$/,
-        include: fonts,
-        loader: "file-loader",
-        query: { name: "static/fonts/[name].[ext]" }
-      },
-      {
-        test: /\.(mp4|webm)(\?.*)?$/,
-        include: [src, nodeModules],
-        loader: "url-loader",
-        query: { limit: 10000, name: "static/media/[name].[hash:8].[ext]" }
-      }
-    ]
+    ],
   },
   plugins: [
     new HtmlWebpackPlugin({
-      inject: "body",
       filename: "index.html",
       template: path.resolve("index_template.html"),
-      favicon: path.resolve("favicon.png"),
-      inlineSource: ".(js|css)$",
-      minify: {
-        removeComments: true,
-        collapseWhitespace: true,
-        removeRedundantAttributes: true,
-        useShortDoctype: true,
-        removeEmptyAttributes: true,
-        removeStyleLinkTypeAttributes: true,
-        keepClosingSlash: true,
-        minifyJS: true,
-        minifyCSS: true,
-        minifyURLs: true
-      }
+      decodeEntities: false,
+      minify: false,
+    }),
+    new CleanWebpackPlugin({
+      verbose: true,
+      protectWebpackAssets: false,
+      cleanAfterEveryBuildPatterns: ["main.js", "main.css"],
+    }),
+    new FaviconsWebpackPlugin({
+      logo: "./favicon.png",
+      prefix: "static/assets/",
+      favicons: {
+        icons: {
+          android: false,
+          appleIcon: false,
+          appleStartup: false,
+          coast: false,
+          firefox: false,
+          windows: false,
+          yandex: false,
+        },
+      },
     }),
-    new HtmlWebpackInlineSourcePlugin(),
     new MiniCssExtractPlugin({
-      filename: "static/css/[name].[contenthash:8].css"
+      filename: "static/[name]-[contenthash].css",
+    }),
+    new CspHashPlugin({
+      filename: "csp-hashes.json",
+    }),
+    new webpack.DefinePlugin({
+      // webpack 5 no longer polyfills NodeJS modules, so fake the one we need
+      "process.env": JSON.stringify({
+        NODE_ENV: "production",
+      }),
     }),
-    new SWPrecacheWebpackPlugin({
-      cacheId: "cellxgene",
-      filename: "service-worker.js"
-    })
   ],
   performance: {
     maxEntrypointSize: 2000000,
-    maxAssetSize: 2000000
-  }
+    maxAssetSize: 2000000,
+  },
 };
+
+module.exports = merge(sharedConfig, prodConfig);

client/configuration/webpack/webpack.config.shared.js

@@ -0,0 +1,54 @@
+const path = require("path");
+const MiniCssExtractPlugin = require("mini-css-extract-plugin");
+// eslint-disable-next-line @blueprintjs/classes-constants -- incorrect match
+
+const src = path.resolve("src");
+const nodeModules = path.resolve("node_modules");
+
+const publicPath = "";
+
+module.exports = {
+  entry: [
+    "core-js",
+    "regenerator-runtime/runtime",
+    "whatwg-fetch",
+    "abort-controller/polyfill",
+    "./src/index",
+  ],
+  output: {
+    path: path.resolve("build"),
+    publicPath,
+  },
+  module: {
+    rules: [
+      {
+        test: /\.css$/,
+        include: src,
+        exclude: [path.resolve(src, "index.css")],
+        use: [
+          MiniCssExtractPlugin.loader,
+          {
+            loader: "css-loader",
+            options: {
+              modules: {
+                localIdentName: "[name]__[local]___[contenthash:base64:5]",
+              },
+              importLoaders: 1,
+            },
+          },
+        ],
+      },
+      {
+        test: /index\.css$/,
+        include: [path.resolve(src, "index.css")],
+        use: [MiniCssExtractPlugin.loader, "css-loader"],
+      },
+      {
+        test: /\.json$/,
+        include: [src, nodeModules],
+        loader: "json-loader",
+        exclude: /manifest.json$/,
+      },
+    ],
+  },
+};

client/index.html

@@ -1,12 +1,27 @@
 <!DOCTYPE html>
 <html lang="en">
   <head>
-    <meta charset="utf-8">
-    <meta name="viewport" content="width=device-width, initial-scale=1">
+    <meta charset="utf-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1" />
     <title>cell&times;gene</title>
     <style>
-      html, body, p, h1, h2, h3, h4, h5, h6, span, button, input, label, text, div {
-        font-family: 'Roboto Condensed','Helvetica Neue','Helvetica','Arial',sans-serif;
+      html,
+      body,
+      p,
+      h1,
+      h2,
+      h3,
+      h4,
+      h5,
+      h6,
+      span,
+      button,
+      input,
+      label,
+      text,
+      div {
+        font-family: "Roboto Condensed", "Helvetica Neue", "Helvetica", "Arial",
+          sans-serif;
         font-size: 14px;
       }
       body {
@@ -20,9 +35,12 @@
     </style>
   </head>
   <body>
-    <noscript>If you're seeing this message, that means <strong>JavaScript has been disabled on your browser</strong>, please <strong>enable JS</strong> to make this app work.</noscript>
+    <noscript
+      >If you're seeing this message, that means
+      <strong>JavaScript has been disabled on your browser</strong>, please
+      <strong>enable JS</strong> to make this app work.</noscript
+    >
 
     <div id="root"></div>
-
   </body>
 </html>

client/index_template.html

@@ -1,41 +1,65 @@
 <!DOCTYPE html>
 <html lang="en">
-<head>
-    <meta charset="utf-8">
-    <meta name="viewport" content="width=device-width, initial-scale=1">
+  <head>
+    <meta charset="utf-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1" />
     <title>cell&times;gene</title>
     <style>
-        html, body, p, h1, h2, h3, h4, h5, h6, span, button, input, label, text, div {
-            font-family: 'Roboto Condensed', 'Helvetica Neue', 'Helvetica', 'Arial', sans-serif;
-            font-size: 14px;
-        }
+      html,
+      body,
+      p,
+      h1,
+      h2,
+      h3,
+      h4,
+      h5,
+      h6,
+      span,
+      button,
+      input,
+      label,
+      text,
+      div {
+        font-family: "Roboto Condensed", "Helvetica Neue", "Helvetica", "Arial",
+          sans-serif;
+        font-size: 14px;
+      }
 
-        body {
-            margin: 0;
-            padding: 0;
-        }
+      body {
+        margin: 0;
+        padding: 0;
+      }
 
-        * {
-            box-sizing: border-box;
-        }
+      * {
+        box-sizing: border-box;
+      }
     </style>
+  </head>
+  <body>
+    <script type="text/javascript">
+      window.CELLXGENE = {};
+      window.CELLXGENE.API = {
+        prefix: `${location.origin}${location.pathname}api/`,
+        version: "v0.2/",
+      };
+    </script>
+    <noscript
+      >If you're seeing this message, that means
+      <strong>JavaScript has been disabled on your browser</strong>, please
+      <strong>enable JS</strong> to make this app work.
+    </noscript>
 
-</head>
-<body>
-<script type="text/javascript">
-  window.CELLXGENE = {};
-  window.CELLXGENE.API = {
-    prefix: window.location.href + "api/",
-    version: "v0.2/"
-  };
-</script>
-<noscript>If you're seeing this message, that means <strong>JavaScript has been disabled on your browser</strong>,
-    please <strong>enable JS</strong> to make this app work.
-</noscript>
+    <div id="root"></div>
+    {% for script in SCRIPTS -%}
+    <script type="text/javascript"
+      {{ ('integrity="%s"' % script.integrity) | safe if script.integrity }}
+      {{ ('crossorigin="%s"' % script.crossorigin) | safe if script.crossorigin }}
+      src="{{script.src | safe}}"
+    ></script>
+    {%- endfor %}
 
-<div id="root"></div>
-{% for script in SCRIPTS %}
-<script type="text/javascript" src="{{script | safe}}"></script>
-{% endfor %}
-</body>
+    {% for ils in INLINE_SCRIPTS %}
+    <script type="text/javascript" no-csp-hash>{% include ils %}</script>
+    {% endfor %}
+  </body>
 </html>

client/jest-puppeteer.config.js

@@ -0,0 +1,46 @@
+/**
+ * `client/jest-puppeteer.config.js` is for configuring Puppeteer's launch config options
+ * `client/__tests__/e2e/puppeteer.setup.js` is for configuring `jest`, `browser`,
+ * and `page` objects
+ */
+
+const ENV_DEFAULT = require("../environment.default.json");
+
+const jestEnv = process.env.JEST_ENV || ENV_DEFAULT.JEST_ENV;
+const isHeadful =
+  process.env.HEADFUL === "true" || process.env.HEADLESS === "false";
+
+const DEFAULT_LAUNCH_CONFIG = {
+  headless: !isHeadful,
+  args: ["--ignore-certificate-errors", "--ignore-ssl-errors"],
+  ignoreHTTPSErrors: true,
+  defaultViewport: {
+    width: 1280,
+    height: 960,
+  },
+};
+
+const LAUNCH_CONFIG_BY_ENV = {
+  [ENV_DEFAULT.DEBUG]: {
+    ...DEFAULT_LAUNCH_CONFIG,
+    headless: false,
+    slowMo: 100,
+    devtools: true,
+    defaultViewport: {
+      width: DEFAULT_LAUNCH_CONFIG.defaultViewport.width,
+      height: DEFAULT_LAUNCH_CONFIG.defaultViewport.height + 560,
+    },
+  },
+  [ENV_DEFAULT.DEV]: {
+    ...DEFAULT_LAUNCH_CONFIG,
+    headless: false,
+    slowMo: 5,
+  },
+};
+
+const launchConfig = LAUNCH_CONFIG_BY_ENV[jestEnv] || DEFAULT_LAUNCH_CONFIG;
+
+module.exports = {
+  browserContext: "incognito",
+  launch: launchConfig,
+};

client/package.json

@@ -1,15 +1,20 @@
 {
   "name": "cellxgene",
-  "version": "0.15.0",
+  "version": "1.0.0",
   "license": "MIT",
   "description": "cellxgene is a web application for the interactive exploration of single cell sequence data.",
   "repository": "https://github.com/chanzuckerberg/cellxgene",
   "scripts": {
+    "build": "npm run clean && webpack --config",
     "clean": "rimraf build",
-    "build": "npm run clean && webpack --config configuration/webpack/webpack.config.prod.js",
-    "dev": "npm run clean && webpack --config configuration/webpack/webpack.config.dev.js",
-    "fmt": "eslint --fix src",
-    "lint": "eslint src"
+    "dev": "npm run build -- configuration/webpack/webpack.config.dev.js",
+    "e2e": "jest --config __tests__/e2e/e2eJestConfig.json e2e/e2e.test.js",
+    "e2e-annotations": "jest --config __tests__/e2e/e2eJestConfig.json e2e/e2eAnnotations.test.js",
+    "fmt": "eslint --fix src __tests__",
+    "lint": "eslint --fix src __tests__",
+    "prod": "npm run build -- configuration/webpack/webpack.config.prod.js",
+    "test": "jest --testPathIgnorePatterns e2e",
+    "prepare": "cd .. && husky install client/.husky"
   },
   "engineStrict": true,
   "engines": {
@@ -24,87 +29,126 @@
   "resolutions": {
     "eslint-scope": "3.7.1"
   },
+  "browserslist": [
+    "Chrome > 60",
+    "iOS >= 10.3",
+    "Firefox >= 60",
+    "Edge >= 79",
+    "not Explorer > 0",
+    "not Safari > 0"
+  ],
   "dependencies": {
-    "@blueprintjs/core": "^3.24.0",
-    "@blueprintjs/icons": "^3.14.0",
-    "@blueprintjs/select": "^3.12.0",
-    "client": "0.0.1",
+    "@babel/eslint-parser": "^7.15.0",
+    "@blueprintjs/core": "^3.44.0",
+    "@blueprintjs/icons": "^3.19.0",
+    "@blueprintjs/popover2": "^0.11.2",
+    "@blueprintjs/select": "^3.16.0",
+    "abort-controller": "^3.0.0",
+    "core-js": "^3.16.3",
     "d3": "^4.10.0",
     "d3-scale-chromatic": "^1.5.0",
     "flatbuffers": "^1.11.0",
     "fuzzysort": "^1.1.4",
     "gl-mat4": "^1.2.0",
-    "gl-matrix": "^3.2.1",
+    "gl-matrix": "^3.3.0",
     "gl-vec3": "^1.1.3",
     "is-number": "^7.0.0",
-    "lodash": "^4.17.15",
+    "lodash.clonedeep": "^4.5.0",
+    "lodash.difference": "^4.5.0",
+    "lodash.foreach": "^4.5.0",
+    "lodash.pull": "^4.1.0",
+    "lodash.sortby": "^4.7.0",
+    "lodash.uniq": "^4.5.0",
     "memoize-one": "^5.1.1",
-    "react": "^16.13.0",
-    "react-dom": "^16.13.0",
-    "react-flip-toolkit": "7.0.6",
-    "react-helmet": "^5.2.1",
-    "react-icons": "^3.9.0",
+    "pako": "^2.0.3",
+    "react": "^17.0.2",
+    "react-async": "^10.0.1",
+    "react-dom": "^17.0.2",
+    "react-flip-toolkit": "^7.0.12",
+    "react-helmet": "^6.1.0",
+    "react-icons": "^4.2.0",
     "react-redux": "^7.2.0",
     "redux": "^4.0.5",
     "redux-thunk": "^2.3.0",
-    "regl": "^1.4.0"
+    "regenerator-runtime": "^0.13.7",
+    "regl": "^2.1.0",
+    "sha1": "^1.1.1",
+    "tinyqueue": "^2.0.3",
+    "whatwg-fetch": "^3.2.0"
   },
   "devDependencies": {
-    "@babel/core": "^7.8.7",
-    "@babel/plugin-proposal-class-properties": "^7.8.3",
-    "@babel/plugin-proposal-decorators": "^7.8.3",
-    "@babel/plugin-proposal-export-namespace-from": "^7.8.3",
-    "@babel/plugin-proposal-function-bind": "^7.8.3",
-    "@babel/plugin-proposal-nullish-coalescing-operator": "^7.8.3",
-    "@babel/plugin-proposal-optional-chaining": "^7.8.3",
-    "@babel/plugin-transform-react-constant-elements": "^7.8.3",
-    "@babel/plugin-transform-runtime": "^7.8.3",
-    "@babel/preset-env": "^7.8.7",
-    "@babel/preset-react": "^7.8.3",
-    "@babel/register": "^7.8.6",
-    "@babel/runtime": "^7.8.7",
-    "babel-eslint": "^10.1.0",
-    "babel-jest": "^25.1.0",
-    "babel-loader": "^8.0.6",
-    "babel-preset-modern-browsers": "^14.1.1",
-    "chalk": "^3.0.0",
-    "codecov": "^3.6.5",
-    "connect-history-api-fallback": "^1.6.0",
-    "copy-webpack-plugin": "^5.1.1",
-    "css-loader": "^3.4.2",
-    "eslint": "^6.8.0",
-    "eslint-config-airbnb": "^18.0.1",
-    "eslint-config-prettier": "^6.10.0",
-    "eslint-loader": "^3.0.3",
+    "@babel/core": "^7.13.16",
+    "@babel/plugin-proposal-class-properties": "^7.10.4",
+    "@babel/plugin-proposal-decorators": "^7.13.15",
+    "@babel/plugin-proposal-export-namespace-from": "^7.10.4",
+    "@babel/plugin-proposal-function-bind": "^7.10.5",
+    "@babel/plugin-proposal-nullish-coalescing-operator": "^7.10.4",
+    "@babel/plugin-proposal-optional-chaining": "^7.10.4",
+    "@babel/plugin-proposal-private-methods": "^7.18.6",
+    "@babel/plugin-proposal-private-property-in-object": "^7.21.11",
+    "@babel/plugin-transform-react-constant-elements": "^7.13.13",
+    "@babel/plugin-transform-runtime": "^7.13.15",
+    "@babel/preset-env": "^7.13.15",
+    "@babel/preset-react": "^7.13.13",
+    "@babel/register": "^7.13.16",
+    "@babel/runtime": "^7.13.16",
+    "@blueprintjs/eslint-plugin": "^0.3.0",
+    "@sentry/webpack-plugin": "^1.15.0",
+    "babel-jest": "^26.1.0",
+    "babel-loader": "^8.1.0",
+    "babel-preset-modern-browsers": "^15.0.2",
+    "chalk": "^4.1.0",
+    "cheerio": "^1.0.0-rc.6",
+    "clean-css": "^5.1.2",
+    "clean-webpack-plugin": "^4.0.0-alpha.0",
+    "codecov": "^3.7.1",
+    "css-loader": "^5.2.4",
+    "eslint": "^7.24.0",
+    "eslint-config-airbnb": "^18.2.0",
+    "eslint-config-prettier": "^8.2.0",
+    "eslint-plugin-compat": "^3.8.0",
+    "eslint-plugin-eslint-comments": "^3.2.0",
     "eslint-plugin-filenames": "^1.3.2",
-    "eslint-plugin-import": "^2.20.1",
-    "eslint-plugin-jest": "^23.8.2",
-    "eslint-plugin-jsx-a11y": "^6.2.3",
-    "eslint-plugin-react": "^7.19.0",
-    "eslint-plugin-react-hooks": "^2.5.0",
+    "eslint-plugin-import": "^2.24.2",
+    "eslint-plugin-jest": "^24.3.5",
+    "eslint-plugin-jsx-a11y": "^6.3.1",
+    "eslint-plugin-react": "^7.23.2",
+    "eslint-plugin-react-hooks": "^4.0.8",
+    "expect-puppeteer": "^5.0.0",
     "express": "^4.17.1",
-    "file-loader": "^5.1.0",
-    "html-webpack-inline-source-plugin": "0.0.10",
-    "html-webpack-plugin": "^3.2.0",
-    "jest": "^25.1.0",
-    "jest-puppeteer": "^4.4.0",
+    "favicons": "^6.2.2",
+    "favicons-webpack-plugin": "^5.0.2",
+    "file-loader": "^6.0.0",
+    "html-webpack-plugin": "^5.3.1",
+    "husky": "^7.0.2",
+    "jest": "^27.0.6",
+    "jest-circus": "^27.0.6",
+    "jest-environment-puppeteer": "^5.0.1",
+    "jest-fetch-mock": "^3.0.3",
+    "jest-puppeteer": "^5.0.1",
     "json-loader": "^0.5.7",
-    "mini-css-extract-plugin": "^0.9.0",
-    "puppeteer": "^2.1.1",
+    "lint-staged": "^10.2.11",
+    "lodash": "^4.17.21",
+    "lodash.every": "^4.6.0",
+    "lodash.filter": "^4.6.0",
+    "lodash.isnumber": "^3.0.3",
+    "lodash.map": "^4.6.0",
+    "lodash.zip": "^4.2.0",
+    "mini-css-extract-plugin": "^1.5.0",
+    "prettier": "^2.0.5",
+    "puppeteer": "^8.0.0",
     "rimraf": "^3.0.2",
     "serve-favicon": "^2.5.0",
-    "style-loader": "^1.1.3",
-    "sw-precache-webpack-plugin": "^1.0.0",
-    "url-loader": "^3.0.0",
-    "webpack": "^4.42.0",
-    "webpack-cli": "^3.3.11",
-    "webpack-dev-middleware": "^3.7.2"
+    "terser-webpack-plugin": "^5.1.1",
+    "webpack": "^5.34.0",
+    "webpack-cli": "^4.6.0",
+    "webpack-dev-middleware": "^4.1.0",
+    "webpack-merge": "^5.0.9"
   },
   "jest": {
     "testMatch": [
       "**/__tests__/**/?(*.)(spec|test).js?(x)"
     ],
-    "testURL": "http://localhost/",
     "setupFiles": [
       "./__tests__/setupMissingGlobals.js"
     ],
@@ -132,6 +176,18 @@
               "loose": true
             }
           ],
+          [
+            "@babel/plugin-proposal-private-methods",
+            {
+              "loose": true
+            }
+          ],
+          [
+            "@babel/plugin-proposal-private-property-in-object",
+            {
+              "loose": true
+            }
+          ],
           "@babel/plugin-proposal-export-namespace-from",
           "@babel/plugin-transform-react-constant-elements",
           "@babel/plugin-transform-runtime",

client/server/development.js

@@ -1,49 +1,53 @@
-/* eslint-disable */
-// jshint esversion: 6
-var path = require("path");
-var historyApiFallback = require("connect-history-api-fallback");
-var chalk = require("chalk");
-var express = require("express");
-var favicon = require("serve-favicon");
-var webpack = require("webpack");
-var config = require("../configuration/webpack/webpack.config.dev");
-var utils = require("./utils");
+const chalk = require("chalk");
+const express = require("express");
+const favicon = require("serve-favicon");
+const webpack = require("webpack");
+const devMiddleware = require("webpack-dev-middleware");
+const config = require("../configuration/webpack/webpack.config.dev");
+const utils = require("./utils");
 
 process.env.NODE_ENV = "development";
 
 const CLIENT_PORT = process.env.CXG_CLIENT_PORT;
+const { CXG_SERVER_PORT } = process.env;
+
+const API = {
+  prefix: `http://localhost:${CXG_SERVER_PORT}/`,
+};
 
 // Set up compiler
-var compiler = webpack(config);
+const compiler = webpack(config);
 
-compiler.plugin("invalid", () => {
+compiler.hooks.invalid.tap("invalid", () => {
   utils.clearConsole();
   console.log("Compiling...");
 });
 
-compiler.plugin("done", stats => {
+compiler.hooks.done.tap("done", (stats) => {
   utils.formatStats(stats, CLIENT_PORT);
 });
 
 // Launch server
-var app = express();
-
-app.use(historyApiFallback({ verbose: false }));
+const app = express();
 
 app.use(
-  require("webpack-dev-middleware")(compiler, {
-    logLevel: "warn",
-    publicPath: config.output.publicPath
+  devMiddleware(compiler, {
+    publicPath: config.output.publicPath,
+    index: true,
   })
 );
 
 app.use(favicon("./favicon.png"));
 
-app.get("*", (req, res) => {
-  res.sendFile(path.resolve("index.html"));
+app.get("/logout", async (req, res) => {
+  try {
+    res.redirect(`${API.prefix}logout?dataset=http://localhost:${CLIENT_PORT}`);
+  } catch (err) {
+    console.error(err);
+  }
 });
 
-app.listen(CLIENT_PORT, err => {
+app.listen(CLIENT_PORT, (err) => {
   if (err) {
     console.log(err);
     return;

client/server/utils.js

@@ -8,7 +8,9 @@ function isLikelyASyntaxError(message) {
   return message.indexOf(friendlySyntaxErrorLabel) !== -1;
 }
 
-function formatMessage(message) {
+function formatMessage(messageObject) {
+  let { message, details } = messageObject;
+  if (details) message = message + ": " + details;
   return message
     .replace("Module build failed: SyntaxError:", friendlySyntaxErrorLabel)
     .replace(
@@ -36,10 +38,10 @@ var formatStats = (stats, port) => {
 
   var json = stats.toJson();
   var formattedErrors = json.errors.map(
-    message => "Error in " + formatMessage(message)
+    (message) => "Error in " + formatMessage(message)
   );
   var formattedWarnings = json.warnings.map(
-    message => "Warning in " + formatMessage(message)
+    (message) => "Warning in " + formatMessage(message)
   );
 
   if (hasErrors) {
@@ -48,7 +50,7 @@ var formatStats = (stats, port) => {
     if (formattedErrors.some(isLikelyASyntaxError)) {
       formattedErrors = formattedErrors.filter(isLikelyASyntaxError);
     }
-    formattedErrors.forEach(message => {
+    formattedErrors.forEach((message) => {
       console.log(message);
       console.log();
     });
@@ -58,7 +60,7 @@ var formatStats = (stats, port) => {
   if (hasWarnings) {
     console.log(chalk.yellow("Compiled with warnings."));
     console.log();
-    formattedWarnings.forEach(message => {
+    formattedWarnings.forEach((message) => {
       console.log(message);
       console.log();
     });

client/src/actions/annotation.js

@@ -0,0 +1,455 @@
+/*
+Action creators for user annotation
+*/
+import difference from "lodash.difference";
+import pako from "pako";
+import * as globals from "../globals";
+import { MatrixFBS, AnnotationsHelpers } from "../util/stateManager";
+
+const { isUserAnnotation } = AnnotationsHelpers;
+
+export const annotationCreateCategoryAction =
+  (newCategoryName, categoryToDuplicate) => async (dispatch, getState) => {
+    /*
+  Add a new user-created category to the obs annotations.
+
+  Arguments:
+    newCategoryName - string name for the category.
+    categoryToDuplicate - obs category to use for initial values, or null.
+  */
+    const { annoMatrix: prevAnnoMatrix, obsCrossfilter: prevObsCrossfilter } =
+      getState();
+    if (!prevAnnoMatrix || !prevObsCrossfilter) return;
+    const { schema } = prevAnnoMatrix;
+
+    /* name must be a string,  non-zero length */
+    if (typeof newCategoryName !== "string" || newCategoryName.length === 0)
+      throw new Error("user annotations require string name");
+
+    /* ensure the name isn't already in use! */
+    if (schema.annotations.obsByName[newCategoryName])
+      throw new Error("name collision on annotation category create");
+
+    let initialValue;
+    let newSchema;
+    let ctor;
+    if (categoryToDuplicate) {
+      /* if we are duplicating a category, retrieve it */
+      const catDupSchema = schema.annotations.obsByName[categoryToDuplicate];
+      const catDupType = catDupSchema?.type;
+      if (catDupType !== "string" && catDupType !== "categorical")
+        throw new Error(
+          "categoryToDuplicate does not exist or has invalid type"
+        );
+
+      const catToDupDf = await prevAnnoMatrix
+        .base()
+        .fetch("obs", categoryToDuplicate);
+      const col = catToDupDf.col(categoryToDuplicate);
+      initialValue = col.asArray();
+      const { categories } = col.summarizeCategorical();
+      // all user-created annotations must have the unassigned category
+      if (!categories.includes(globals.unassignedCategoryLabel)) {
+        categories.push(globals.unassignedCategoryLabel);
+      }
+      ctor = initialValue.constructor;
+      newSchema = {
+        ...catDupSchema,
+        name: newCategoryName,
+        categories,
+        writable: true,
+      };
+    } else {
+      /* else assign to the standard default value */
+      initialValue = globals.unassignedCategoryLabel;
+      ctor = Array;
+      newSchema = {
+        name: newCategoryName,
+        type: "categorical",
+        categories: [globals.unassignedCategoryLabel],
+        writable: true,
+      };
+    }
+
+    const obsCrossfilter = prevObsCrossfilter.addObsColumn(
+      newSchema,
+      ctor,
+      initialValue
+    );
+
+    dispatch({
+      type: "annotation: create category",
+      data: newCategoryName,
+      categoryToDuplicate,
+      annoMatrix: obsCrossfilter.annoMatrix,
+      obsCrossfilter,
+    });
+  };
+
+export const annotationRenameCategoryAction =
+  (oldCategoryName, newCategoryName) => (dispatch, getState) => {
+    /*
+  Rename a user-created annotation category
+  */
+    const { annoMatrix: prevAnnoMatrix, obsCrossfilter: prevObsCrossfilter } =
+      getState();
+    if (!prevAnnoMatrix || !prevObsCrossfilter) return;
+    if (!isUserAnnotation(prevAnnoMatrix, oldCategoryName))
+      throw new Error("not a user annotation");
+
+    /* name must be a string,  non-zero length */
+    if (typeof newCategoryName !== "string" || newCategoryName.length === 0)
+      throw new Error("user annotations require string name");
+
+    if (oldCategoryName === newCategoryName) return;
+
+    const obsCrossfilter = prevObsCrossfilter.renameObsColumn(
+      oldCategoryName,
+      newCategoryName
+    );
+
+    dispatch({
+      type: "annotation: category edited",
+      annoMatrix: obsCrossfilter.annoMatrix,
+      obsCrossfilter,
+      metadataField: oldCategoryName,
+      newCategoryText: newCategoryName,
+      data: newCategoryName,
+    });
+  };
+
+export const annotationDeleteCategoryAction =
+  (categoryName) => (dispatch, getState) => {
+    /*
+  Delete a user-created category
+  */
+    const { annoMatrix: prevAnnoMatrix, obsCrossfilter: prevObsCrossfilter } =
+      getState();
+    if (!prevAnnoMatrix || !prevObsCrossfilter) return;
+    if (!isUserAnnotation(prevAnnoMatrix, categoryName))
+      throw new Error("not a user annotation");
+
+    const obsCrossfilter = prevObsCrossfilter.dropObsColumn(categoryName);
+    dispatch({
+      type: "annotation: delete category",
+      annoMatrix: obsCrossfilter.annoMatrix,
+      obsCrossfilter,
+      metadataField: categoryName,
+    });
+  };
+
+export const annotationCreateLabelInCategory =
+  (categoryName, labelName, assignSelected) => async (dispatch, getState) => {
+    /*
+  Add a new label to a user-defined category.  If assignSelected is true, assign
+  the label to all currently selected cells.
+  */
+    const { annoMatrix: prevAnnoMatrix, obsCrossfilter: prevObsCrossfilter } =
+      getState();
+    if (!prevAnnoMatrix || !prevObsCrossfilter) return;
+    if (!isUserAnnotation(prevAnnoMatrix, categoryName))
+      throw new Error("not a user annotation");
+
+    let obsCrossfilter = prevObsCrossfilter.addObsAnnoCategory(
+      categoryName,
+      labelName
+    );
+    if (assignSelected) {
+      obsCrossfilter = await obsCrossfilter.setObsColumnValues(
+        categoryName,
+        prevObsCrossfilter.allSelectedLabels(),
+        labelName
+      );
+    }
+
+    dispatch({
+      type: "annotation: add new label to category",
+      annoMatrix: obsCrossfilter.annoMatrix,
+      obsCrossfilter,
+      metadataField: categoryName,
+      newLabelText: labelName,
+      assignSelectedCells: assignSelected,
+    });
+  };
+
+export const annotationDeleteLabelFromCategory =
+  (categoryName, labelName) => async (dispatch, getState) => {
+    /*
+  delete a label from a user-defined category
+  */
+    const { annoMatrix: prevAnnoMatrix, obsCrossfilter: prevObsCrossfilter } =
+      getState();
+    if (!prevAnnoMatrix || !prevObsCrossfilter) return;
+    if (!isUserAnnotation(prevAnnoMatrix, categoryName))
+      throw new Error("not a user annotation");
+
+    const obsCrossfilter = await prevObsCrossfilter.removeObsAnnoCategory(
+      categoryName,
+      labelName,
+      globals.unassignedCategoryLabel
+    );
+
+    dispatch({
+      type: "annotation: delete label",
+      metadataField: categoryName,
+      label: labelName,
+      annoMatrix: obsCrossfilter.annoMatrix,
+      obsCrossfilter,
+    });
+  };
+
+export const annotationRenameLabelInCategory =
+  (categoryName, oldLabelName, newLabelName) => async (dispatch, getState) => {
+    /*
+  label name change
+  */
+    const { annoMatrix: prevAnnoMatrix, obsCrossfilter: prevObsCrossfilter } =
+      getState();
+    if (!prevAnnoMatrix || !prevObsCrossfilter) return;
+    if (!isUserAnnotation(prevAnnoMatrix, categoryName))
+      throw new Error("not a user annotation");
+
+    let obsCrossfilter = await prevObsCrossfilter.resetObsColumnValues(
+      categoryName,
+      oldLabelName,
+      newLabelName
+    );
+    obsCrossfilter = await obsCrossfilter.removeObsAnnoCategory(
+      categoryName,
+      oldLabelName,
+      globals.unassignedCategoryLabel
+    );
+
+    dispatch({
+      type: "annotation: label edited",
+      editedLabel: newLabelName,
+      metadataField: categoryName,
+      label: oldLabelName,
+      annoMatrix: obsCrossfilter.annoMatrix,
+      obsCrossfilter,
+    });
+  };
+
+export const annotationLabelCurrentSelection =
+  (categoryName, labelName) => async (dispatch, getState) => {
+    /*
+  set the label on all currently selected
+  */
+    const { annoMatrix: prevAnnoMatrix, obsCrossfilter: prevObsCrossfilter } =
+      getState();
+    if (!prevAnnoMatrix || !prevObsCrossfilter) return;
+    if (!isUserAnnotation(prevAnnoMatrix, categoryName))
+      throw new Error("not a user annotation");
+
+    const obsCrossfilter = await prevObsCrossfilter.setObsColumnValues(
+      categoryName,
+      prevObsCrossfilter.allSelectedLabels(),
+      labelName
+    );
+
+    dispatch({
+      type: "annotation: label current cell selection",
+      metadataField: categoryName,
+      label: labelName,
+      obsCrossfilter,
+      annoMatrix: obsCrossfilter.annoMatrix,
+    });
+  };
+
+function writableAnnotations(annoMatrix) {
+  return annoMatrix.schema.annotations.obs.columns
+    .filter((s) => s.writable)
+    .map((s) => s.name);
+}
+
+export const needToSaveObsAnnotations = (annoMatrix, lastSavedAnnoMatrix) => {
+  /*
+  Return true if there are LIKELY user-defined annotation modifications between the two
+  annoMatrices.  Technically not an action creator, but intimately intertwined
+  with the save process.
+
+  Two conditions will trigger a need to save:
+    * the collection of user-defined columns have changed
+    * the contents of the user-defined columns have change
+  */
+
+  annoMatrix = annoMatrix.base();
+
+  // if the annoMatrix hasn't changed, we are guaranteed no changes to the matrix schema or contents.
+  if (annoMatrix === lastSavedAnnoMatrix) return false;
+
+  // if the schema has changed, we need to save
+  const currentWritable = writableAnnotations(annoMatrix);
+  if (difference(currentWritable, writableAnnotations(lastSavedAnnoMatrix))) {
+    return true;
+  }
+
+  // no schema changes; check for change in contents
+  return currentWritable.some(
+    (col) => annoMatrix.col(col) !== lastSavedAnnoMatrix.col(col)
+  );
+};
+
+export const saveObsAnnotationsAction = () => async (dispatch, getState) => {
+  /*
+  Save the user-created obs annotations IF any have changed.
+  */
+  const state = getState();
+  const { annotations, autosave } = state;
+  const { dataCollectionNameIsReadOnly, dataCollectionName } = annotations;
+  const { lastSavedAnnoMatrix, saveInProgress } = autosave;
+
+  const annoMatrix = state.annoMatrix.base();
+
+  if (saveInProgress || annoMatrix === lastSavedAnnoMatrix) return;
+  if (!needToSaveObsAnnotations(annoMatrix, lastSavedAnnoMatrix)) {
+    dispatch({
+      type: "writable obs annotations - save complete",
+      lastSavedAnnoMatrix: annoMatrix,
+    });
+    return;
+  }
+
+  /*
+  Else, we really do need to save
+  */
+
+  dispatch({
+    type: "writable obs annotations - save started",
+  });
+
+  const df = await annoMatrix.fetch("obs", writableAnnotations(annoMatrix));
+  const matrix = MatrixFBS.encodeMatrixFBS(df);
+  const compressedMatrix = pako.deflate(matrix);
+  try {
+    const queryString =
+      !dataCollectionNameIsReadOnly && !!dataCollectionName
+        ? `?annotation-collection-name=${encodeURIComponent(
+            dataCollectionName
+          )}`
+        : "";
+    const res = await fetch(
+      `${globals.API.prefix}${globals.API.version}annotations/obs${queryString}`,
+      {
+        method: "PUT",
+        body: compressedMatrix,
+        headers: new Headers({
+          "Content-Type": "application/octet-stream",
+        }),
+        credentials: "include",
+      }
+    );
+    if (res.ok) {
+      dispatch({
+        type: "writable obs annotations - save complete",
+        lastSavedAnnoMatrix: annoMatrix,
+      });
+    } else {
+      dispatch({
+        type: "writable obs annotations - save error",
+        message: `HTTP error ${res.status} - ${res.statusText}`,
+        res,
+      });
+    }
+  } catch (error) {
+    dispatch({
+      type: "writable obs annotations - save error",
+      message: error.toString(),
+      error,
+    });
+  }
+};
+
+export const saveGenesetsAction = () => async (dispatch, getState) => {
+  const state = getState();
+
+  // bail if gene sets not available, or in readonly mode.
+  const { config } = state;
+  const { lastTid, genesets } = state.genesets;
+
+  const genesetsAreAvailable =
+    config?.parameters?.annotations_genesets ?? false;
+  const genesetsReadonly =
+    config?.parameters?.annotations_genesets_readonly ?? true;
+  if (!genesetsAreAvailable || genesetsReadonly) {
+    // our non-save was completed!
+    return dispatch({
+      type: "autosave: genesets complete",
+      lastSavedGenesets: genesets,
+    });
+  }
+
+  dispatch({
+    type: "autosave: genesets started",
+  });
+
+  /* Create the JSON OTA data structure */
+  const tid = (lastTid ?? 0) + 1;
+  const genesetsOTA = [];
+  for (const [name, gs] of genesets) {
+    const genes = [];
+    for (const g of gs.genes.values()) {
+      genes.push({
+        gene_symbol: g.geneSymbol,
+        gene_description: g.geneDescription,
+      });
+    }
+    genesetsOTA.push({
+      geneset_name: name,
+      geneset_description: gs.genesetDescription,
+      genes,
+    });
+  }
+  const ota = {
+    tid,
+    genesets: genesetsOTA,
+  };
+
+  /* Save to server */
+  try {
+    const { dataCollectionNameIsReadOnly, dataCollectionName } =
+      state.annotations;
+    const queryString =
+      !dataCollectionNameIsReadOnly && !!dataCollectionName
+        ? `?annotation-collection-name=${encodeURIComponent(
+            dataCollectionName
+          )}`
+        : "";
+
+    const res = await fetch(
+      `${globals.API.prefix}${globals.API.version}genesets${queryString}`,
+      {
+        method: "PUT",
+        headers: new Headers({
+          Accept: "application/json",
+          "Content-Type": "application/json",
+        }),
+        body: JSON.stringify(ota),
+        credentials: "include",
+      }
+    );
+    if (!res.ok) {
+      return dispatch({
+        type: "autosave: genesets error",
+        message: `HTTP error ${res.status} - ${res.statusText}`,
+        res,
+      });
+    }
+    return Promise.all([
+      dispatch({
+        type: "autosave: genesets complete",
+        lastSavedGenesets: genesets,
+      }),
+      dispatch({
+        type: "geneset: set tid",
+        tid,
+      }),
+    ]);
+  } catch (error) {
+    return dispatch({
+      type: "autosave: genesets error",
+      message: error.toString(),
+      error,
+    });
+  }
+};

client/src/actions/embedding.js

@@ -0,0 +1,47 @@
+/*
+action creators related to embeddings choice
+*/
+
+import { AnnoMatrixObsCrossfilter } from "../annoMatrix";
+import { _setEmbeddingSubset } from "../util/stateManager/viewStackHelpers";
+
+export async function _switchEmbedding(
+  prevAnnoMatrix,
+  prevCrossfilter,
+  newEmbeddingName
+) {
+  /*
+  DRY helper used by embedding action creators
+  */
+  const base = prevAnnoMatrix.base();
+  const embeddingDf = await base.fetch("emb", newEmbeddingName);
+  const annoMatrix = _setEmbeddingSubset(prevAnnoMatrix, embeddingDf);
+  const obsCrossfilter = await new AnnoMatrixObsCrossfilter(
+    annoMatrix,
+    prevCrossfilter.obsCrossfilter
+  ).select("emb", newEmbeddingName, {
+    mode: "all",
+  });
+  return [annoMatrix, obsCrossfilter];
+}
+
+export const layoutChoiceAction =
+  (newLayoutChoice) => async (dispatch, getState) => {
+    /*
+  On layout choice, make sure we have selected all on the previous layout, AND the new
+  layout.
+  */
+    const { annoMatrix: prevAnnoMatrix, obsCrossfilter: prevCrossfilter } =
+      getState();
+    const [annoMatrix, obsCrossfilter] = await _switchEmbedding(
+      prevAnnoMatrix,
+      prevCrossfilter,
+      newLayoutChoice
+    );
+    dispatch({
+      type: "set layout choice",
+      layoutChoice: newLayoutChoice,
+      obsCrossfilter,
+      annoMatrix,
+    });
+  };

client/src/actions/geneset.js

@@ -0,0 +1,149 @@
+import { postUserErrorToast } from "../components/framework/toasters";
+/*
+Action creators for gene sets
+
+Primarily used to keep the crossfilter and underlying data in sync with the UI.
+
+The behavior manifest in these action creators:
+
+    Delete a gene set, will
+      * drop index & clear selection state on the gene set summary
+      * drop index & clear selection state of each gene in the geneset
+
+    Delete a gene from a gene set, will:
+      * drop index & clear selection state on the gene set summary
+      * drop index & clear selection state on the gene
+
+    Add a gene to a gene set, will:
+      * drop index & clear selection state on the gene set summary
+      * will NOT touch the selection state for the gene
+
+Note that crossfilter indices are lazy created, as needed.
+*/
+
+export const genesetDelete = (genesetName) => (dispatch, getState) => {
+  const state = getState();
+  const { genesets } = state;
+  const gs = genesets?.genesets?.get(genesetName) ?? {};
+  const geneSymbols = Array.from(gs.genes.keys());
+  const obsCrossfilter = dropGeneset(dispatch, state, genesetName, geneSymbols);
+  if (genesetName === state.colors.colorAccessor) {
+    dispatch({
+      type: "reset colorscale",
+    });
+  }
+  dispatch({
+    type: "geneset: delete",
+    genesetName,
+    obsCrossfilter,
+    annoMatrix: obsCrossfilter.annoMatrix,
+  });
+};
+
+export const genesetAddGenes =
+  (genesetName, genes) => async (dispatch, getState) => {
+    const state = getState();
+    const { obsCrossfilter: prevObsCrossfilter, annoMatrix } = state;
+    const { schema } = annoMatrix;
+    const varIndex = schema.annotations.var.index;
+    const df = await annoMatrix.fetch("var", varIndex);
+    const geneNames = df.col(varIndex).asArray();
+    genes = genes.reduce((acc, gene) => {
+      if (geneNames.indexOf(gene.geneSymbol) === -1) {
+        postUserErrorToast(
+          `${gene.geneSymbol} doesn't appear to be a valid gene name.`
+        );
+      } else acc.push(gene);
+      return acc;
+    }, []);
+
+    const obsCrossfilter = dropGenesetSummaryDimension(
+      prevObsCrossfilter,
+      state,
+      genesetName
+    );
+    dispatch({
+      type: "continuous metadata histogram cancel",
+      continuousNamespace: { isGeneSetSummary: true },
+      selection: genesetName,
+    });
+    return dispatch({
+      type: "geneset: add genes",
+      genesetName,
+      genes,
+      obsCrossfilter,
+      annoMatrix: obsCrossfilter.annoMatrix,
+    });
+  };
+
+export const genesetDeleteGenes =
+  (genesetName, geneSymbols) => (dispatch, getState) => {
+    const state = getState();
+    const obsCrossfilter = dropGeneset(
+      dispatch,
+      state,
+      genesetName,
+      geneSymbols
+    );
+    return dispatch({
+      type: "geneset: delete genes",
+      genesetName,
+      geneSymbols,
+      obsCrossfilter,
+      annoMatrix: obsCrossfilter.annoMatrix,
+    });
+  };
+
+/*
+Private
+*/
+
+function dropGenesetSummaryDimension(obsCrossfilter, state, genesetName) {
+  const { annoMatrix, genesets } = state;
+  const varIndex = annoMatrix.schema.annotations?.var?.index;
+  const gs = genesets?.genesets?.get(genesetName) ?? {};
+  const genes = Array.from(gs.genes.keys());
+  const query = {
+    summarize: {
+      method: "mean",
+      field: "var",
+      column: varIndex,
+      values: genes,
+    },
+  };
+  return obsCrossfilter.dropDimension("X", query);
+}
+
+function dropGeneDimension(obsCrossfilter, state, gene) {
+  const { annoMatrix } = state;
+  const varIndex = annoMatrix.schema.annotations?.var?.index;
+  const query = {
+    where: {
+      field: "var",
+      column: varIndex,
+      value: gene,
+    },
+  };
+  return obsCrossfilter.dropDimension("X", query);
+}
+
+function dropGeneset(dispatch, state, genesetName, geneSymbols) {
+  const { obsCrossfilter: prevObsCrossfilter } = state;
+  const obsCrossfilter = geneSymbols.reduce(
+    (crossfilter, gene) => dropGeneDimension(crossfilter, state, gene),
+    dropGenesetSummaryDimension(prevObsCrossfilter, state, genesetName)
+  );
+  dispatch({
+    type: "continuous metadata histogram cancel",
+    continuousNamespace: { isGeneSetSummary: true },
+    selection: genesetName,
+  });
+  geneSymbols.forEach((g) =>
+    dispatch({
+      type: "continuous metadata histogram cancel",
+      continuousNamespace: { isUserDefined: true },
+      selection: g,
+    })
+  );
+  return obsCrossfilter;
+}

client/src/actions/index.js

@@ -1,294 +1,142 @@
-import _ from "lodash";
 import * as globals from "../globals";
-import { Universe, MatrixFBS } from "../util/stateManager";
+import { AnnoMatrixLoader, AnnoMatrixObsCrossfilter } from "../annoMatrix";
 import {
   catchErrorsWrap,
   doJsonRequest,
-  doBinaryRequest,
-  dispatchNetworkErrorMessageToUser
+  dispatchNetworkErrorMessageToUser,
 } from "../util/actionHelpers";
-import { requestReembed, reembedResetWorldToUniverse } from "./reembed";
+import { loadUserColorConfig } from "../util/stateManager/colorHelpers";
+import * as selnActions from "./selection";
+import * as annoActions from "./annotation";
+import * as spatialActions from "./spatial";
+import * as viewActions from "./viewStack";
+import * as embActions from "./embedding";
+import * as genesetActions from "./geneset";
 
-/*
-return promise to fetch the OBS annotations we need to load.  Omit anything
-we don't need.
-*/
-function obsAnnotationFetchAndLoad(dispatch, schema) {
-  const obsAnnotations = schema?.schema?.annotations?.obs ?? {};
-  const columns = obsAnnotations.columns ?? [];
-  const index = obsAnnotations.index ?? false;
-  return Promise.all(
-    columns
-      .filter(col => col.name !== index)
-      .map(col => {
-        const path = `annotations/obs?annotation-name=${encodeURIComponent(
-          col.name
-        )}`;
-        const url = `${globals.API.prefix}${globals.API.version}${path}`;
-        return doBinaryRequest(url);
-      })
-      .map(rqst => rqst.then(buffer => Universe.matrixFBSToDataframe(buffer)))
-      .map(resp =>
-        resp.then(df =>
-          dispatch({
-            type: "universe: column load success",
-            dim: "obsAnnotations",
-            dataframe: df
-          })
-        )
-      )
-  );
+function setGlobalConfig(config) {
+  /**
+   * Set any global run-time config not _exclusively_ managed by the config reducer.
+   * This should only set fields defined in globals.globalConfig.
+   */
+  globals.globalConfig.maxCategoricalOptionsToDisplay =
+    config?.parameters?.["max-category-items"] ??
+    globals.globalConfig.maxCategoricalOptionsToDisplay;
 }
 
 /*
-return promise fetching VAR annotations we need to load.  Only index is currently used.
+return promise fetching user-configured colors
 */
-function varAnnotationFetchAndLoad(dispatch, schema) {
-  const varAnnotations = schema?.schema?.annotations?.var ?? {};
-  const index = varAnnotations.index ?? false;
-  const names = index ? [index] : [];
-  return Promise.all(
-    names
-      .map(name => {
-        const path = `annotations/var?annotation-name=${encodeURIComponent(
-          name
-        )}`;
-        const url = `${globals.API.prefix}${globals.API.version}${path}`;
-        return doBinaryRequest(url);
-      })
-      .map(rqst => rqst.then(buffer => Universe.matrixFBSToDataframe(buffer)))
-      .map(resp =>
-        resp.then(df =>
-          dispatch({
-            type: "universe: column load success",
-            dim: "varAnnotations",
-            dataframe: df
-          })
-        )
-      )
+async function userColorsFetchAndLoad(dispatch) {
+  return fetchJson("colors").then((response) =>
+    dispatch({
+      type: "universe: user color load success",
+      userColors: loadUserColorConfig(response),
+    })
   );
 }
 
-/*
-return promise fetching layout we need
-*/
-function layoutFetchAndLoad(dispatch) {
-  return Promise.all(
-    ["layout/obs"]
-      .map(path => {
-        const url = `${globals.API.prefix}${globals.API.version}${path}`;
-        return doBinaryRequest(url);
-      })
-      .map(rqst => rqst.then(buffer => Universe.matrixFBSToDataframe(buffer)))
-      .map(resp =>
-        resp.then(df =>
-          dispatch({
-            type: "universe: column load success",
-            dim: "obsLayout",
-            dataframe: df
-          })
-        )
-      )
-  );
+async function schemaFetch() {
+  return fetchJson("schema");
+}
+
+async function configFetch(dispatch) {
+  return fetchJson("config").then((response) => {
+    const config = { ...globals.configDefaults, ...response.config };
+
+    setGlobalConfig(config);
+
+    dispatch({
+      type: "configuration load complete",
+      config,
+    });
+    return config;
+  });
+}
+
+async function genesetsFetch(dispatch, config) {
+  /* request genesets ONLY if the backend supports the feature */
+  const defaultResponse = {
+    genesets: [],
+    tid: 0,
+  };
+  if (config?.parameters?.annotations_genesets ?? false) {
+    fetchJson("genesets").then((response) => {
+      dispatch({
+        type: "geneset: initial load",
+        data: response ?? defaultResponse,
+      });
+    });
+  } else {
+    dispatch({
+      type: "geneset: initial load",
+      data: defaultResponse,
+    });
+  }
+}
+
+function prefetchEmbeddings(annoMatrix) {
+  /*
+  prefetch requests for all embeddings
+  */
+  const { schema } = annoMatrix;
+  const available = schema.layout.obs.map((v) => v.name);
+  available.forEach((embName) => annoMatrix.prefetch("emb", embName));
 }
 
 /*
-Bootstrap application with the initial data loading.
-  * /config - application configuration
-  * /schema - schema of dataframe
-  * /annotations - all metadata annotation
-  * /layout - all default layout
+Application bootstrap
 */
 const doInitialDataLoad = () =>
-  catchErrorsWrap(async dispatch => {
+  catchErrorsWrap(async (dispatch) => {
     dispatch({ type: "initial data load start" });
 
     try {
-      /*
-      Step 1 - config & schema, all JSON
-      */
-      const requestJson = ["config", "schema"]
-        .map(r => `${globals.API.prefix}${globals.API.version}${r}`)
-        .map(url => doJsonRequest(url));
-      const stepOneResults = await Promise.all(requestJson);
-      /* set config defaults */
-      const config = { ...globals.configDefaults, ...stepOneResults[0].config };
-      const schema = stepOneResults[1];
-      const universe = Universe.createUniverseFromResponse(config, schema);
-      dispatch({
-        type: "universe exists, but loading is still in progress",
-        universe
-      });
-      dispatch({
-        type: "configuration load complete",
-        config
-      });
-
-      /*
-      Step 2 - load the minimum stuff required to display.
-      */
-      await Promise.all([
-        layoutFetchAndLoad(dispatch),
-        varAnnotationFetchAndLoad(dispatch, schema)
+      const [config, schema] = await Promise.all([
+        configFetch(dispatch),
+        schemaFetch(dispatch),
+        userColorsFetchAndLoad(dispatch),
       ]);
 
-      /*
-      Step 3 - load everything else
-      */
-      await obsAnnotationFetchAndLoad(dispatch, schema);
+      genesetsFetch(dispatch, config);
+
+      const baseDataUrl = `${globals.API.prefix}${globals.API.version}`;
+      const annoMatrix = new AnnoMatrixLoader(baseDataUrl, schema.schema);
+      const obsCrossfilter = new AnnoMatrixObsCrossfilter(annoMatrix);
+      prefetchEmbeddings(annoMatrix);
 
       dispatch({
-        type: "initial data load complete (universe exists)",
-        universe
+        type: "annoMatrix: init complete",
+        annoMatrix,
+        obsCrossfilter,
       });
+      dispatch({ type: "initial data load complete" });
+
+      const defaultEmbedding = config?.parameters?.default_embedding;
+      const layoutSchema = schema?.schema?.layout?.obs ?? [];
+      if (
+        defaultEmbedding &&
+        layoutSchema.some((s) => s.name === defaultEmbedding)
+      ) {
+        dispatch(embActions.layoutChoiceAction(defaultEmbedding));
+      }
     } catch (error) {
       dispatch({ type: "initial data load error", error });
     }
   }, true);
 
-/*
-Set the view (world) to current selection.   Placeholder for an async action
-which also does re-layout.
-*/
-const setWorldToSelection = () => (dispatch, getState) => {
-  const { universe, world, crossfilter } = getState();
-  dispatch({
-    type: "set World to current selection",
-    universe,
-    world,
-    crossfilter
-  });
-};
-
-// Throws
-const dispatchExpressionErrors = (dispatch, res) => {
-  const msg = `Unexpected HTTP response while fetching expression data ${res.status}, ${res.statusText}`;
-  dispatchNetworkErrorMessageToUser(msg);
-  throw new Error(msg);
-};
-
-/*
-Fetch expression vectors for each gene in genes.  This is NOT an action
-function, but rather a helper to be called from an action helper that
-needs expression data.
-
-Transparently utilizes cached data if it is already present.
-*/
-async function _doRequestExpressionData(dispatch, getState, genes) {
-  const state = getState();
-  const { universe } = state;
-  const varIndexName = universe.schema.annotations.var.index;
-
-  /* helper for this function only */
-  const fetchData = async geneNames => {
-    const res = await fetch(
-      `${globals.API.prefix}${globals.API.version}data/var`,
-      {
-        method: "PUT",
-        body: JSON.stringify({
-          filter: {
-            var: {
-              annotation_value: [{ name: varIndexName, values: geneNames }]
-            }
-          }
-        }),
-        headers: new Headers({
-          accept: "application/octet-stream",
-          "Content-Type": "application/json"
-        }),
-        credentials: "include"
-      }
-    );
-
-    if (
-      !res.ok ||
-      res.headers.get("Content-Type") !== "application/octet-stream"
-    ) {
-      // WILL throw
-      return dispatchExpressionErrors(dispatch, res);
-    }
-
-    const data = await res.arrayBuffer();
-    return Universe.convertDataFBStoObject(universe, data);
-  };
-
-  /* preload data already in cache */
-  let expressionData = _.transform(
-    genes,
-    (expData, g) => {
-      const data = universe.varData.col(g);
-      if (data) {
-        expData[g] = data.asArray();
-      }
-    },
-    {}
-  ); // --> { gene: data }
-  /* make a list of genes for which we do not have data */
-  const genesToFetch = _.filter(genes, g => expressionData[g] === undefined);
-
-  dispatch({ type: "expression load start" });
-
-  /* Fetch data for any genes not in cache */
-  if (genesToFetch.length) {
-    try {
-      const newExpressionData = await fetchData(genesToFetch);
-      expressionData = {
-        ...expressionData,
-        ...newExpressionData
-      };
-    } catch (error) {
-      dispatch({ type: "expression load error", error });
-      throw error; // rethrow
-    }
-  }
-
-  dispatch({ type: "expression load success", expressionData });
-  return expressionData;
-}
-
 function requestSingleGeneExpressionCountsForColoringPOST(gene) {
-  return async (dispatch, getState) => {
-    dispatch({ type: "get single gene expression for coloring started" });
-    try {
-      await _doRequestExpressionData(dispatch, getState, [gene]);
-      const { world } = getState();
-      dispatch({
-        type: "color by expression",
-        gene,
-        data: {
-          [gene]: world.varData.col(gene).asArray()
-        }
-      });
-    } catch (error) {
-      dispatch({
-        type: "get single gene expression for coloring error",
-        error
-      });
-    }
+  return {
+    type: "color by expression",
+    gene,
   };
 }
 
-const requestUserDefinedGene = gene => async (dispatch, getState) => {
-  dispatch({ type: "request user defined gene started" });
-  try {
-    await await _doRequestExpressionData(dispatch, getState, [gene]);
-    const { world } = getState();
-
-    /* then send the success case action through */
-    return dispatch({
-      type: "request user defined gene success",
-      data: {
-        genes: [gene],
-        expression: world.varData.col(gene).asArray()
-      }
-    });
-  } catch (error) {
-    return dispatch({
-      type: "request user defined gene error",
-      error
-    });
-  }
-};
+const requestUserDefinedGene = (gene) => ({
+  type: "request user defined gene success",
+  data: {
+    genes: [gene],
+  },
+});
 
 const dispatchDiffExpErrors = (dispatch, response) => {
   switch (response.status) {
@@ -307,154 +155,123 @@ const dispatchDiffExpErrors = (dispatch, response) => {
       dispatchNetworkErrorMessageToUser(msg);
       dispatch({
         type: "request differential expression error",
-        error: new Error(msg)
+        error: new Error(msg),
       });
     }
   }
 };
 
-const requestDifferentialExpression = (set1, set2, num_genes = 10) => async (
-  dispatch,
-  getState
-) => {
-  dispatch({ type: "request differential expression started" });
-  try {
-    /*
+const requestDifferentialExpression =
+  (set1, set2, num_genes = 50) =>
+  async (dispatch, getState) => {
+    dispatch({ type: "request differential expression started" });
+    try {
+      /*
     Steps:
     1. get the most differentially expressed genes
     2. get expression data for each
     */
-    const state = getState();
-    const { universe } = state;
-    const varIndexName = universe.schema.annotations.var.index;
+      const { annoMatrix } = getState();
+      const varIndexName = annoMatrix.schema.annotations.var.index;
 
-    // Legal values are null, Array or TypedArray.  Null is initial state.
-    if (!set1) set1 = [];
-    if (!set2) set2 = [];
+      // Legal values are null, Array or TypedArray.  Null is initial state.
+      if (!set1) set1 = [];
+      if (!set2) set2 = [];
 
-    // These lines ensure that we convert any TypedArray to an Array.
-    // This is necessary because JSON.stringify() does some very strange
-    // things with TypedArrays (they are marshalled to JSON objects, rather
-    // than being marshalled as a JSON array).
-    set1 = Array.isArray(set1) ? set1 : Array.from(set1);
-    set2 = Array.isArray(set2) ? set2 : Array.from(set2);
+      // These lines ensure that we convert any TypedArray to an Array.
+      // This is necessary because JSON.stringify() does some very strange
+      // things with TypedArrays (they are marshalled to JSON objects, rather
+      // than being marshalled as a JSON array).
+      set1 = Array.isArray(set1) ? set1 : Array.from(set1);
+      set2 = Array.isArray(set2) ? set2 : Array.from(set2);
 
-    const res = await fetch(
-      `${globals.API.prefix}${globals.API.version}diffexp/obs`,
-      {
-        method: "POST",
-        headers: new Headers({
-          Accept: "application/json",
-          "Content-Type": "application/json"
-        }),
-        body: JSON.stringify({
-          mode: "topN",
-          count: num_genes,
-          set1: { filter: { obs: { index: set1 } } },
-          set2: { filter: { obs: { index: set2 } } }
-        }),
-        credentials: "include"
+      const res = await fetch(
+        `${globals.API.prefix}${globals.API.version}diffexp/obs`,
+        {
+          method: "POST",
+          headers: new Headers({
+            Accept: "application/json",
+            "Content-Type": "application/json",
+          }),
+          body: JSON.stringify({
+            mode: "topN",
+            count: num_genes,
+            set1: { filter: { obs: { index: set1 } } },
+            set2: { filter: { obs: { index: set2 } } },
+          }),
+          credentials: "include",
+        }
+      );
+
+      if (!res.ok || res.headers.get("Content-Type") !== "application/json") {
+        return dispatchDiffExpErrors(dispatch, res);
       }
-    );
 
-    if (!res.ok || res.headers.get("Content-Type") !== "application/json") {
-      return dispatchDiffExpErrors(dispatch, res);
-    }
-
-    const data = await res.json();
-    // result is [ [varIdx, ...], ... ]
-    const topNGenes = _.map(data, r =>
-      universe.varAnnotations.at(r[0], varIndexName)
-    );
-
-    /*
-    Kick off secondary action to fetch all of the expression data for the
-    topN expressed genes.
-    */
-    await _doRequestExpressionData(dispatch, getState, topNGenes);
-
-    /* then send the success case action through */
-    return dispatch({
-      type: "request differential expression success",
-      data
-    });
-  } catch (error) {
-    return dispatch({
-      type: "request differential expression error",
-      error
-    });
-  }
-};
-
-const resetWorldToUniverse = () => (dispatch, getState) => {
-  const { universe } = getState();
-  reembedResetWorldToUniverse(dispatch, getState);
-  dispatch({
-    type: "reset World to eq Universe",
-    universe
-  });
-};
-
-const saveObsAnnotations = () => async (dispatch, getState) => {
-  const { universe, annotations } = getState();
-  const { obsAnnotations, schema } = universe;
-  const { dataCollectionNameIsReadOnly, dataCollectionName } = annotations;
-
-  dispatch({
-    type: "writable obs annotations - save started"
-  });
-
-  const writableAnnotations = schema.annotations.obs.columns
-    .filter(s => s.writable)
-    .map(s => s.name);
-  const df = obsAnnotations.subset(null, writableAnnotations);
-  const matrix = MatrixFBS.encodeMatrixFBS(df);
-  try {
-    const queryString =
-      !dataCollectionNameIsReadOnly && !!dataCollectionName
-        ? `?annotation-collection-name=${encodeURIComponent(
-            dataCollectionName
-          )}`
-        : "";
-    const res = await fetch(
-      `${globals.API.prefix}${globals.API.version}annotations/obs${queryString}`,
-      {
-        method: "PUT",
-        body: matrix,
-        headers: new Headers({
-          "Content-Type": "application/octet-stream"
-        }),
-        credentials: "include"
+      const response = await res.json();
+      const varIndex = await annoMatrix.fetch("var", varIndexName);
+      const diffexpLists = { negative: [], positive: [] };
+      for (const polarity of Object.keys(diffexpLists)) {
+        diffexpLists[polarity] = response[polarity].map((v) => [
+          varIndex.at(v[0], varIndexName),
+          ...v.slice(1),
+        ]);
       }
-    );
-    if (res.ok) {
-      dispatch({
-        type: "writable obs annotations - save complete",
-        obsAnnotations
+
+      /* then send the success case action through */
+      return dispatch({
+        type: "request differential expression success",
+        data: diffexpLists,
       });
-    } else {
-      dispatch({
-        type: "writable obs annotations - save error",
-        message: `HTTP error ${res.status} - ${res.statusText}`,
-        res
+    } catch (error) {
+      return dispatch({
+        type: "request differential expression error",
+        error,
       });
     }
-  } catch (error) {
-    dispatch({
-      type: "writable obs annotations - save error",
-      message: error.toString(),
-      error
-    });
-  }
-};
+  };
+
+function fetchJson(pathAndQuery) {
+  return doJsonRequest(
+    `${globals.API.prefix}${globals.API.version}${pathAndQuery}`
+  );
+}
 
 export default {
   doInitialDataLoad,
   requestDifferentialExpression,
   requestSingleGeneExpressionCountsForColoringPOST,
   requestUserDefinedGene,
-  requestReembed,
-  resetWorldToUniverse,
-  saveObsAnnotations,
-  setWorldToSelection
+  selectContinuousMetadataAction: selnActions.selectContinuousMetadataAction,
+  selectCategoricalMetadataAction: selnActions.selectCategoricalMetadataAction,
+  selectCategoricalAllMetadataAction:
+    selnActions.selectCategoricalAllMetadataAction,
+  graphBrushStartAction: selnActions.graphBrushStartAction,
+  graphBrushChangeAction: selnActions.graphBrushChangeAction,
+  graphBrushDeselectAction: selnActions.graphBrushDeselectAction,
+  graphBrushCancelAction: selnActions.graphBrushCancelAction,
+  graphBrushEndAction: selnActions.graphBrushEndAction,
+  graphLassoStartAction: selnActions.graphLassoStartAction,
+  graphLassoEndAction: selnActions.graphLassoEndAction,
+  graphLassoCancelAction: selnActions.graphLassoCancelAction,
+  graphLassoDeselectAction: selnActions.graphLassoDeselectAction,
+  clipAction: viewActions.clipAction,
+  subsetAction: viewActions.subsetAction,
+  resetSubsetAction: viewActions.resetSubsetAction,
+  annotationCreateCategoryAction: annoActions.annotationCreateCategoryAction,
+  annotationRenameCategoryAction: annoActions.annotationRenameCategoryAction,
+  annotationDeleteCategoryAction: annoActions.annotationDeleteCategoryAction,
+  annotationCreateLabelInCategory: annoActions.annotationCreateLabelInCategory,
+  annotationDeleteLabelFromCategory:
+    annoActions.annotationDeleteLabelFromCategory,
+  annotationRenameLabelInCategory: annoActions.annotationRenameLabelInCategory,
+  annotationLabelCurrentSelection: annoActions.annotationLabelCurrentSelection,
+  saveObsAnnotationsAction: annoActions.saveObsAnnotationsAction,
+  saveGenesetsAction: annoActions.saveGenesetsAction,
+  needToSaveObsAnnotations: annoActions.needToSaveObsAnnotations,
+  layoutChoiceAction: embActions.layoutChoiceAction,
+  setCellSetFromSelection: selnActions.setCellSetFromSelection,
+  genesetDelete: genesetActions.genesetDelete,
+  genesetAddGenes: genesetActions.genesetAddGenes,
+  genesetDeleteGenes: genesetActions.genesetDeleteGenes,
+  requestSpatialMetadata: spatialActions.requestSpatialMetadata,
 };

client/src/actions/reembed.js

@@ -1,113 +0,0 @@
-import { API } from "../globals";
-import { Universe } from "../util/stateManager";
-import {
-  postNetworkErrorToast,
-  postAsyncSuccessToast,
-  postAsyncFailureToast
-} from "../components/framework/toasters";
-
-function abortableFetch(request, opts, timeout = 0) {
-  const controller = new AbortController();
-  const { signal } = controller;
-
-  return {
-    abort: () => controller.abort(),
-    isAborted: () => signal.aborted,
-    ready: () => {
-      if (timeout) {
-        setTimeout(() => controller.abort(), timeout);
-      }
-      return fetch(request, { ...opts, signal });
-    }
-  };
-}
-
-async function doReembedFetch(dispatch, getState) {
-  const state = getState();
-  let cells = state.world.obsAnnotations.rowIndex.keys();
-
-  // These lines ensure that we convert any TypedArray to an Array.
-  // This is necessary because JSON.stringify() does some very strange
-  // things with TypedArrays (they are marshalled to JSON objects, rather
-  // than being marshalled as a JSON array).
-  cells = Array.isArray(cells) ? cells : Array.from(cells);
-
-  const af = abortableFetch(
-    `${API.prefix}${API.version}layout/obs`,
-    {
-      method: "PUT",
-      headers: new Headers({
-        Accept: "application/octet-stream",
-        "Content-Type": "application/json"
-      }),
-      body: JSON.stringify({
-        method: "umap",
-        filter: { obs: { index: cells } }
-      }),
-      credentials: "include"
-    },
-    60000 // 1 minute timeout
-  );
-  dispatch({
-    type: "reembed: request start",
-    abortableFetch: af
-  });
-  const res = await af.ready();
-
-  if (
-    res.ok &&
-    res.headers.get("Content-Type").includes("application/octet-stream")
-  ) {
-    return res;
-  }
-
-  // else an error
-  let msg = `Unexpected HTTP response ${res.status}, ${res.statusText}`;
-  const body = await res.text();
-  if (body && body.length > 0) {
-    msg = `${msg} -- ${body}`;
-  }
-  postNetworkErrorToast(msg);
-  throw new Error(msg);
-}
-
-/*
-functions below are dispatch-able
-*/
-export function requestReembed() {
-  return async (dispatch, getState) => {
-    try {
-      const res = await doReembedFetch(dispatch, getState);
-      const schema = JSON.parse(res.headers.get("CxG-Schema"));
-      const buffer = await res.arrayBuffer();
-      const df = Universe.matrixFBSToDataframe(buffer);
-      dispatch({
-        type: "reembed: request completed"
-      });
-      dispatch({
-        type: "reembed: add reembedding",
-        embedding: df,
-        schema
-      });
-      postAsyncSuccessToast("Re-embedding has completed.");
-    } catch (error) {
-      dispatch({
-        type: "reembed: request aborted"
-      });
-      if (error.name === "AbortError") {
-        postAsyncFailureToast("Re-embedding calculation was aborted.");
-      } else {
-        postNetworkErrorToast(`Re-embedding: ${error.message}`);
-      }
-      console.log("Reembed exception:", error, error.name, error.message);
-    }
-  };
-}
-
-export function reembedResetWorldToUniverse(dispatch, getState) {
-  const { reembedController } = getState();
-  if (reembedController.pendingFetch) reembedController.pendingFetch.abort();
-  dispatch({
-    type: "reembed: clear all reembeddings"
-  });
-}

client/src/actions/selection.js

@@ -0,0 +1,192 @@
+/*
+Action creators for selection 
+*/
+export const selectContinuousMetadataAction =
+  (type, query, range, oldProps = {}) =>
+  async (dispatch, getState) => {
+    const { obsCrossfilter: prevObsCrossfilter } = getState();
+
+    const selection = range
+      ? {
+          mode: "range",
+          lo: range[0],
+          hi: range[1],
+          inclusive: true, // [lo, hi] incluisve selection
+        }
+      : { mode: "all" };
+
+    const obsCrossfilter = await prevObsCrossfilter.select(...query, selection);
+
+    dispatch({
+      type,
+      obsCrossfilter,
+      range,
+      ...oldProps,
+    });
+  };
+
+export const selectCategoricalMetadataAction =
+  (
+    type, // action type
+    metadataField, // annotation category name
+    labels,
+    label, // the label being selected/deselected
+    isSelected, // bool
+    oldProps = {}
+  ) =>
+  async (dispatch, getState) => {
+    const { obsCrossfilter: prevObsCrossfilter, categoricalSelection } =
+      getState();
+
+    const labelSelectionState = new Map(categoricalSelection[metadataField]);
+    labels.forEach(
+      (l) => labelSelectionState.has(l) || labelSelectionState.set(l, true)
+    );
+    labelSelectionState.set(label, isSelected);
+
+    const values = Array.from(labelSelectionState.keys()).filter((k) =>
+      labelSelectionState.get(k)
+    );
+    const selection = {
+      mode: "exact",
+      values,
+    };
+    const obsCrossfilter = await prevObsCrossfilter.select(
+      "obs",
+      metadataField,
+      selection
+    );
+
+    dispatch({
+      type,
+      obsCrossfilter,
+      metadataField,
+      labelSelectionState,
+      ...oldProps,
+    });
+  };
+
+export const selectCategoricalAllMetadataAction =
+  (
+    type, // action type
+    metadataField, // annotation category name
+    labels,
+    isSelected, // bool, select all or none
+    oldProps = {}
+  ) =>
+  async (dispatch, getState) => {
+    const { obsCrossfilter: prevObsCrossfilter, categoricalSelection } =
+      getState();
+
+    const labelSelectionState = new Map(categoricalSelection[metadataField]);
+    labels.forEach((label) => labelSelectionState.set(label, isSelected));
+
+    const selection = { mode: isSelected ? "all" : "none" };
+    const obsCrossfilter = await prevObsCrossfilter.select(
+      "obs",
+      metadataField,
+      selection
+    );
+
+    dispatch({
+      type,
+      obsCrossfilter,
+      metadataField,
+      labelSelectionState,
+      ...oldProps,
+    });
+  };
+
+/**
+ ** Graph selection-related actions
+ **/
+
+export const graphBrushStartAction = () =>
+  /* no change to crossfilter until a change fires */
+  ({ type: "graph brush start" });
+
+const _graphBrushWithinRectAction =
+  (type, embName, brushCoords) => async (dispatch, getState) => {
+    const { obsCrossfilter: prevObsCrossfilter } = getState();
+
+    const selection = { mode: "within-rect", ...brushCoords };
+    const obsCrossfilter = await prevObsCrossfilter.select(
+      "emb",
+      embName,
+      selection
+    );
+
+    dispatch({
+      type,
+      obsCrossfilter,
+      brushCoords,
+    });
+  };
+
+const _graphAllAction = (type, embName) => async (dispatch, getState) => {
+  const { obsCrossfilter: prevObsCrossfilter } = getState();
+
+  const obsCrossfilter = await prevObsCrossfilter.select("emb", embName, {
+    mode: "all",
+  });
+
+  dispatch({
+    type,
+    obsCrossfilter,
+  });
+};
+
+export const graphBrushChangeAction = (embName, brushCoords) =>
+  _graphBrushWithinRectAction("graph brush change", embName, brushCoords);
+
+export const graphBrushEndAction = (embName, brushCoords) =>
+  _graphBrushWithinRectAction("graph brush end", embName, brushCoords);
+
+export const graphBrushCancelAction = (embName) =>
+  _graphAllAction("graph brush cancel", embName);
+export const graphBrushDeselectAction = (embName) =>
+  _graphAllAction("graph brush deselect", embName);
+
+export const graphLassoStartAction = () =>
+  /* no change to crossfilter until a change fires */
+  ({ type: "graph lasso start" });
+
+export const graphLassoCancelAction = (embName) =>
+  _graphAllAction("graph lasso cancel", embName);
+
+export const graphLassoDeselectAction = (embName) =>
+  _graphAllAction("graph lasso cancel", embName);
+
+export const graphLassoEndAction =
+  (embName, polygon) => async (dispatch, getState) => {
+    const { obsCrossfilter: prevObsCrossfilter } = getState();
+
+    const selection = {
+      mode: "within-polygon",
+      polygon,
+    };
+    const obsCrossfilter = await prevObsCrossfilter.select(
+      "emb",
+      embName,
+      selection
+    );
+
+    dispatch({
+      type: "graph lasso end",
+      obsCrossfilter,
+      polygon,
+    });
+  };
+
+/*
+Differential expression set selection
+*/
+export const setCellSetFromSelection = (cellSetId) => (dispatch, getState) => {
+  const { obsCrossfilter } = getState();
+  const selected = obsCrossfilter.allSelectedLabels();
+
+  dispatch({
+    type: `store current cell selection as differential set ${cellSetId}`,
+    data: selected.length > 0 ? selected : null,
+  });
+};

client/src/actions/spatial.js

@@ -0,0 +1,35 @@
+import * as globals from "../globals";
+
+export const requestSpatialMetadata = () => async (dispatch) => {
+  dispatch({ type: "request spatial metadata started" });
+  try {
+    const res = await fetch(
+      `${globals.API.prefix}${globals.API.version}spatial/meta`,
+      {
+        method: "GET",
+        headers: new Headers({
+          Accept: "application/json",
+          "Content-Type": "application/json",
+        }),
+        credentials: "include",
+      }
+    );
+
+    if (!res.ok || res.headers.get("Content-Type") !== "application/json") {
+      return null; // TODO need a dispatch //dispatchDiffExpErrors(dispatch, res);
+    }
+
+    const response = await res.json();
+
+    /* then send the success case action through */
+    return dispatch({
+      type: "request spatial metadata success",
+      data: response,
+    });
+  } catch (error) {
+    return dispatch({
+      type: "request spatial metadata error",
+      error,
+    });
+  }
+};

client/src/actions/viewStack.js

@@ -0,0 +1,74 @@
+/*
+The following actions manage the view stack for annoMatrix.
+
+Conventions used and assumed elsewhere in the code base:
+  * there will be zero or one clip view, and it will be the TOP view always.
+  * there will be zero or more subset views
+
+In other words, in our current use, we do not stack multiple clip views but we do
+stack multiple subsets.
+
+If these conventions change, code elsewhere (eg. menubar/clip.js) will need to
+change as well.
+*/
+import { AnnoMatrixObsCrossfilter } from "../annoMatrix";
+import {
+  _clipAnnoMatrix,
+  _userSubsetAnnoMatrix,
+  _userResetSubsetAnnoMatrix,
+} from "../util/stateManager/viewStackHelpers";
+
+export const clipAction = (min, max) => (dispatch, getState) => {
+  /*
+  apply a clip to the current annoMatrix.  By convention, the clip
+  view is ALWAYS the top view.
+  */
+  const { annoMatrix: prevAnnoMatrix } = getState();
+  const annoMatrix = _clipAnnoMatrix(prevAnnoMatrix, min, max);
+  const obsCrossfilter = new AnnoMatrixObsCrossfilter(annoMatrix);
+  dispatch({
+    type: "set clip quantiles",
+    clipQuantiles: { min, max },
+    annoMatrix,
+    obsCrossfilter,
+  });
+};
+
+export const subsetAction = () => (dispatch, getState) => {
+  /*
+  Subset the annoMatrix to the current crossfilter selection by pushing a 
+  subset view.
+
+  By convention, a clip view is ALWAYS the top view, so if present, pop
+  off and re-apply
+  */
+  const { annoMatrix: prevAnnoMatrix, obsCrossfilter: prevObsCrossfilter } =
+    getState();
+  const annoMatrix = _userSubsetAnnoMatrix(
+    prevAnnoMatrix,
+    prevObsCrossfilter.allSelectedMask()
+  );
+  const obsCrossfilter = new AnnoMatrixObsCrossfilter(annoMatrix);
+  dispatch({
+    type: "subset to selection",
+    annoMatrix,
+    obsCrossfilter,
+  });
+};
+
+export const resetSubsetAction = () => (dispatch, getState) => {
+  /*
+  Reset the annoMatrix to all data.  Because we may have multiple views
+  stacked, we pop them all.  By convention, any clip transformation will
+  be the top of the stack, and must be preserved.
+  */
+
+  const { annoMatrix: prevAnnoMatrix } = getState();
+  const annoMatrix = _userResetSubsetAnnoMatrix(prevAnnoMatrix);
+  const obsCrossfilter = new AnnoMatrixObsCrossfilter(annoMatrix);
+  dispatch({
+    type: "reset subset",
+    annoMatrix,
+    obsCrossfilter,
+  });
+};

client/src/annoMatrix/annoMatrix.js

@@ -0,0 +1,650 @@
+import {
+  Dataframe,
+  IdentityInt32Index,
+  dataframeMemo,
+} from "../util/dataframe";
+import {
+  _getColumnDimensionNames,
+  _getColumnSchema,
+  _schemaColumns,
+  _getWritableColumns,
+} from "./schema";
+import { indexEntireSchema } from "../util/stateManager/schemaHelpers";
+import { _whereCacheGet, _whereCacheMerge } from "./whereCache";
+import _shallowClone from "./clone";
+import { _queryValidate, _queryCacheKey } from "./query";
+
+const _dataframeCache = dataframeMemo(128);
+
+export default class AnnoMatrix {
+  /*
+  Abstract base class for all AnnoMatrix objects.  This class provides a proxy
+  to the annotated matrix data authoritatively served by the server/back-end.
+
+  AnnoMatrix instances are immutable, meaning that their schema and dimensionality
+  will not change, and simple object equality can be used to detect structural
+  changes. The actual data is cached, and not guaranteed to be present -- any
+  request to access data must be resolved by a fetch() call, which is async, and
+  may involve a server round-trip.
+
+  Guarantees made by the immutabilty, ie, any of these can be detected by
+  simple annoMatrix compare:
+    * schema is the same, including all fields and columns
+    * dimensionality is the same (nObs, nVar)
+    * data mapping/transformation, such as clipping, are the same
+
+  AnnoMatrixes also "stack" like filters, allowing for the construction of
+  views which transform the data in some manner.
+
+  The bootstrap class is AnnoMatrixLoader, which is the caching server proxy, and
+  is bootstrapped with a API URL:
+    new AnnoMatirx(url, schema) -> annoMatrix
+
+  There are various "views", such as AnnoMatrixRowSubsetView, which provide
+  the same interface but with a transformed view of the server data.  Utilities in
+  viewCreators.js can be used to create these views:
+      clip(annoMatrix, min, max) -> annoMatrix
+      subset(annoMatrix, rowLabels) -> annoMatrix
+  etc.
+  */
+  static fields() {
+    /*
+    return the fields present in the AnnoMatrix instance.
+    */
+    return ["obs", "var", "emb", "X"];
+  }
+
+  constructor(schema, nObs, nVar, rowIndex = null) {
+    /*
+    Private constructor - this is an abstract base class.  Do not use.
+    */
+
+    /*
+    Public instance fields:
+      * schema - the matrix schema.  IMPORTANT: always the entire schema, for the
+        base (unfiltered, unclipped, unsubset) annotated matrix, as the server
+        presents it.
+      * nObs, nVar - size of each dimension.  These will accurately reflect the
+        size of the current annoMatrix view.  For example, if you subset the view,
+        the nObs will be smaller.
+      * rowIndex - a rowIndex shared by all data on this view (ie, the list of cells).
+        The row index labels are as defined by the base dataset from the server.
+      * isView - true if this is a view, false if not.
+      * viewOf - pointer to parent annomatrix if a view, undefined/null if not a view.
+      * userFlags - container for any additional state a user of this API wants to hang
+        off of an annoMatrix, and have propagated by the (shallow) cloning protocol.
+    */
+    this.schema = indexEntireSchema(schema);
+    this.nObs = nObs;
+    this.nVar = nVar;
+    this.rowIndex = rowIndex || new IdentityInt32Index(nObs);
+    this.isView = false;
+    this.viewOf = undefined;
+    this.userFlags = {};
+
+    /*
+		Private instance variables.
+
+		These are caches - lazily loaded. The only guarantee is that if they
+		are loaded, they will conform to the schema & dimensionality constraints.
+
+		Do NOT use directly - instead, use the fetch() and preload() API.
+		*/
+    this._cache = {
+      obs: Dataframe.empty(this.rowIndex),
+      var: Dataframe.empty(this.rowIndex),
+      emb: Dataframe.empty(this.rowIndex),
+      X: Dataframe.empty(this.rowIndex),
+    };
+    this._pendingLoad = {
+      obs: {},
+      var: {},
+      emb: {},
+      X: {},
+    };
+    this._whereCache = {};
+    this._gcInfo = new Map();
+  }
+
+  /**
+   ** Schema helper/accessors
+   **/
+  getMatrixColumns(field) {
+    /*
+    Return array of column names in the field.  ONLY supported on the
+    obs, var and emb fields.  X currently unimplemented and will throw.
+
+    For exmaple:
+
+      annoMatrix.getMatrixColumns("obs") -> ["louvain", "n_genes"]
+    */
+    return _schemaColumns(this.schema, field);
+  }
+
+  // eslint-disable-next-line class-methods-use-this -- need to be able to call this on instances
+  getMatrixFields() {
+    /*
+    Return array of fields in this annoMatrix.  Currently hard-wired to
+    return:  ["X", "obs", "var", "emb"].
+
+    These are the fields from data may be requested.
+    */
+    return AnnoMatrix.fields();
+  }
+
+  getColumnSchema(field, col) {
+    /*
+    Return the schema for the field & column ,eg,
+
+      anonMatrix.getColumnSchema("obs", "n_genes") -> { type: "int32", name: "n_genes" }
+
+    This is identical to the information in the annoMatrix.schema
+    instance variable.
+    */
+    return _getColumnSchema(this.schema, field, col);
+  }
+
+  getColumnDimensions(field, col) {
+    /*
+    Return the dimensions on this field / column.  For most fields, which are 1D,
+    this just return the column name.  Multi-dimensional columns, such as embeddings,
+    will return >1 name.
+
+    Examples:
+
+      getColumnDimensions("obs", "louvain") -> ["louvain"]
+      getColumnDimensions("emb", "umap") -> ["umap_0", "umap_1"]
+
+    */
+    return _getColumnDimensionNames(this.schema, field, col);
+  }
+
+  /**
+   ** General utility methods
+   **/
+  base() {
+    /*
+    return the base of view, or `this` if not a view.
+    */
+    let annoMatrix = this;
+    while (annoMatrix.isView) annoMatrix = annoMatrix.viewOf;
+    return annoMatrix;
+  }
+
+  /**
+   ** Load / read interfaces
+   **/
+  fetch(field, q) {
+    /*
+		Return the given query on a single matrix field as a single dataframe.
+		Currently supports ONLY full column query.
+
+		Returns a Promise for the query result, which will resolve to a dataframe.
+
+		Field must be one of the matrix fields: 'obs', 'var', 'X', 'emb'.  Value
+		represents the underlying object upon which the query is occurring.
+
+		Query is one of:
+			* a string, representing a single column name from the field, eg,
+				"n_genes"
+			* an object, containing an "value" query (see below).
+			* an array, containing one or more of the above.
+
+		Columns may have more than one dimension, and all will be fetched
+		and returned together.  This is most commonly seen in an embedding,
+    which usually has two dimensions.
+
+		A value query allows for fetching based upon the value in another 
+		field/column, similar to a join.  Currently only supported on the var
+		dimension, allowing query of X columns by var value (eg, gene name)
+
+		Examples:
+
+    1. Fetch the "n_genes" column the "obs":
+
+			const df = await fetch("obs", "n_genes")
+      console.log("Largest number of genes is: ", df.summarize().max);
+
+    2. Fetch two separate columns from obs.  Returns a single dataframe containing
+       the columns:
+
+			const df = await fetch("obs", ["n_genes", "louvain"])
+      console.log("Cell 0 has category: ", df.at(0, "louvain"));
+
+    3. Fetch an entire X (expression counts) column that has a var annotation
+       value "TYMP" in the var index.
+
+			fetch("X", { 
+				where: {
+          field: "var", column: this.schema.annotations.var.index, value: "TYMP"
+        }
+			})
+
+      In AnnData & Pandas DataFrame API, this is equivalent to:
+        adata.X[:, adata.var.index.get_loc("SUMO3")]
+
+		The value query is a recodification and subset of the server REST API 
+		value filter JSON.  Range queries and multiple filters are not currently
+		supported.
+
+		*/
+    return this._fetch(field, q);
+  }
+
+  prefetch(field, q) {
+    /*
+		Start a data fetch & cache fill.  Identical to fetch() except it does
+		not return a value.
+
+    Primary use is to being a cache load as early as is possible, reducing
+    overall component rendering latency.
+		*/
+    this._fetch(field, q);
+    return undefined;
+  }
+
+  /**
+   ** Save / mutate interfaces - manipulation of "writable" OBS annotations.
+   **
+   ** These are all present to support client-side creation of OBS annotations, aka
+   ** "user annotations".
+   **
+   ** They implement common manipulations to the AnnoMatrix, maintaining the
+   ** norma guarantees around correctness of public API, eg,
+   **   - schema will be correct, including the "writable" attribute
+   **   - fetch() will return the latest data, even from views
+   **   - immutability guranteeds
+   **
+   ** As most of these interfaces mutate the annoMatrix, they return a new
+   ** annoMatrix
+   **
+   ** The actual implementation is in the sub-classes, which MUST override these.
+   **/
+
+  // eslint-disable-next-line class-methods-use-this, no-unused-vars -- make sure subclass implements
+  addObsAnnoCategory(col, category) {
+    /*
+    Add a new category value (aka "label") to a writable obs column, and return the new AnnoMatrix.
+    Typical use is to add a new user-created label to a user-created obs categorical
+    annotation.
+
+    Will throw column does not exist or is not writable.
+
+    Example:
+
+      addObsAnnoCategory("my cell type", "left toenail") -> AnnoMatrix
+
+    */
+    _subclassResponsibility();
+  }
+
+  // eslint-disable-next-line class-methods-use-this, no-unused-vars -- make sure subclass implements
+  async removeObsAnnoCategory(col, category, unassignedCategory) {
+    /*
+    Remove a category value from an obs column, reassign any obs having that value
+    to the 'unassignedCategory' value, and return a promise for a new AnnoMatrix.
+    Typical use is to remove a user-created label from a user-created obs categorical
+    annotation.
+
+    Will throw column does not exist or is not writable.
+
+    An `unassignedCategory` value must be provided, for assignment to any obs/cells
+    that had the now-delete category label as their value.
+
+    Example:
+      await removeObsAnnoCategory("my-tissue-type", "right earlobe", "unassigned") -> AnnoMatrix
+
+    NOTE: method is async as it may need to fetch data to provide the reassignment.
+    */
+    _subclassResponsibility();
+  }
+
+  // eslint-disable-next-line class-methods-use-this, no-unused-vars -- make sure subclass implements
+  dropObsColumn(col) {
+    /*
+    Drop an entire writable column, eg a user-created obs annotation.  Typical use
+    is to provide the "Delete Category" implementation.  Returns the new AnnoMatrix.
+    Will throw if not a writable annotation.
+
+    Will throw column does not exist or is not writable.
+
+    Example:
+
+      dropObsColumn("old annotations") ->  AnnoMatrix
+    */
+    _subclassResponsibility();
+  }
+
+  // eslint-disable-next-line class-methods-use-this, no-unused-vars -- make sure subclass implements
+  addObsColumn(colSchema, Ctor, value) {
+    /*
+    Add a new writable OBS annotation column, with the caller-specified schema, initial value
+    type and value.
+
+    Value may be any one of:
+      * an array of values
+      * a primitive type, including null or undefined.
+    If an array, length must be the same as 'this.nObs', and constructor must equal 'Ctor'.
+    If a primitive, 'Ctor' will be used to create the initial value, which will be filled
+    with 'value'.
+
+    Throws if the name specified in 'colSchema' duplicates an existing obs column.
+
+    Returns a new AnnoMatrix.
+
+    Examples:
+
+      addObsColumn(
+        { name: "foo", type: "categorical", categories: "unassigned" },
+        Array,
+        "unassigned"
+      ) -> AnnoMatrix
+
+    */
+    _subclassResponsibility();
+  }
+
+  // eslint-disable-next-line class-methods-use-this, no-unused-vars -- make sure subclass implements
+  renameObsColumn(oldCol, newCol) {
+    /*
+    Rename the obs column 'oldCol' to have name 'newCol' and returns new AnnoMatrix.
+
+    Will throw column does not exist or is not writable, or if 'newCol' is not unique.
+
+    Example:
+
+      renameObsColumn('cell type', 'old cell type') -> AnnoMatrix.
+
+    */
+    _subclassResponsibility();
+  }
+
+  // eslint-disable-next-line class-methods-use-this, no-unused-vars -- make sure subclass implements
+  async setObsColumnValues(col, obsLabels, value) {
+    /*
+    Set all obs with label in array 'obsLabels' to have 'value'.  Typical use would be
+    to set a group of cells to have a label on a user-created categorical anntoation
+    (eg set all selected cells to have a label).
+
+    NOTE: async method, as it may need to fetch.
+
+    Will throw column does not exist or is not writable.
+
+    Example:
+      await setObsColmnValues("flavor", [383, 400], "tasty") -> AnnoMtarix
+
+    */
+    _subclassResponsibility();
+  }
+
+  // eslint-disable-next-line class-methods-use-this, no-unused-vars -- make sure subclass implements
+  async resetObsColumnValues(col, oldValue, newValue) {
+    /*
+    Set by value - all elements in the column with value 'oldValue' are set to 'newValue'.
+    Async method - returns a promise for a new AnnoMatrix.
+
+    Typical use would be to set all labels of one value to another.
+
+    Will throw column does not exist or is not writable.
+
+    Example:
+      await resetObsColumnValues("my notes", "good", "not-good") -> AnnoMatrix
+
+    */
+    _subclassResponsibility();
+  }
+
+  // eslint-disable-next-line class-methods-use-this, no-unused-vars -- make sure subclass implements
+  addEmbedding(colSchema) {
+    /*
+    Add a new obs embedding to the AnnoMatrix, with provided schema.
+    Returns a new annomatrix.
+
+    Typical use will be to add a re-embedding that the server has calculated.
+
+    Will throw if the column schema is invalid (eg, duplicate name).
+    */
+    _subclassResponsibility();
+  }
+
+  getCacheKeys(field, query) {
+    /*
+    Return cache keys for columns associated with this query.  May return
+    [unknown] if no keys are known (ie, nothing is or was cached).
+    */
+    return _whereCacheGet(this._whereCache, this.schema, field, query);
+  }
+
+  /**
+   ** Private interfaces below.
+   **/
+  _resolveCachedQueries(field, queries) {
+    return queries
+      .map((query) =>
+        _whereCacheGet(this._whereCache, this.schema, field, query).filter(
+          (cacheKey) =>
+            cacheKey !== undefined && this._cache[field].hasCol(cacheKey)
+        )
+      )
+      .flat();
+  }
+
+  async _fetch(field, q) {
+    if (!AnnoMatrix.fields().includes(field)) return undefined;
+    const queries = Array.isArray(q) ? q : [q];
+    queries.forEach(_queryValidate);
+
+    /* find cached columns we need, and GC the rest */
+    const cachedColumns = this._resolveCachedQueries(field, queries);
+    this._gcFetchCleanup(field, cachedColumns);
+
+    /* find any query not already cached */
+    const uncachedQueries = queries.filter((query) =>
+      _whereCacheGet(this._whereCache, this.schema, field, query).some(
+        (cacheKey) =>
+          cacheKey === undefined || !this._cache[field].hasCol(cacheKey)
+      )
+    );
+
+    /* load uncached queries */
+    if (uncachedQueries.length > 0) {
+      await Promise.all(
+        uncachedQueries.map((query) =>
+          this._getPendingLoad(field, query, async (_field, _query) => {
+            /* fetch, then index.  _doLoad is subclass interface */
+            const [whereCacheUpdate, df] = await this._doLoad(_field, _query);
+            this._cache[_field] = this._cache[_field].withColsFrom(df);
+            this._whereCache = _whereCacheMerge(
+              this._whereCache,
+              whereCacheUpdate
+            );
+          })
+        )
+      );
+    }
+
+    /* everything we need is in the cache, so just cherry-pick requested columns */
+    const requestedCacheKeys = this._resolveCachedQueries(field, queries);
+    const response = _dataframeCache(
+      this._cache[field].subset(null, requestedCacheKeys)
+    );
+    this._gcUpdateStats(field, response);
+    return response;
+  }
+
+  async _getPendingLoad(field, query, fetchFn) {
+    /*
+    Given a query on a field, ensure that we only have a single outstanding
+    fetch at any given time.  If multiple requests occur while a fetch is
+    outstanding, just wait for the original.
+
+    This is implemented by returning a promise that will await the singular
+    fetch promise.
+    */
+    const key = _queryCacheKey(field, query);
+    if (!this._pendingLoad[field][key]) {
+      this._pendingLoad[field][key] = fetchFn(field, query);
+      try {
+        await this._pendingLoad[field][key];
+      } finally {
+        delete this._pendingLoad[field][key];
+      }
+    }
+    return this._pendingLoad[field][key];
+  }
+
+  // eslint-disable-next-line class-methods-use-this -- make sure subclass implements
+  async _doLoad() {
+    _subclassResponsibility();
+  }
+
+  /**
+   ** Garbage collection of annomatrix cache to manage memory use.
+   **/
+
+  /*
+  These callbacks implement a GC policy for the cache.  Background:
+
+    * For the Loader (base) annomatrix, re-filling the cache is expensive as
+      it requires an HTTP fetch.
+    * user-defined / writable columns must not be GC'ed as they may be
+      still pending a save/commit.
+    * For views, cost is less and (roughly) proportional with nObs
+    * obs, var and emb do not grow without bounds, and are needed constantly 
+      for rendering.  
+      a) There is no upside to GC'ing these in the base (loader)
+      b) The undo/redo cache can hold a large number in views, which is worth GC'ing
+    * X is often much larger than memory, and the UI allows add/del from
+      this.  Most of the GC potential is here in both the base and views.
+
+  Current policy:
+    * if in active use ("hot") do not GC obs, var or emb.
+    * never, ever GC writable obs columns
+    * For base/loader set a numeric limit on maximum X column count
+    * For views, apply a fixed limit to the number of columns cached in any field.
+      Limit will be lower if not hot.
+
+  To be effective, the GC callback needs to be invoked from the undo/redo code,
+  as much of the cache is pinned by that data structure.
+  */
+  _gcField(field, isHot, pinnedColumns) {
+    const maxColumns = isHot ? 256 : 10; // maybe to aggressive?
+
+    const cache = this._cache[field];
+    if (cache.colIndex.size() < maxColumns) return; // trivial rejection
+
+    const candidates = cache.colIndex
+      .labels()
+      .filter((col) => !pinnedColumns.includes(col));
+
+    const excessCount = candidates.length + pinnedColumns.length - maxColumns;
+    if (excessCount > 0) {
+      const { _gcInfo } = this;
+      candidates.sort((a, b) => {
+        let atime = _gcInfo.get(_columnCacheKey(field, a));
+        if (atime === undefined) atime = 0;
+
+        let btime = _gcInfo.get(_columnCacheKey(field, b));
+        if (btime === undefined) btime = 0;
+
+        return atime - btime;
+      });
+
+      const toDrop = candidates.slice(0, excessCount);
+      // helpful debugging - please leave in place.
+      // console.log(
+      //   `GC: dropping from ${field} hot:${isHot}, columns [${toDrop.join(
+      //     ", "
+      //   )}]`
+      // );
+      this._cache[field] = toDrop.reduce(
+        (df, col) => df.dropCol(col),
+        this._cache[field]
+      );
+      toDrop.forEach((col) => _gcInfo.delete(_columnCacheKey(field, col)));
+    }
+  }
+
+  _gcFetchCleanup(field, pinnedColumns) {
+    /*
+    Called during data load/fetch.  By definition, this is 'hot', so we
+    only want to gc X.
+    */
+    if (field === "X") {
+      this._gcField(
+        field,
+        true,
+        pinnedColumns.concat(_getWritableColumns(this.schema, field))
+      );
+    }
+  }
+
+  _gc(hints) {
+    /*
+    Called from middleware, or elsewhere.  isHot is true if we are in the active store, 
+    or false if we are in some other context (eg, history state).
+    */
+    const { isHot } = hints;
+    const candidateFields = isHot ? ["X"] : ["X", "emb", "var", "obs"];
+    candidateFields.forEach((field) =>
+      this._gcField(field, isHot, _getWritableColumns(this.schema, field))
+    );
+  }
+
+  _gcUpdateStats(field, dataframe) {
+    /*
+    called each time a query is performed, allowing the gc to update any bookkeeping
+    information.  Currently, this is just a simple last-fetched timestamp, stored
+    in a Map.
+    */
+    const cols = dataframe.colIndex.labels();
+    const { _gcInfo } = this;
+    const now = Date.now();
+    cols.forEach((c) => {
+      _gcInfo.set(_columnCacheKey(field, c), now);
+    });
+  }
+
+  /**
+  Cloning subclass protocol - we rely in cloning to preserve immutable
+  semantics while not causing races or other side effects in internal
+  cache management.
+
+  Subclasses must override _cloneDeeper() if they have state which requires 
+  something other than a shallow copy.  Overrides MUST call super()._cloneDeepr(), 
+  and return its result (after any required modification).  _cloneDeeper()
+  will be called on the OLD object, with the NEW object as an argument.
+
+  Do not override _clone();
+  **/
+  _cloneDeeper(clone) {
+    clone._cache = _shallowClone(this._cache);
+    clone._gcInfo = new Map();
+    clone._pendingLoad = {
+      obs: {},
+      var: {},
+      emb: {},
+      X: {},
+    };
+    return clone;
+  }
+
+  _clone() {
+    const clone = _shallowClone(this);
+    this._cloneDeeper(clone);
+    Object.seal(clone);
+    return clone;
+  }
+}
+
+/*
+private utility functions below
+*/
+function _columnCacheKey(field, column) {
+  return `${field}/${column}`;
+}
+
+function _subclassResponsibility() {
+  /* protect against bugs in subclass */
+  throw new Error("subclass failed to implement required method");
+}

client/src/annoMatrix/clone.js

@@ -0,0 +1,6 @@
+/*
+Shallow clone an object, correctly handling prototype
+*/
+export default function _shallowClone(orig) {
+  return Object.assign(Object.create(Object.getPrototypeOf(orig)), orig);
+}

client/src/annoMatrix/crossfilter.js

@@ -0,0 +1,284 @@
+/*
+Row crossfilter proxy for an AnnoMatrix.  This wraps Crossfilter,
+providing a number of services, and ensuring that the crossfilter and
+AnnoMatrix stay in sync:
+	- on-demand index creation as data is loaded
+	- transparently mapping between queries and crossfilter index names.
+  - for mutation of the matrix by user annotations, maintain synchronization
+    between Crossfilter and AnnoMatrix.
+*/
+import Crossfilter from "../util/typedCrossfilter";
+import { _getColumnSchema } from "./schema";
+
+function _dimensionNameFromDf(field, df) {
+  const colNames = df.colIndex.labels();
+  return _dimensionName(field, colNames);
+}
+
+function _dimensionName(field, colNames) {
+  if (!Array.isArray(colNames)) return `${field}/${colNames}`;
+  return `${field}/${colNames.join(":")}`;
+}
+
+export default class AnnoMatrixObsCrossfilter {
+  constructor(annoMatrix, _obsCrossfilter = null) {
+    this.annoMatrix = annoMatrix;
+    this.obsCrossfilter =
+      _obsCrossfilter || new Crossfilter(annoMatrix._cache.obs);
+    this.obsCrossfilter = this.obsCrossfilter.setData(annoMatrix._cache.obs);
+  }
+
+  size() {
+    return this.obsCrossfilter.size();
+  }
+
+  /**
+  Managing the associated annoMatrix.  These wrappers are necessary to 
+  make coordinated changes to BOTH the crossfilter and annoMatrix, and
+  ensure that all state stays synchronized.
+
+  See API documentation in annoMatrix.js.
+  **/
+  addObsColumn(colSchema, Ctor, value) {
+    const annoMatrix = this.annoMatrix.addObsColumn(colSchema, Ctor, value);
+    const obsCrossfilter = this.obsCrossfilter.setData(annoMatrix._cache.obs);
+    return new AnnoMatrixObsCrossfilter(annoMatrix, obsCrossfilter);
+  }
+
+  dropObsColumn(col) {
+    const annoMatrix = this.annoMatrix.dropObsColumn(col);
+    let { obsCrossfilter } = this;
+    const dimName = _dimensionName("obs", col);
+    if (obsCrossfilter.hasDimension(dimName)) {
+      obsCrossfilter = obsCrossfilter.delDimension(dimName);
+    }
+    return new AnnoMatrixObsCrossfilter(annoMatrix, obsCrossfilter);
+  }
+
+  renameObsColumn(oldCol, newCol) {
+    const annoMatrix = this.annoMatrix.renameObsColumn(oldCol, newCol);
+    const oldDimName = _dimensionName("obs", oldCol);
+    const newDimName = _dimensionName("obs", newCol);
+    let { obsCrossfilter } = this;
+    if (obsCrossfilter.hasDimension(oldDimName)) {
+      obsCrossfilter = obsCrossfilter.renameDimension(oldDimName, newDimName);
+    }
+    return new AnnoMatrixObsCrossfilter(annoMatrix, obsCrossfilter);
+  }
+
+  addObsAnnoCategory(col, category) {
+    const annoMatrix = this.annoMatrix.addObsAnnoCategory(col, category);
+    const dimName = _dimensionName("obs", col);
+    let { obsCrossfilter } = this;
+    if (obsCrossfilter.hasDimension(dimName)) {
+      obsCrossfilter = obsCrossfilter.delDimension(dimName);
+    }
+    return new AnnoMatrixObsCrossfilter(annoMatrix, obsCrossfilter);
+  }
+
+  async removeObsAnnoCategory(col, category, unassignedCategory) {
+    const annoMatrix = await this.annoMatrix.removeObsAnnoCategory(
+      col,
+      category,
+      unassignedCategory
+    );
+    const dimName = _dimensionName("obs", col);
+    let { obsCrossfilter } = this;
+    if (obsCrossfilter.hasDimension(dimName)) {
+      obsCrossfilter = obsCrossfilter.delDimension(dimName);
+    }
+    return new AnnoMatrixObsCrossfilter(annoMatrix, obsCrossfilter);
+  }
+
+  async setObsColumnValues(col, rowLabels, value) {
+    const annoMatrix = await this.annoMatrix.setObsColumnValues(
+      col,
+      rowLabels,
+      value
+    );
+    const dimName = _dimensionName("obs", col);
+    let { obsCrossfilter } = this;
+    if (obsCrossfilter.hasDimension(dimName)) {
+      obsCrossfilter = obsCrossfilter.delDimension(dimName);
+    }
+    return new AnnoMatrixObsCrossfilter(annoMatrix, obsCrossfilter);
+  }
+
+  async resetObsColumnValues(col, oldValue, newValue) {
+    const annoMatrix = await this.annoMatrix.resetObsColumnValues(
+      col,
+      oldValue,
+      newValue
+    );
+    const dimName = _dimensionName("obs", col);
+    let { obsCrossfilter } = this;
+    if (obsCrossfilter.hasDimension(dimName)) {
+      obsCrossfilter = obsCrossfilter.delDimension(dimName);
+    }
+    return new AnnoMatrixObsCrossfilter(annoMatrix, obsCrossfilter);
+  }
+
+  addEmbedding(colSchema) {
+    const annoMatrix = this.annoMatrix.addEmbedding(colSchema);
+    return new AnnoMatrixObsCrossfilter(annoMatrix, this.obsCrossfilter);
+  }
+
+  /**
+   * Drop the crossfilter dimension. Do not change the annoMatrix. Useful when we
+   * want to stop trackin the selection state, but aren't sure we want to blow the
+   * annomatrix cache.
+   */
+  dropDimension(field, query) {
+    const { annoMatrix } = this;
+    let { obsCrossfilter } = this;
+    const keys = annoMatrix
+      .getCacheKeys(field, query)
+      .filter((k) => k !== undefined);
+    const dimName = _dimensionName(field, keys);
+    if (obsCrossfilter.hasDimension(dimName)) {
+      obsCrossfilter = obsCrossfilter.delDimension(dimName);
+    }
+    return new AnnoMatrixObsCrossfilter(annoMatrix, obsCrossfilter);
+  }
+
+  /**
+  Selection state - API is identical to ImmutableTypedCrossfilter, as these
+  are just wrappers to lazy create indices.
+  **/
+
+  async select(field, query, spec) {
+    const { annoMatrix } = this;
+    let { obsCrossfilter } = this;
+
+    if (!annoMatrix?._cache?.[field]) {
+      throw new Error("Unknown field name");
+    }
+    if (field === "var") {
+      throw new Error("unable to obsSelect upon the var dimension");
+    }
+
+    // grab the data, so we can grab the index.
+    const df = await annoMatrix.fetch(field, query);
+
+    const dimName = _dimensionNameFromDf(field, df);
+    if (!obsCrossfilter.hasDimension(dimName)) {
+      // lazy index generation - add dimension when first used
+      obsCrossfilter = this._addObsCrossfilterDimension(
+        annoMatrix,
+        obsCrossfilter,
+        field,
+        df
+      );
+    }
+
+    // select
+    obsCrossfilter = obsCrossfilter.select(dimName, spec);
+    return new AnnoMatrixObsCrossfilter(annoMatrix, obsCrossfilter);
+  }
+
+  selectAll() {
+    /*
+		Select all on any dimension in this field.
+		*/
+    const { annoMatrix } = this;
+    const currentDims = this.obsCrossfilter.dimensionNames();
+    const obsCrossfilter = currentDims.reduce((xfltr, dim) => xfltr.select(dim, { mode: "all" }), this.obsCrossfilter);
+    return new AnnoMatrixObsCrossfilter(annoMatrix, obsCrossfilter);
+  }
+
+  countSelected() {
+    /* if no data yet indexed in the crossfilter, just say everything is selected */
+    if (this.obsCrossfilter.size() === 0) return this.annoMatrix.nObs;
+    return this.obsCrossfilter.countSelected();
+  }
+
+  allSelectedMask() {
+    /* if no data yet indexed in the crossfilter, just say everything is selected */
+    if (
+      this.obsCrossfilter.size() === 0 ||
+      this.obsCrossfilter.dimensionNames().length === 0
+    ) {
+      /* fake the mask */
+      return new Uint8Array(this.annoMatrix.nObs).fill(1);
+    }
+    return this.obsCrossfilter.allSelectedMask();
+  }
+
+  allSelectedLabels() {
+    /* if no data yet indexed in the crossfilter, just say everything is selected */
+    if (
+      this.obsCrossfilter.size() === 0 ||
+      this.obsCrossfilter.dimensionNames().length === 0
+    ) {
+      return this.annoMatrix.rowIndex.labels();
+    }
+
+    const mask = this.obsCrossfilter.allSelectedMask();
+    const index = this.annoMatrix.rowIndex.isubsetMask(mask);
+    return index.labels();
+  }
+
+  fillByIsSelected(array, selectedValue, deselectedValue) {
+    /* if no data yet indexed in the crossfilter, just say everything is selected */
+    if (
+      this.obsCrossfilter.size() === 0 ||
+      this.obsCrossfilter.dimensionNames().length === 0
+    ) {
+      return array.fill(selectedValue);
+    }
+    return this.obsCrossfilter.fillByIsSelected(
+      array,
+      selectedValue,
+      deselectedValue
+    );
+  }
+
+  /**
+   ** Private below
+   **/
+
+  _addObsCrossfilterDimension(annoMatrix, obsCrossfilter, field, df) {
+    if (field === "var") return obsCrossfilter;
+    const dimName = _dimensionNameFromDf(field, df);
+    const dimParams = this._getObsDimensionParams(field, df);
+    obsCrossfilter = obsCrossfilter.setData(annoMatrix._cache.obs);
+    obsCrossfilter = obsCrossfilter.addDimension(dimName, ...dimParams);
+    return obsCrossfilter;
+  }
+
+  _getColumnBaseType(field, col) {
+    /* Look up the primitive type for this field/col */
+    const colSchema = _getColumnSchema(this.annoMatrix.schema, field, col);
+    return colSchema.type;
+  }
+
+  _getObsDimensionParams(field, df) {
+    /* return the crossfilter dimensiontype type and params for this field/dataframe */
+
+    if (field === "emb") {
+      /* assumed to be 2D */
+      return ["spatial", df.icol(0).asArray(), df.icol(1).asArray()];
+    }
+
+    /* assumed to be 1D */
+    const col = df.icol(0);
+    const colName = df.colIndex.getLabel(0);
+    const type = this._getColumnBaseType(field, colName);
+    if (type === "string" || type === "categorical" || type === "boolean") {
+      return ["enum", col.asArray()];
+    }
+    if (type === "int32") {
+      return ["scalar", col.asArray(), Int32Array];
+    }
+    if (type === "float32") {
+      return ["scalar", col.asArray(), Float32Array];
+    }
+    // Currently not supporting boolean and categorical types.
+    console.error(
+      `Warning - unknown metadata schema (${type}) for field ${field} ${colName}.`
+    );
+    // skip it - we don't know what to do with this type
+
+    return undefined;
+  }
+}

client/src/annoMatrix/fetchHelpers.js

@@ -0,0 +1,25 @@
+export { doBinaryRequest, doFetch } from "../util/actionHelpers";
+
+/* double URI encode - needed for query-param filters */
+export function _dubEncURIComp(s) {
+  return encodeURIComponent(encodeURIComponent(s));
+}
+
+/* currently unused, consider deleting */
+export function _fetchResult(promise) {
+  let _status = "pending";
+  const res = promise.then(
+    (r) => {
+      _status = "success";
+      return r;
+    },
+    (e) => {
+      _status = "error";
+      throw e;
+    }
+  );
+
+  res.status = () => _status;
+
+  return res;
+}

client/src/annoMatrix/index.js

@@ -0,0 +1,15 @@
+/*
+AnnoMatrix -- Annotated Matrix exported interface
+
+Public API is defined in:
+
+	annoMatrix.js
+	viewCreators.js
+	crossfilter.js
+
+*/
+
+export { default as AnnoMatrixLoader } from "./loader";
+export * from "./viewCreators";
+export { default as AnnoMatrixObsCrossfilter } from "./crossfilter";
+export { default as gcMiddleware } from "./middleware";

client/src/annoMatrix/loader.js

@@ -0,0 +1,346 @@
+import { doBinaryRequest, doFetch } from "./fetchHelpers";
+import { matrixFBSToDataframe } from "../util/stateManager/matrix";
+import { _getColumnSchema } from "./schema";
+import {
+  addObsAnnoColumn,
+  removeObsAnnoColumn,
+  addObsAnnoCategory,
+  removeObsAnnoCategory,
+  addObsLayout,
+} from "../util/stateManager/schemaHelpers";
+import { isArrayOrTypedArray } from "../util/typeHelpers";
+import { _whereCacheCreate } from "./whereCache";
+import AnnoMatrix from "./annoMatrix";
+import PromiseLimit from "../util/promiseLimit";
+import {
+  _expectSimpleQuery,
+  _expectComplexQuery,
+  _urlEncodeLabelQuery,
+  _urlEncodeComplexQuery,
+  _hashStringValues,
+} from "./query";
+import {
+  normalizeResponse,
+  normalizeWritableCategoricalSchema,
+} from "./normalize";
+
+const promiseThrottle = new PromiseLimit(5);
+
+export default class AnnoMatrixLoader extends AnnoMatrix {
+  /*
+  AnnoMatrix implementation which proxies to HTTP server using the CXG REST API.
+  Used as the base (non-view) instance.
+
+  Public API is same as AnnoMatrix class (refer there for API description),
+  with the addition of the constructor which bootstraps:
+
+    new AnnoMatrixLoader(serverBaseURL, schema) -> instance
+
+  */
+  constructor(baseURL, schema) {
+    const { nObs, nVar } = schema.dataframe;
+    super(schema, nObs, nVar);
+
+    if (baseURL[baseURL.length - 1] !== "/") {
+      // must have trailing slash
+      baseURL += "/";
+    }
+    this.baseURL = baseURL;
+    Object.seal(this);
+  }
+
+  /**
+   ** Public.  API described in base class.
+   **/
+  addObsAnnoCategory(col, category) {
+    /*
+    Add a new category (aka label) to the schema for an obs column.
+    */
+    const colSchema = _getColumnSchema(this.schema, "obs", col);
+    _writableCategoryTypeCheck(colSchema); // throws on error
+
+    const newAnnoMatrix = this._clone();
+    newAnnoMatrix.schema = addObsAnnoCategory(this.schema, col, category);
+    return newAnnoMatrix;
+  }
+
+  async removeObsAnnoCategory(col, category, unassignedCategory) {
+    /*
+    Remove a single "category" (aka "label") from the data & schema of an obs column.
+    */
+    const colSchema = _getColumnSchema(this.schema, "obs", col);
+    _writableCategoryTypeCheck(colSchema); // throws on error
+
+    const newAnnoMatrix = await this.resetObsColumnValues(
+      col,
+      category,
+      unassignedCategory
+    );
+    newAnnoMatrix.schema = removeObsAnnoCategory(
+      newAnnoMatrix.schema,
+      col,
+      category
+    );
+    return newAnnoMatrix;
+  }
+
+  dropObsColumn(col) {
+    /*
+		drop column from field
+		*/
+    const colSchema = _getColumnSchema(this.schema, "obs", col);
+    _writableCheck(colSchema); // throws on error
+
+    const newAnnoMatrix = this._clone();
+    newAnnoMatrix._cache.obs = this._cache.obs.dropCol(col);
+    newAnnoMatrix.schema = removeObsAnnoColumn(this.schema, col);
+    return newAnnoMatrix;
+  }
+
+  addObsColumn(colSchema, Ctor, value) {
+    /*
+		add a column to field, initializing with value.  Value may 
+    be one of:
+      * an array of values
+      * a primitive type, including null or undefined.
+    If an array, it must be of same size as nObs and same type as Ctor
+		*/
+    colSchema.writable = true;
+    const colName = colSchema.name;
+    if (
+      _getColumnSchema(this.schema, "obs", colName) ||
+      this._cache.obs.hasCol(colName)
+    ) {
+      throw new Error("column already exists");
+    }
+
+    const newAnnoMatrix = this._clone();
+    let data;
+    if (isArrayOrTypedArray(value)) {
+      if (value.constructor !== Ctor)
+        throw new Error("Mismatched value array type");
+      if (value.length !== this.nObs)
+        throw new Error("Value array has incorrect length");
+      data = value.slice();
+    } else {
+      data = new Ctor(this.nObs).fill(value);
+    }
+    newAnnoMatrix._cache.obs = this._cache.obs.withCol(colName, data);
+    normalizeWritableCategoricalSchema(
+      colSchema,
+      newAnnoMatrix._cache.obs.col(colName)
+    );
+    newAnnoMatrix.schema = addObsAnnoColumn(this.schema, colName, colSchema);
+    return newAnnoMatrix;
+  }
+
+  renameObsColumn(oldCol, newCol) {
+    /*
+    Rename the obs oldColName to newColName.  oldCol must be writable.
+    */
+    const oldColSchema = _getColumnSchema(this.schema, "obs", oldCol);
+    _writableCheck(oldColSchema); // throws on error
+
+    const value = this._cache.obs.hasCol(oldCol)
+      ? this._cache.obs.col(oldCol).asArray()
+      : undefined;
+    return this.dropObsColumn(oldCol).addObsColumn(
+      {
+        ...oldColSchema,
+        name: newCol,
+      },
+      value.constructor,
+      value
+    );
+  }
+
+  async setObsColumnValues(col, rowLabels, value) {
+    /*
+		Set all rows identified by rowLabels to value.
+		*/
+    const colSchema = _getColumnSchema(this.schema, "obs", col);
+    _writableCategoryTypeCheck(colSchema); // throws on error
+
+    // ensure that we have the data in cache before we manipulate it
+    await this.fetch("obs", col);
+    if (!this._cache.obs.hasCol(col))
+      throw new Error("Internal error - user annotation data missing");
+
+    const rowIndices = this.rowIndex.getOffsets(rowLabels);
+    const data = this._cache.obs.col(col).asArray().slice();
+    for (let i = 0, len = rowIndices.length; i < len; i += 1) {
+      const idx = rowIndices[i];
+      if (idx === undefined) throw new Error("Unknown row label");
+      data[idx] = value;
+    }
+
+    const newAnnoMatrix = this._clone();
+    newAnnoMatrix._cache.obs = this._cache.obs.replaceColData(col, data);
+    const { categories } = colSchema;
+    if (!categories?.includes(value)) {
+      newAnnoMatrix.schema = addObsAnnoCategory(this.schema, col, value);
+    }
+    return newAnnoMatrix;
+  }
+
+  async resetObsColumnValues(col, oldValue, newValue) {
+    /*
+    Set all rows with value 'oldValue' to 'newValue'.
+    */
+    const colSchema = _getColumnSchema(this.schema, "obs", col);
+    _writableCategoryTypeCheck(colSchema); // throws on error
+
+    if (!colSchema.categories.includes(oldValue)) {
+      throw new Error("unknown category");
+    }
+
+    // ensure that we have the data in cache before we manipulate it
+    await this.fetch("obs", col);
+    if (!this._cache.obs.hasCol(col))
+      throw new Error("Internal error - user annotation data missing");
+
+    const data = this._cache.obs.col(col).asArray().slice();
+    for (let i = 0, l = data.length; i < l; i += 1) {
+      if (data[i] === oldValue) data[i] = newValue;
+    }
+
+    const newAnnoMatrix = this._clone();
+    newAnnoMatrix._cache.obs = this._cache.obs.replaceColData(col, data);
+    const { categories } = colSchema;
+    if (!categories?.includes(newValue)) {
+      newAnnoMatrix.schema = addObsAnnoCategory(this.schema, col, newValue);
+    }
+    return newAnnoMatrix;
+  }
+
+  addEmbedding(colSchema) {
+    /*
+    add new layout to the obs embeddings
+    */
+    const { name: colName } = colSchema;
+    if (_getColumnSchema(this.schema, "emb", colName)) {
+      throw new Error("column already exists");
+    }
+
+    const newAnnoMatrix = this._clone();
+    newAnnoMatrix.schema = addObsLayout(this.schema, colSchema);
+    return newAnnoMatrix;
+  }
+
+  /**
+   ** Private below
+   **/
+  async _doLoad(field, query) {
+    /*
+    _doLoad - evaluates the query against the field. Returns:
+      * whereCache update: column query map mapping the query to the column labels
+      * Dataframe containing the new columns (one per dimension)
+    */
+    let doRequest;
+    let priority = 10; // default fetch priority
+
+    switch (field) {
+      case "obs":
+      case "var": {
+        doRequest = _obsOrVarLoader(this.baseURL, field, query);
+        break;
+      }
+      case "X": {
+        doRequest = _XLoader(this.baseURL, field, query);
+        break;
+      }
+      case "emb": {
+        doRequest = _embLoader(this.baseURL, field, query);
+        priority = 0; // high prio load for embeddings
+        break;
+      }
+      default:
+        throw new Error("Unknown field name");
+    }
+
+    const buffer = await promiseThrottle.priorityAdd(priority, doRequest);
+    let result = matrixFBSToDataframe(buffer);
+    if (!result || result.isEmpty()) throw Error("Unknown field/col");
+
+    const whereCacheUpdate = _whereCacheCreate(
+      field,
+      query,
+      result.colIndex.labels()
+    );
+
+    result = normalizeResponse(field, query, this.schema, result);
+
+    return [whereCacheUpdate, result];
+  }
+}
+
+/*
+Utility functions below
+*/
+
+function _writableCheck(colSchema) {
+  if (!colSchema?.writable) {
+    throw new Error("Unknown or readonly obs column");
+  }
+}
+
+function _writableCategoryTypeCheck(colSchema) {
+  _writableCheck(colSchema);
+  if (colSchema.type !== "categorical") {
+    throw new Error("column must be categorical");
+  }
+}
+
+function _embLoader(baseURL, _field, query) {
+  _expectSimpleQuery(query);
+
+  const urlBase = `${baseURL}layout/obs`;
+  const urlQuery = _urlEncodeLabelQuery("layout-name", query);
+  const url = `${urlBase}?${urlQuery}`;
+  return () => doBinaryRequest(url);
+}
+
+function _obsOrVarLoader(baseURL, field, query) {
+  _expectSimpleQuery(query);
+
+  const urlBase = `${baseURL}annotations/${field}`;
+  const urlQuery = _urlEncodeLabelQuery("annotation-name", query);
+  const url = `${urlBase}?${urlQuery}`;
+  return () => doBinaryRequest(url);
+}
+
+function _XLoader(baseURL, field, query) {
+  _expectComplexQuery(query);
+
+  if (query.where) {
+    const urlBase = `${baseURL}data/var`;
+    const urlQuery = _urlEncodeComplexQuery(query);
+    const url = `${urlBase}?${urlQuery}`;
+    return () => doBinaryRequest(url);
+  }
+
+  if (query.summarize) {
+    const urlBase = `${baseURL}summarize/var`;
+    const urlQuery = _urlEncodeComplexQuery(query);
+
+    if (urlBase.length + urlQuery.length < 2000) {
+      const url = `${urlBase}?${urlQuery}`;
+      return () => doBinaryRequest(url);
+    }
+
+    const url = `${urlBase}?key=${_hashStringValues([urlQuery])}`;
+    return async () => {
+      const res = await doFetch(url, {
+        method: "POST",
+        body: urlQuery,
+        headers: new Headers({
+          Accept: "application/octet-stream",
+          "Content-Type": "application/x-www-form-urlencoded",
+        }),
+      });
+      return res.arrayBuffer();
+    };
+  }
+
+  throw new Error("Unknown query structure");
+}