chore: CCIE-4984 conform to open sourcing guidelines

Trademark

.bumpversion.cfg

@@ -1,14 +1,21 @@
 [bumpversion]
-current_version = 0.13.0
+current_version = 1.3.0
+commit = True
+parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(?:-(?P<prerel>rc)\.(?P<prerelversion>\d+))?
+serialize = 
+	{major}.{minor}.{patch}-{prerel}.{prerelversion}
+	{major}.{minor}.{patch}
+
+[bumpversion:part:prerel]
+optional_value = release
+values = 
+	rc
+	release
 
 [bumpversion:file:setup.py]
 search = version="{current_version}"
 replace = version="{new_version}"
 
-[bumpversion:file:server/cli/cli.py]
-search = version="{current_version}"
-replace = version="{new_version}"
-
 [bumpversion:file:client/package.json]
 search = "version": "{current_version}"
 replace = "version": "{new_version}"
@@ -16,4 +23,3 @@ replace = "version": "{new_version}"
 [bumpversion:file:server/__init__.py]
 search = __version__ = "{current_version}"
 replace = __version__ = "{new_version}"
-

.codecov.yml

@@ -0,0 +1,8 @@
+coverage:
+  status:
+    project:
+      default:
+        informational: true
+    patch:
+      default:
+        informational: true

.github/ISSUE_TEMPLATE/---bug-report.md

@@ -0,0 +1,32 @@
+---
+name: "\U0001F41E Bug report"
+about: Create a report to identify a bug in cellxgene
+title: "[BUG]"
+labels: bug
+assignees: ''
+
+---
+
+**Describe the bug**
+A clear and concise description of what the bug is.
+
+**To Reproduce**
+Steps to reproduce the behavior:
+1. Go to '...'
+2. Click on '....'
+3. Scroll down to '....'
+4. See error
+
+**Expected behavior**
+A clear and concise description of what you expected to happen.
+
+**Screenshots**
+If applicable, add screenshots to help explain your problem.
+
+**Version (please complete the following information):**
+ - Desktop or hosted?:
+ - Browser (if hosted) [e.g. chrome, safari]:
+ - Version [e.g. 0.13.0]:
+
+**Additional context**
+Add any other context about the problem here.

.github/ISSUE_TEMPLATE/---feature-request.md

@@ -0,0 +1,20 @@
+---
+name: "\U0001F4A1 Feature request"
+about: Suggest an idea for this project
+title: "[FEATURE REQUEST]"
+labels: user request
+assignees: ''
+
+---
+
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Describe alternatives you've considered**
+A clear and concise description of any alternative solutions or features you've considered.
+
+**Additional context**
+Add any other context or screenshots about the feature request here.

.github/ISSUE_TEMPLATE/---question-clarification.md

@@ -0,0 +1,10 @@
+---
+name: "\U0001F9D0 Question/Clarification"
+about: Ask a question or for a clarification
+title: "[QUESTION]"
+labels: question
+assignees: ''
+
+---
+
+

.github/ISSUE_TEMPLATE/tech-issue.md

@@ -0,0 +1,23 @@
+---
+name: Tech Issue
+about: Engineering-specific technical work that is not product-specific. Engineering team "owns" these issues.
+title: ""
+labels: tech
+assignees: ""
+---
+
+## Motivation
+
+Why is this work important to engineers?
+
+## Definition of Done
+
+What should the end result look like? What will have been changed?
+
+## Tasks
+
+Detail the specific tasks that can be used to accomplish the desired changes.
+If detailed steps cannot be provided at this time, please file a [Tech Proposal](https://docs.google.com/document/d/1o2vuvl-kXwRJN1nBoPzJS_MAQgDGYnjmPZWa4qRDi-I/edit#heading=h.7dvzhm7gqc3v) instead.
+
+- [ ]
+- [ ]

.github/workflows/close-stale-prs.yml

@@ -0,0 +1,23 @@
+name: Close inactive pull requests
+on:
+  schedule:
+    - cron: "30 1 * * *"
+
+jobs:
+  close-issues:
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write
+      pull-requests: write
+    steps:
+      - uses: actions/stale@v5
+        with:
+          days-before-issue-stale: -1 # Do not mark any issues as stale
+          days-before-pr-stale: 14
+          days-before-pr-close: 3
+          stale-pr-message: "This PR has not seen any activity in the past 2 weeks; if no one comments or reviews it in the next 3 days, this PR will be closed."
+          close-pr-message: "This PR was closed because it has been inactive for 17 days, 3 days since being marked as stale. Please re-open if you still need this to be addressed."
+          stale-pr-label: "stale"
+          close-pr-label: "autoclosed"
+          exempt-draft-pr: true
+          repo-token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/codeql-analysis.yml

@@ -0,0 +1,67 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL Scan"
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [ main ]
+  schedule:
+    - cron: '0 8 * * *'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'javascript', 'python' ]
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ]
+        # Learn more:
+        # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v1
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+        # queries: ./path/to/local/query, your-org/your-repo/queries@main
+
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v1
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 https://git.io/JvXDl
+
+    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
+    #    and modify them (or add more) to build your code if your project
+    #    uses a compiled language
+
+    #- run: |
+    #   make bootstrap
+    #   make release
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v1

.github/workflows/compatibility_tests.yml

@@ -0,0 +1,112 @@
+name: Compatibility Tests
+
+on:
+  schedule:
+    - cron: "0 8 7 * 2"
+  push:
+    branches:
+      - main
+
+env:
+  JEST_ENV: prod
+
+jobs:
+  docker-build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Build docker image
+        run: docker build .
+
+  matrix-compatibility-test:
+    name: cxg:${{ matrix.cellxgene_build }} os:${{ matrix.os }} py:${{ matrix.python-version }} anndata:${{ matrix.anndata_version || 'latest' }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest, macos-13]
+        python-version: ["3.10", "3.11", "3.12"]
+        cellxgene_build: [main, latest]
+        # add anndata pinned version test for subset of matrix configurations,
+        # in order to reduce matrix cross-product explosion
+        include:
+          - python-version: 3.12
+            cellxgene_build: latest
+            # TODO: dynamically use the literal version in requirements.txt,
+            # to avoid having to update this in manually in the future
+            # TODO: Do not bother running this if anndata latest version
+            # matches this pinned version, to avoid a redundant test
+            anndata_version: "==0.10.9"
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Cache env vars
+        run: echo "PIP_CACHE=`python -m pip cache dir`" >> $GITHUB_ENV
+      - name: Cache env vars (MacOS)
+        if: startsWith(matrix.os, 'macos')
+        run: echo "BREW_CACHE=`brew --cache`" >> $GITHUB_ENV
+      # FIXME: Only working for Linux
+      - name: Python cache
+        uses: actions/cache@v4
+        with:
+          path: ${{ env.PIP_CACHE }}
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+      - name: Node cache
+        uses: actions/cache@v4
+        with:
+          path: ~/.npm
+          key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-node-
+      - name: Brew cache (MacOS)
+        if: startsWith(matrix.os, 'macos')
+        uses: actions/cache@v4
+        with:
+          path: ${{ env.BREW_CACHE }}
+          key: ${{ runner.os }}-brew-
+      - name: Install dependencies (Ubuntu Linux)
+        if: startsWith(matrix.os, 'ubuntu')
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y libhdf5-serial-dev
+      - name: Install dependencies (MacOS)
+        if: startsWith(matrix.os, 'macos')
+        run: brew install hdf5
+      - name: Install cellxgene from `main` branch
+        if: matrix.cellxgene_build == 'main'
+        run: |
+          pip install -r server/requirements-dev.txt
+          make pydist install-dist
+      - name: Install cellxgene from latest release (pypi.org)
+        if: matrix.cellxgene_build == 'latest'
+        run: |
+          pip install --upgrade cellxgene
+          # install the additional dev requirements on top of what is in the
+          # cellxgene pip package, which are needed for testing, but otherwise
+          # keep same pip pkg versions as in the cxg release
+          sed -i'' -e 's/-r requirements.txt//' server/requirements-dev.txt
+          pip install -r server/requirements-dev.txt
+          pip install --force-reinstall numpy==2.0.1 numba>=0.60.0 pandas flatbuffers==2.0.7
+      - name: Install anndata version per matrix variable
+        run: pip install anndata${{ matrix.anndata_version }}
+      - name: Install node
+        run: make dev-env-client
+      # Run different types of test separately, to facilitate troubleshooting
+      - name: Unit Tests - client
+        run: make unit-test-client
+      - name: Unit Tests - server
+        run: make unit-test-server
+      - name: Smoke Tests
+        run: make smoke-test
+    # FIXME: Fails intermittently. See https://app.zenhub.com/workspaces/single-cell-5e2a191dad828d52cc78b028/issues/chanzuckerberg/cellxgene/2415
+    # - name: Smoke Tests with Annotations
+    #   run: make smoke-test-annotations

.github/workflows/lint-pr-commit-message.yml

@@ -0,0 +1,19 @@
+
+name: "Lint PR commit message"
+
+on:
+  pull_request_target:
+    types:
+      - opened
+      - edited
+      - synchronize
+
+jobs:
+  main:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: amannn/action-semantic-pull-request@v3.4.1
+        with:
+          validateSingleCommit: true
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/push_tests.yml

@@ -0,0 +1,138 @@
+name: Push Tests
+
+on:
+  push:
+    branches: main
+  pull_request:
+    branches: "*"
+
+env:
+  JEST_ENV: prod
+  CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - run: |
+          git fetch --depth=1 origin +${{github.base_ref}}
+      - name: Set up Python 3.12
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.12
+      - name: Node cache
+        uses: actions/cache@v4
+        with:
+          path: ~/.npm
+          key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-node-
+      - name: Install dependencies
+        run: |
+          pip install flake8
+          pip install black
+          cd client
+          npm install
+      - name: Format with black and lint with flake8
+        run: |
+          make lint-server
+      - name: Lint src with eslint
+        working-directory: ./client
+        run: |
+          npx eslint src __tests__
+
+  unit-test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python 3.12 (pyenv) # pyenv needed for mlflow in cli annotate tests
+        uses: gabrielfalcao/pyenv-action@v9
+        with:
+          default: 3.12
+          command: pip install -U pip # upgrade pip after installing python
+      - run: pip install virtualenv # virtualenv needed for mlflow in cli annotate tests
+      - name: Python cache
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+      - name: Node cache
+        uses: actions/cache@v4
+        with:
+          path: ~/.npm
+          key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-node-
+      - name: Install dependencies
+        run: make pydist install-dist dev-env-server
+      - name: Unit tests
+        run: |
+          make unit-test-server unit-test-client
+          bash <(curl -s https://codecov.io/bash) -y .codecov.yml -k server -cF server,python,unitTest
+          cd client && ./node_modules/codecov/bin/codecov --yml=../.codecov.yml --root=../ --gcov-root=../ -C -F frontend,javascript,unitTest
+
+  smoke-tests:
+    runs-on: macos-latest
+    timeout-minutes: 20
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python 3.12
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.12
+      - name: Python cache
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+      - name: Node cache
+        uses: actions/cache@v4
+        with:
+          path: ~/.npm
+          key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-node-
+      - name: Install dependencies
+        run: |
+          pip install setuptools
+          make pydist install-dist
+      - name: Smoke tests (without annotations feature)
+        run: |
+          cd client && make smoke-test
+          ./node_modules/codecov/bin/codecov --yml=../.codecov.yml --root=../ --gcov-root=../ -C -F frontend,javascript,smokeTest
+
+  # TODO: reinstate: https://github.com/chanzuckerberg/cellxgene/issues/2544
+  # smoke-tests-annotations:
+  #   runs-on: ubuntu-latest
+  #   timeout-minutes: 20
+  #   steps:
+  #     - uses: actions/checkout@v2
+  #     - name: Set up Python 3.9
+  #       uses: actions/setup-python@v4
+  #       with:
+  #         python-version: 3.9
+  #     - name: Python cache
+  #       uses: actions/cache@v1
+  #       with:
+  #         path: ~/.cache/pip
+  #         key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}
+  #         restore-keys: |
+  #           ${{ runner.os }}-pip-
+  #     - name: Node cache
+  #       uses: actions/cache@v1
+  #       with:
+  #         path: ~/.npm
+  #         key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
+  #         restore-keys: |
+  #           ${{ runner.os }}-node-
+  #     - name: Install dependencies
+  #       run: make pydist install-dist
+  #     - name: Smoke tests (with annotations feature)
+  #       run: |
+  #         cd client && make smoke-test-annotations
+  #         ./node_modules/codecov/bin/codecov --yml=../.codecov.yml --root=../ --gcov-root=../ -C -F frontend,javascript,smokeTestAnnotations

.gitignore

@@ -3,6 +3,8 @@ node_modules/
 
 # coverage
 coverage
+coverage.xml
+.coverage
 .nyc_output
 
 .pytest_cache/
@@ -13,15 +15,18 @@ dist/
 *.egg-info
 
 # Environments
-venv/
+venv*/
 cellxgene/
 
 # client build
-server/app/web/static/css/
-server/app/web/static/img/
-server/app/web/static/media/
-server/app/web/static/js/
-server/app/web/templates/index\.html
+server/common/web/static/*
+server/common/web/templates/
+server/common/web/csp-hashes.json
+
+# eb build
+artifact.dir
+artifact.zip
+customize
 
 # Jupyter Notebook
 .ipynb_checkpoints
@@ -38,7 +43,17 @@ npm-debug.log
 __pycache__
 *.DS_Store*
 data
+tags
+myconfig.yaml
 
 # Jekyll
 docs/_site/
 docs/Gemfile.lock
+
+client/.eslintcache
+
+# E2E Testing
+ignoreE2E*
+
+# annotate subcmd
+.models_cache

.travis.yml

@@ -1,31 +0,0 @@
-language: python
-dist: xenial
-sudo: required
-node_js:
-  - 8
-cache:
-  - pip
-  - npm
-install:
-  - set -eo pipefail
-  - pip install flake8
-  - make pydist
-  - make install-dist
-  - pip install -r server/requirements-dev.txt
-
-jobs:
-  include:
-  - name: "Branch Tests 3.7"
-    python: "3.7"
-    script: ./travis-build.sh
-  - name: "Branch Tests 3.6"
-    python: "3.6"
-    script: ./travis-build.sh
-  - name: "Docker Build"
-    install: skip
-    python: "3.6"
-    script: docker build .
-  - name: "Smoke Tests"
-    python: "3.6"
-    script:
-      - npm run --prefix client/ smoke-test

CONTRIBUTING.md

@@ -1,3 +1,3 @@
 We warmly welcome contributions from the community!
 
-Whether you want to contribute ideas, requests, documentation, or code, you can get started by visiting our [contribution guide](https://chanzuckerberg.github.io/cellxgene/posts/contribute).
+Whether you want to contribute ideas, requests, documentation, or code, you can get started by visiting our [contribution guide](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/contribute.md).

Dockerfile

@@ -4,7 +4,8 @@ ENV LC_ALL=C.UTF-8
 ENV LANG=C.UTF-8
 
 RUN apt-get update && \
-    apt-get install -y build-essential libxml2-dev python3-dev python3-pip zlib1g-dev python3-requests && \
+    apt-get install -y build-essential libxml2-dev python3-dev python3-pip zlib1g-dev python3-requests python3-aiohttp && \
+    python3 -m pip install --upgrade pip && \
     pip3 install cellxgene
 
 ENTRYPOINT ["cellxgene"]

LICENSE

@@ -0,0 +1,20 @@
+The MIT License (MIT)
+
+Copyright (c) 2017-2023 Chan Zuckerberg Initiative
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

LICENSE.txt

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2013
+Copyright (c) 2017-2023 Chan Zuckerberg Initiative
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of
 this software and associated documentation files (the "Software"), to deal in
@@ -17,4 +17,4 @@ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
 FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
 COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
 IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
-CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

MANIFEST.in

@@ -1,5 +1,8 @@
-recursive-include server/app/web/templates *
-recursive-include server/app/web/static *
+recursive-include server/common/web/templates *
+recursive-include server/common/web/static *
 
 include server/requirements.txt
 include server/requirements-prepare.txt
+include server/requirements-annotate.txt
+include server/converters/schema/hgnc_complete_set.txt.gz
+include server/converters/schema/schema_definitions/*

Makefile

@@ -0,0 +1,220 @@
+include common.mk
+
+BUILDDIR := build
+CLIENTBUILD := $(BUILDDIR)/client
+SERVERBUILD := $(BUILDDIR)/server
+CLEANFILES :=  $(BUILDDIR)/ client/build build dist cellxgene.egg-info
+
+PART ?= patch
+
+# CLEANING
+.PHONY: clean
+clean: clean-lite clean-server clean-client
+
+# cleaning the client's node_modules is the longest one, so we avoid that if possible
+.PHONY: clean-lite
+clean-lite:
+	rm -rf $(CLEANFILES)
+
+.PHONY: clean-client
+clean-client:
+	cd client && $(MAKE) clean
+
+.PHONY: clean-server
+clean-server:
+	cd server && $(MAKE) clean
+
+
+# BUILDING PACKAGE
+
+.PHONY: build-client
+build-client:
+	cd client && $(MAKE) ci build
+
+.PHONY: build
+build: clean build-client
+	git ls-files server/ | cpio -pdm $(BUILDDIR)
+	cp -r client/build/  $(CLIENTBUILD)
+	$(call copy_client_assets,$(CLIENTBUILD),$(SERVERBUILD))
+	cp MANIFEST.in README.md setup.cfg setup.py $(BUILDDIR)
+
+# If you are actively developing in the server folder use this, dirties the source tree
+.PHONY: build-for-server-dev
+build-for-server-dev: clean-server build-client copy-client-assets
+
+.PHONY: copy-client-assets
+copy-client-assets:
+	$(call copy_client_assets,client/build,server)
+
+
+# TESTING
+.PHONY: test
+test: unit-test smoke-test
+
+.PHONY: unit-test
+unit-test: unit-test-server unit-test-client
+
+.PHONY: test-server
+test-server: unit-test-server smoke-test
+
+.PHONY: unit-test-client
+unit-test-client:
+	cd client && $(MAKE) unit-test
+
+.PHONY: unit-test-server
+unit-test-server:
+	PYTHONWARNINGS=ignore:ResourceWarning coverage run \
+		--source=server \
+		--omit=.coverage,venv \
+		-m unittest discover \
+		--start-directory test/unit \
+		--verbose; test_result=$$?; \
+	exit $$test_result \
+
+.PHONY: smoke-test
+smoke-test:
+	cd client && $(MAKE) smoke-test
+
+.PHONY: smoke-test-annotations
+smoke-test-annotations:
+	cd client && $(MAKE) smoke-test-annotations
+
+# FORMATTING CODE
+
+.PHONY: fmt
+fmt: fmt-client fmt-py
+
+.PHONY: fmt-client
+fmt-client:
+	cd client && $(MAKE) fmt
+
+.PHONY: fmt
+fmt-py:
+	black .
+
+.PHONY: lint
+lint: lint-server lint-client
+
+
+.PHONY: lint-server
+lint-server: fmt-py
+	flake8 server --per-file-ignores='test/fixtures/dataset_config_outline.py:F821 test/fixtures/server_config_outline.py:F821 test/performance/scale_test_annotations.py:E501'
+
+.PHONY: lint-client
+lint-client:
+	cd client && $(MAKE) lint
+
+# CREATING DISTRIBUTION RELEASE
+
+.PHONY: pydist
+pydist: build
+	cd $(BUILDDIR); python setup.py sdist -d ../dist
+	@echo "done"
+
+# RELEASE HELPERS
+
+# Set PART=[major, minor, patch] as param to make bump.
+# This will create a release candidate. (i.e. 0.16.1 -> 0.16.2-rc.0 for a patch bump)
+.PHONY: bump-version
+bump-version:
+	bumpversion --config-file .bumpversion.cfg $(PART)
+
+# Create new version to commit to main
+.PHONY: create-release-candidate
+create-release-candidate: bump-version clean-lite gen-package-lock
+	@echo "Version bumped part:$(PART) and client built. Ready to commit and push"
+
+# Bump the release candidate version if needed (i.e. the previous release candidate had errors).
+.PHONY: recreate-release-candidate
+recreate-release-candidate: bump-release-candidate clean-lite gen-package-lock
+	@echo "Version bumped part:$(PART) and client built. Ready to commit and push"
+
+# Build dist and release to Test PyPI
+.PHONY: release-candidate-to-test-pypi
+release-candidate-to-test-pypi: pydist twine
+	@echo "Dist built and uploaded to test.pypi.org"
+	@echo "Test the install:"
+	@echo "    make install-release-test"
+
+# Build final dist (gets rid of the rc tag) and release final candidate to TestPyPI
+.PHONY: release-final-to-test-pypi
+release-final-to-test-pypi: bump-release clean-lite gen-package-lock pydist twine
+	@echo "Final release dist built and uploaded to test.pypi.org"
+	@echo "Test the install:"
+	@echo "    make install-release-test"
+
+.PHONY: release-final
+release-final: twine-prod
+	@echo "Release uploaded to pypi.org"
+
+# DANGER: releases directly to prod
+# use this if you accidentally burned a test release version number,
+.PHONY: release-directly-to-prod
+release-directly-to-prod: pydist twine-prod
+	@echo "Dist built and uploaded to pypi.org"
+	@echo "Test the install:"
+	@echo "    make install-release"
+
+.PHONY: dev-env
+dev-env: dev-env-client dev-env-server
+
+.PHONY: dev-env-client
+dev-env-client:
+	cd client && $(MAKE) ci
+
+.PHONY: dev-env-server
+dev-env-server:
+	pip install -r server/requirements-dev.txt
+
+# Increments the release candidate version (i.e. 0.16.2-rc.1 -> 0.16.2-rc.2)
+.PHONY: bump-release-candidate
+bump-release-candidate:
+	bumpversion --config-file .bumpversion.cfg prerelversion --allow-dirty
+
+# Finalizes the release candidate by removing the release candidate tag (i.e. 0.16.2-rc.2 -> 0.16.2).
+.PHONY: bump-release
+bump-release:
+	bumpversion --config-file .bumpversion.cfg prerel --allow-dirty
+
+.PHONY: twine
+twine:
+	twine upload --repository-url https://test.pypi.org/legacy/ dist/*
+
+.PHONY: twine-prod
+twine-prod:
+	twine upload dist/*
+
+# quicker than re-building client
+.PHONY: gen-package-lock
+gen-package-lock:
+	cd client && $(MAKE) install
+
+
+# INSTALL
+
+# install from source tree for development
+.PHONY: install-dev
+install-dev: uninstall
+	pip install -e .
+
+# install from test.pypi to test your release
+.PHONY: install-release-test
+install-release-test: uninstall
+	pip install --no-cache-dir --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple cellxgene==$(VERSION)
+	@echo "Installed cellxgene from test.pypi.org, now run and smoke test"
+
+# install from pypi to test your release
+.PHONY: install-release
+install-release: uninstall
+	pip install --no-cache-dir cellxgene
+	@echo "Installed cellxgene from pypi.org"
+
+# install from dist
+.PHONY: install-dist
+install-dist: uninstall
+	pip install dist/cellxgene*.tar.gz
+
+.PHONY: uninstall
+uninstall:
+	pip uninstall -y cellxgene || :
+

PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,11 @@
+#### Reviewers
+**Functional:** 
+
+**Readability:** 
+
+---
+
+## Changes
+- add
+- remove
+- modify

README.md

@@ -3,81 +3,99 @@
 _an interactive explorer for single-cell transcriptomics data_
 
 [![DOI](https://zenodo.org/badge/105615409.svg)](https://zenodo.org/badge/latestdoi/105615409) [![PyPI](https://img.shields.io/pypi/v/cellxgene)](https://pypi.org/project/cellxgene/) [![PyPI - Downloads](https://img.shields.io/pypi/dm/cellxgene)](https://pypistats.org/packages/cellxgene) [![GitHub last commit](https://img.shields.io/github/last-commit/chanzuckerberg/cellxgene)](https://github.com/chanzuckerberg/cellxgene/pulse)
+[![Push Tests](https://github.com/chanzuckerberg/cellxgene/workflows/Push%20Tests/badge.svg)](https://github.com/chanzuckerberg/cellxgene/actions?query=workflow%3A%22Push+Tests%22)
+[![Compatibility Tests](https://github.com/chanzuckerberg/cellxgene/workflows/Compatibility%20Tests/badge.svg)](https://github.com/chanzuckerberg/cellxgene/actions?query=workflow%3A%22Compatibility+Tests%22)
+![Code Coverage](https://codecov.io/gh/chanzuckerberg/cellxgene/branch/main/graph/badge.svg)
 
-cellxgene (pronounced "cell-by-gene") is an interactive data explorer for single-cell transcriptomics datasets, such as those coming from the [Human Cell Atlas](https://humancellatlas.org). Leveraging modern web development techniques to enable fast visualizations of at least 1 million cells, we hope to enable biologists and computational researchers to explore their data.
+CZ CELLxGENE Annotate (pronounced "cell-by-gene") is an interactive data explorer for single-cell datasets, such as those coming from the [Human Cell Atlas](https://humancellatlas.org). Leveraging modern web development techniques to enable fast visualizations of at least 1 million cells, we hope to enable biologists and computational researchers to explore their data.
 
-Whether you need to visualize one thousand cells or one million, cellxgene helps you gain insight into your single-cell data.
+Whether you need to visualize one thousand cells or one million, CELLxGENE Annotate helps you gain insight into your single-cell data.
 
-<img src="https://github.com/chanzuckerberg/cellxgene/raw/master/docs/images/crossfilter.gif" width="350" height="200" hspace="30"><img src="https://github.com/chanzuckerberg/cellxgene/raw/master/docs/images/category-breakdown.gif" width="350" height="200" hspace="30">
+<img src="https://github.com/chanzuckerberg/cellxgene/raw/main/docs/images/crossfilter.gif" width="350" height="200" hspace="30"><img src="https://github.com/chanzuckerberg/cellxgene/raw/main/docs/images/category-breakdown.gif" width="350" height="200" hspace="30">
 
 # Getting started
-### The comprehensive guide to cellxgene
-[The cellxgene documentation is your one-stop-shop for information about cellxgene](https://chanzuckerberg.github.io/cellxgene/)! You may be particularly interested in:
-* Seeing [what cellxgene can do](https://chanzuckerberg.github.io/cellxgene/posts/gallery)  
-* Learning more about cellxgene [installation](https://chanzuckerberg.github.io/cellxgene/posts/install) and [usage](https://chanzuckerberg.github.io/cellxgene/posts/launch)
-* [Preparing your own data](https://chanzuckerberg.github.io/cellxgene/posts/prepare) for use in cellxgene
-* Checking out [our roadmap](https://chanzuckerberg.github.io/cellxgene/posts/roadmap) for future development
-* [Contributing](https://chanzuckerberg.github.io/cellxgene/posts/contribute) to cellxgene
+
+### The comprehensive guide to CZ CELLxGENE Annotate
+
+[The CZ CELLxGENE Annotate documentation is your one-stop-shop for information about CELLxGENE Annotate](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/README.md)! You may be particularly interested in:
+
+- Seeing [what Annotate can do](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/explore-data/explorer-tutorials.md)
+- Learning more about Annotate [installation](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/desktop/install.md) and [usage](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/desktop/quick-start.md#quick-start-1)
+- [Preparing your own data](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/desktop/data-reqs.md) for use in Annotate
+- Checking out [our roadmap](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/roadmap.md) for future development
+- [Contributing](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/contribute.md) to Annotate
 
 ### Quick start
 
-To install cellxgene you need Python 3.6+. We recommend [installing cellxgene into a conda or virtual environment.](https://chanzuckerberg.github.io/cellxgene/posts/install)
+To install CELLxGENE Annotate you need Python 3.10+. We recommend [installing Annotate into a conda or virtual environment.](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/desktop/install.md)
 
 Install the package.
-``` bash
+
+```bash
 pip install cellxgene
 ```
 
-Launch cellxgene with an example [anndata](https://anndata.readthedocs.io/en/latest/) file
+Launch Annotate with an example [anndata](https://anndata.readthedocs.io/en/latest/) file
 
-``` bash
+```bash
 cellxgene launch https://cellxgene-example-data.czi.technology/pbmc3k.h5ad
 ```
 
-To explore more datasets already formatted for cellxgene, check out the [Demo data](https://chanzuckerberg.github.io/cellxgene/posts/demo-data) or
-see [Preparing your data](https://chanzuckerberg.github.io/cellxgene/posts/prepare) to learn more about formatting your own
-data for cellxgene.
+To explore more datasets already formatted for Annotate, check out the [Demo data](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/desktop/quick-start.md#example-datasets) or
+see [Preparing your data](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/desktop/data-reqs.md) to learn more about formatting your own
+data for CELLxGENE Annotate.
+
+### Supported browsers
+
+CELLxGENE Annotate currently supports the following browsers:
+
+- Google Chrome 61+
+- Edge 15+
+- Firefox 60+
+
+Please [file an issue](https://github.com/chanzuckerberg/cellxgene/issues/new/choose) if you would like us to add support for an unsupported browser.
 
 ### Finding help
 
 We'd love to hear from you!
-For questions, suggestions, or accolades, [join the `#cellxgene-users` channel on the CZI Science Slack](https://join-cellxgene-users.herokuapp.com/) and say "hi!".
+For questions, suggestions, or accolades, join the `#cellxgene-users` channel on the [CZI Science Community Slack](https://czi.co/science-slack) and say "hi!".
 
 For any errors, [report bugs on Github](https://github.com/chanzuckerberg/cellxgene/issues).
 
-# Developing with cellxgene
+# Developing with CZ CELLxGENE Annotate
 
 ### Contributing
-We warmly welcome contributions from the community! Please see our [contributing guide](https://chanzuckerberg.github.io/cellxgene/posts/contribute) and don't hesitate to open an issue or send a pull request to improve cellxgene.
+
+We warmly welcome contributions from the community! Please see our [contributing guide](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/contribute.md) and don't hesitate to open an issue or send a pull request to improve CELLxGENE Annotate. Please see the [dev_docs](https://github.com/chanzuckerberg/cellxgene/tree/main/dev_docs) for pull request suggestions, unit test details, local documentation preview, and other development specifics.
 
 This project adheres to the Contributor Covenant [code of conduct](https://github.com/chanzuckerberg/.github/blob/master/CODE_OF_CONDUCT.md). By participating, you are expected to uphold this code. Please report unacceptable behavior to opensource@chanzuckerberg.com.
 
 ### Reuse
 
-This project was started with the sole goal of empowering the scientific community to explore and understand their data. As such, we encourage other scientific tool builders in academia or industry to adopt the patterns, tools, and code from this project, and reach out to us with ideas or questions. All code is freely available for reuse under the [MIT license](https://opensource.org/licenses/MIT).
+This project was started with the sole goal of empowering the scientific community to explore and understand their data.
+As such, we encourage other scientific tool builders in academia or industry to adopt the patterns, tools, and code from
+this project. All code is freely available for reuse under the [MIT license](https://opensource.org/licenses/MIT).
+
+Before extending CELLxGENE Annotate, we encourage you to reach out to us with ideas or questions. It might be possible that an
+extension could be directly contributed, which would make it available for a wider audience, or that it's on our
+[roadmap](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/roadmap.md) and under active development.
+
+See the [CELLxGENE extensions](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/community-extensions.md) section of our documentation for examples of community use and CELLxGENE extensions.
+
+### Trademarks
+
+CZ CELLXGENE, CZ CELLXGENE DISCOVER, and CZ CELLXGENE ANNOTATE are trademarks of the Chan Zuckerberg Initiative. All rights reserved.
+
+Use, reuse, modification, and re-distribution of the source code in this repository is subject to the terms of the applicable open source [license](LICENSE.txt). However, that license does not grant permission to use the trademarks without separate, express permission from the Chan Zuckerberg Initiative.
+
 
 ### Security
 
 If you believe you have found a security issue, we would appreciate notification. Please send email to <security@chanzuckerberg.com>.
 
-# About
-### Core team
+# Inspiration
 
-The current core team:
-
-- Colin Megill, frontend & product design
-- Bruce Martin, software engineer
-- Sidney Bell, computational biologist
-- Lia Prins, designer
-- Severiano Badajoz, software engineer
-
-We would also like to gratefully acknowledge contributions from past core team members:
-
-- Charlotte Weaver, software engineer
-
-### Inspiration
-
-We've been heavily inspired by several other related single-cell visualization projects, including the [UCSC Cell Browswer](http://cells.ucsc.edu/), [Cytoscape](http://www.cytoscape.org/), [Xena](https://xena.ucsc.edu/), [ASAP](https://asap.epfl.ch/), [Gene Pattern](http://genepattern-notebook.org/), and many others. We hope to explore collaborations where useful as this community works together on improving interactive visualization for single-cell data.
+We've been heavily inspired by several other related single-cell visualization projects, including the [UCSC Cell Browser](http://cells.ucsc.edu/), [Cytoscape](http://www.cytoscape.org/), [Xena](https://xena.ucsc.edu/), [ASAP](https://asap.epfl.ch/), [GenePattern](http://genepattern-notebook.org/), and many others. We hope to explore collaborations where useful as this community works together on improving interactive visualization for single-cell data.
 
 We were inspired by Mike Bostock and the [crossfilter](https://github.com/crossfilter) team for the design of our filtering implementation.
 

SECURITY.md

@@ -0,0 +1,2 @@
+## Reporting Security Issues
+If you believe you have found a security issue, please responsibly disclose by contacting us at [security@chanzuckerberg.com](mailto:security@chanzuckerberg.com).

app.json

@@ -1,21 +0,0 @@
-{
-  "name": "cellxgene",
-  "description": "An interactive explorer for single-cell transcriptomics data",
-  "repository": "https://github.com/chanzuckerberg/cellxgene/",
-  "logo": "https://cellxgene-example-data.czi.technology/favicon.png",
-  "keywords": [
-    "scientific",
-    "visualization",
-    "scrna-seq",
-    "transcriptomics",
-    "dataviz"
-  ],
-  "stack": "container",
-  "env": {
-    "DATASET": {
-      "description": "Link to dataset",
-      "value": "https://cellxgene-example-data.czi.technology/pbmc3k.h5ad",
-      "required": "true"
-    }
-  }
-}

cellxgene-osx.spec

@@ -1,41 +0,0 @@
-# -*- mode: python ; coding: utf-8 -*-
-
-block_cipher = None
-
-
-a = Analysis(['server/gui/main.py'],
-             pathex=['/Users/charlotteweaver/Documents/Git/cellxgene'],
-             binaries=[('/System/Library/Frameworks/Tk.framework/Tk', 'tk'), ('/System/Library/Frameworks/Tcl.framework/Tcl', 'tcl')],
-             datas=[('server/app/web/templates/', 'server/app/web/templates/'), ('server/app/web/static/', 'server/app/web/static/')],
-             hiddenimports=['sklearn', 'sklearn.utils._cython_blas', 'sklearn.neighbors.typedefs', 'sklearn.neighbors.quad_tree', 'sklearn.tree', 'sklearn.tree._utils'],
-             hookspath=['server/gui/'],
-             runtime_hooks=[],
-             excludes=[],
-             win_no_prefer_redirects=False,
-             win_private_assemblies=False,
-             cipher=block_cipher,
-             noarchive=False)
-pyz = PYZ(a.pure, a.zipped_data,
-             cipher=block_cipher)
-exe = EXE(pyz,
-          a.scripts,
-          [],
-          exclude_binaries=True,
-          name='cellxgene',
-          debug=False,
-          bootloader_ignore_signals=False,
-          strip=False,
-          upx=True,
-          console=False , icon='server/gui/images/cxg_icons.icns')
-coll = COLLECT(exe,
-               a.binaries,
-               a.zipfiles,
-               a.datas,
-               strip=False,
-               upx=True,
-               upx_exclude=[],
-               name='cellxgene')
-app = BUNDLE(coll,
-             name='cellxgene.app',
-             icon='server/gui/images/cxg_icons.icns',
-             bundle_identifier=None)

cellxgene-windows.spec

@@ -1,36 +0,0 @@
-# -*- mode: python -*-
-
-block_cipher = None
-
-
-a = Analysis(['server\\gui\\main.py'],
-             pathex=['C:\\Users\\Charlotte\\Documents\\git\\cellxgene'],
-             binaries=[],
-             datas=[('server/app/web/templates/', 'server/app/web/templates'), ('server/app/web/static/', 'server/app/web/static')],
-             hiddenimports=[],
-             hookspath=['server/gui/'],
-             runtime_hooks=[],
-             excludes=[],
-             win_no_prefer_redirects=False,
-             win_private_assemblies=False,
-             cipher=block_cipher,
-             noarchive=False)
-pyz = PYZ(a.pure, a.zipped_data,
-             cipher=block_cipher)
-exe = EXE(pyz,
-          a.scripts,
-          [],
-          exclude_binaries=True,
-          name='cellxgene',
-          debug=False,
-          bootloader_ignore_signals=False,
-          strip=False,
-          upx=True,
-          console=False , icon='server\\gui\\images\\icon.ico')
-coll = COLLECT(exe,
-               a.binaries,
-               a.zipfiles,
-               a.datas,
-               strip=False,
-               upx=True,
-               name='cellxgene')

client/.husky/pre-commit

@@ -0,0 +1,5 @@
+#!/bin/sh
+. "$(dirname "$0")/_/husky.sh"
+
+cd client
+npx --no-install lint-staged --config "./configuration/lint-staged/lint-staged.config.js"

client/.nvmrc

@@ -0,0 +1 @@
+18.17.0

client/Makefile

@@ -0,0 +1,62 @@
+include ../common.mk
+
+ANNOTATIONS := $(if $(ANNOTATIONS),$(ANNOTATIONS),../test/fixtures/pbmc3k-annotations.csv)
+GENE_SETS := $(if $(GENE_SETS),$(GENE_SETS),../test/fixtures/pbmc3k-genesets.csv)
+ANNOTATIONS_FILENAME := $(shell basename $(ANNOTATIONS))
+GENE_SETS_FILENAME := $(shell basename $(GENE_SETS))
+
+CXG_CONFIG := $(if $(CXG_CONFIG),$(CXG_CONFIG),./__tests__/e2e/test_config.yaml)
+
+
+
+# Packaging
+.PHONY: clean
+clean:
+	rm -rf node_modules
+	rm -f __tests__/screenshots/*.png
+
+.PHONY: ci
+ci:
+	npm ci
+
+.PHONY: install
+install:
+	npm install
+
+.PHONY: build
+WEBPACK_CONFIG ?= configuration/webpack/webpack.config.prod.js
+build:
+	npm run build $(WEBPACK_CONFIG)
+
+# Development convenience methods
+.PHONY: start-frontend
+start-frontend:
+	node server/development.js
+
+# start an instance of cellxgene and run the end-to-end tests
+.PHONY: smoke-test
+smoke-test:
+	start_server_and_test \
+		'CXG_OPTIONS="--config-file $(CXG_CONFIG)" $(MAKE) start-server' \
+		$(CXG_SERVER_PORT) \
+		'CXG_URL_BASE="http://localhost:$(CXG_SERVER_PORT)" npm run e2e -- --verbose false'
+
+# start an instance of cellxgene and run the end-to-end annotations tests
+.PHONY: smoke-test-annotations
+smoke-test-annotations:
+	$(eval TMP_DIR := $(shell  mktemp -d /tmp/cellxgene_XXXXXX))
+	cp $(ANNOTATIONS) $(TMP_DIR)/ && \
+	cp $(GENE_SETS) $(TMP_DIR)/ && \
+	start_server_and_test \
+		'CXG_OPTIONS="--annotations-file $(TMP_DIR)/$(ANNOTATIONS_FILENAME) --gene-sets-file $(TMP_DIR)/$(GENE_SETS_FILENAME)" $(MAKE) start-server' \
+		$(CXG_SERVER_PORT) \
+		'CXG_URL_BASE="http://localhost:$(CXG_SERVER_PORT)" npm run e2e-annotations -- --verbose false'
+	rm -rf $(TMP_DIR)
+
+.PHONY: unit-test
+unit-test:
+	node node_modules/jest/bin/jest.js --testPathIgnorePatterns e2e
+
+# pass remaining commands through to npm run
+%:
+	npm run $(*)

client/__tests__/e2e/__snapshots__/e2e.test.js.snap

@@ -0,0 +1,5 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`did launch page launched 1`] = `"<span style=\\"max-width: 155px; display: flex; overflow: hidden; justify-content: flex-start; width: 100%; padding: 0px;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">pbm</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">c3k</span><span style=\\"position: absolute; right: 0px; color: inherit;\\">c3k</span></span></span>"`;
+
+exports[`metadata loads categories and values from dataset appear 1`] = `"<div style=\\"display: flex; justify-content: space-between; align-items: baseline;\\"><div style=\\"display: flex; justify-content: flex-start; align-items: flex-start;\\"><label class=\\"bp3-control bp3-checkbox\\" for=\\"category-select-louvain\\"><input id=\\"category-select-louvain\\" data-testclass=\\"category-select\\" data-testid=\\"louvain:category-select\\" type=\\"checkbox\\" checked=\\"\\"><span class=\\"bp3-control-indicator\\"></span></label><span role=\\"menuitem\\" tabindex=\\"0\\" data-testclass=\\"category-expand\\" data-testid=\\"louvain:category-expand\\" style=\\"cursor: pointer;\\"><span aria-haspopup=\\"true\\" class=\\"bp3-popover2-target\\"><span data-testid=\\"louvain:category-label\\" tabindex=\\"-1\\" aria-label=\\"louvain\\" class=\\"\\" style=\\"max-width: 265px;\\"><span style=\\"max-width: 265px; display: flex; overflow: hidden; justify-content: flex-start; width: 100%; padding: 0px;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">lou</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">vain</span><span style=\\"position: absolute; right: 0px; color: inherit;\\">vain</span></span></span></span></span><svg stroke=\\"currentColor\\" fill=\\"currentColor\\" stroke-width=\\"0\\" viewBox=\\"0 0 320 512\\" data-testclass=\\"category-expand-is-not-expanded\\" height=\\"1em\\" width=\\"1em\\" xmlns=\\"http://www.w3.org/2000/svg\\" style=\\"font-size: 10px; margin-left: 5px;\\"><path d=\\"M285.476 272.971L91.132 467.314c-9.373 9.373-24.569 9.373-33.941 0l-22.667-22.667c-9.357-9.357-9.375-24.522-.04-33.901L188.505 256 34.484 101.255c-9.335-9.379-9.317-24.544.04-33.901l22.667-22.667c9.373-9.373 24.569-9.373 33.941 0L285.475 239.03c9.373 9.372 9.373 24.568.001 33.941z\\"></path></svg></span></div><div><span class=\\"bp3-popover-wrapper\\"><span aria-haspopup=\\"true\\" class=\\"bp3-popover-target\\"><a role=\\"button\\" data-testclass=\\"colorby\\" data-testid=\\"colorby-louvain\\" class=\\"bp3-button\\" tabindex=\\"0\\"><span icon=\\"tint\\" aria-hidden=\\"true\\" tabindex=\\"0\\" class=\\"bp3-icon bp3-icon-tint\\"><svg data-icon=\\"tint\\" width=\\"16\\" height=\\"16\\" viewBox=\\"0 0 16 16\\"><path d=\\"M7.88 1s-4.9 6.28-4.9 8.9c.01 2.82 2.34 5.1 4.99 5.1 2.65-.01 5.03-2.3 5.03-5.13C12.99 7.17 7.88 1 7.88 1z\\" fill-rule=\\"evenodd\\"></path></svg></span></a></span></span></div></div><div style=\\"margin-left: 26px;\\"></div>"`;

client/__tests__/e2e/cellxgeneActions.js

@@ -0,0 +1,445 @@
+/* eslint-disable no-await-in-loop -- await in loop is needed to emulate sequential user actions  */
+import { strict as assert } from "assert";
+import {
+  clearInputAndTypeInto,
+  clickOn,
+  getAllByClass,
+  getOneElementInnerText,
+  typeInto,
+  waitByID,
+  waitByClass,
+  waitForAllByIds,
+  clickOnUntil,
+  getTestClass,
+  getTestId,
+  isElementPresent,
+} from "./puppeteerUtils";
+
+export async function drag(testId, start, end, lasso = false) {
+  const layout = await waitByID(testId);
+  const elBox = await layout.boxModel();
+  const x1 = elBox.content[0].x + start.x;
+  const x2 = elBox.content[0].x + end.x;
+  const y1 = elBox.content[0].y + start.y;
+  const y2 = elBox.content[0].y + end.y;
+  await page.mouse.move(x1, y1);
+  await page.mouse.down();
+  if (lasso) {
+    await page.mouse.move(x2, y1);
+    await page.mouse.move(x2, y2);
+    await page.mouse.move(x1, y2);
+    await page.mouse.move(x1, y1);
+  } else {
+    await page.mouse.move(x2, y2);
+  }
+  await page.mouse.up();
+}
+
+export async function clickOnCoordinate(testId, coord) {
+  const layout = await expect(page).toMatchElement(getTestId(testId));
+  const elBox = await layout.boxModel();
+
+  if (!elBox) {
+    throw Error("Layout's boxModel is not available!");
+  }
+
+  const x = elBox.content[0].x + coord.x;
+  const y = elBox.content[0].y + coord.y;
+  await page.mouse.click(x, y);
+}
+
+export async function getAllHistograms(testclass, testIds) {
+  const histTestIds = testIds.map((tid) => `histogram-${tid}`);
+
+  // these load asynchronously, so we need to wait for each histogram individually,
+  // and they may be quite slow in some cases.
+  await waitForAllByIds(histTestIds, { timeout: 4 * 60 * 1000 });
+
+  const allHistograms = await getAllByClass(testclass);
+
+  const testIDs = await Promise.all(
+    allHistograms.map((hist) =>
+      page.evaluate((elem) => elem.dataset.testid, hist)
+    )
+  );
+
+  return testIDs.map((id) => id.replace(/^histogram-/, ""));
+}
+
+export async function getAllCategoriesAndCounts(category) {
+  // these load asynchronously, so we have to wait for the specific category.
+  await waitByID(`category-${category}`);
+
+  return page.$$eval(
+    `[data-testid="category-${category}"] [data-testclass='categorical-row']`,
+    (rows) =>
+      Object.fromEntries(
+        rows.map((row) => {
+          const cat = row
+            .querySelector("[data-testclass='categorical-value']")
+            .getAttribute("aria-label");
+
+          const count = row.querySelector(
+            "[data-testclass='categorical-value-count']"
+          ).innerText;
+
+          return [cat, count];
+        })
+      )
+  );
+}
+
+export async function getCellSetCount(num) {
+  await clickOn(`cellset-button-${num}`);
+  return getOneElementInnerText(`[data-testid='cellset-count-${num}']`);
+}
+
+export async function resetCategory(category) {
+  const checkboxId = `${category}:category-select`;
+  await waitByID(checkboxId);
+  const checkedPseudoclass = await page.$eval(
+    `[data-testid='${checkboxId}']`,
+    (el) => el.matches(":checked")
+  );
+  if (!checkedPseudoclass) await clickOn(checkboxId);
+
+  const categoryRow = await waitByID(`${category}:category-expand`);
+
+  const isExpanded = await categoryRow.$(
+    "[data-testclass='category-expand-is-expanded']"
+  );
+
+  if (isExpanded) await clickOn(`${category}:category-expand`);
+}
+
+export async function calcCoordinate(testId, xAsPercent, yAsPercent) {
+  const el = await waitByID(testId);
+  const size = await el.boxModel();
+  return {
+    x: Math.floor(size.width * xAsPercent),
+    y: Math.floor(size.height * yAsPercent),
+  };
+}
+
+export async function calcDragCoordinates(testId, coordinateAsPercent) {
+  return {
+    start: await calcCoordinate(
+      testId,
+      coordinateAsPercent.x1,
+      coordinateAsPercent.y1
+    ),
+    end: await calcCoordinate(
+      testId,
+      coordinateAsPercent.x2,
+      coordinateAsPercent.y2
+    ),
+  };
+}
+
+export async function selectCategory(category, values, reset = true) {
+  if (reset) await resetCategory(category);
+
+  await clickOn(`${category}:category-expand`);
+  await clickOn(`${category}:category-select`);
+
+  for (const value of values) {
+    await clickOn(`categorical-value-select-${category}-${value}`);
+  }
+}
+
+export async function expandCategory(category) {
+  const expand = await waitByID(`${category}:category-expand`);
+  const notExpanded = await expand.$(
+    "[data-testclass='category-expand-is-not-expanded']"
+  );
+  if (notExpanded) await clickOn(`${category}:category-expand`);
+}
+
+export async function clip(min = 0, max = 100) {
+  await clickOn("visualization-settings");
+  await clearInputAndTypeInto("clip-min-input", min);
+  await clearInputAndTypeInto("clip-max-input", max);
+  await clickOn("clip-commit");
+}
+
+export async function createCategory(categoryName) {
+  await clickOnUntil("open-annotation-dialog", async () => {
+    await expect(page).toMatchElement(getTestId("new-category-name"));
+  });
+
+  await typeInto("new-category-name", categoryName);
+  await clickOn("submit-category");
+}
+
+/*
+
+  GENESET
+
+*/
+
+export async function colorByGeneset(genesetName) {
+  await clickOn(`${genesetName}:colorby-entire-geneset`);
+}
+
+export async function colorByGene(gene) {
+  await clickOn(`colorby-${gene}`);
+}
+
+export async function assertColorLegendLabel(label) {
+  const handle = await waitByID("continuous_legend_color_by_label");
+
+  const result = await handle.evaluate((node) =>
+    node.getAttribute("aria-label")
+  );
+
+  return expect(result).toBe(label);
+}
+
+export async function expandGeneset(genesetName) {
+  const expand = await waitByID(`${genesetName}:geneset-expand`);
+  const notExpanded = await expand.$(
+    "[data-testclass='geneset-expand-is-not-expanded']"
+  );
+  if (notExpanded) await clickOn(`${genesetName}:geneset-expand`);
+}
+
+export async function createGeneset(genesetName) {
+  await clickOnUntil("open-create-geneset-dialog", async () => {
+    await expect(page).toMatchElement(getTestId("create-geneset-input"));
+  });
+
+  await typeInto("create-geneset-input", genesetName);
+  await clickOn("submit-geneset");
+  await waitByClass("autosave-complete");
+}
+
+export async function editGenesetName(genesetName, editText) {
+  const editButton = `${genesetName}:edit-genesetName-mode`;
+  const submitButton = `${genesetName}:submit-geneset`;
+  await clickOnUntil(`${genesetName}:see-actions`, async () => {
+    await expect(page).toMatchElement(getTestId(editButton));
+  });
+  await clickOn(editButton);
+  await typeInto("rename-geneset-modal", editText);
+  await clickOn(submitButton);
+}
+
+export async function deleteGeneset(genesetName) {
+  const targetId = `${genesetName}:delete-geneset`;
+
+  await clickOnUntil(`${genesetName}:see-actions`, async () => {
+    await expect(page).toMatchElement(getTestId(targetId));
+  });
+
+  await clickOn(targetId);
+
+  await assertGenesetDoesNotExist(genesetName);
+  await waitByClass("autosave-complete");
+}
+
+export async function assertGenesetDoesNotExist(genesetName) {
+  const result = await isElementPresent(
+    getTestId(`${genesetName}:geneset-name`)
+  );
+  await expect(result).toBe(false);
+}
+
+export async function assertGenesetExists(genesetName) {
+  const handle = await waitByID(`${genesetName}:geneset-name`);
+
+  const result = await handle.evaluate((node) =>
+    node.getAttribute("aria-label")
+  );
+
+  return expect(result).toBe(genesetName);
+}
+
+/*
+
+  GENE
+
+*/
+
+export async function addGeneToSet(genesetName, geneToAddToSet) {
+  const submitButton = `${genesetName}:submit-gene`;
+
+  await clickOn(`${genesetName}:add-new-gene-to-geneset`);
+  await typeInto("add-genes", geneToAddToSet);
+  await clickOn(submitButton);
+}
+
+export async function removeGene(geneSymbol) {
+  const targetId = `delete-from-geneset:${geneSymbol}`;
+
+  await clickOn(targetId);
+
+  await waitByClass("autosave-complete");
+}
+
+export async function assertGeneExistsInGeneset(geneSymbol) {
+  const handle = await waitByID(`${geneSymbol}:gene-label`);
+
+  const result = await handle.evaluate((node) =>
+    node.getAttribute("aria-label")
+  );
+
+  return expect(result).toBe(geneSymbol);
+}
+
+export async function assertGeneDoesNotExist(geneSymbol) {
+  const result = await isElementPresent(getTestId(`${geneSymbol}:gene-label`));
+
+  await expect(result).toBe(false);
+}
+
+export async function expandGene(geneSymbol) {
+  await clickOn(`maximize-${geneSymbol}`);
+}
+
+/*
+
+  CATEGORY
+
+*/
+
+export async function duplicateCategory(categoryName) {
+  await clickOn("open-annotation-dialog");
+
+  await typeInto("new-category-name", categoryName);
+
+  const dropdownOptionClass = "duplicate-category-dropdown-option";
+
+  await clickOnUntil("duplicate-category-dropdown", async () => {
+    await expect(page).toMatchElement(getTestClass(dropdownOptionClass));
+  });
+
+  const option = await expect(page).toMatchElement(
+    getTestClass(dropdownOptionClass)
+  );
+
+  await option.click();
+
+  await clickOnUntil("submit-category", async () => {
+    await expect(page).toMatchElement(
+      getTestId(`${categoryName}:category-expand`)
+    );
+  });
+
+  await waitByClass("autosave-complete");
+}
+
+export async function renameCategory(oldCategoryName, newCategoryName) {
+  await clickOn(`${oldCategoryName}:see-actions`);
+  await clickOn(`${oldCategoryName}:edit-category-mode`);
+  await clearInputAndTypeInto(
+    `${oldCategoryName}:edit-category-name-text`,
+    newCategoryName
+  );
+  await clickOn(`${oldCategoryName}:submit-category-edit`);
+}
+
+export async function deleteCategory(categoryName) {
+  const targetId = `${categoryName}:delete-category`;
+
+  await clickOnUntil(`${categoryName}:see-actions`, async () => {
+    await expect(page).toMatchElement(getTestId(targetId));
+  });
+
+  await clickOn(targetId);
+
+  await assertCategoryDoesNotExist();
+}
+
+export async function createLabel(categoryName, labelName) {
+  /**
+   * (thuang): This explicit wait is needed, since currently showing
+   * the modal again quickly after the previous action dismissing the
+   * modal will persist the input value from the previous action.
+   *
+   * To reproduce:
+   * 1. Click on the plus sign to show the modal to add a new label to the category
+   * 2. Type `123` in the input box
+   * 3. Hover over your mouse over the plus sign and double click to quickly dismiss and
+   * invoke the modal again
+   * 4. You will see `123` is persisted in the input box
+   * 5. Expected behavior is to get an empty input box
+   */
+  await page.waitForTimeout(500);
+
+  await clickOn(`${categoryName}:see-actions`);
+
+  await clickOn(`${categoryName}:add-new-label-to-category`);
+
+  await typeInto(`${categoryName}:new-label-name`, labelName);
+
+  await clickOn(`${categoryName}:submit-label`);
+}
+
+export async function deleteLabel(categoryName, labelName) {
+  await expandCategory(categoryName);
+  await clickOn(`${categoryName}:${labelName}:see-actions`);
+  await clickOn(`${categoryName}:${labelName}:delete-label`);
+}
+
+export async function renameLabel(categoryName, oldLabelName, newLabelName) {
+  await expandCategory(categoryName);
+  await clickOn(`${categoryName}:${oldLabelName}:see-actions`);
+  await clickOn(`${categoryName}:${oldLabelName}:edit-label`);
+  await clearInputAndTypeInto(
+    `${categoryName}:${oldLabelName}:edit-label-name`,
+    newLabelName
+  );
+  await clickOn(`${categoryName}:${oldLabelName}:submit-label-edit`);
+}
+
+export async function addGeneToSearch(geneName) {
+  await typeInto("gene-search", geneName);
+  await page.keyboard.press("Enter");
+  await page.waitForSelector(`[data-testid='histogram-${geneName}']`);
+}
+
+export async function subset(coordinatesAsPercent) {
+  // In order to deselect the selection after the subset, make sure we have some clear part
+  // of the scatterplot we can click on
+  assert(coordinatesAsPercent.x2 < 0.99 || coordinatesAsPercent.y2 < 0.99);
+  const lassoSelection = await calcDragCoordinates(
+    "layout-graph",
+    coordinatesAsPercent
+  );
+  await drag("layout-graph", lassoSelection.start, lassoSelection.end, true);
+  await clickOn("subset-button");
+  const clearCoordinate = await calcCoordinate("layout-graph", 0.5, 0.99);
+  await clickOnCoordinate("layout-graph", clearCoordinate);
+}
+
+export async function setSellSet(cellSet, cellSetNum) {
+  const selections = cellSet.filter((sel) => sel.kind === "categorical");
+
+  for (const selection of selections) {
+    await selectCategory(selection.metadata, selection.values, true);
+  }
+
+  await getCellSetCount(cellSetNum);
+}
+
+export async function runDiffExp(cellSet1, cellSet2) {
+  await setSellSet(cellSet1, 1);
+  await setSellSet(cellSet2, 2);
+  await clickOn("diffexp-button");
+}
+
+export async function bulkAddGenes(geneNames) {
+  await clickOn("section-bulk-add");
+  await typeInto("input-bulk-add", geneNames.join(","));
+  await page.keyboard.press("Enter");
+}
+
+export async function assertCategoryDoesNotExist(categoryName) {
+  const result = await isElementPresent(
+    getTestId(`${categoryName}:category-label`)
+  );
+
+  await expect(result).toBe(false);
+}
+
+/* eslint-enable no-await-in-loop -- await in loop is needed to emulate sequential user actions */

client/__tests__/e2e/config.js

@@ -1,6 +1,10 @@
-export const jest_env = process.env.JEST_ENV || "dev";
-export const appPort = process.env.JEST_CXG_PORT || 3000;
-export const appUrlBase = `http://localhost:${appPort}`;
-export const DEV = jest_env === "dev";
-export const DEBUG = jest_env === "debug";
+import * as ENV_DEFAULT from "../../../environment.default.json";
+
+export const jestEnv = process.env.JEST_ENV || ENV_DEFAULT.JEST_ENV;
+export const appUrlBase =
+  process.env.CXG_URL_BASE || `http://localhost:${ENV_DEFAULT.CXG_CLIENT_PORT}`;
 export const DATASET = "pbmc3k";
+export const isDev = jestEnv === ENV_DEFAULT.DEV;
+export const isDebug = jestEnv === ENV_DEFAULT.DEBUG;
+export const TEST_EMAIL = "user@example.com";
+export const TEST_PASSWORD = process.env.TEST_ACCOUNT_PASS ?? "";

client/__tests__/e2e/data.js

@@ -4,7 +4,7 @@ export const datasets = {
     dataframe: {
       nObs: "2638",
       nVar: "1838",
-      type: "float32"
+      type: "float32",
     },
     categorical: {
       louvain: {
@@ -15,47 +15,47 @@ export const datasets = {
         "Dendritic cells": "37",
         "FCGR3A+ Monocytes": "150",
         Megakaryocytes: "15",
-        "NK cells": "154"
-      }
+        "NK cells": "154",
+      },
     },
     continuous: {
       n_genes: "int32",
       percent_mito: "float32",
-      n_counts: "float32"
+      n_counts: "float32",
     },
     cellsets: {
       lasso: [
         {
           "coordinates-as-percent": { x1: 0.1, y1: 0.25, x2: 0.7, y2: 0.75 },
-          count: "1181"
-        }
+          count: "1131",
+        },
       ],
       categorical: [
         {
           metadata: "louvain",
           values: ["B cells", "Megakaryocytes"],
-          count: "357"
-        }
+          count: "357",
+        },
       ],
       continuous: [
         {
           metadata: "n_genes",
           "coordinates-as-percent": { x1: 0.25, y1: 0.5, x2: 0.55, y2: 0.5 },
-          count: "1537"
-        }
-      ]
+          count: "1537",
+        },
+      ],
     },
 
     diffexp: {
       cellset1: [
-        { kind: "categorical", metadata: "louvain", values: ["B cells"] }
+        { kind: "categorical", metadata: "louvain", values: ["B cells"] },
       ],
       cellset2: [
         {
           kind: "categorical",
           metadata: "louvain",
-          values: ["CD4 T cells", "NK cells"]
-        }
+          values: ["CD4 T cells", "NK cells"],
+        },
       ],
       "gene-results": [
         "HLA-DRB1",
@@ -67,21 +67,21 @@ export const datasets = {
         "HLA-DQB1",
         "MS4A1",
         "IL32",
-        "CD37"
-      ]
+        "CD37",
+      ],
     },
 
     genes: {
       bulkadd: ["S100A8", "FCGR3A", "LGALS2", "GSTP1"],
-      search: "ACD"
+      search: "ACD",
     },
     subset: {
       cellset1: [
         {
           kind: "categorical",
           metadata: "louvain",
-          values: ["B cells", "Megakaryocytes"]
-        }
+          values: ["B cells", "Megakaryocytes"],
+        },
       ],
       count: "357",
       categorical: {
@@ -93,27 +93,46 @@ export const datasets = {
           "Dendritic cells": "0",
           "FCGR3A+ Monocytes": "0",
           Megakaryocytes: "15",
-          "NK cells": "0"
-        }
+          "NK cells": "0",
+        },
       },
       lasso: {
         "coordinates-as-percent": { x1: 0.25, y1: 0.05, x2: 0.75, y2: 0.55 },
-        count: "329"
-      }
+        count: "331",
+      },
     },
     scatter: {
-      genes: { x: "S100A8", y: "FCGR3A" }
+      genes: { x: "S100A8", y: "FCGR3A" },
     },
     pan: {
-      "coordinates-as-percent": { x1: 0.75, y1: 0.75, x2: 0.35, y2: 0.35 }
+      "coordinates-as-percent": { x1: 0.75, y1: 0.75, x2: 0.35, y2: 0.35 },
     },
     features: {
       panzoom: {
         lasso: {
           "coordinates-as-percent": { x1: 0.3, y1: 0.3, x2: 0.5, y2: 0.5 },
-          count: "24"
-        }
-      }
+          count: "38",
+        },
+      },
+    },
+    categoryLabel: {
+      lasso: {
+        "coordinates-as-percent": { x1: 0.05, y1: 0.3, x2: 0.5, y2: 0.5 },
+      },
+      newCount: {
+        bySubsetConfig: {
+          false: "668",
+          true: "659",
+        },
+      },
+    },
+    annotationsFromFile: {
+      count: {
+        bySubsetConfig: {
+          false: "1161",
+          true: "852",
+        },
+      },
     },
     clip: {
       min: "30",
@@ -121,8 +140,8 @@ export const datasets = {
       metadata: "n_genes",
       gene: "S100A8",
       "coordinates-as-percent": { x1: 0.25, y1: 0.5, x2: 0.55, y2: 0.5 },
-      count: "392",
-      "gene-cell-count": "421"
-    }
-  }
+      count: "386",
+      "gene-cell-count": "416",
+    },
+  },
 };

client/__tests__/e2e/diffexpGeneSets.js

@@ -0,0 +1,105 @@
+export const diffexpPop1Genes = [
+  "CD79A",
+  "HLA-DRB1",
+  "HLA-DQA1",
+  "HLA-DPB1",
+  "HLA-DQB1",
+  "HLA-DPA1",
+  "MS4A1",
+  "LTB",
+  "CD79B",
+  "CD37",
+  "HLA-DMA",
+  "TCL1A",
+  "LINC00926",
+  "HLA-DMB",
+  "HVCN1",
+  "EAF2",
+  "FCRLA",
+  "IRF8",
+  "PKIG",
+  "P2RX5",
+  "HLA-DOB",
+  "SPIB",
+  "BLNK",
+  "SWAP70",
+  "PNOC",
+  "CD19",
+  "SMIM14",
+  "CD72",
+  "KIAA0125",
+  "IGLL5",
+  "ARHGAP24",
+  "COTL1",
+  "C16orf74",
+  "BTK",
+  "SNX29P2",
+  "ADAM28",
+  "FCGR2B",
+  "PLD4",
+  "PPP1R14A",
+  "MZB1",
+  "KIAA0040",
+  "PHACTR1",
+  "FCRL2",
+  "RIC3",
+  "P2RY10",
+  "SCPEP1",
+  "DRAM2",
+  "RP5-887A10.1",
+  "CD82",
+  "GPX1",
+];
+
+export const diffexpPop2Genes = [
+  "NKG7",
+  "GZMB",
+  "CTSW",
+  "PRF1",
+  "GNLY",
+  "GZMA",
+  "CST7",
+  "FGFBP2",
+  "SRGN",
+  "CD247",
+  "FCGR3A",
+  "TYROBP",
+  "FCER1G",
+  "ID2",
+  "SPON2",
+  "CCL4",
+  "CCL5",
+  "GZMH",
+  "GIMAP7",
+  "CLIC3",
+  "HOPX",
+  "XCL2",
+  "LGALS1",
+  "IGFBP7",
+  "AKR1C3",
+  "IL32",
+  "EFHD2",
+  "PRSS23",
+  "TTC38",
+  "ZAP70",
+  "S1PR5",
+  "SAMD3",
+  "GIMAP4",
+  "CCL3",
+  "ABI3",
+  "XCL1",
+  "S100A6",
+  "UBB",
+  "GPR56",
+  "PDIA3",
+  "S100A11",
+  "APOBEC3G",
+  "HAVCR2",
+  "PLEKHF1",
+  "LITAF",
+  "ARPC5L",
+  "PTGDR",
+  "PRMT2",
+  "GSTP1",
+  "FCRL6",
+];

client/__tests__/e2e/e2e.test.js

@@ -1,79 +1,63 @@
-/*
-Smoke test suite that will be run in Travis CI
-
-Tests included in this file are expected to be relatively stable and test core features
+/**
+ * Smoke test suite that will be run in Travis CI
+ * Tests included in this file are expected to be relatively stable and test core features
  */
-import puppeteer from "puppeteer";
-import { appUrlBase, DEBUG, DEV, DATASET } from "./config";
-import { puppeteerUtils, cellxgeneActions } from "./puppeteerUtils";
+
+/* eslint-disable no-await-in-loop -- await in loop is needed to emulate sequential user actions  */
+import { appUrlBase, DATASET } from "./config";
+
 import { datasets } from "./data";
 
-let browser, page, utils, cxgActions, spy;
-const browserViewport = { width: 1280, height: 960 };
-let data = datasets[DATASET];
+import {
+  clickOn,
+  getAllByClass,
+  getElementCoordinates,
+  getOneElementInnerHTML,
+  getTestId,
+  goToPage,
+  waitByID,
+} from "./puppeteerUtils";
 
-if (DEBUG) jest.setTimeout(100000);
-if (DEV) jest.setTimeout(10000);
+import {
+  calcDragCoordinates,
+  clip,
+  drag,
+  getAllCategoriesAndCounts,
+  getCellSetCount,
+  selectCategory,
+} from "./cellxgeneActions";
 
-beforeAll(async () => {
-  const browserParams = DEV
-    ? { headless: false, slowMo: 5 }
-    : DEBUG
-    ? { headless: false, slowMo: 100, devtools: true }
-    : {};
-  browser = await puppeteer.launch(browserParams);
-  page = await browser.newPage();
-  await page.setViewport(browserViewport);
-  if (DEV || DEBUG) {
-    page.on("console", async msg => {
-      // If there is a console.error but an error is not thrown, this will ensure the test fails
-      if (msg.type() === "error") {
-        const errorMsgText = await Promise.all(
-          // TODO can we do this without internal properties?
-          msg.args().map(arg => arg._remoteObject.description)
-        );
-        throw new Error(`Console error: ${errorMsgText}`);
-      }
-      console.log(`PAGE LOG: ${msg.text()}`);
-    });
-  }
-  page.on("pageerror", err => {
-    throw new Error(`Console error: ${err}`);
-  });
-  utils = puppeteerUtils(page);
-  cxgActions = cellxgeneActions(page);
-});
-
-beforeEach(async () => {
-  await page.goto(appUrlBase);
-});
-
-afterAll(() => {
-  if (!DEBUG) {
-    browser.close();
-  }
-});
+const data = datasets[DATASET];
 
 describe("did launch", () => {
   test("page launched", async () => {
-    let el = await utils.getOneElementInnerHTML("[data-testid='header']");
-    expect(el).toBe(data.title);
+    await goToPage(appUrlBase);
+
+    const element = await getOneElementInnerHTML(getTestId("header"));
+
+    expect(element).toMatchSnapshot();
   });
 });
 
 describe("metadata loads", () => {
   test("categories and values from dataset appear", async () => {
-    for (const label in data.categorical) {
-      await utils.waitByID(`category-${label}`);
-      const categoryName = await utils.getOneElementInnerText(
-        `[data-testid="category-${label}"]`
+    await goToPage(appUrlBase);
+
+    for (const label of Object.keys(data.categorical)) {
+      const element = await getOneElementInnerHTML(
+        getTestId(`category-${label}`)
       );
-      expect(categoryName).toMatch(label);
-      await utils.clickOn(`category-expand-${label}`);
-      const categories = await cxgActions.getAllCategoriesAndCounts(label);
+
+      expect(element).toMatchSnapshot();
+
+      await clickOn(`${label}:category-expand`);
+
+      const categories = await getAllCategoriesAndCounts(label);
+
       expect(Object.keys(categories)).toMatchObject(
         Object.keys(data.categorical[label])
       );
+
       expect(Object.values(categories)).toMatchObject(
         Object.values(data.categorical[label])
       );
@@ -81,263 +65,287 @@ describe("metadata loads", () => {
   });
 
   test("continuous data appears", async () => {
-    for (const label in data.continuous) {
-      await utils.waitByID(`histogram-${label}`);
+    await goToPage(appUrlBase);
+
+    for (const label of Object.keys(data.continuous)) {
+      await waitByID(`histogram-${label}`);
     }
   });
 });
 
 describe("cell selection", () => {
   test("selects all cells cellset 1", async () => {
-    const cellCount = await cxgActions.cellSet(1);
+    await goToPage(appUrlBase);
+
+    const cellCount = await getCellSetCount(1);
     expect(cellCount).toBe(data.dataframe.nObs);
   });
 
   test("selects all cells cellset 2", async () => {
-    const cellCount = await cxgActions.cellSet(2);
+    await goToPage(appUrlBase);
+
+    const cellCount = await getCellSetCount(2);
     expect(cellCount).toBe(data.dataframe.nObs);
   });
 
   test("selects cells via lasso", async () => {
+    await goToPage(appUrlBase);
+
     for (const cellset of data.cellsets.lasso) {
-      const cellset1 = await cxgActions.calcDragCoordinates(
+      const cellset1 = await calcDragCoordinates(
         "layout-graph",
         cellset["coordinates-as-percent"]
       );
-      await cxgActions.drag("layout-graph", cellset1.start, cellset1.end, true);
-      const cellCount = await cxgActions.cellSet(1);
+
+      await drag("layout-graph", cellset1.start, cellset1.end, true);
+      const cellCount = await getCellSetCount(1);
       expect(cellCount).toBe(cellset.count);
     }
   });
 
   test("selects cells via categorical", async () => {
+    await goToPage(appUrlBase);
+
     for (const cellset of data.cellsets.categorical) {
-      await utils.clickOn(`category-expand-${cellset.metadata}`);
-      await utils.clickOn(`category-select-${cellset.metadata}`);
-      for (const val of cellset.values) {
-        await utils.clickOn(
-          `categorical-value-select-${cellset.metadata}-${val}`
-        );
+      await clickOn(`${cellset.metadata}:category-expand`);
+      await clickOn(`${cellset.metadata}:category-select`);
+
+      for (const value of cellset.values) {
+        await clickOn(`categorical-value-select-${cellset.metadata}-${value}`);
       }
-      const cellCount = await cxgActions.cellSet(1);
+
+      const cellCount = await getCellSetCount(1);
+
       expect(cellCount).toBe(cellset.count);
     }
   });
 
   test("selects cells via continuous", async () => {
+    await goToPage(appUrlBase);
+
     for (const cellset of data.cellsets.continuous) {
-      const histId = `histogram-${cellset.metadata}-plot-brush`;
-      const coords = await cxgActions.calcDragCoordinates(
-        histId,
+      const histBrushableAreaId = `histogram-${cellset.metadata}-plot-brushable-area`;
+
+      const coords = await calcDragCoordinates(
+        histBrushableAreaId,
         cellset["coordinates-as-percent"]
       );
-      await cxgActions.drag(histId, coords.start, coords.end);
-      const cellCount = await cxgActions.cellSet(1);
+
+      await drag(histBrushableAreaId, coords.start, coords.end);
+
+      const cellCount = await getCellSetCount(1);
+
       expect(cellCount).toBe(cellset.count);
     }
   });
 });
 
-describe("gene entry", () => {
-  test("search for single gene", async () => {
-    // blueprint's  typeahead is treating typing weird, clicking & waiting first solves this
-    await utils.typeInto("gene-search", data.genes.search);
-    await page.keyboard.press("Enter");
-    await page.waitForSelector(
-      `[data-testid='histogram-${data.genes.search}']`
-    );
-  });
-
-  test("bulk add genes", async () => {
-    await cxgActions.reset();
-    const testGenes = data.genes.bulkadd;
-    await utils.clickOn("section-bulk-add");
-    await utils.typeInto("input-bulk-add", testGenes.join(","));
-    await page.keyboard.press("Enter");
-
-    const allHistograms = await cxgActions.getAllHistograms(
-      "histogram-user-gene",
-      testGenes
-    );
-    expect(allHistograms).toEqual(expect.arrayContaining(testGenes));
-    expect(allHistograms.length).toEqual(testGenes.length);
-  });
-});
-
-describe("diffexp", () => {
-  test("selects cells, saves them and performs diffexp", async () => {
-    for (const select of data.diffexp.cellset1) {
-      if (select.kind === "categorical") {
-        await cxgActions.selectCategory(select.metadata, select.values, true);
-      }
-    }
-    await cxgActions.cellSet(1);
-    for (const select of data.diffexp.cellset2) {
-      if (select.kind === "categorical") {
-        await cxgActions.selectCategory(select.metadata, select.values, true);
-      }
-    }
-    await cxgActions.cellSet(2);
-    await utils.clickOn("diffexp-button");
-    const allHistograms = await cxgActions.getAllHistograms(
-      "histogram-diffexp",
-      data.diffexp["gene-results"]
-    );
-    expect(allHistograms).toEqual(
-      expect.arrayContaining(data.diffexp["gene-results"])
-    );
-    expect(allHistograms.length).toEqual(data.diffexp["gene-results"].length);
-  });
-});
-
-describe("subset/reset", () => {
+describe("subset", () => {
   test("subset - cell count matches", async () => {
-    for (const select of data.subset.cellset1) {
-      if (select.kind === "categorical") {
-        await cxgActions.selectCategory(select.metadata, select.values, true);
-      }
-    }
-    await utils.clickOn("subset-button");
-    for (const label in data.subset.categorical) {
-      const categories = await cxgActions.getAllCategoriesAndCounts(label);
-      expect(Object.keys(categories)).toMatchObject(
-        Object.keys(data.subset.categorical[label])
-      );
-      expect(Object.values(categories)).toMatchObject(
-        Object.values(data.subset.categorical[label])
-      );
-    }
-  });
+    await goToPage(appUrlBase);
 
-  test("reset after subset", async () => {
     for (const select of data.subset.cellset1) {
       if (select.kind === "categorical") {
-        await cxgActions.selectCategory(select.metadata, select.values, true);
+        await selectCategory(select.metadata, select.values, true);
       }
     }
-    await utils.clickOn("subset-button");
-    for (const label in data.subset.categorical) {
-      const categories = await cxgActions.getAllCategoriesAndCounts(label);
+
+    await clickOn("subset-button");
+
+    for (const label of Object.keys(data.subset.categorical)) {
+      const categories = await getAllCategoriesAndCounts(label);
+
       expect(Object.keys(categories)).toMatchObject(
         Object.keys(data.subset.categorical[label])
       );
+
       expect(Object.values(categories)).toMatchObject(
         Object.values(data.subset.categorical[label])
       );
     }
-    await cxgActions.reset();
-    for (const label in data.categorical) {
-      await utils.waitByID(`category-${label}`);
-      const categoryName = await utils.getOneElementInnerText(
-        `[data-testid="category-${label}"]`
-      );
-      expect(categoryName).toMatch(label);
-      const categories = await cxgActions.getAllCategoriesAndCounts(label);
-      expect(Object.keys(categories)).toMatchObject(
-        Object.keys(data.categorical[label])
-      );
-      expect(Object.values(categories)).toMatchObject(
-        Object.values(data.categorical[label])
-      );
-    }
   });
 
   test("lasso after subset", async () => {
+    await goToPage(appUrlBase);
+
     for (const select of data.subset.cellset1) {
       if (select.kind === "categorical") {
-        await cxgActions.selectCategory(select.metadata, select.values, true);
+        await selectCategory(select.metadata, select.values, true);
       }
     }
-    await utils.clickOn("subset-button");
-    const lassoSelection = await cxgActions.calcDragCoordinates(
+
+    await clickOn("subset-button");
+
+    const lassoSelection = await calcDragCoordinates(
       "layout-graph",
       data.subset.lasso["coordinates-as-percent"]
     );
-    await cxgActions.drag(
-      "layout-graph",
-      lassoSelection.start,
-      lassoSelection.end,
-      true
-    );
-    const cellCount = await cxgActions.cellSet(1);
-    expect(cellCount).toBe(data.subset.lasso.count);
-  });
-});
 
-describe("scatter plot", () => {
-  test("scatter plot appears", async () => {
-    await cxgActions.reset();
-    const testGenes = data.scatter.genes;
-    await utils.clickOn("section-bulk-add");
-    await utils.typeInto("input-bulk-add", Object.values(testGenes).join(","));
-    await page.keyboard.press("Enter");
-    await utils.clickOn(`plot-x-${data.scatter.genes.x}`);
-    await utils.clickOn(`plot-y-${data.scatter.genes.y}`);
-    await utils.waitByID("scatterplot");
+    await drag("layout-graph", lassoSelection.start, lassoSelection.end, true);
+
+    const cellCount = await getCellSetCount(1);
+    expect(cellCount).toBe(data.subset.lasso.count);
   });
 });
 
 describe("clipping", () => {
   test("clip continuous", async () => {
-    await cxgActions.clip(data.clip.min, data.clip.max);
-    const histId = `histogram-${data.clip.metadata}-plot-brush`;
-    const coords = await cxgActions.calcDragCoordinates(
-      histId,
-      data.clip["coordinates-as-percent"]
-    );
-    await cxgActions.drag(histId, coords.start, coords.end);
-    const cellCount = await cxgActions.cellSet(1);
-    expect(cellCount).toBe(data.clip.count);
-  });
+    await goToPage(appUrlBase);
 
-  test("clip gene", async () => {
-    await utils.typeInto("gene-search", data.clip.gene);
-    await page.keyboard.press("Enter");
-    await page.waitForSelector(`[data-testid='histogram-${data.clip.gene}']`);
-    await cxgActions.clip(data.clip.min, data.clip.max);
-    const histId = `histogram-${data.clip.gene}-plot-brush`;
-    const coords = await cxgActions.calcDragCoordinates(
-      histId,
+    await clip(data.clip.min, data.clip.max);
+    const histBrushableAreaId = `histogram-${data.clip.metadata}-plot-brushable-area`;
+    const coords = await calcDragCoordinates(
+      histBrushableAreaId,
       data.clip["coordinates-as-percent"]
     );
-    await cxgActions.drag(histId, coords.start, coords.end);
-    const cellCount = await cxgActions.cellSet(1);
-    expect(cellCount).toBe(data.clip["gene-cell-count"]);
+    await drag(histBrushableAreaId, coords.start, coords.end);
+    const cellCount = await getCellSetCount(1);
+    expect(cellCount).toBe(data.clip.count);
   });
 });
 
 // interact with UI elements just that they do not break
 describe("ui elements don't error", () => {
   test("color by", async () => {
-    for (const label in data.categorical) {
-      await utils.clickOn(`colorby-${label}`);
-    }
-    for (const label in data.continuous) {
-      await utils.clickOn(`colorby-${label}`);
-    }
-  });
+    await goToPage(appUrlBase);
 
-  test("color by for gene", async () => {
-    await utils.typeInto("gene-search", data.genes.search);
-    await page.keyboard.press("Enter");
-    await page.waitForSelector(
-      `[data-testid='histogram-${data.genes.search}']`
-    );
-    await utils.clickOn(`colorby-${data.genes.search}`);
+    const allLabels = [
+      ...Object.keys(data.categorical),
+      ...Object.keys(data.continuous),
+    ];
+
+    for (const label of allLabels) {
+      await clickOn(`colorby-${label}`);
+    }
   });
 
   test("pan and zoom", async () => {
-    await utils.clickOn("mode-pan-zoom");
-    const panCoords = await cxgActions.calcDragCoordinates(
+    await goToPage(appUrlBase);
+
+    await clickOn("mode-pan-zoom");
+    const panCoords = await calcDragCoordinates(
       "layout-graph",
       data.pan["coordinates-as-percent"]
     );
-    await cxgActions.drag(
-      "layout-graph",
-      panCoords.start,
-      panCoords.end,
-      false
-    );
-    await page.evaluate(`window.scrollBy(0, 1000);`);
+
+    await drag("layout-graph", panCoords.start, panCoords.end, false);
+
+    await page.evaluate("window.scrollBy(0, 1000);");
   });
 });
+
+describe("centroid labels", () => {
+  test("labels are created", async () => {
+    await goToPage(appUrlBase);
+
+    const labels = Object.keys(data.categorical);
+
+    await clickOn(`colorby-${labels[0]}`);
+    await clickOn("centroid-label-toggle");
+
+    // Toggle colorby for each category and check to see if labels are generated
+    for (let i = 0, { length } = labels; i < length; i += 1) {
+      const label = labels[i];
+      // first label is already enabled
+      if (i !== 0) await clickOn(`colorby-${label}`);
+      const generatedLabels = await getAllByClass("centroid-label");
+      // Number of labels generated should be equal to size of the object
+      expect(generatedLabels).toHaveLength(
+        Object.keys(data.categorical[label]).length
+      );
+    }
+  });
+});
+
+describe("graph overlay", () => {
+  test("transform centroids correctly", async () => {
+    await goToPage(appUrlBase);
+
+    const category = Object.keys(data.categorical)[0];
+
+    await clickOn(`colorby-${category}`);
+    await clickOn("centroid-label-toggle");
+    await clickOn("mode-pan-zoom");
+
+    const panCoords = await calcDragCoordinates(
+      "layout-graph",
+      data.pan["coordinates-as-percent"]
+    );
+
+    const categoryValue = Object.keys(data.categorical[category])[0];
+    const initialCoordinates = await getElementCoordinates(
+      `${categoryValue}-centroid-label`
+    );
+
+    await drag("layout-graph", panCoords.start, panCoords.end, false);
+    const terminalCoordinates = await getElementCoordinates(
+      `${categoryValue}-centroid-label`
+    );
+
+    expect(terminalCoordinates[0] - initialCoordinates[0]).toBeCloseTo(
+      panCoords.end.x - panCoords.start.x
+    );
+    expect(terminalCoordinates[1] - initialCoordinates[1]).toBeCloseTo(
+      panCoords.end.y - panCoords.start.y
+    );
+  });
+});
+
+test("pan zoom mode resets lasso selection", async () => {
+  await goToPage(appUrlBase);
+
+  const panzoomLasso = data.features.panzoom.lasso;
+
+  const lassoSelection = await calcDragCoordinates(
+    "layout-graph",
+    panzoomLasso["coordinates-as-percent"]
+  );
+
+  await drag("layout-graph", lassoSelection.start, lassoSelection.end, true);
+  await waitByID("lasso-element", { visible: true });
+
+  const initialCount = await getCellSetCount(1);
+
+  expect(initialCount).toBe(panzoomLasso.count);
+
+  await clickOn("mode-pan-zoom");
+  await clickOn("mode-lasso");
+
+  const modeSwitchCount = await getCellSetCount(1);
+
+  expect(modeSwitchCount).toBe(initialCount);
+});
+
+test("lasso moves after pan", async () => {
+  await goToPage(appUrlBase);
+
+  const panzoomLasso = data.features.panzoom.lasso;
+  const coordinatesAsPercent = panzoomLasso["coordinates-as-percent"];
+
+  const lassoSelection = await calcDragCoordinates(
+    "layout-graph",
+    coordinatesAsPercent
+  );
+
+  await drag("layout-graph", lassoSelection.start, lassoSelection.end, true);
+  await waitByID("lasso-element", { visible: true });
+
+  const initialCount = await getCellSetCount(1);
+
+  expect(initialCount).toBe(panzoomLasso.count);
+
+  await clickOn("mode-pan-zoom");
+
+  const panCoords = await calcDragCoordinates(
+    "layout-graph",
+    coordinatesAsPercent
+  );
+
+  await drag("layout-graph", panCoords.start, panCoords.end, false);
+  await clickOn("mode-lasso");
+
+  const panCount = await getCellSetCount(2);
+
+  expect(panCount).toBe(initialCount);
+});
+/* eslint-enable no-await-in-loop -- await in loop is needed to emulate sequential user actions */

client/__tests__/e2e/e2eAnnotations.test.js

@@ -0,0 +1,595 @@
+/*
+Tests included in this file are specific to annotation features
+*/
+import { appUrlBase, DATASET } from "./config";
+import { datasets } from "./data";
+
+import {
+  clickOn,
+  goToPage,
+  waitByClass,
+  waitByID,
+  getTestId,
+  getTestClass,
+  getAllByClass,
+  clickOnUntil,
+  getOneElementInnerHTML,
+} from "./puppeteerUtils";
+
+import {
+  assertCategoryDoesNotExist,
+  calcDragCoordinates,
+  createCategory,
+  createLabel,
+  deleteCategory,
+  deleteLabel,
+  drag,
+  expandCategory,
+  renameCategory,
+  renameLabel,
+  subset,
+  duplicateCategory,
+  createGeneset,
+  deleteGeneset,
+  assertGenesetExists,
+  assertGenesetDoesNotExist,
+  getCellSetCount,
+  expandGeneset,
+  editGenesetName,
+  addGeneToSet,
+  assertGeneExistsInGeneset,
+  removeGene,
+  assertGeneDoesNotExist,
+  expandGene,
+  colorByGeneset,
+  assertColorLegendLabel,
+  colorByGene,
+} from "./cellxgeneActions";
+
+const data = datasets[DATASET];
+
+const perTestCategoryName = "TEST-CATEGORY";
+const perTestLabelName = "TEST-LABEL";
+
+// geneset CRUD
+const genesetToDeleteName = "geneset_to_delete";
+const preExistingGenesetName = "fifth_dataset";
+const meanExpressionBrushGenesetName = "second_gene_set";
+const meanExpressionBrushCellsSelected = "557";
+const subsetMeanExpressionBrushCellsSelected = "452";
+
+// initial text, the text we type in, the result
+const editableGenesetName = "geneset_to_edit";
+const editText = "_111";
+const newGenesetName = "geneset_to_edit_111";
+
+// add gene to set
+const geneToAddToSet = "RER1";
+const setToAddGeneTo = "fill_this_geneset";
+
+// remove gene from set
+const geneToRemove = "SIK1";
+const setToRemoveFrom = "empty_this_geneset";
+
+// brush a gene
+const geneToBrushAndColorBy = "SIK1";
+const brushThisGeneGeneset = "brush_this_gene";
+const geneBrushedCellCount = "109";
+const subsetGeneBrushedCellCount = "96";
+
+const genesetDescriptionID =
+  "geneset-description-tooltip-fourth_gene_set: fourth description";
+const genesetDescriptionString = "fourth_gene_set: fourth description";
+const genesetToCheckForDescription = "fourth_gene_set";
+
+async function setup(config) {
+  await goToPage(appUrlBase);
+
+  if (config.categoricalAnno) {
+    // setup the test fixtures
+    await createCategory(perTestCategoryName);
+    await createLabel(perTestCategoryName, perTestLabelName);
+  }
+
+  if (config.withSubset) {
+    await subset({ x1: 0.1, y1: 0.1, x2: 0.8, y2: 0.8 });
+  }
+
+  await waitByClass("autosave-complete");
+}
+
+describe.each([
+  { withSubset: true, tag: "subset" },
+  { withSubset: false, tag: "whole" },
+])("geneSET crud operations and interactions", (config) => {
+  test("genesets load from csv", async () => {
+    await setup(config);
+
+    await assertGenesetExists(preExistingGenesetName);
+  });
+  test("brush on geneset mean", async () => {
+    await setup(config);
+
+    await expandGeneset(meanExpressionBrushGenesetName);
+
+    const histBrushableAreaId = `histogram-${meanExpressionBrushGenesetName}-plot-brushable-area`;
+
+    const coords = await calcDragCoordinates(histBrushableAreaId, {
+      x1: 0.25,
+      y1: 0.5,
+      x2: 0.55,
+      y2: 0.5,
+    });
+
+    await drag(histBrushableAreaId, coords.start, coords.end);
+
+    const cellCount = await getCellSetCount(1);
+    if (config.withSubset) {
+      expect(cellCount).toBe(subsetMeanExpressionBrushCellsSelected);
+    } else {
+      expect(cellCount).toBe(meanExpressionBrushCellsSelected);
+    }
+  });
+  test("color by mean expression", async () => {
+    await setup(config);
+
+    await colorByGeneset(meanExpressionBrushGenesetName);
+    await assertColorLegendLabel(meanExpressionBrushGenesetName);
+  });
+  test("diffexp", async () => {
+    if (config.withSubset) return;
+
+    await setup(config);
+
+    // set the two cell sets to b cells vs nk cells
+    await expandCategory(`louvain`);
+    await clickOn(`louvain:category-select`);
+    await clickOn(`categorical-value-select-louvain-B cells`);
+    await clickOn(`cellset-button-1`);
+    await clickOn(`categorical-value-select-louvain-B cells`);
+    await clickOn(`categorical-value-select-louvain-NK cells`);
+    await clickOn(`cellset-button-2`);
+
+    // run diffexp
+    await clickOn(`diffexp-button`);
+    await waitByClass("pop-1-geneset-expand");
+    await expect(page).toClick(getTestClass("pop-1-geneset-expand"));
+
+    await page.waitForFunction(
+      (selector) => !document.querySelector(selector),
+      {},
+      getTestClass("gene-loading-spinner")
+    );
+
+    let genesHTML = await getOneElementInnerHTML(
+      getTestClass("gene-set-genes")
+    );
+
+    expect(genesHTML).toMatchSnapshot();
+
+    await expect(page).toClick(getTestClass("pop-1-geneset-expand"));
+    await expect(page).toClick(getTestClass("pop-2-geneset-expand"));
+
+    await page.waitForFunction(
+      (selector) => !document.querySelector(selector),
+      {},
+      getTestClass("gene-loading-spinner")
+    );
+
+    genesHTML = await getOneElementInnerHTML(getTestClass("gene-set-genes"));
+
+    expect(genesHTML).toMatchSnapshot();
+  });
+  test("create a new geneset and undo/redo", async () => {
+    if (config.withSubset) return;
+
+    await setup(config);
+
+    const genesetName = `test-geneset-foo-123`;
+    await assertGenesetDoesNotExist(genesetName);
+    await createGeneset(genesetName);
+    /* note: as of June 2021, the aria label is in the truncate component which clones the element */
+    await assertGenesetExists(genesetName);
+    await clickOn("undo");
+    await assertGenesetDoesNotExist(genesetName);
+    await clickOn("redo");
+    await assertGenesetExists(genesetName);
+  });
+  test("edit geneset name and undo/redo", async () => {
+    await setup(config);
+
+    await editGenesetName(editableGenesetName, editText);
+    await assertGenesetExists(newGenesetName);
+    await clickOn("undo");
+    await assertGenesetExists(editableGenesetName);
+    await clickOn("redo");
+    await assertGenesetExists(newGenesetName);
+  });
+  test("delete a geneset and undo/redo", async () => {
+    if (config.withSubset) return;
+
+    await setup(config);
+
+    await deleteGeneset(genesetToDeleteName);
+    await clickOn("undo");
+    await assertGenesetExists(genesetToDeleteName);
+    await clickOn("redo");
+    await assertGenesetDoesNotExist(genesetToDeleteName);
+  });
+  test("geneset description", async () => {
+    if (config.withSubset) return;
+
+    await setup(config);
+
+    await clickOnUntil(
+      `${genesetToCheckForDescription}:geneset-expand`,
+      async () => {
+        expect(page).toMatchElement(getTestId(genesetDescriptionID), {
+          text: genesetDescriptionString,
+        });
+      }
+    );
+  });
+});
+
+describe.each([
+  { withSubset: true, tag: "subset" },
+  { withSubset: false, tag: "whole" },
+])("GENE crud operations and interactions", (config) => {
+  test("add a gene to geneset and undo/redo", async () => {
+    await setup(config);
+
+    await addGeneToSet(setToAddGeneTo, geneToAddToSet);
+    await expandGeneset(setToAddGeneTo);
+    await assertGeneExistsInGeneset(geneToAddToSet);
+    await clickOn("undo");
+    await assertGeneDoesNotExist(geneToAddToSet);
+    await clickOn("redo");
+    await assertGeneExistsInGeneset(geneToAddToSet);
+  });
+  test("expand gene and brush", async () => {
+    await setup(config);
+
+    await expandGeneset(brushThisGeneGeneset);
+    await expandGene(geneToBrushAndColorBy);
+    const histBrushableAreaId = `histogram-${geneToBrushAndColorBy}-plot-brushable-area`;
+
+    const coords = await calcDragCoordinates(histBrushableAreaId, {
+      x1: 0.25,
+      y1: 0.5,
+      x2: 0.55,
+      y2: 0.5,
+    });
+    await drag(histBrushableAreaId, coords.start, coords.end);
+    const cellCount = await getCellSetCount(1);
+    if (config.withSubset) {
+      expect(cellCount).toBe(subsetGeneBrushedCellCount);
+    } else {
+      expect(cellCount).toBe(geneBrushedCellCount);
+    }
+  });
+  test("color by gene in geneset", async () => {
+    await setup(config);
+
+    await expandGeneset(meanExpressionBrushGenesetName);
+
+    await colorByGene(geneToBrushAndColorBy);
+    await assertColorLegendLabel(geneToBrushAndColorBy);
+  });
+  test("delete gene from geneset and undo/redo", async () => {
+    // We've already deleted the gene
+    if (config.withSubset) return;
+
+    await setup(config);
+
+    await expandGeneset(setToRemoveFrom);
+    await removeGene(geneToRemove);
+    await assertGeneDoesNotExist(geneToRemove);
+    await clickOn("undo");
+    await assertGeneExistsInGeneset(geneToRemove);
+    await clickOn("redo");
+    await assertGeneDoesNotExist(geneToRemove);
+  });
+});
+
+describe.each([
+  { withSubset: true, tag: "subset", categoricalAnno: true },
+  { withSubset: false, tag: "whole", categoricalAnno: true },
+])("annotations", (config) => {
+  test("create a category", async () => {
+    await setup(config);
+
+    const categoryName = `category-created-${config.tag}`;
+
+    await assertCategoryDoesNotExist(categoryName);
+
+    await createCategory(categoryName);
+
+    await assertCategoryExists(categoryName);
+  });
+
+  test("delete a category", async () => {
+    await setup(config);
+
+    await deleteCategory(perTestCategoryName);
+    await assertCategoryDoesNotExist(perTestCategoryName);
+  });
+
+  test("rename a category", async () => {
+    await setup(config);
+
+    const newCategoryName = `NEW-${config.tag}`;
+
+    await renameCategory(perTestCategoryName, newCategoryName);
+    await assertCategoryDoesNotExist(perTestCategoryName);
+    await assertCategoryExists(newCategoryName);
+  });
+
+  test("create a label", async () => {
+    await setup(config);
+
+    const labelName = `new-label-${config.tag}`;
+
+    await assertLabelDoesNotExist(perTestCategoryName, labelName);
+
+    await createLabel(perTestCategoryName, labelName);
+
+    await assertLabelExists(perTestCategoryName, labelName);
+  });
+
+  test("delete a label", async () => {
+    await setup(config);
+
+    await deleteLabel(perTestCategoryName, perTestLabelName);
+    await assertLabelDoesNotExist(perTestCategoryName, perTestLabelName);
+  });
+
+  test("rename a label", async () => {
+    await setup(config);
+
+    const newLabelName = "my-cool-new-label";
+
+    await assertLabelDoesNotExist(perTestCategoryName, newLabelName);
+    await renameLabel(perTestCategoryName, perTestLabelName, newLabelName);
+    await assertLabelDoesNotExist(perTestCategoryName, perTestLabelName);
+    await assertLabelExists(perTestCategoryName, newLabelName);
+  });
+
+  test("check cell count for a label loaded from file", async () => {
+    await setup(config);
+
+    const duplicateCategoryName = "duplicate";
+    await duplicateCategory(duplicateCategoryName);
+
+    await page.reload({ waitUntil: ["networkidle0", "domcontentloaded"] });
+
+    const firstCategoryExpandIcon = await expect(page).toMatchElement(
+      getTestClass("category-expand")
+    );
+
+    await firstCategoryExpandIcon.click();
+
+    const expectedCategoryRow = await expect(page).toMatchElement(
+      getTestClass("categorical-row")
+    );
+    const expectedLabelName = await getInnerText(
+      expectedCategoryRow,
+      "categorical-value"
+    );
+    const expectedLabelCount = await getInnerText(
+      expectedCategoryRow,
+      "categorical-value-count"
+    );
+
+    await expandCategory(duplicateCategoryName);
+
+    const expectedCategory = await expect(page).toMatchElement(
+      getTestClass("category")
+    );
+
+    const actualCategoryRow = await expect(expectedCategory).toMatchElement(
+      getTestClass("categorical-row")
+    );
+    const actualLabelName = await getInnerText(
+      actualCategoryRow,
+      "categorical-value"
+    );
+    const actualLabelCount = await getInnerText(
+      actualCategoryRow,
+      "categorical-value-count"
+    );
+
+    expect(actualLabelName).toBe(expectedLabelName);
+    expect(actualLabelCount).toBe(expectedLabelCount);
+
+    async function getInnerText(element, className) {
+      return element.$eval(getTestClass(className), (node) => node?.innerText);
+    }
+  });
+
+  test("assign cells to a label", async () => {
+    await setup(config);
+
+    await expandCategory(perTestCategoryName);
+
+    const lassoSelection = await calcDragCoordinates(
+      "layout-graph",
+      data.categoryLabel.lasso["coordinates-as-percent"]
+    );
+
+    await drag("layout-graph", lassoSelection.start, lassoSelection.end, true);
+    await waitByID("lasso-element", { visible: true });
+    await clickOn(`${perTestCategoryName}:${perTestLabelName}:see-actions`);
+    await clickOn(
+      `${perTestCategoryName}:${perTestLabelName}:add-current-selection-to-this-label`
+    );
+
+    const result = await waitByID(
+      `categorical-value-count-${perTestCategoryName}-${perTestLabelName}`
+    );
+
+    expect(await result.evaluate((node) => node.innerText)).toBe(
+      data.categoryLabel.newCount.bySubsetConfig[config.withSubset]
+    );
+  });
+
+  test("undo/redo category creation", async () => {
+    await setup(config);
+
+    const categoryName = `category-created-undo-${config.tag}`;
+
+    await assertCategoryDoesNotExist(categoryName);
+    await createCategory(categoryName);
+    await assertCategoryExists(categoryName);
+    await clickOn("undo");
+    await assertCategoryDoesNotExist(categoryName);
+    await clickOn("redo");
+    await assertCategoryExists(categoryName);
+  });
+
+  test("undo/redo category deletion", async () => {
+    await setup(config);
+
+    const categoryName = `category-deleted-undo-${config.tag}`;
+
+    await createCategory(categoryName);
+    await assertCategoryExists(categoryName);
+    await deleteCategory(categoryName);
+    await assertCategoryDoesNotExist(categoryName);
+    await clickOn("undo");
+    await assertCategoryExists(categoryName);
+    await clickOn("redo");
+    await assertCategoryDoesNotExist(categoryName);
+  });
+
+  test("undo/redo category rename", async () => {
+    await setup(config);
+
+    const newCategoryName = `category-renamed-undo-${config.tag}`;
+
+    await assertCategoryDoesNotExist(newCategoryName);
+    await renameCategory(perTestCategoryName, newCategoryName);
+    await assertCategoryExists(newCategoryName);
+    await assertCategoryDoesNotExist(perTestCategoryName);
+    await clickOn("undo");
+    await assertCategoryExists(perTestCategoryName);
+    await assertCategoryDoesNotExist(newCategoryName);
+    await clickOn("redo");
+    await assertCategoryExists(newCategoryName);
+    await assertCategoryDoesNotExist(perTestCategoryName);
+  });
+
+  test("undo/redo label creation", async () => {
+    await setup(config);
+
+    const labelName = `label-created-undo-${config.tag}`;
+
+    await assertLabelDoesNotExist(perTestCategoryName, labelName);
+    await createLabel(perTestCategoryName, labelName);
+    await assertLabelExists(perTestCategoryName, labelName);
+    await clickOn("undo");
+    await assertLabelDoesNotExist(perTestCategoryName);
+    await clickOn("redo");
+    await assertLabelExists(perTestCategoryName, labelName);
+  });
+
+  test("undo/redo label deletion", async () => {
+    await setup(config);
+
+    await deleteLabel(perTestCategoryName, perTestLabelName);
+    await assertLabelDoesNotExist(perTestCategoryName);
+    await clickOn("undo");
+    await assertLabelExists(perTestCategoryName, perTestLabelName);
+    await clickOn("redo");
+    await assertLabelDoesNotExist(perTestCategoryName);
+  });
+
+  test("undo/redo label rename", async () => {
+    await setup(config);
+
+    const newLabelName = `label-renamed-undo-${config.tag}`;
+
+    await assertLabelDoesNotExist(perTestCategoryName, newLabelName);
+    await renameLabel(perTestCategoryName, perTestLabelName, newLabelName);
+    await assertLabelExists(perTestCategoryName, newLabelName);
+    await assertLabelDoesNotExist(perTestCategoryName, perTestLabelName);
+    await clickOn("undo");
+    await assertLabelExists(perTestCategoryName, perTestLabelName);
+    await assertLabelDoesNotExist(perTestCategoryName, newLabelName);
+    await clickOn("redo");
+    await assertLabelExists(perTestCategoryName, newLabelName);
+    await assertLabelDoesNotExist(perTestCategoryName, perTestLabelName);
+  });
+
+  test("stacked bar graph renders", async () => {
+    await setup(config);
+
+    await expandCategory(perTestCategoryName);
+
+    await clickOn(`colorby-louvain`);
+
+    const labels = await getAllByClass("categorical-row");
+
+    const result = await Promise.all(
+      labels.map((label) => page.evaluate((element) => element.outerHTML, label))
+    );
+
+    expect(result).toMatchSnapshot();
+  });
+
+  test("truncate midpoint whitespace", async () => {
+    await setup(config);
+    const newLabelName = "123 456";
+    await renameLabel(perTestCategoryName, perTestLabelName, newLabelName);
+    const value = await waitByID(
+      `categorical-value-${perTestCategoryName}-${newLabelName}`
+    );
+    const result = await page.evaluate((elem) => elem.outerHTML, value);
+    expect(result).toMatchSnapshot();
+  });
+
+  test("truncate single character", async () => {
+    await setup(config);
+    const newLabelName = "T";
+    await renameLabel(perTestCategoryName, perTestLabelName, newLabelName);
+    const value = await waitByID(
+      `categorical-value-${perTestCategoryName}-${newLabelName}`
+    );
+    const result = await page.evaluate((elem) => elem.outerHTML, value);
+    expect(result).toMatchSnapshot();
+  });
+
+  async function assertCategoryExists(categoryName) {
+    const handle = await waitByID(`${categoryName}:category-label`);
+
+    const result = await handle.evaluate((node) =>
+      node.getAttribute("aria-label")
+    );
+
+    return expect(result).toBe(categoryName);
+  }
+
+  async function assertLabelExists(categoryName, labelName) {
+    await expect(page).toMatchElement(
+      getTestId(`${categoryName}:category-expand`)
+    );
+
+    await expandCategory(categoryName);
+
+    const previous = await waitByID(
+      `categorical-value-${categoryName}-${labelName}`
+    );
+
+    expect(
+      await previous.evaluate((node) => node.getAttribute("aria-label"))
+    ).toBe(labelName);
+  }
+
+  async function assertLabelDoesNotExist(categoryName, labelName) {
+    await expandCategory(categoryName);
+    const result = await page.$(
+      `[data-testid='categorical-value-${categoryName}-${labelName}']`
+    );
+    expect(result).toBeNull();
+  }
+});

client/__tests__/e2e/e2eJestConfig.json

@@ -1,10 +1,10 @@
 {
+  "testRunner": "jest-circus/runner",
   "preset": "jest-puppeteer",
-  "testMatch": [
-    "**/__tests__/**/?(*.)(spec|test).js?(x)"
-  ],
-  "testURL": "http://localhost/",
-  "setupFiles": [
-    "../setupMissingGlobals.js"
-  ]
+  "testMatch": ["**/__tests__/**/?(*.)(spec|test).js?(x)"],
+  "setupFiles": ["../setupMissingGlobals.js"],
+  "setupFilesAfterEnv": ["expect-puppeteer", "./puppeteer.setup.js"],
+  "globalSetup": "../globalSetup.js",
+  "globalTeardown": "jest-environment-puppeteer/teardown",
+  "testEnvironment": "./screenshot_env.js"
 }

client/__tests__/e2e/feature.test.js

@@ -1,118 +0,0 @@
-/*
-NOT run in Travis CI
-
-UX tests using puppeteer to be run locally.
-
-To run locally, ensure you are running the client is running on port 3000.
-Then run jest --verbose false --config __tests__/e2e/e2eJestConfig.json feature.
- */
-
-import puppeteer from "puppeteer";
-import { appUrlBase, DEBUG, DEV, DATASET } from "./config";
-import { puppeteerUtils, cellxgeneActions } from "./puppeteerUtils";
-import { datasets } from "./data";
-
-let browser, page, utils, cxgActions, spy;
-const browserViewport = { width: 1280, height: 960 };
-let data = datasets[DATASET].features;
-
-if (DEBUG) jest.setTimeout(100000);
-if (DEV) jest.setTimeout(10000);
-
-beforeAll(async () => {
-  const browserParams = DEV
-    ? { headless: false, slowMo: 5 }
-    : DEBUG
-    ? { headless: false, slowMo: 100, devtools: true }
-    : {};
-  browser = await puppeteer.launch(browserParams);
-  page = await browser.newPage();
-  await page.setViewport(browserViewport);
-  if (DEV || DEBUG) {
-    page.on("console", msg => console.log(`PAGE LOG: ${msg.text()}`));
-  }
-  page.on("pageerror", err => {
-    throw new Error(`Console error: ${err}`);
-  });
-  utils = puppeteerUtils(page);
-  cxgActions = cellxgeneActions(page);
-});
-
-beforeEach(async () => {
-  await page.goto(appUrlBase);
-});
-
-afterAll(() => {
-  if (!DEBUG) {
-    browser.close();
-  }
-});
-
-describe("zoom interaction", async () => {
-  // Skip this test since UI is to hide lasso path when switching modes
-  test.skip("lasso visible after switching modes to pan/zoom", async () => {
-    const lassoSelection = await cxgActions.calcDragCoordinates(
-      "layout-graph",
-      data.panzoom.lasso["coordinates-as-percent"]
-    );
-    await cxgActions.drag(
-      "layout-graph",
-      lassoSelection.start,
-      lassoSelection.end,
-      true
-    );
-    await utils.waitByID("lasso-element", { visible: true });
-    await utils.clickOn("mode-pan-zoom");
-    await utils.waitByID("lasso-element", { visible: true });
-  });
-
-  test("pan zoom mode resets lasso selection", async () => {
-    const lassoSelection = await cxgActions.calcDragCoordinates(
-      "layout-graph",
-      data.panzoom.lasso["coordinates-as-percent"]
-    );
-    await cxgActions.drag(
-      "layout-graph",
-      lassoSelection.start,
-      lassoSelection.end,
-      true
-    );
-    await utils.waitByID("lasso-element", { visible: true });
-    const initialCount = await cxgActions.cellSet(1);
-    expect(initialCount).toBe(data.panzoom.lasso.count);
-    await utils.clickOn("mode-pan-zoom");
-    await utils.clickOn("mode-lasso");
-    const modeSwitchCount = await cxgActions.cellSet(1);
-    expect(modeSwitchCount).toBe(initialCount);
-  });
-
-  test("lasso moves after pan", async () => {
-    const lassoSelection = await cxgActions.calcDragCoordinates(
-      "layout-graph",
-      data.panzoom.lasso["coordinates-as-percent"]
-    );
-    await cxgActions.drag(
-      "layout-graph",
-      lassoSelection.start,
-      lassoSelection.end,
-      true
-    );
-    await utils.waitByID("lasso-element", { visible: true });
-    const initialCount = await cxgActions.cellSet(1);
-    expect(initialCount).toBe(data.panzoom.lasso.count);
-    await utils.clickOn("mode-pan-zoom");
-    const panCoords = await cxgActions.calcDragCoordinates(
-      "layout-graph",
-      data.panzoom.lasso["coordinates-as-percent"]
-    );
-    await cxgActions.drag(
-      "layout-graph",
-      panCoords.start,
-      panCoords.end,
-      false
-    );
-    await utils.clickOn("mode-lasso");
-    const panCount = await cxgActions.cellSet(2);
-    expect(panCount).toBe(initialCount);
-  });
-});

client/__tests__/e2e/puppeteer.setup.js

@@ -0,0 +1,58 @@
+/**
+ * `client/jest-puppeteer.config.js` is for configuring Puppeteer's launch config options
+ * `client/__tests__/e2e/puppeteer.setup.js` is for configuring `jest`, `browser`,
+ * and `page` objects
+ */
+
+import { setDefaultOptions } from "expect-puppeteer";
+import { isDebug, isDev } from "./config";
+import * as ENV_DEFAULT from "../../../environment.default.json";
+
+// (thuang): This is the max time a test can take to run.
+// Since when debugging, we run slowMo and !headless, this means
+// a test can take more time to finish, so we don't want
+// jest to shut off the test too soon
+jest.setTimeout(2 * 60 * 1000);
+setDefaultOptions({ timeout: 60 * 1000 });
+
+jest.retryTimes(ENV_DEFAULT.RETRY_ATTEMPTS);
+
+beforeEach(async () => {
+  await jestPuppeteer.resetBrowser();
+
+  const userAgent = await browser.userAgent();
+  await page.setUserAgent(`${userAgent}bot`);
+
+  await page._client.send("Animation.setPlaybackRate", { playbackRate: 12 });
+
+  page.on("pageerror", (err) => {
+    throw new Error(`Console error: ${err}`);
+  });
+
+  page.on("error", (err) => {
+    throw new Error(`Console error: ${err}`);
+  });
+
+  page.on("console", async (msg) => {
+    if (isDev || isDebug) {
+      // If there is a console.error but an error is not thrown, this will ensure the test fails
+      console.log(`PAGE LOG: ${msg.text()}`);
+      if (msg.type() === "error") {
+        // TODO: chromium does not currently support the CSP directive on the
+        // line below, so we swallow this error. Remove this when the test
+        // suite uses a browser version that supports this directive.
+        if (
+          msg.text() ===
+          "Unrecognized Content-Security-Policy directive 'require-trusted-types-for'.\n"
+        ) {
+          return;
+        }
+        const errorMsgText = await Promise.all(
+          // TODO can we do this without internal properties?
+          msg.args().map((arg) => arg._remoteObject.description)
+        );
+        throw new Error(`Console error: ${errorMsgText}`);
+      }
+    }
+  });
+});

client/__tests__/e2e/puppeteerUtils.js

@@ -1,211 +1,124 @@
-export const puppeteerUtils = puppeteerPage => ({
-  async waitByID(testid, props = {}) {
-    return await puppeteerPage.waitForSelector(
-      `[data-testid='${testid}']`,
-      props
-    );
-  },
+/* eslint-disable no-await-in-loop -- await in loop is needed to emulate sequential user actions */
+export function getTestId(id) {
+  return `[data-testid='${id}']`;
+}
 
-  async waitByClass(testclass, props = {}) {
-    return await puppeteerPage.waitForSelector(
-      `[data-testclass='${testclass}']`,
-      props
-    );
-  },
+export function getTestClass(className) {
+  return `[data-testclass='${className}']`;
+}
 
-  async waitForAllByIds(testids, props = {}) {
-    await Promise.all(
-      testids.map(testid =>
-        puppeteerPage.waitForSelector(`[data-testid='${testid}']`)
-      )
-    );
-  },
+export async function waitByID(testId, props = {}) {
+  return page.waitForSelector(getTestId(testId), props);
+}
 
-  async getAllByClass(testclass, props = {}) {
-    const elements = await puppeteerPage.$$eval(
-      `[data-testclass=${testclass}]`,
-      els => {
-        return els.map(el => {
-          return el.dataset.testid;
-        });
-      }
-    );
-    return elements;
-  },
+export async function waitByClass(testClass, props = {}) {
+  return page.waitForSelector(`[data-testclass='${testClass}']`, props);
+}
 
-  async typeInto(testid, text) {
-    // only works for text without special characters
-    await this.waitByID(testid);
-    const selector = `[data-testid='${testid}']`;
-    // type ahead can be annoying if you don't pause before you type
-    await puppeteerPage.click(selector);
-    await puppeteerPage.waitFor(200);
-    await puppeteerPage.type(selector, text);
-  },
+export async function waitForAllByIds(testIds) {
+  await Promise.all(
+    testIds.map((testId) => page.waitForSelector(getTestId(testId)))
+  );
+}
 
-  async clearInputAndTypeInto(testid, text) {
-    await this.waitByID(testid);
-    const selector = `[data-testid='${testid}']`;
-    // only works for text without special characters
-    // type ahead can be annoying if you don't pause before you type
-    await puppeteerPage.click(selector);
-    await puppeteerPage.waitFor(200);
-    // select all
+export async function getAllByClass(testClass) {
+  return page.$$(`[data-testclass=${testClass}]`);
+}
 
-    await puppeteerPage.click(selector, { clickCount: 3 });
-    await puppeteerPage.keyboard.type("Backspace");
-    await puppeteerPage.type(selector, text);
-  },
+export async function typeInto(testId, text) {
+  // blueprint's  typeahead is treating typing weird, clicking & waiting first solves this
+  // only works for text without special characters
+  await waitByID(testId);
+  const selector = getTestId(testId);
+  // type ahead can be annoying if you don't pause before you type
+  await page.click(selector);
+  await page.waitForTimeout(200);
+  await page.type(selector, text);
+}
 
-  async clickOn(testid) {
-    await this.waitByID(testid);
-    await puppeteerPage.click(`[data-testid='${testid}']`);
-    await puppeteerPage.waitFor(50);
-  },
+export async function clearInputAndTypeInto(testId, text) {
+  await waitByID(testId);
+  const selector = getTestId(testId);
+  // only works for text without special characters
+  // type ahead can be annoying if you don't pause before you type
+  await page.click(selector);
+  await page.waitForTimeout(200);
+  // select all
+  await page.click(selector, { clickCount: 3 });
+  await page.keyboard.press("Backspace");
+  await page.type(selector, text);
+}
 
-  async getOneElementInnerHTML(selector) {
-    await puppeteerPage.waitForSelector(selector);
-    let text = await puppeteerPage.$eval(selector, el => el.innerHTML);
-    return text;
-  },
+export async function clickOn(testId, options = {}) {
+  await expect(page).toClick(getTestId(testId), options);
+}
 
-  async getOneElementInnerText(selector) {
-    await puppeteerPage.waitForSelector(selector);
-    let text = await puppeteerPage.$eval(selector, el => el.innerText);
-    return text;
-  }
-});
+/**
+ * (thuang): There are times when Puppeteer clicks on a button and the page doesn't respond.
+ * So I added clickOnUntil() to retry clicking until a given condition is met.
+ */
+export async function clickOnUntil(testId, assert) {
+  const MAX_RETRY = 10;
+  const WAIT_FOR_MS = 200;
 
-export const cellxgeneActions = puppeteerPage => ({
-  async drag(testid, start, end, lasso = false) {
-    const layout = await puppeteerUtils(puppeteerPage).waitByID(testid);
-    const elBox = await layout.boxModel();
-    const x1 = elBox.content[0].x + start.x;
-    const x2 = elBox.content[0].x + end.x;
-    const y1 = elBox.content[0].y + start.y;
-    const y2 = elBox.content[0].y + end.y;
-    await puppeteerPage.mouse.move(x1, y1);
-    await puppeteerPage.mouse.down();
-    if (lasso) {
-      await puppeteerPage.mouse.move(x2, y1);
-      await puppeteerPage.mouse.move(x2, y2);
-      await puppeteerPage.mouse.move(x1, y2);
-      await puppeteerPage.mouse.move(x1, y1);
-    } else {
-      await puppeteerPage.mouse.move(x2, y2);
-    }
-    await puppeteerPage.mouse.up();
-  },
+  let retry = 0;
 
-  async getAllHistograms(testclass, testids) {
-    const histTestIds = testids.map(tid => `histogram-${tid}`);
-    // these load asynchronously, so we need to wait for each histogram individually
-    await puppeteerUtils(puppeteerPage).waitForAllByIds(histTestIds);
-    const allHistograms = await puppeteerUtils(puppeteerPage).getAllByClass(
-      testclass
-    );
-    return allHistograms.map(hist =>
-      hist.substr("histogram_".length, hist.length)
-    );
-  },
-
-  async getAllCategoriesAndCounts(category) {
-    await puppeteerUtils(puppeteerPage).waitByClass("categorical-row");
-    const categories = await puppeteerPage.$$eval(
-      `[data-testid="category-${category}"] [data-testclass='categorical-row']`,
-      els => {
-        let result = {};
-        els.forEach(el => {
-          const cat = el.querySelector("[data-testclass='categorical-value']")
-            .innerText;
-          const count = el.querySelector(
-            "[data-testclass='categorical-value-count']"
-          ).innerText;
-          result[cat] = count;
-        });
-        return result;
-      }
-    );
-    return categories;
-  },
-
-  async cellSet(num) {
-    await puppeteerUtils(puppeteerPage).clickOn(`cellset-button-${num}`);
-    return await puppeteerUtils(puppeteerPage).getOneElementInnerText(
-      `[data-testid='cellset-count-${num}']`
-    );
-  },
-
-  async resetCategory(category) {
-    const checkboxId = `category-select-${category}`;
-    await puppeteerUtils(puppeteerPage).waitByID(checkboxId);
-    const checkedPseudoclass = await puppeteerPage.$eval(
-      `[data-testid='${checkboxId}']`,
-      el => {
-        return el.matches(":checked");
-      }
-    );
-    if (!checkedPseudoclass) {
-      await puppeteerUtils(puppeteerPage).clickOn(checkboxId);
-    }
+  while (retry < MAX_RETRY) {
     try {
-      const categoryRow = await puppeteerUtils(puppeteerPage).waitByID(
-        `category-expand-${category}`
-      );
-      const isExpanded = await categoryRow.$(
-        "[data-testclass='category-expand-is-expanded']"
-      );
-      if (isExpanded) {
-        await puppeteerUtils(puppeteerPage).clickOn(
-          `category-expand-${category}`
-        );
-      }
-    } catch {}
-  },
+      await clickOn(testId);
+      await assert();
 
-  async calcDragCoordinates(testid, coordinateAsPercent) {
-    const el = await puppeteerUtils(puppeteerPage).waitByID(testid);
-    const size = await el.boxModel();
-    const coords = {
-      start: {
-        x: Math.floor(size.width * coordinateAsPercent.x1),
-        y: Math.floor(size.height * coordinateAsPercent.y1)
-      },
-      end: {
-        x: Math.floor(size.width * coordinateAsPercent.x2),
-        y: Math.floor(size.height * coordinateAsPercent.y2)
-      }
-    };
-    return coords;
-  },
+      break;
+    } catch (error) {
+      retry += 1;
 
-  async selectCategory(category, values, reset = true) {
-    if (reset) await this.resetCategory(category);
-    await puppeteerUtils(puppeteerPage).clickOn(`category-expand-${category}`);
-    await puppeteerUtils(puppeteerPage).clickOn(`category-select-${category}`);
-    for (const val of values) {
-      await puppeteerUtils(puppeteerPage).clickOn(
-        `categorical-value-select-${category}-${val}`
-      );
+      await page.waitForTimeout(WAIT_FOR_MS);
     }
-  },
-
-  async reset() {
-    await puppeteerUtils(puppeteerPage).clickOn("reset");
-    // loading state never actually happens, reset is too fast
-    await page.waitFor(200);
-  },
-
-  async clip(min = 0, max = 100) {
-    await puppeteerUtils(puppeteerPage).clickOn("visualization-settings");
-    await puppeteerUtils(puppeteerPage).clearInputAndTypeInto(
-      "clip-min-input",
-      min
-    );
-    await puppeteerUtils(puppeteerPage).clearInputAndTypeInto(
-      "clip-max-input",
-      max
-    );
-    await puppeteerUtils(puppeteerPage).clickOn("clip-commit");
   }
-});
+
+  if (retry === MAX_RETRY) {
+    throw Error("clickOnUntil() assertion failed!");
+  }
+}
+
+export async function getOneElementInnerHTML(selector, options = {}) {
+  await page.waitForSelector(selector, options);
+
+  return page.$eval(selector, (el) => el.innerHTML);
+}
+
+export async function getOneElementInnerText(selector) {
+  expect(page).toMatchElement(selector);
+
+  return page.$eval(selector, (el) => el.innerText);
+}
+
+export async function getElementCoordinates(testId) {
+  return page.$eval(getTestId(testId), (elem) => {
+    const { left, top } = elem.getBoundingClientRect();
+    return [left, top];
+  });
+}
+
+async function nameNewAnnotation() {
+  if (await isElementPresent(getTestId("annotation-dialog"))) {
+    await typeInto("new-annotation-name", "ignoreE2E");
+    await clickOn("submit-annotation");
+
+    // wait for the page to load
+    await waitByClass("autosave-complete");
+  }
+}
+
+export async function goToPage(url) {
+  await page.goto(url, {
+    waitUntil: "networkidle0",
+  });
+
+  await nameNewAnnotation();
+}
+
+export async function isElementPresent(selector, options) {
+  return Boolean(await page.$(selector, options));
+}
+/* eslint-enable no-await-in-loop -- await in loop is needed to emulate sequential user actions */

client/__tests__/e2e/screenshot_env.js

@@ -0,0 +1,36 @@
+const PuppeteerEnvironment = require("jest-environment-puppeteer");
+require("jest-circus");
+const ENV_DEFAULT = require("../../../environment.default.json");
+
+const takeScreenshot = require("./takeScreenshot");
+
+class ScreenshotEnvironment extends PuppeteerEnvironment {
+  async handleTestEvent(event, state) {
+    if (["test_start", "test_done"].includes(event.name)) {
+      console.log("------------------event name:\n", event.name);
+      console.log("~~~~ Current test errors\n", new Date(), event.test.errors);
+      console.log("~~~~ Current test\n", new Date(), event.test);
+    }
+
+    if (event.name === "error") {
+      console.log("error event:", JSON.stringify(event));
+    }
+
+    if (event.name === "test_fn_failure" || event.name === "hook_failure") {
+      console.log("------------------event name:\n", event.name);
+      console.log(">>>> Current state\n", new Date(), state);
+      console.log("===> Failure event\n", new Date(), event);
+
+      // (thuang): We only want to take screenshot on the last try
+      if (
+        state.currentlyRunningTest.invocations <= ENV_DEFAULT.RETRY_ATTEMPTS
+      ) {
+        return;
+      }
+
+      await takeScreenshot(state.currentlyRunningTest.name, this.global.page);
+    }
+  }
+}
+
+module.exports = ScreenshotEnvironment;

client/__tests__/e2e/takeScreenshot.js

@@ -0,0 +1,17 @@
+function toFilename(name) {
+  return name.replace(/[^a-z0-9.-]+/gi, "-");
+}
+
+async function takeScreenshot(currentTestName, page) {
+  const testName = toFilename(currentTestName);
+
+  // Take a screenshot at the point of failure
+  const date = new Date().toISOString();
+  const screenshotName = `${date}-${testName}.png`;
+
+  await page.screenshot({
+    path: `./__tests__/screenshots/ignoreE2E-screenshot-${screenshotName}`,
+  });
+}
+
+module.exports = takeScreenshot;

client/__tests__/e2e/test_config.yaml

@@ -0,0 +1,19 @@
+server:
+  app:
+    force_https: true
+    port: 5005
+
+dataset:
+  presentation:
+    max_categories: 1000
+    custom_colors: true
+
+  user_annotations:
+    enable: false
+    type: local_file_csv
+    local_file_csv:
+      directory: null
+      file: null
+
+  embeddings:
+    names: []

client/__tests__/globalSetup.js

@@ -0,0 +1,5 @@
+const { setup } = require("jest-environment-puppeteer");
+
+module.exports = async () => {
+  await setup();
+};

client/__tests__/reducers/cascade.test.js

@@ -26,7 +26,7 @@ describe("cascade", () => {
           expect(nextSharedState).toStrictEqual({});
           expect(prevSharedState).toBe(topLevelState);
           return 0;
-        }
+        },
       ],
       [
         "bar",
@@ -36,8 +36,8 @@ describe("cascade", () => {
           expect(nextSharedState).toStrictEqual({ foo: 0 });
           expect(prevSharedState).toBe(topLevelState);
           return 99;
-        }
-      ]
+        },
+      ],
     ]);
 
     const nextState = reducer(topLevelState, topLevelAction);

client/__tests__/reducers/genesets.test.js

@@ -0,0 +1,524 @@
+import genesetsReducer from "../../src/reducers/genesets";
+
+describe("initial reducer state", () => {
+  test("some other action", () => {
+    expect(genesetsReducer(undefined, { type: "foo" })).toMatchObject({
+      initialized: false,
+      lastTid: undefined,
+      genesets: new Map(),
+    });
+  });
+});
+
+describe("geneset: initial load", () => {
+  test("missing JSON response", () => {
+    expect(() =>
+      genesetsReducer(undefined, {
+        type: "geneset: initial load",
+      })
+    ).toThrow("missing or malformed JSON response");
+  });
+
+  test("empty geneset", () => {
+    expect(
+      genesetsReducer(undefined, {
+        type: "geneset: initial load",
+        data: {
+          tid: 0,
+          genesets: [],
+        },
+      })
+    ).toMatchObject({
+      initialized: true,
+      lastTid: 0,
+      genesets: new Map(),
+    });
+  });
+
+  test("non-empty geneset", () => {
+    expect(
+      genesetsReducer(undefined, {
+        type: "geneset: initial load",
+        data: {
+          tid: 99,
+          genesets: [
+            {
+              geneset_name: "G1",
+              genes: [{ gene_symbol: "F5" }],
+            },
+            {
+              geneset_name: "G2",
+              geneset_description: "G2 desc",
+              genes: [{ gene_symbol: "F6" }],
+            },
+            {
+              geneset_name: "G3",
+              geneset_description: "G3 desc",
+              genes: [{ gene_symbol: "F7", gene_description: "gene desc" }],
+            },
+          ],
+        },
+      })
+    ).toMatchObject({
+      initialized: true,
+      lastTid: 99,
+      genesets: new Map([
+        [
+          "G1",
+          {
+            genesetName: "G1",
+            genesetDescription: "",
+            genes: new Map([["F5", { geneSymbol: "F5", geneDescription: "" }]]),
+          },
+        ],
+        [
+          "G2",
+          {
+            genesetName: "G2",
+            genesetDescription: "G2 desc",
+            genes: new Map([["F6", { geneSymbol: "F6", geneDescription: "" }]]),
+          },
+        ],
+        [
+          "G3",
+          {
+            genesetName: "G3",
+            genesetDescription: "G3 desc",
+            genes: new Map([
+              ["F7", { geneSymbol: "F7", geneDescription: "gene desc" }],
+            ]),
+          },
+        ],
+      ]),
+    });
+  });
+});
+
+describe("geneset: create", () => {
+  const initialState = genesetsReducer(undefined, {
+    type: "geneset: initial load",
+    data: {
+      tid: 0,
+      genesets: [],
+    },
+  });
+
+  test("simple create", () => {
+    expect(
+      genesetsReducer(initialState, {
+        type: "geneset: create",
+        genesetName: "a geneset",
+        genesetDescription: "",
+      })
+    ).toMatchObject({
+      ...initialState,
+      genesets: new Map([
+        [
+          "a geneset",
+          {
+            genesetName: "a geneset",
+            genesetDescription: "",
+            genes: new Map(),
+          },
+        ],
+      ]),
+    });
+  });
+
+  test("error - duplicate name", () => {
+    expect(() => {
+      genesetsReducer(
+        genesetsReducer(initialState, {
+          type: "geneset: create",
+          genesetName: "foo",
+          genesetDescription: "foo",
+        }),
+        {
+          type: "geneset: create",
+          genesetName: "foo",
+          genesetDescription: "bar",
+        }
+      );
+    }).toThrow("name already defined");
+  });
+
+  test("error - missing required action values", () => {
+    expect(() => {
+      genesetsReducer(initialState, {
+        type: "geneset: create",
+        genesetDescription: "foo",
+      });
+    }).toThrow();
+    expect(() => {
+      genesetsReducer(initialState, {
+        type: "geneset: create",
+        genesetName: "foo",
+      });
+    }).toThrow("name or description unspecified");
+  });
+});
+
+describe("geneset: delete", () => {
+  const initialState = genesetsReducer(undefined, {
+    type: "geneset: initial load",
+    data: {
+      tid: 0,
+      genesets: [],
+    },
+  });
+
+  test("simple delete", () => {
+    expect(
+      genesetsReducer(
+        genesetsReducer(initialState, {
+          type: "geneset: create",
+          genesetName: "foo",
+          genesetDescription: "foo",
+        }),
+        {
+          type: "geneset: delete",
+          genesetName: "foo",
+        }
+      )
+    ).toMatchObject({
+      initialized: true,
+      lastTid: 0,
+      genesets: new Map(),
+    });
+  });
+
+  test("error - missing name", () => {
+    expect(() => {
+      genesetsReducer(initialState, {
+        type: "geneset: delete",
+        genesetName: "foo",
+      });
+    }).toThrow("name does not exist");
+  });
+});
+
+describe("geneset: update", () => {
+  const initialState = genesetsReducer(undefined, {
+    type: "geneset: initial load",
+    data: {
+      tid: 0,
+      genesets: [],
+    },
+  });
+
+  test("simple update", () => {
+    expect(
+      genesetsReducer(
+        genesetsReducer(
+          genesetsReducer(initialState, {
+            type: "geneset: create",
+            genesetName: "foo1",
+            genesetDescription: "foo1",
+          }),
+          {
+            type: "geneset: create",
+            genesetName: "foo2",
+            genesetDescription: "foo2",
+          }
+        ),
+        {
+          type: "geneset: update",
+          genesetName: "foo1",
+          update: {
+            genesetName: "bar",
+            genesetDescription: "bar",
+          },
+        }
+      )
+    ).toMatchObject({
+      initialized: true,
+      lastTid: 0,
+      genesets: new Map([
+        [
+          "bar",
+          { genesetName: "bar", genesetDescription: "bar", genes: new Map() },
+        ],
+        [
+          "foo2",
+          { genesetName: "foo2", genesetDescription: "foo2", genes: new Map() },
+        ],
+      ]),
+    });
+  });
+
+  test("error - unknown name", () => {
+    expect(() => {
+      genesetsReducer(initialState, {
+        type: "geneset: update",
+        genesetName: "foo",
+        update: {
+          genesetName: "foo",
+          genesetDescription: "bar",
+        },
+      });
+    }).toThrow("name unspecified or does not exist");
+  });
+
+  test("error - duplicate name", () => {
+    expect(() => {
+      genesetsReducer(
+        genesetsReducer(initialState, {
+          type: "geneset: create",
+          genesetName: "foo",
+          genesetDescription: "foo",
+        }),
+        {
+          type: "geneset: update",
+          genesetName: "foo",
+          update: {
+            genesetName: "foo",
+            genesetDescription: "foo",
+          },
+        }
+      );
+    }).toThrow("update specified existing name and description");
+  });
+});
+
+describe("geneset: add genes", () => {
+  const initialState = genesetsReducer(
+    genesetsReducer(undefined, {
+      type: "geneset: initial load",
+      data: {
+        tid: 0,
+        genesets: [],
+      },
+    }),
+    {
+      type: "geneset: create",
+      genesetName: "test",
+      genesetDescription: "",
+    }
+  );
+
+  test("add a gene", () => {
+    expect(
+      genesetsReducer(initialState, {
+        type: "geneset: add genes",
+        genesetName: "test",
+        genes: [{ geneSymbol: "F5" }],
+      })
+    ).toMatchObject({
+      ...initialState,
+      genesets: new Map([
+        [
+          "test",
+          {
+            genesetName: "test",
+            genesetDescription: "",
+            genes: new Map([["F5", { geneSymbol: "F5", geneDescription: "" }]]),
+          },
+        ],
+      ]),
+    });
+
+    expect(
+      genesetsReducer(initialState, {
+        type: "geneset: add genes",
+        genesetName: "test",
+        genes: [
+          { geneSymbol: "F5", geneDescription: "desc" },
+          { geneSymbol: "SET1", geneDescription: "" },
+        ],
+      })
+    ).toMatchObject({
+      ...initialState,
+      genesets: new Map([
+        [
+          "test",
+          {
+            genesetName: "test",
+            genesetDescription: "",
+            genes: new Map([
+              ["F5", { geneSymbol: "F5", geneDescription: "desc" }],
+              ["SET1", { geneSymbol: "SET1", geneDescription: "" }],
+            ]),
+          },
+        ],
+      ]),
+    });
+  });
+
+  test("no such geneset error", () => {
+    expect(() => {
+      genesetsReducer(initialState, {
+        type: "geneset: add genes",
+        genesetName: "mumble",
+        genes: [],
+      });
+    }).toThrow("geneset name does not exist");
+  });
+});
+
+describe("geneset: delete genes", () => {
+  const initialState = genesetsReducer(
+    genesetsReducer(
+      genesetsReducer(undefined, {
+        type: "geneset: initial load",
+        data: {
+          tid: 0,
+          genesets: [],
+        },
+      }),
+      {
+        type: "geneset: create",
+        genesetName: "test",
+        genesetDescription: "",
+      }
+    ),
+    {
+      type: "geneset: add genes",
+      genesetName: "test",
+      genes: [{ geneSymbol: "F5" }],
+    }
+  );
+
+  test("simple", () => {
+    expect(
+      genesetsReducer(initialState, {
+        type: "geneset: delete genes",
+        genesetName: "test",
+        geneSymbols: ["F5"],
+      })
+    ).toMatchObject({
+      ...initialState,
+      genesets: new Map([
+        [
+          "test",
+          {
+            genesetName: "test",
+            genesetDescription: "",
+            genes: new Map(),
+          },
+        ],
+      ]),
+    });
+  });
+
+  test("no such geneset error", () => {
+    expect(() => {
+      genesetsReducer(initialState, {
+        type: "geneset: delete genes",
+        genesetName: "mumble",
+        geneSymbols: [],
+      });
+    }).toThrow("name does not exist");
+  });
+});
+
+describe("geneset: set gene description", () => {
+  const initialState = genesetsReducer(
+    genesetsReducer(
+      genesetsReducer(undefined, {
+        type: "geneset: initial load",
+        data: {
+          tid: 0,
+          genesets: [],
+        },
+      }),
+      {
+        type: "geneset: create",
+        genesetName: "test",
+        genesetDescription: "",
+      }
+    ),
+    {
+      type: "geneset: add genes",
+      genesetName: "test",
+      genes: [{ geneSymbol: "F5" }],
+    }
+  );
+
+  test("simple set", () => {
+    expect(
+      genesetsReducer(initialState, {
+        type: "geneset: set gene description",
+        genesetName: "test",
+        update: {
+          geneSymbol: "F5",
+          geneDescription: "mumble",
+        },
+      })
+    ).toMatchObject({
+      ...initialState,
+      genesets: new Map([
+        [
+          "test",
+          {
+            genesetName: "test",
+            genesetDescription: "",
+            genes: new Map([
+              ["F5", { geneSymbol: "F5", geneDescription: "mumble" }],
+            ]),
+          },
+        ],
+      ]),
+    });
+  });
+
+  test("no such geneset error", () => {
+    expect(() => {
+      genesetsReducer(initialState, {
+        type: "geneset: set gene description",
+        genesetName: "does not exist",
+        update: {
+          geneSymbol: "F5",
+          geneDescription: "mumble",
+        },
+      });
+    }).toThrow("geneset name does not exist");
+  });
+
+  test("no such gene error", () => {
+    expect(() => {
+      genesetsReducer(initialState, {
+        type: "geneset: set gene description",
+        genesetName: "test",
+        update: {
+          geneSymbol: "NO SUCH GENE",
+          geneDescription: "mumble",
+        },
+      });
+    }).toThrow("no such gene");
+  });
+});
+
+describe("geneset: set tid", () => {
+  test("simple set", () => {
+    expect(
+      genesetsReducer(undefined, {
+        type: "geneset: set tid",
+        tid: 1,
+      })
+    ).toMatchObject({ lastTid: 1 });
+  });
+
+  test("not a number error", () => {
+    expect(() => {
+      genesetsReducer(
+        { lastTid: 1 },
+        {
+          type: "geneset: set tid",
+          tid: "0",
+        }
+      );
+    }).toThrow("must be a positive integer");
+  });
+
+  test("decrement error", () => {
+    expect(() => {
+      genesetsReducer(
+        { lastTid: 1 },
+        {
+          type: "geneset: set tid",
+          tid: 0,
+        }
+      );
+    }).toThrow("may not be decremented");
+  });
+});

client/__tests__/reducers/genesetsUI.test.js

@@ -0,0 +1,75 @@
+import genesetsUIReducer from "../../src/reducers/genesetsUI";
+
+// Format: GeneSetsUI(state,action)
+
+const initialState = {
+  createGenesetModeActive: false,
+  isEditingGenesetName: false,
+  isAddingGenesToGeneset: false,
+};
+
+/* initial */
+describe("geneset UI states", () => {
+  test("initial state, some other action", () => {
+    expect(
+      genesetsUIReducer(undefined, {
+        type: "foo",
+      })
+    ).toMatchObject(initialState);
+  });
+  test("geneset: activate add new geneset mode", () => {
+    expect(
+      genesetsUIReducer(initialState, {
+        type: "geneset: activate add new geneset mode",
+      })
+    ).toMatchObject({
+      createGenesetModeActive: true,
+      isEditingGenesetName: false,
+      isAddingGenesToGeneset: false,
+    });
+  });
+  test("geneset: disable create geneset mode", () => {
+    expect(
+      genesetsUIReducer(undefined, { isEditingGenesetName: false })
+    ).toMatchObject(initialState);
+  });
+
+  test("activate add new genes mode", () => {
+    expect(
+      genesetsUIReducer(undefined, {
+        type: "geneset: activate add new genes mode",
+        geneset: "a geneset name",
+      })
+    ).toMatchObject({
+      createGenesetModeActive: false,
+      isEditingGenesetName: false,
+      isAddingGenesToGeneset: "a geneset name",
+    });
+  });
+  test("disable create geneset mode", () => {
+    expect(
+      genesetsUIReducer(undefined, {
+        type: "geneset: disable create geneset mode",
+      })
+    ).toMatchObject(initialState);
+  });
+  test("activate rename geneset mode", () => {
+    expect(
+      genesetsUIReducer(undefined, {
+        type: "geneset: activate rename geneset mode",
+        data: "a geneset name",
+      })
+    ).toMatchObject({
+      createGenesetModeActive: false,
+      isEditingGenesetName: "a geneset name",
+      isAddingGenesToGeneset: false,
+    });
+  });
+  test("disable rename geneset mode", () => {
+    expect(
+      genesetsUIReducer(undefined, {
+        type: "geneset: disable rename geneset mode",
+      })
+    ).toMatchObject(initialState);
+  });
+});

client/__tests__/reducers/undoable.test.js

@@ -23,9 +23,7 @@ describe("create", () => {
 describe("undo", () => {
   test("expected state modifications", () => {
     const initialState = { a: 0, b: 1000 };
-    const reducer = state => {
-      return { a: state.a + 1, b: state.b + 1 };
-    };
+    const reducer = (state) => ({ a: state.a + 1, b: state.b + 1 });
     const undoableReducer = undoable(reducer, ["a"]);
 
     const s1 = undoableReducer(initialState, { type: "test" });
@@ -43,9 +41,7 @@ describe("undo", () => {
 
 describe("redo", () => {
   const initialState = { a: 0, b: 1000 };
-  const reducer = state => {
-    return { a: state.a + 1, b: state.b + 1 };
-  };
+  const reducer = (state) => ({ a: state.a + 1, b: state.b + 1 });
   let UR;
 
   beforeEach(() => {
@@ -58,7 +54,7 @@ describe("redo", () => {
 
     // verify undo->redo reverts state.
     const s2 = UR(UR(s1, { type: "@@undoable/undo" }), {
-      type: "@@undoable/redo"
+      type: "@@undoable/redo",
     });
     expect(s2).toMatchObject({ a: 1, b: 1001 });
 

client/__tests__/screenshots/.gitignore

@@ -0,0 +1,4 @@
+# Ignore everything in this directory
+*
+# Except this file
+!.gitignore

client/__tests__/util/actionHelpers.test.js

@@ -11,14 +11,10 @@ describe("rangeEncodeIndices", () => {
 
   test("sorted flag", () => {
     expect(rangeEncodeIndices([1, 9, 432], 10, true)).toMatchObject([
-      1,
-      9,
-      432
+      1, 9, 432,
     ]);
     expect(rangeEncodeIndices([1, 9, 432], 10, false)).toMatchObject([
-      1,
-      9,
-      432
+      1, 9, 432,
     ]);
     expect(rangeEncodeIndices([0, 1, 2, 3, 9, 10, 432], 2, true)).toMatchObject(
       [[0, 3], [9, 10], 432]
@@ -34,7 +30,10 @@ describe("rangeEncodeIndices", () => {
     ).toMatchObject([[3, 7], [10, 12], 99]);
     expect(
       rangeEncodeIndices([3, 4, 5, 6, 7, 10, 11, 12], 3, false)
-    ).toMatchObject([[3, 7], [10, 12]]);
+    ).toMatchObject([
+      [3, 7],
+      [10, 12],
+    ]);
     expect(
       rangeEncodeIndices([0, 3, 4, 5, 6, 7, 10, 11, 12], 3, false)
     ).toMatchObject([0, [3, 7], [10, 12]]);

client/__tests__/util/annoMatrix/annoMatrix.test.js

@@ -0,0 +1,322 @@
+// these TWO statements MUST be first in the file, before any other imports
+import { enableFetchMocks } from "jest-fetch-mock";
+import * as serverMocks from "./serverMocks";
+// OK, continue on!
+
+import {
+  AnnoMatrixLoader,
+  clip,
+  isubset,
+  isubsetMask,
+} from "../../../src/annoMatrix";
+import { Dataframe } from "../../../src/util/dataframe";
+
+enableFetchMocks();
+
+describe("AnnoMatrix", () => {
+  let annoMatrix;
+
+  beforeEach(async () => {
+    fetch.resetMocks(); // reset all fetch mocking state
+    annoMatrix = new AnnoMatrixLoader(
+      serverMocks.baseDataURL,
+      serverMocks.schema.schema
+    );
+  });
+
+  describe("basics", () => {
+    test("annomatrix static checks", () => {
+      expect(annoMatrix).toBeDefined();
+      expect(annoMatrix.schema).toMatchObject(serverMocks.schema.schema);
+      expect(annoMatrix.nObs).toEqual(serverMocks.schema.schema.dataframe.nObs);
+      expect(annoMatrix.nVar).toEqual(serverMocks.schema.schema.dataframe.nVar);
+      expect(annoMatrix.isView).toBeFalsy();
+      expect(annoMatrix.viewOf).toBeUndefined();
+      expect(annoMatrix.rowIndex).toBeDefined();
+    });
+
+    test("simple single column fetch", async () => {
+      fetch.once(serverMocks.annotationsObs(["name_0"]));
+
+      const df = await annoMatrix.fetch("obs", "name_0");
+      expect(df).toBeInstanceOf(Dataframe);
+      expect(df.colIndex.labels()).toEqual(["name_0"]);
+      expect(df.dims).toEqual([annoMatrix.nObs, 1]);
+    });
+
+    test("simple multi column fetch", async () => {
+      fetch
+        .once(serverMocks.annotationsObs(["name_0"]))
+        .once(serverMocks.annotationsObs(["n_genes"]));
+
+      await expect(
+        annoMatrix.fetch("obs", ["name_0", "n_genes"])
+      ).resolves.toBeInstanceOf(Dataframe);
+    });
+
+    describe("fetch from field", () => {
+      const getLastTwo = async (field) => {
+        const columnNames = annoMatrix.getMatrixColumns(field).slice(-2);
+        fetch.mockResponses(...columnNames.map(() => serverMocks.responder));
+        await expect(
+          annoMatrix.fetch(field, columnNames)
+        ).resolves.toBeInstanceOf(Dataframe);
+      };
+
+      test("obs", async () => getLastTwo("obs"));
+      test("var", async () => getLastTwo("var"));
+      test("emb", async () => getLastTwo("emb"));
+    });
+
+    test("fetch - test all query forms", async () => {
+      // single string is a column name
+      fetch.once(serverMocks.annotationsObs(["n_genes"]));
+      await expect(annoMatrix.fetch("obs", "n_genes")).resolves.toBeInstanceOf(
+        Dataframe
+      );
+
+      // array of column names, expecting n_genes to be cached.
+      fetch.once(serverMocks.annotationsObs(["percent_mito"]));
+      await expect(
+        annoMatrix.fetch("obs", ["n_genes", "percent_mito"])
+      ).resolves.toBeInstanceOf(Dataframe);
+
+      // more complex value filter query, enumerated
+      fetch.once(serverMocks.responder);
+      await expect(
+        annoMatrix.fetch("X", {
+          where: {
+            field: "var",
+            column: annoMatrix.schema.annotations.var.index,
+            value: "TYMP",
+          },
+        })
+      ).resolves.toBeInstanceOf(Dataframe);
+
+      // more complex value filter query, range
+      const varIndex = annoMatrix.schema.annotations.var.index;
+      fetch
+        .once(
+          serverMocks.withExpected("/data/var", [[`var:${varIndex}`, "SUMO3"]])
+        )
+        .once(
+          serverMocks.withExpected("/data/var", [[`var:${varIndex}`, "TYMP"]])
+        );
+      await expect(
+        annoMatrix.fetch("X", [
+          {
+            where: {
+              field: "var",
+              column: varIndex,
+              value: "SUMO3",
+            },
+          },
+          {
+            where: {
+              field: "var",
+              column: varIndex,
+              value: "TYMP",
+            },
+          },
+        ])
+      ).resolves.toBeInstanceOf(Dataframe);
+      // XXX inspect the wherecache?
+    });
+
+    test("push and pop views", async () => {
+      const am1 = clip(annoMatrix, 0.1, 0.9);
+      expect(am1.viewOf).toBe(annoMatrix);
+      expect(am1.nObs).toEqual(annoMatrix.nObs);
+      expect(am1.nVar).toEqual(annoMatrix.nVar);
+      expect(am1.rowIndex).toBe(annoMatrix.rowIndex);
+
+      const am2 = clip(annoMatrix, 0.1, 0.9);
+      expect(am2.viewOf).toBe(annoMatrix);
+      expect(am2).not.toBe(am1);
+      expect(am2.rowIndex).toBe(annoMatrix.rowIndex);
+    });
+
+    test("schema accessors", () => {
+      expect(annoMatrix.getMatrixFields()).toEqual(
+        expect.arrayContaining(["X", "obs", "emb", "var"])
+      );
+      expect(annoMatrix.getMatrixColumns("obs")).toEqual(
+        expect.arrayContaining(["name_0", "n_genes", "louvain"])
+      );
+      expect(annoMatrix.getColumnSchema("emb", "umap")).toEqual({
+        name: "umap",
+        dims: ["umap_0", "umap_1"],
+        type: "float32",
+      });
+      expect(annoMatrix.getColumnDimensions("emb", "umap")).toEqual([
+        "umap_0",
+        "umap_1",
+      ]);
+    });
+
+    /*
+  	test the mask & label access to subset via isubset and isubsetMask
+  	*/
+    test("isubset", async () => {
+      const rowList = [0, 10];
+      const rowMask = new Uint8Array(annoMatrix.nObs);
+      for (let i = 0; i < rowList.length; i += 1) {
+        rowMask[rowList[i]] = 1;
+      }
+
+      const am1 = isubset(annoMatrix, rowList);
+      const am2 = isubsetMask(annoMatrix, rowMask);
+      expect(am1).not.toBe(am2);
+      expect(am1.nObs).toEqual(2);
+      expect(am1.nObs).toEqual(am2.nObs);
+      expect(am1.nVar).toEqual(am2.nVar);
+
+      fetch
+        .once(serverMocks.annotationsObs(["n_genes"]))
+        .once(serverMocks.annotationsObs(["n_genes"]));
+      const ng1 = await am1.fetch("obs", "n_genes");
+      const ng2 = await am2.fetch("obs", "n_genes");
+      expect(ng1).toHaveLength(ng2.length);
+      expect(ng1.colIndex.labels()).toEqual(ng2.colIndex.labels());
+      expect(ng1.col("n_genes").asArray()).toEqual(
+        ng2.col("n_genes").asArray()
+      );
+    });
+  });
+
+  describe("add/drop column", () => {
+    async function addDrop(base) {
+      expect(base.getMatrixColumns("obs")).not.toContain("foo");
+      fetch.mockRejectOnce(new Error("unknown column name"));
+      await expect(base.fetch("obs", "foo")).rejects.toThrow(
+        "unknown column name"
+      );
+
+      /* add */
+      const am1 = base.addObsColumn(
+        { name: "foo", type: "float32", writable: true },
+        Float32Array,
+        0
+      );
+      expect(base.getMatrixColumns("obs")).not.toContain("foo");
+      expect(am1.getMatrixColumns("obs")).toContain("foo");
+      const foo = await am1.fetch("obs", "foo");
+      expect(foo).toBeDefined();
+      expect(foo).toBeInstanceOf(Dataframe);
+      expect(foo).toHaveLength(am1.nObs);
+      expect(foo.col("foo").asArray()).toEqual(
+        new Float32Array(am1.nObs).fill(0)
+      );
+
+      /* drop */
+      const am2 = am1.dropObsColumn("foo");
+      expect(base.getMatrixColumns("obs")).not.toContain("foo");
+      expect(am2.getMatrixColumns("obs")).not.toContain("foo");
+      fetch.mockRejectOnce(new Error("unknown column name"));
+      await expect(am2.fetch("obs", "foo")).rejects.toThrow(
+        "unknown column name"
+      );
+    }
+
+    test("add/drop column, without view", async () => {
+      await addDrop(annoMatrix);
+    });
+
+    test("add/drop column, with view", async () => {
+      const am1 = clip(annoMatrix, 0.1, 0.9);
+      await addDrop(am1);
+
+      const am2 = isubset(am1, [0, 1, 2, 20, 30, 400]);
+      await addDrop(am2);
+
+      const am3 = isubset(annoMatrix, [10, 0, 7, 3]);
+      await addDrop(am3);
+
+      const am4 = clip(am3, 0, 1);
+      await addDrop(am4);
+
+      fetch.mockResponse(serverMocks.responder);
+
+      await am1.fetch("obs", am1.getMatrixColumns("obs"));
+      await am2.fetch("obs", am2.getMatrixColumns("obs"));
+      await am3.fetch("obs", am3.getMatrixColumns("obs"));
+      await am4.fetch("obs", am4.getMatrixColumns("obs"));
+
+      fetch.resetMocks();
+
+      await addDrop(am1);
+      await addDrop(am2);
+      await addDrop(am3);
+      await addDrop(am4);
+    });
+  });
+
+  describe("setObsColumnValues", () => {
+    async function addSetDrop(base) {
+      /* add column */
+      let am = base.addObsColumn(
+        {
+          name: "test",
+          type: "categorical",
+          categories: ["unassigned", "red", "green"],
+          writable: true,
+        },
+        Array,
+        "unassigned"
+      );
+
+      const testVal = await am.fetch("obs", "test");
+      expect(testVal.col("test").asArray()).toEqual(
+        new Array(am.nObs).fill("unassigned")
+      );
+
+      /* set values in column */
+      const whichRows = [1, 2, 10];
+      const am1 = await am.setObsColumnValues("test", whichRows, "yo");
+      const testVal1 = await am1.fetch("obs", "test");
+      const expt = new Array(am1.nObs).fill("unassigned");
+      for (let i = 0; i < whichRows.length; i += 1) {
+        const offset = am1.rowIndex.getOffset(whichRows[i]);
+        expt[offset] = "yo";
+      }
+      expect(testVal1).not.toBe(testVal);
+      expect(testVal1.col("test").asArray()).toEqual(expt);
+      expect(am1.getColumnSchema("obs", "test").type).toBe("categorical");
+      expect(am1.getColumnSchema("obs", "test").categories).toEqual(
+        expect.arrayContaining(["unassigned", "red", "green", "yo"])
+      );
+
+      /* drop column */
+      fetch.mockRejectOnce(new Error("unknown column name"));
+      am = am1.dropObsColumn("test");
+      await expect(am.fetch("obs", "test")).rejects.toThrow(
+        "unknown column name"
+      );
+    }
+
+    test("set, without a view", async () => {
+      await addSetDrop(annoMatrix);
+    });
+
+    test("set, with a view", async () => {
+      const am1 = clip(annoMatrix, 0.1, 0.9);
+      await addSetDrop(am1);
+
+      const am2 = isubset(am1, [0, 1, 2, 10, 20, 30, 400]);
+      await addSetDrop(am2);
+
+      const am3 = isubset(annoMatrix, [10, 1, 0, 30, 2]);
+      await addSetDrop(am3);
+
+      fetch.mockResponse(serverMocks.responder);
+
+      await am1.fetch("obs", am1.getMatrixColumns("obs"));
+      await am2.fetch("obs", am2.getMatrixColumns("obs"));
+      await am3.fetch("obs", am3.getMatrixColumns("obs"));
+
+      await addSetDrop(am1);
+      await addSetDrop(am2);
+      await addSetDrop(am3);
+    });
+  });
+});

client/__tests__/util/annoMatrix/crossfilter.test.js

@@ -0,0 +1,692 @@
+// these TWO statements MUST be first in the file, before any other imports
+import { enableFetchMocks } from "jest-fetch-mock";
+import * as serverMocks from "./serverMocks";
+// OK, continue on!
+
+import obsLouvain from "./louvain.json";
+import obsNGenes from "./n_genes.json";
+import embUmap from "./umap.json";
+
+import {
+  AnnoMatrixLoader,
+  AnnoMatrixObsCrossfilter,
+  isubsetMask,
+} from "../../../src/annoMatrix";
+import { rangeFill } from "../../../src/util/range";
+
+enableFetchMocks();
+
+describe("AnnoMatrixCrossfilter", () => {
+  let annoMatrix;
+  let crossfilter;
+
+  beforeEach(async () => {
+    fetch.resetMocks(); // reset all fetch mocking state
+    annoMatrix = new AnnoMatrixLoader(
+      serverMocks.baseDataURL,
+      serverMocks.schema.schema
+    );
+    crossfilter = new AnnoMatrixObsCrossfilter(annoMatrix);
+  });
+
+  test("initial state of crossfilter", () => {
+    const { nObs } = annoMatrix;
+
+    expect(crossfilter).toBeDefined();
+    expect(crossfilter.size()).toEqual(nObs);
+    expect(crossfilter.annoMatrix).toBe(annoMatrix);
+
+    /* by default, everything should be selected, even if no data in cache */
+    expect(crossfilter.countSelected()).toEqual(nObs);
+    expect(crossfilter.allSelectedLabels()).toEqual(
+      rangeFill(new Int32Array(nObs))
+    );
+    expect(crossfilter.allSelectedMask()).toEqual(new Uint8Array(nObs).fill(1));
+    expect(crossfilter.fillByIsSelected(new Uint8Array(nObs), 2, 1)).toEqual(
+      new Uint8Array(nObs).fill(2)
+    );
+  });
+
+  describe("select", () => {
+    /*
+    test the selection state via crossfilter proxy
+    */
+
+    test("select loads index", async () => {
+      /*
+      Select should transparently load/create dimension index.
+
+      Internal dimension names are field/col:col:col..., eg,
+
+        obs:louvain
+        emb:umap_0:umap_1
+
+      */
+      expect(crossfilter.obsCrossfilter.dimensionNames()).toEqual([]);
+      expect(
+        crossfilter.obsCrossfilter.hasDimension("obs/louvain")
+      ).toBeFalsy();
+
+      fetch.once(serverMocks.dataframeResponse(["louvain"], [obsLouvain]));
+      let newCrossfilter = await crossfilter.select("obs", "louvain", {
+        mode: "none",
+      });
+
+      expect(newCrossfilter.countSelected()).toEqual(0);
+      expect(
+        newCrossfilter.obsCrossfilter.hasDimension("obs/louvain")
+      ).toBeTruthy();
+      expect(fetch.mock.calls).toHaveLength(1);
+
+      newCrossfilter = await crossfilter.select("obs", "louvain", {
+        mode: "all",
+      });
+      expect(newCrossfilter.countSelected()).toEqual(annoMatrix.nObs);
+    });
+
+    test("simple column select", async () => {
+      let xfltr;
+
+      fetch.once(serverMocks.dataframeResponse(["louvain"], [obsLouvain]));
+      xfltr = await crossfilter.select("obs", "louvain", {
+        mode: "exact",
+        values: ["NK cells", "B cells"],
+      });
+
+      expect(xfltr).toBeDefined();
+      expect(xfltr.countSelected()).toEqual(496);
+      expect(xfltr.allSelectedMask()).toEqual(
+        Uint8Array.from(
+          obsLouvain.map((val) =>
+            val === "NK cells" || val === "B cells" ? 1 : 0
+          )
+        )
+      );
+      expect(xfltr.allSelectedLabels()).toEqual(
+        Int32Array.from(
+          obsLouvain.reduce((acc, val, idx) => {
+            if (val === "NK cells" || val === "B cells") acc.push(idx);
+            return acc;
+          }, [])
+        )
+      );
+      expect(
+        xfltr.fillByIsSelected(new Uint8Array(annoMatrix.nObs), 3, 1)
+      ).toEqual(
+        Uint8Array.from(
+          obsLouvain.map((val) =>
+            val === "NK cells" || val === "B cells" ? 3 : 1
+          )
+        )
+      );
+
+      const df = await annoMatrix.fetch("obs", "louvain");
+      const values = df.col("louvain").asArray();
+      const selected = xfltr.allSelectedMask();
+      values.every(
+        (val, idx) => !["NK cells", "B cells"].includes(val) !== !selected[idx]
+      );
+
+      fetch.once(
+        serverMocks.dataframeResponse(["n_genes"], [new Int32Array(obsNGenes)])
+      );
+      xfltr = await xfltr.select("obs", "n_genes", {
+        mode: "range",
+        lo: 0,
+        hi: 500,
+        inclusive: false,
+      });
+      expect(xfltr.countSelected()).toEqual(33);
+      expect(xfltr.allSelectedLabels()).toEqual(
+        Int32Array.from(
+          obsNGenes.reduce((acc, val, idx) => {
+            const louvain = obsLouvain[idx];
+            if (
+              val >= 0 &&
+              val < 500 &&
+              (louvain === "NK cells" || louvain === "B cells")
+            )
+              acc.push(idx);
+            return acc;
+          }, [])
+        )
+      );
+
+      xfltr = await xfltr.selectAll();
+      expect(xfltr.countSelected()).toEqual(annoMatrix.nObs);
+    });
+
+    test("join column select", async () => {
+      const varIndex = annoMatrix.schema.annotations.var.index;
+
+      const { nObs } = annoMatrix.schema.dataframe;
+      fetch.once(
+        serverMocks.dataframeResponse(
+          ["TEST"],
+          [rangeFill(new Float32Array(nObs), 0, 0.1)]
+        )
+      );
+
+      const xfltr = await crossfilter.select(
+        "X",
+        {
+          where: {
+            field: "var",
+            column: varIndex,
+            value: "TYMP",
+          },
+        },
+        {
+          mode: "range",
+          lo: 0,
+          hi: 50,
+          inclusive: true,
+        }
+      );
+
+      expect(xfltr).toBeDefined();
+      expect(xfltr.countSelected()).toEqual(501);
+
+      const df = await annoMatrix.fetch("X", {
+        where: {
+          field: "var",
+          column: varIndex,
+          value: "TYMP",
+        },
+      });
+      const values = df.icol(0).asArray();
+      const selected = xfltr.allSelectedMask();
+      values.every((val, idx) => !(val >= 0 && val <= 50) !== !selected[idx]);
+      expect(selected.reduce((acc, val) => (val ? acc + 1 : acc), 0)).toEqual(
+        xfltr.countSelected()
+      );
+    });
+
+    test("spatial column select", async () => {
+      fetch.once(
+        serverMocks.dataframeResponse(
+          ["umap_0", "umap_1"],
+          [Float32Array.from(embUmap[0]), Float32Array.from(embUmap[1])]
+        )
+      );
+      const xfltr = await crossfilter.select("emb", "umap", {
+        mode: "within-rect",
+        minX: 0,
+        minY: 0,
+        maxX: 0.5,
+        maxY: 0.5,
+      });
+      expect(xfltr.countSelected()).toEqual(16);
+    });
+
+    test("select on subset", async () => {
+      const mask = new Uint8Array(annoMatrix.nObs).fill(0);
+      for (let i = 0; i < mask.length; i += 2) {
+        mask[i] = true;
+      }
+      const annoMatrixSubset = isubsetMask(annoMatrix, mask);
+      expect(annoMatrixSubset.nObs).toEqual(Math.floor(annoMatrix.nObs / 2));
+
+      let xfltr = new AnnoMatrixObsCrossfilter(annoMatrixSubset);
+      expect(xfltr.countSelected()).toEqual(annoMatrixSubset.nObs);
+
+      fetch.once(serverMocks.dataframeResponse(["louvain"], [obsLouvain]));
+      xfltr = await xfltr.select("obs", "louvain", {
+        mode: "exact",
+        values: ["NK cells", "B cells"],
+      });
+
+      expect(xfltr).toBeDefined();
+      expect(xfltr.countSelected()).toEqual(240);
+
+      const df = await annoMatrixSubset.fetch("obs", "louvain");
+      const values = df.col("louvain").asArray();
+      const selected = xfltr.allSelectedMask();
+      values.every(
+        (val, idx) => !["NK cells", "B cells"].includes(val) !== !selected[idx]
+      );
+    });
+
+    test("select catches errors", async () => {
+      await expect(crossfilter.select("NADA", "foo")).rejects.toThrow(
+        "Unknown field name"
+      );
+
+      await expect(crossfilter.select("var", "foo")).rejects.toThrow(
+        "unable to obsSelect upon the var dimension"
+      );
+
+      fetch.mockRejectOnce(new Error("unknown column name"));
+      await expect(crossfilter.select("obs", "foo")).rejects.toThrow(
+        "unknown column name"
+      );
+    });
+  });
+
+  describe("mutate matrix", () => {
+    /*
+    test the matrix mutators via crossfilter proxy
+    */
+    async function helperAddTestCol(cf, colName, colSchema = null) {
+      expect(
+        cf.annoMatrix.getMatrixColumns("obs").includes(colName)
+      ).toBeFalsy();
+
+      if (colSchema === null) {
+        colSchema = {
+          name: colName,
+          type: "categorical",
+          categories: ["toasty"],
+        };
+      }
+      colSchema.name = colName;
+      const initValue = colSchema.categories[0];
+      const xfltr = cf.addObsColumn(colSchema, Array, initValue);
+      expect(
+        xfltr.annoMatrix.schema.annotations.obs.columns.filter(
+          (v) => v.name === colName
+        )
+      ).toHaveLength(1);
+      const df = await xfltr.annoMatrix.fetch("obs", colName);
+      expect(df.hasCol(colName)).toBeTruthy();
+      return xfltr;
+    }
+
+    test("addObsColumn", async () => {
+      expect(crossfilter.countSelected()).toBe(annoMatrix.nObs);
+      expect(
+        crossfilter.annoMatrix.getMatrixColumns("obs").includes("foo")
+      ).toBeFalsy();
+      const xfltr = crossfilter.addObsColumn(
+        { name: "foo", type: "categorical", categories: ["A"] },
+        Array,
+        "A"
+      );
+
+      // check schema updates correctly.
+      expect(xfltr.countSelected()).toBe(annoMatrix.nObs);
+      expect(
+        xfltr.annoMatrix.getMatrixColumns("obs").includes("foo")
+      ).toBeTruthy();
+      expect(xfltr.annoMatrix.schema.annotations.obsByName.foo).toMatchObject({
+        name: "foo",
+        type: "categorical",
+      });
+      expect(
+        xfltr.annoMatrix.schema.annotations.obs.columns.filter(
+          (v) => v.name === "foo"
+        )
+      ).toHaveLength(1);
+
+      // check data update.
+      const df = await xfltr.annoMatrix.fetch("obs", "foo");
+      expect(
+        df
+          .col("foo")
+          .asArray()
+          .every((v) => v === "A")
+      ).toBeTruthy();
+
+      // check that we catch dups
+      expect(() =>
+        xfltr.addObsColumn(
+          { name: "foo", type: "categorical" },
+          Array,
+          "toasty"
+        )
+      ).toThrow("column already exists");
+      expect(() =>
+        xfltr.addObsColumn(
+          { name: "louvain", type: "categorical" },
+          Array,
+          "toasty"
+        )
+      ).toThrow("column already exists");
+    });
+
+    test("dropObsColumn", async () => {
+      let xfltr;
+
+      /* check that we catch attempt to drop readonly dimension */
+      expect(() => crossfilter.dropObsColumn("louvain")).toThrow(
+        "Unknown or readonly obs column"
+      );
+      /* non-existent column */
+      expect(() => crossfilter.dropObsColumn("does-not-exist")).toThrow(
+        "Unknown or readonly obs column"
+      );
+
+      // add a column, then drop it.
+      xfltr = await helperAddTestCol(crossfilter, "foo");
+      xfltr = xfltr.dropObsColumn("foo");
+      expect(
+        xfltr.annoMatrix.schema.annotations.obs.columns.filter(
+          (v) => v.name === "foo"
+        )
+      ).toHaveLength(0);
+      expect(xfltr.annoMatrix.schema.annotations.obsByName.foo).toBeUndefined();
+      fetch.mockRejectOnce(new Error("unknown column name"));
+      await expect(xfltr.annoMatrix.fetch("obs", "foo")).rejects.toThrow(
+        "unknown column name"
+      );
+
+      // now same, but ensure we have built an index before doing the drop
+      xfltr = await helperAddTestCol(crossfilter, "bar");
+      xfltr = await xfltr.select("obs", "bar", {
+        mode: "exact",
+        values: "whatever",
+      });
+      xfltr = xfltr.dropObsColumn("bar");
+
+      fetch.mockRejectOnce(new Error("unknown column name"));
+      await expect(xfltr.select("obs", "bar", { mode: "all" })).rejects.toThrow(
+        "unknown column name"
+      );
+    });
+
+    test("renameObsColumn", async () => {
+      let xfltr;
+
+      /* catch attempts to rename non-existent or readonly columns */
+      expect(() =>
+        crossfilter.renameObsColumn("does-not-exist", "foo")
+      ).toThrow("Unknown or readonly obs column");
+      expect(() => crossfilter.renameObsColumn("louvain", "foo")).toThrow(
+        "Unknown or readonly obs column"
+      );
+
+      // add a column, then rename it.
+      xfltr = await helperAddTestCol(crossfilter, "foo");
+      xfltr = xfltr.renameObsColumn("foo", "bar");
+      expect(xfltr.annoMatrix.getColumnSchema("obs", "foo")).toBeUndefined();
+      expect(xfltr.annoMatrix.getColumnSchema("obs", "bar")).toMatchObject({
+        name: "bar",
+        type: "categorical",
+      });
+
+      fetch.mockRejectOnce(new Error("unknown column name"));
+      await expect(xfltr.annoMatrix.fetch("obs", "foo")).rejects.toThrow(
+        "unknown column name"
+      );
+      const df = await xfltr.annoMatrix.fetch("obs", "bar");
+      expect(df.hasCol("bar")).toBeTruthy();
+
+      // now same, but ensure we have built an index before doing the rename
+      xfltr = await helperAddTestCol(crossfilter, "bar");
+      xfltr = await xfltr.select("obs", "bar", {
+        mode: "exact",
+        values: "whatever",
+      });
+      xfltr = xfltr.renameObsColumn("bar", "xyz");
+
+      fetch.mockRejectOnce(new Error("unknown column name"));
+      await expect(xfltr.select("obs", "bar", { mode: "all" })).rejects.toThrow(
+        "unknown column name"
+      );
+      await expect(
+        xfltr.select("obs", "xyz", { mode: "none" })
+      ).resolves.toBeInstanceOf(AnnoMatrixObsCrossfilter);
+    });
+
+    test("addObsAnnoCategory", async () => {
+      let xfltr;
+
+      // catch unknown or readonly columns
+      expect(() => crossfilter.addObsAnnoCategory("louvain", "mumble")).toThrow(
+        "Unknown or readonly obs column"
+      );
+      expect(() =>
+        crossfilter.addObsAnnoCategory("undefined-name", "mumble")
+      ).toThrow("Unknown or readonly obs column");
+
+      // add a column and then add category to it
+      xfltr = await helperAddTestCol(crossfilter, "foo", {
+        name: "foo",
+        type: "categorical",
+        categories: ["unassigned"],
+      });
+      xfltr = xfltr.addObsAnnoCategory("foo", "a-new-label");
+      expect(xfltr.annoMatrix.getColumnSchema("obs", "foo")).toMatchObject({
+        name: "foo",
+        type: "categorical",
+        categories: expect.arrayContaining(["a-new-label", "unassigned"]),
+      });
+
+      // do it again, dup; should throw
+      expect(() => xfltr.addObsAnnoCategory("foo", "a-new-label")).toThrow(
+        "category already exists"
+      );
+
+      // now same, but ensure we have built an index before doing the operation
+      xfltr = await helperAddTestCol(crossfilter, "bar", {
+        name: "bar",
+        type: "categorical",
+        categories: ["unassigned"],
+      });
+      xfltr = await xfltr.select("obs", "bar", {
+        mode: "exact",
+        values: "something",
+      });
+      xfltr = xfltr.addObsAnnoCategory("bar", "a-new-label");
+      expect(xfltr.annoMatrix.getColumnSchema("obs", "bar")).toMatchObject({
+        name: "bar",
+        type: "categorical",
+        categories: expect.arrayContaining(["a-new-label", "unassigned"]),
+      });
+    });
+
+    test("removeObsAnnoCategory", async () => {
+      let xfltr;
+
+      // catch unknown or readonly categories
+      await expect(() =>
+        crossfilter.removeObsAnnoCategory("louvain", "mumble", "unassigned")
+      ).rejects.toThrow("Unknown or readonly obs column");
+      await expect(() =>
+        crossfilter.removeObsAnnoCategory("undefined-name", "mumble")
+      ).rejects.toThrow("Unknown or readonly obs column");
+
+      xfltr = await helperAddTestCol(crossfilter, "foo", {
+        name: "foo",
+        type: "categorical",
+        categories: ["unassigned", "red", "green", "blue"],
+      });
+      xfltr = await xfltr.select("obs", "foo", { mode: "all" });
+      expect(
+        (await xfltr.annoMatrix.fetch("obs", "foo"))
+          .col("foo")
+          .asArray()
+          .every((v) => v === "unassigned")
+      ).toBeTruthy();
+      expect(xfltr.annoMatrix.getColumnSchema("obs", "foo")).toMatchObject({
+        name: "foo",
+        type: "categorical",
+        categories: expect.arrayContaining([
+          "unassigned",
+          "red",
+          "green",
+          "blue",
+        ]),
+      });
+
+      // remove an unused category
+      const xfltr1 = await xfltr.removeObsAnnoCategory("foo", "red", "mumble");
+      expect(
+        (await xfltr1.annoMatrix.fetch("obs", "foo"))
+          .col("foo")
+          .asArray()
+          .every((v) => v === "unassigned")
+      ).toBeTruthy();
+      expect(xfltr1.annoMatrix.getColumnSchema("obs", "foo")).toMatchObject({
+        name: "foo",
+        type: "categorical",
+        categories: expect.arrayContaining([
+          "unassigned",
+          "green",
+          "blue",
+          "mumble",
+        ]),
+      });
+
+      // remove a used category
+      const xfltr2 = await xfltr.removeObsAnnoCategory(
+        "foo",
+        "unassigned",
+        "red"
+      );
+      expect(
+        (await xfltr2.annoMatrix.fetch("obs", "foo"))
+          .col("foo")
+          .asArray()
+          .every((v) => v === "red")
+      ).toBeTruthy();
+      expect(xfltr2.annoMatrix.getColumnSchema("obs", "foo")).toMatchObject({
+        name: "foo",
+        type: "categorical",
+        categories: expect.arrayContaining(["green", "blue", "red"]),
+      });
+    });
+
+    test("setObsColumnValues", async () => {
+      // catch unknown or readonly categories
+      await expect(() =>
+        crossfilter.setObsColumnValues("louvain", [0, 1], "unassigned")
+      ).rejects.toThrow("Unknown or readonly obs column");
+      await expect(() =>
+        crossfilter.setObsColumnValues("undefined-name", [0], "mumble")
+      ).rejects.toThrow("Unknown or readonly obs column");
+
+      let xfltr = await helperAddTestCol(crossfilter, "foo", {
+        name: "foo",
+        type: "categorical",
+        categories: ["unassigned", "red", "green", "blue"],
+      });
+      xfltr = await xfltr.select("obs", "foo", { mode: "all" });
+
+      // catch unknown row label
+      await expect(() =>
+        xfltr.setObsColumnValues("foo", [-1], "red")
+      ).rejects.toThrow("Unknown row label");
+
+      // set a few rows
+      expect(
+        (await xfltr.annoMatrix.fetch("obs", "foo"))
+          .col("foo")
+          .asArray()
+          .every((v) => v === "unassigned")
+      ).toBeTruthy();
+      const xfltr1 = await xfltr.setObsColumnValues("foo", [0, 10], "purple");
+      expect(
+        (await xfltr1.annoMatrix.fetch("obs", "foo"))
+          .col("foo")
+          .asArray()
+          .every(
+            (v, i) =>
+              v === "unassigned" || (v === "purple" && (i === 0 || i === 10))
+          )
+      ).toBeTruthy();
+      expect(xfltr1.annoMatrix.getColumnSchema("obs", "foo")).toMatchObject({
+        name: "foo",
+        type: "categorical",
+        categories: expect.arrayContaining([
+          "unassigned",
+          "red",
+          "green",
+          "blue",
+          "purple",
+        ]),
+      });
+
+      expect(xfltr1.countSelected()).toEqual(xfltr1.annoMatrix.nObs);
+      const xfltr2 = await xfltr1.select("obs", "foo", {
+        mode: "exact",
+        values: ["purple"],
+      });
+      expect(xfltr2.countSelected()).toEqual(2);
+      expect(xfltr2.allSelectedLabels()).toEqual(Int32Array.from([0, 10]));
+    });
+
+    test("resetObsColumnValues", async () => {
+      // catch unknown or readonly categories
+      await expect(() =>
+        crossfilter.resetObsColumnValues("louvain", "red", "blue")
+      ).rejects.toThrow("Unknown or readonly obs column");
+      await expect(() =>
+        crossfilter.resetObsColumnValues("undefined-name", "red", "blue")
+      ).rejects.toThrow("Unknown or readonly obs column");
+
+      let xfltr = await helperAddTestCol(crossfilter, "foo", {
+        name: "foo",
+        type: "categorical",
+        categories: ["unassigned", "red", "green", "blue"],
+      });
+      xfltr = await xfltr.select("obs", "foo", {
+        mode: "exact",
+        values: "red",
+      });
+
+      // catch unknown category name label
+      await expect(() =>
+        xfltr.resetObsColumnValues("foo", "unknown-label", "red")
+      ).rejects.toThrow("unknown category");
+
+      let xfltr1 = await xfltr.setObsColumnValues("foo", [0, 10], "purple");
+      xfltr1 = await xfltr1.select("obs", "foo", {
+        mode: "exact",
+        values: "purple",
+      });
+      expect(
+        (await xfltr1.annoMatrix.fetch("obs", "foo"))
+          .col("foo")
+          .asArray()
+          .filter((v) => v === "purple")
+      ).toHaveLength(2);
+
+      xfltr1 = await xfltr1.resetObsColumnValues("foo", "purple", "magenta");
+      expect(
+        (await xfltr1.annoMatrix.fetch("obs", "foo"))
+          .col("foo")
+          .asArray()
+          .filter((v) => v === "magenta")
+      ).toHaveLength(2);
+      expect(
+        (await xfltr1.annoMatrix.fetch("obs", "foo"))
+          .col("foo")
+          .asArray()
+          .filter((v) => v === "purple")
+      ).toHaveLength(0);
+      expect(xfltr1.annoMatrix.getColumnSchema("obs", "foo")).toMatchObject({
+        name: "foo",
+        type: "categorical",
+        categories: expect.arrayContaining([
+          "unassigned",
+          "red",
+          "green",
+          "blue",
+          "purple",
+          "magenta",
+        ]),
+      });
+    });
+  });
+
+  describe("edge cases", () => {
+    test("transition from empty annoMatrix", async () => {
+      // select before fetch needs to work
+      fetch.once(serverMocks.dataframeResponse(["louvain"], [obsLouvain]));
+      const xfltr = await crossfilter.select("obs", "louvain", {
+        mode: "exact",
+        values: "B cells",
+      });
+      expect(fetch.mock.calls).toHaveLength(1);
+      expect(xfltr.obsCrossfilter.hasDimension("obs/louvain")).toBeTruthy();
+      expect(xfltr.obsCrossfilter.all()).toBe(xfltr.annoMatrix._cache.obs);
+      expect(xfltr.countSelected()).toEqual(
+        obsLouvain.reduce(
+          (count, v) => (v === "B cells" ? count + 1 : count),
+          0
+        )
+      );
+    });
+  });
+});

client/__tests__/util/annoMatrix/n_genes.json

@@ -0,0 +1,180 @@
+[
+  781, 1352, 1131, 960, 522, 782, 783, 790, 533, 550, 1116, 751, 866, 1059, 458,
+  335, 1424, 1014, 1446, 446, 1020, 417, 878, 789, 510, 824, 1545, 996, 937,
+  1368, 428, 406, 1020, 786, 1019, 750, 822, 982, 876, 930, 838, 1014, 732, 877,
+  782, 787, 791, 880, 801, 1215, 343, 1460, 1250, 756, 836, 824, 827, 1238,
+  1243, 1652, 843, 825, 656, 776, 766, 1465, 790, 871, 803, 965, 800, 876, 690,
+  988, 906, 741, 620, 867, 916, 969, 803, 732, 555, 790, 862, 900, 674, 397,
+  663, 563, 786, 859, 568, 412, 1043, 1206, 702, 1263, 929, 1079, 938, 316, 900,
+  919, 862, 903, 390, 1717, 819, 1877, 660, 791, 478, 769, 481, 819, 866, 600,
+  1185, 650, 775, 699, 642, 857, 832, 388, 710, 341, 894, 935, 604, 1008, 985,
+  679, 603, 864, 1031, 887, 603, 610, 1119, 669, 794, 963, 756, 637, 1032, 776,
+  860, 825, 852, 745, 858, 925, 1228, 806, 715, 668, 779, 1197, 888, 1273, 873,
+  847, 781, 959, 805, 554, 604, 785, 978, 910, 936, 997, 961, 1314, 799, 1112,
+  677, 775, 1298, 657, 626, 1313, 467, 936, 977, 780, 1311, 432, 579, 850, 736,
+  800, 892, 860, 720, 822, 681, 954, 889, 1265, 919, 799, 833, 496, 1476, 848,
+  869, 1059, 490, 897, 832, 864, 419, 856, 907, 791, 756, 771, 957, 1190, 680,
+  524, 908, 506, 851, 775, 793, 748, 951, 643, 1277, 828, 480, 969, 1112, 648,
+  805, 1223, 1023, 669, 489, 390, 350, 1113, 837, 1547, 840, 581, 748, 1861,
+  735, 488, 1016, 585, 797, 769, 490, 1307, 895, 686, 602, 772, 704, 892, 1169,
+  1375, 1189, 892, 2455, 355, 1856, 1317, 703, 825, 736, 1997, 892, 1034, 545,
+  1188, 659, 1056, 819, 979, 632, 598, 690, 310, 803, 743, 560, 1073, 844, 882,
+  841, 815, 771, 976, 986, 820, 957, 640, 1012, 927, 794, 753, 791, 366, 539,
+  752, 769, 650, 947, 771, 824, 744, 837, 723, 640, 923, 1174, 1597, 699, 618,
+  1418, 820, 1047, 981, 866, 527, 762, 717, 860, 603, 828, 476, 1071, 775, 614,
+  913, 836, 669, 942, 792, 871, 1046, 859, 793, 822, 751, 435, 1142, 781, 718,
+  471, 1750, 892, 841, 1156, 1031, 912, 873, 824, 1233, 1312, 575, 605, 717,
+  1019, 1215, 928, 1780, 657, 718, 646, 808, 1120, 750, 390, 1100, 456, 755,
+  1118, 571, 867, 728, 916, 491, 960, 625, 1090, 772, 968, 480, 810, 725, 1016,
+  1011, 1075, 808, 672, 950, 862, 766, 963, 507, 570, 678, 768, 1037, 885, 1426,
+  1496, 587, 879, 924, 783, 696, 1057, 783, 867, 919, 1251, 1023, 727, 645,
+  1217, 929, 792, 994, 1025, 946, 600, 881, 975, 1609, 758, 772, 682, 998, 979,
+  1045, 706, 808, 855, 819, 1147, 742, 914, 969, 704, 1398, 581, 809, 921, 805,
+  542, 888, 519, 1092, 762, 698, 752, 771, 899, 1101, 760, 881, 1124, 809, 445,
+  1703, 789, 641, 819, 890, 767, 806, 1323, 942, 807, 981, 888, 726, 1190, 826,
+  661, 713, 816, 822, 806, 864, 464, 664, 931, 860, 674, 803, 464, 788, 1068,
+  781, 843, 779, 873, 707, 492, 669, 982, 749, 789, 780, 561, 655, 432, 801,
+  945, 770, 503, 766, 776, 970, 989, 654, 762, 882, 1236, 1499, 626, 1380, 1170,
+  491, 833, 924, 581, 842, 596, 811, 542, 1572, 758, 854, 274, 1413, 872, 559,
+  907, 951, 1175, 946, 905, 737, 604, 843, 606, 1079, 668, 785, 726, 978, 941,
+  994, 655, 1013, 987, 591, 1041, 625, 582, 814, 570, 775, 822, 715, 738, 956,
+  1178, 743, 1861, 841, 944, 783, 643, 924, 936, 431, 490, 532, 524, 620, 749,
+  1334, 834, 790, 702, 892, 693, 784, 944, 471, 839, 529, 729, 1084, 802, 886,
+  815, 856, 746, 1318, 545, 696, 872, 1154, 467, 725, 1027, 479, 728, 926, 1282,
+  907, 833, 1024, 838, 900, 737, 367, 459, 1030, 1279, 756, 662, 1323, 1003,
+  359, 770, 813, 634, 924, 1184, 901, 816, 1421, 771, 706, 953, 348, 716, 870,
+  715, 550, 689, 947, 1157, 690, 383, 374, 882, 697, 246, 833, 1006, 1181, 974,
+  856, 978, 1551, 965, 907, 565, 417, 907, 927, 966, 658, 727, 743, 381, 385,
+  905, 645, 1167, 936, 724, 618, 1038, 853, 808, 1403, 762, 741, 687, 932, 1096,
+  601, 652, 895, 682, 1553, 604, 823, 803, 646, 917, 942, 850, 795, 771, 949,
+  516, 664, 1001, 637, 619, 907, 961, 812, 793, 1043, 1343, 1326, 981, 675, 937,
+  631, 1026, 1135, 499, 948, 801, 848, 741, 604, 864, 1076, 1106, 1111, 624,
+  1008, 908, 815, 346, 1062, 803, 749, 779, 1027, 1032, 1040, 654, 631, 755,
+  854, 850, 798, 864, 1078, 690, 864, 1523, 838, 966, 389, 1654, 808, 885, 1665,
+  920, 855, 807, 859, 1276, 987, 1079, 678, 626, 831, 829, 1009, 547, 893, 722,
+  656, 415, 773, 1262, 1218, 365, 661, 805, 1409, 1094, 779, 898, 830, 1242,
+  864, 576, 901, 1274, 852, 1006, 719, 763, 683, 957, 831, 724, 354, 763, 910,
+  749, 626, 870, 795, 1078, 736, 835, 402, 1203, 699, 755, 697, 1229, 637, 666,
+  846, 1036, 1027, 831, 999, 942, 891, 1007, 712, 990, 725, 745, 921, 333, 1042,
+  895, 873, 1612, 724, 929, 601, 862, 908, 658, 775, 724, 753, 741, 690, 379,
+  608, 927, 777, 969, 827, 709, 385, 690, 769, 554, 892, 761, 367, 731, 1103,
+  944, 832, 675, 652, 418, 727, 745, 872, 1336, 863, 934, 844, 721, 432, 782,
+  1006, 834, 840, 840, 819, 699, 489, 665, 576, 1291, 1102, 826, 880, 738, 904,
+  686, 874, 887, 873, 560, 766, 710, 1135, 1054, 805, 724, 973, 1201, 575, 838,
+  865, 546, 811, 884, 886, 791, 1026, 2000, 644, 763, 969, 800, 359, 624, 993,
+  800, 1167, 833, 1871, 616, 822, 647, 1000, 618, 734, 618, 1938, 861, 945,
+  1032, 723, 984, 994, 771, 738, 1583, 1113, 614, 1146, 615, 848, 983, 677, 972,
+  791, 827, 804, 395, 843, 493, 741, 941, 1659, 742, 1517, 559, 937, 740, 781,
+  819, 813, 578, 1022, 1191, 824, 1146, 757, 638, 830, 713, 609, 1271, 680, 769,
+  1119, 731, 804, 781, 916, 735, 835, 1257, 472, 879, 851, 1023, 661, 1008, 748,
+  845, 393, 675, 843, 876, 939, 932, 760, 735, 1561, 752, 940, 705, 405, 690,
+  1071, 544, 927, 817, 388, 560, 1322, 640, 886, 1075, 689, 524, 606, 802, 868,
+  939, 753, 770, 1105, 841, 786, 445, 703, 593, 875, 901, 927, 798, 1221, 415,
+  1381, 949, 1322, 1169, 745, 727, 799, 490, 767, 943, 808, 926, 664, 569, 843,
+  727, 1222, 457, 1515, 1138, 1174, 525, 878, 525, 999, 778, 772, 819, 1015,
+  939, 856, 715, 793, 837, 1193, 764, 834, 677, 625, 420, 837, 874, 679, 811,
+  546, 787, 587, 821, 669, 813, 780, 649, 924, 1322, 701, 792, 817, 667, 804,
+  593, 740, 1243, 859, 685, 596, 1193, 859, 775, 947, 689, 907, 734, 621, 336,
+  922, 802, 812, 941, 943, 868, 947, 767, 701, 692, 423, 775, 500, 1100, 812,
+  728, 616, 928, 835, 454, 812, 769, 1058, 914, 628, 649, 452, 754, 1327, 776,
+  670, 1017, 599, 431, 967, 1077, 2033, 731, 533, 926, 725, 559, 870, 964, 1341,
+  1981, 1103, 326, 428, 808, 821, 554, 596, 680, 1034, 849, 566, 879, 1091, 823,
+  447, 1688, 868, 1254, 942, 462, 1055, 852, 738, 804, 775, 726, 993, 1462,
+  1007, 798, 1036, 786, 948, 743, 761, 838, 1040, 859, 867, 1188, 846, 687, 672,
+  629, 725, 660, 809, 469, 600, 812, 856, 397, 786, 895, 882, 449, 890, 823,
+  1051, 823, 1055, 741, 999, 1241, 790, 878, 778, 1066, 815, 465, 1079, 743,
+  1098, 807, 1120, 1025, 805, 676, 828, 763, 997, 852, 866, 1118, 508, 928, 958,
+  932, 892, 905, 494, 710, 1068, 795, 787, 951, 720, 842, 890, 1355, 1005, 872,
+  1185, 912, 869, 894, 997, 770, 554, 806, 1426, 1012, 452, 896, 426, 239, 829,
+  895, 787, 1139, 925, 1015, 1360, 1097, 650, 853, 549, 1052, 307, 1152, 907,
+  1628, 731, 897, 1749, 762, 712, 1195, 851, 864, 968, 845, 331, 840, 734, 948,
+  842, 1543, 661, 981, 912, 912, 1063, 683, 823, 996, 695, 1483, 927, 574, 1052,
+  571, 1028, 1263, 671, 958, 747, 866, 896, 489, 643, 923, 820, 1466, 550, 1112,
+  1006, 1448, 727, 899, 998, 563, 870, 903, 516, 754, 879, 588, 740, 798, 798,
+  653, 902, 990, 724, 953, 891, 1437, 653, 714, 956, 877, 1012, 824, 1077, 740,
+  692, 1063, 771, 808, 1389, 1264, 952, 816, 795, 795, 760, 886, 349, 868, 842,
+  819, 626, 418, 903, 838, 723, 436, 1112, 724, 1299, 719, 843, 1090, 696, 885,
+  627, 809, 423, 729, 853, 855, 608, 627, 823, 1063, 575, 743, 1528, 681, 544,
+  422, 731, 920, 761, 884, 982, 784, 496, 573, 521, 663, 794, 975, 856, 978,
+  590, 905, 695, 816, 976, 816, 753, 791, 858, 813, 841, 1085, 1692, 716, 955,
+  1467, 741, 296, 738, 1573, 1119, 918, 283, 703, 842, 1253, 676, 1636, 1273,
+  380, 799, 1491, 878, 939, 725, 1365, 818, 719, 1343, 905, 837, 803, 990, 1084,
+  976, 1630, 795, 1408, 771, 650, 779, 648, 817, 1127, 882, 954, 830, 732, 783,
+  756, 708, 976, 718, 887, 809, 795, 662, 912, 1550, 1509, 1021, 1751, 776, 910,
+  714, 530, 846, 631, 1152, 1118, 755, 573, 1176, 267, 918, 1132, 849, 938,
+  1140, 909, 840, 806, 904, 788, 778, 715, 869, 714, 883, 767, 858, 788, 553,
+  634, 1230, 1131, 849, 811, 827, 1753, 713, 484, 783, 722, 596, 514, 372, 925,
+  747, 840, 673, 955, 796, 719, 718, 857, 578, 1063, 594, 997, 1268, 341, 388,
+  753, 834, 593, 1189, 911, 738, 476, 816, 641, 746, 635, 953, 801, 326, 849,
+  499, 865, 1420, 487, 876, 797, 981, 756, 850, 1097, 998, 829, 1159, 955, 1061,
+  696, 786, 743, 1211, 893, 491, 744, 1447, 767, 1050, 853, 1118, 1428, 555,
+  612, 854, 789, 889, 784, 712, 1323, 921, 682, 794, 846, 1012, 839, 807, 587,
+  881, 781, 1010, 1182, 1149, 812, 607, 1155, 714, 642, 1088, 1291, 1196, 325,
+  804, 370, 359, 855, 1165, 836, 923, 863, 1284, 1011, 889, 984, 512, 939, 883,
+  697, 1211, 362, 969, 1135, 1239, 580, 1103, 975, 825, 1170, 921, 640, 1180,
+  378, 982, 916, 1122, 792, 619, 750, 913, 775, 661, 766, 768, 675, 944, 940,
+  761, 727, 767, 873, 1043, 850, 995, 680, 595, 700, 753, 736, 891, 685, 780,
+  986, 989, 830, 810, 685, 784, 642, 742, 961, 906, 829, 621, 822, 717, 1210,
+  800, 1963, 749, 757, 570, 831, 721, 336, 802, 1001, 886, 631, 759, 631, 550,
+  984, 767, 835, 777, 639, 860, 1413, 747, 779, 540, 367, 1629, 1380, 689, 1001,
+  809, 337, 1103, 796, 966, 782, 1018, 642, 967, 436, 826, 779, 1000, 601, 796,
+  945, 1679, 1123, 596, 995, 720, 588, 759, 452, 780, 836, 515, 846, 392, 283,
+  710, 1158, 796, 895, 585, 559, 859, 879, 858, 842, 643, 1308, 595, 1181, 909,
+  710, 821, 817, 841, 1197, 640, 1425, 947, 900, 852, 460, 1100, 824, 780, 932,
+  542, 1137, 1225, 997, 572, 780, 765, 906, 793, 753, 772, 854, 936, 1048, 819,
+  645, 619, 314, 726, 737, 1162, 1081, 868, 1032, 913, 476, 490, 799, 1201, 997,
+  898, 212, 1586, 427, 947, 937, 724, 380, 715, 739, 931, 973, 773, 1497, 906,
+  798, 953, 471, 821, 806, 714, 828, 727, 773, 976, 856, 727, 761, 1084, 1557,
+  693, 559, 627, 795, 750, 838, 803, 453, 734, 607, 1029, 805, 669, 505, 858,
+  832, 1019, 585, 1225, 1287, 903, 752, 2020, 774, 666, 843, 857, 887, 1082,
+  656, 674, 911, 734, 910, 672, 802, 539, 699, 941, 828, 800, 642, 733, 607,
+  992, 379, 562, 847, 787, 1461, 732, 941, 785, 696, 795, 809, 828, 651, 882,
+  972, 1323, 679, 774, 784, 766, 520, 671, 796, 644, 1549, 756, 723, 788, 643,
+  856, 825, 730, 831, 653, 429, 641, 637, 812, 1527, 859, 972, 744, 869, 508,
+  624, 923, 976, 801, 1014, 1429, 586, 692, 704, 1176, 806, 883, 1249, 765, 743,
+  907, 666, 669, 364, 794, 959, 766, 937, 1398, 942, 1469, 905, 812, 572, 1378,
+  1058, 1215, 697, 531, 676, 1819, 503, 801, 943, 874, 813, 694, 494, 660, 1467,
+  976, 833, 689, 921, 625, 1428, 817, 909, 956, 765, 1207, 829, 1648, 554, 1500,
+  953, 647, 537, 786, 814, 761, 862, 838, 1102, 1392, 1042, 372, 971, 1364,
+  1137, 847, 935, 710, 1070, 914, 855, 759, 654, 981, 1193, 397, 1123, 616, 747,
+  876, 949, 965, 789, 752, 717, 1198, 952, 625, 784, 914, 659, 913, 863, 959,
+  1637, 796, 1508, 906, 714, 1195, 867, 819, 375, 656, 1047, 745, 866, 1186,
+  1669, 539, 942, 839, 927, 796, 734, 831, 967, 620, 814, 605, 1391, 655, 1512,
+  625, 719, 547, 864, 902, 853, 1143, 990, 858, 604, 1291, 701, 859, 768, 1621,
+  715, 594, 783, 1608, 927, 740, 805, 705, 491, 514, 768, 880, 993, 356, 748,
+  993, 801, 843, 1194, 794, 606, 810, 882, 682, 1126, 792, 829, 1657, 786, 610,
+  664, 790, 852, 678, 589, 930, 690, 840, 647, 622, 427, 856, 522, 1262, 722,
+  641, 997, 983, 1099, 607, 704, 947, 886, 1062, 659, 995, 492, 754, 676, 837,
+  532, 598, 1274, 921, 700, 828, 828, 543, 815, 709, 842, 1200, 1126, 1327, 849,
+  1160, 707, 675, 709, 990, 270, 445, 892, 1040, 832, 815, 437, 1401, 1341, 480,
+  1517, 704, 622, 823, 731, 781, 849, 1209, 472, 838, 643, 1101, 879, 895, 408,
+  832, 845, 928, 920, 1269, 652, 1463, 840, 706, 956, 1219, 612, 970, 789, 382,
+  533, 627, 388, 655, 1103, 782, 1231, 617, 589, 758, 882, 909, 661, 558, 947,
+  491, 1386, 1481, 823, 1064, 681, 652, 363, 824, 933, 611, 886, 806, 531, 864,
+  648, 1142, 736, 955, 826, 725, 1174, 841, 1006, 1418, 955, 829, 1038, 632,
+  750, 846, 1033, 859, 890, 893, 1030, 913, 1385, 750, 416, 703, 522, 856, 914,
+  742, 381, 1079, 671, 1101, 858, 663, 581, 845, 881, 823, 835, 1290, 598, 1130,
+  793, 838, 765, 768, 815, 810, 912, 818, 790, 665, 703, 875, 657, 1567, 688,
+  749, 1076, 782, 543, 1100, 841, 809, 789, 1263, 758, 368, 1022, 730, 1101,
+  478, 488, 618, 940, 771, 784, 847, 1303, 821, 1111, 752, 1128, 958, 742, 782,
+  824, 751, 872, 939, 606, 698, 705, 963, 607, 621, 1650, 1093, 545, 670, 617,
+  723, 1194, 1019, 1291, 758, 855, 1549, 743, 1372, 802, 337, 1121, 1028, 1524,
+  645, 847, 866, 941, 751, 583, 796, 793, 975, 936, 524, 659, 607, 1433, 562,
+  696, 927, 517, 719, 599, 639, 977, 1019, 816, 672, 1903, 1162, 964, 936, 947,
+  989, 790, 902, 982, 673, 856, 629, 692, 843, 940, 795, 780, 821, 471, 702,
+  631, 1557, 868, 901, 798, 1020, 885, 881, 719, 1043, 1238, 565, 776, 696, 725,
+  365, 811, 1212, 1178, 1132, 661, 831, 786, 471, 835, 564, 929, 958, 706, 388,
+  842, 965, 1088, 511, 794, 900, 865, 789, 504, 701, 817, 796, 972, 906, 871,
+  922, 724, 628, 1479, 533, 1101, 1913, 855, 1266, 884, 817, 619, 591, 685, 887,
+  1336, 656, 1227, 980, 817, 582, 1370, 460, 638, 471, 650, 414, 907, 1147, 732,
+  992, 801, 822, 529, 737, 806, 816, 889, 1305, 588, 657, 1154, 713, 326, 1129,
+  1603, 879, 1156, 642, 285, 825, 823, 719, 1253, 971, 853, 916, 1053, 515,
+  1017, 953, 832, 645, 667, 1326, 547, 636, 1783, 1211, 788, 807, 1104, 884,
+  848, 788, 1013, 1003, 916, 818, 828, 882, 959, 395, 368, 787, 929, 1379, 711,
+  733, 752, 464, 626, 735, 946, 876, 647, 536, 954, 486, 712, 786, 438, 807,
+  1016, 551, 841, 929, 757, 971, 708, 567, 881, 801, 873, 805, 1359, 866, 945,
+  1068, 819, 815, 1058, 845, 881, 1051, 1179, 718, 657, 882, 709, 754, 735, 603,
+  944, 1794, 712, 721, 1097, 813, 788, 917, 656, 1104, 1268, 1239, 862, 739,
+  858, 1066, 752, 615, 721, 571, 861, 933, 807, 1082, 820, 887, 850, 1567, 803,
+  1156, 721, 692, 700, 458, 637, 873, 1544, 1155, 1227, 622, 454, 724
+]

client/__tests__/util/annoMatrix/serverMocks/index.js

@@ -0,0 +1,4 @@
+export const baseDataURL = "https://a.fake.url/api/v0.2";
+
+export { schema } from "./schema";
+export * from "./routes";

client/__tests__/util/annoMatrix/serverMocks/routes.js

@@ -0,0 +1,211 @@
+import { schema } from "./schema";
+import { Dataframe, KeyIndex } from "../../../../src/util/dataframe";
+import { encodeMatrixFBS } from "../../../../src/util/stateManager/matrix";
+
+const indexedSchema = {
+  obsByName: Object.fromEntries(
+    schema.schema.annotations.obs.columns.map((v) => [v.name, v]) ?? []
+  ),
+  varByName: Object.fromEntries(
+    schema.schema.annotations.var.columns.map((v) => [v.name, v]) ?? []
+  ),
+  embByName: Object.fromEntries(
+    schema.schema.layout.obs.map((v) => [v.name, v]) ?? []
+  ),
+};
+
+function makeMockColumn(s, length) {
+  const { type } = s;
+  switch (type) {
+    case "int32":
+      return new Int32Array(length).fill(Math.floor(99 * Math.random()));
+
+    case "string":
+      return new Array(length).fill("test");
+
+    case "float32":
+      return new Float32Array(length).fill(99 * Math.random());
+
+    case "boolean":
+      return new Array(length).fill(false);
+
+    case "categorical":
+      return new Array(length).fill(s.categories[0]);
+
+    default:
+      throw new Error("unkonwn type");
+  }
+}
+
+function getEncodedDataframe(colNames, length, colSchemas) {
+  const colIndex = new KeyIndex(colNames);
+  const columns = colSchemas.map((s) => makeMockColumn(s, length));
+  const df = new Dataframe([length, colNames.length], columns, null, colIndex);
+  const body = encodeMatrixFBS(df);
+  return body;
+}
+
+export function dataframeResponse(colNames, columns) {
+  const colIndex = new KeyIndex(colNames);
+  const df = new Dataframe(
+    [columns[0].length, colNames.length],
+    columns,
+    null,
+    colIndex
+  );
+  const body = encodeMatrixFBS(df);
+  const headers = new Headers({
+    "Content-Type": "application/octet-stream",
+  });
+  return () => Promise.resolve({ body, init: { status: 200, headers } });
+}
+
+function annotationObsResponse(request) {
+  const url = new URL(request.url);
+  const params = Array.from(url.searchParams.entries());
+  const names = params
+    .filter(([k]) => k === "annotation-name")
+    .map(([, v]) => v);
+  if (!names.every((n) => indexedSchema.obsByName[n])) {
+    return Promise.reject(new Error("bad obs annotation name in URL"));
+  }
+  const colSchemas = names.map((n) => indexedSchema.obsByName[n]);
+  const body = getEncodedDataframe(
+    names,
+    schema.schema.dataframe.nObs,
+    colSchemas
+  );
+
+  const headers = new Headers({
+    "Content-Type": "application/octet-stream",
+  });
+  return Promise.resolve({
+    body,
+    init: { status: 200, headers },
+  });
+}
+
+function annotationVarResponse(request) {
+  const url = new URL(request.url);
+  const params = Array.from(url.searchParams.entries());
+  const names = params
+    .filter(([k]) => k === "annotation-name")
+    .map(([, v]) => v);
+  if (!names.every((n) => indexedSchema.varByName[n])) {
+    return Promise.reject(new Error("bad var annotation name in URL"));
+  }
+  const colSchemas = names.map((n) => indexedSchema.varByName[n]);
+  const body = getEncodedDataframe(
+    names,
+    schema.schema.dataframe.nVar,
+    colSchemas
+  );
+
+  const headers = new Headers({
+    "Content-Type": "application/octet-stream",
+  });
+  return Promise.resolve({
+    body,
+    init: { status: 200, headers },
+  });
+}
+
+function layoutObsResponse(request) {
+  const url = new URL(request.url);
+  const params = Array.from(url.searchParams.entries());
+  const names = params.filter(([k]) => k === "layout-name").map(([, v]) => v);
+  if (!names.every((n) => indexedSchema.embByName[n])) {
+    return Promise.reject(new Error("bad layout name in URL"));
+  }
+  const dims = names.map((n) => indexedSchema.embByName[n].dims).flat();
+  const colSchemas = names
+    .map((n) => [indexedSchema.embByName[n], indexedSchema.embByName[n]])
+    .flat();
+  const body = getEncodedDataframe(
+    dims,
+    schema.schema.dataframe.nObs,
+    colSchemas
+  );
+
+  const headers = new Headers({
+    "Content-Type": "application/octet-stream",
+  });
+  return Promise.resolve({
+    body,
+    init: { status: 200, headers },
+  });
+}
+
+function dataVarResponse(request) {
+  const url = new URL(request.url);
+  const params = Array.from(url.searchParams.entries());
+
+  const colNames = params.map((v) => `${v[0]}/${v[1]}`);
+  const colSchemas = colNames.map(() => schema.schema.dataframe);
+  const body = getEncodedDataframe(
+    colNames,
+    schema.schema.dataframe.nObs,
+    colSchemas
+  );
+
+  const headers = new Headers({
+    "Content-Type": "application/octet-stream",
+  });
+  return Promise.resolve({
+    body,
+    init: { status: 200, headers },
+  });
+}
+
+export function responder(request) {
+  const url = new URL(request.url);
+  const { pathname } = url;
+  if (pathname.endsWith("/annotations/obs")) {
+    return annotationObsResponse(request);
+  }
+  if (pathname.endsWith("/annotations/var")) {
+    return annotationVarResponse(request);
+  }
+  if (pathname.endsWith("/layout/obs")) {
+    return layoutObsResponse(request);
+  }
+  if (pathname.endsWith("/data/var")) {
+    return dataVarResponse(request);
+  }
+  return Promise.reject(new Error("bad URL"));
+}
+
+export function withExpected(expectedURL, expectedParams) {
+  /*
+  Do some additional error checking
+  */
+  return (request) => {
+    // if URL is bogus, reject the promise
+    const url = new URL(request.url);
+    if (!url.pathname.endsWith(expectedURL)) {
+      return Promise.reject(new Error("Unexpected URL!"));
+    }
+    const params = Array.from(url.searchParams.entries()).sort(
+      (a, b) => a[0] < b[0]
+    );
+    expectedParams = expectedParams.slice().sort((a, b) => a[0] < b[0]);
+
+    if (
+      params.length !== expectedParams.length ||
+      !params.every(
+        (p, i) => p[0] === expectedParams[i][0] && p[1] === expectedParams[i][1]
+      )
+    ) {
+      return Promise.reject(new Error("unexpected name requested in URL"));
+    }
+
+    return responder(request);
+  };
+}
+
+export function annotationsObs(names) {
+  return withExpected(
+    "/annotations/obs",
+    names.map((name) => ["annotation-name", name])
+  );
+}

client/__tests__/util/annoMatrix/serverMocks/schema.js

@@ -0,0 +1,80 @@
+export const schema = {
+  schema: {
+    annotations: {
+      obs: {
+        columns: [
+          {
+            name: "name_0",
+            type: "string",
+            writable: false,
+          },
+          {
+            name: "n_genes",
+            type: "int32",
+            writable: false,
+          },
+          {
+            name: "percent_mito",
+            type: "float32",
+            writable: false,
+          },
+          {
+            name: "n_counts",
+            type: "float32",
+            writable: false,
+          },
+          {
+            name: "louvain",
+            type: "string",
+            writable: false,
+          },
+        ],
+        index: "name_0",
+      },
+      var: {
+        columns: [
+          {
+            name: "name_0",
+            type: "string",
+            writable: false,
+          },
+          {
+            name: "n_cells",
+            type: "int32",
+            writable: false,
+          },
+        ],
+        index: "name_0",
+      },
+    },
+    dataframe: {
+      nObs: 2638,
+      nVar: 1838,
+      type: "float32",
+    },
+    layout: {
+      obs: [
+        {
+          dims: ["draw_graph_fr_0", "draw_graph_fr_1"],
+          name: "draw_graph_fr",
+          type: "float32",
+        },
+        {
+          dims: ["pca_0", "pca_1"],
+          name: "pca",
+          type: "float32",
+        },
+        {
+          dims: ["tsne_0", "tsne_1"],
+          name: "tsne",
+          type: "float32",
+        },
+        {
+          dims: ["umap_0", "umap_1"],
+          name: "umap",
+          type: "float32",
+        },
+      ],
+    },
+  },
+};

client/__tests__/util/annoMatrix/whereCache.test.js

@@ -0,0 +1,382 @@
+import sha1 from "sha1";
+import {
+  _whereCacheGet,
+  _whereCacheCreate,
+  _whereCacheMerge,
+} from "../../../src/annoMatrix/whereCache";
+
+const schema = {};
+
+describe("whereCache", () => {
+  test("whereCacheGet - where query, missing cache values", () => {
+    expect(
+      _whereCacheGet({}, schema, "X", {
+        where: {
+          field: "var",
+          column: "foo",
+          value: "bar",
+        },
+      })
+    ).toEqual([undefined]);
+    expect(
+      _whereCacheGet({}, schema, "X", {
+        summarize: {
+          field: "var",
+          column: "foo",
+          values: ["bar"],
+        },
+      })
+    ).toEqual([undefined]);
+    expect(
+      _whereCacheGet({ where: { X: {} } }, schema, "X", {
+        where: {
+          field: "var",
+          column: "foo",
+          value: "bar",
+        },
+      })
+    ).toEqual([undefined]);
+    expect(
+      _whereCacheGet({ where: { X: { var: new Map() } } }, schema, "X", {
+        where: {
+          field: "var",
+          column: "foo",
+          value: "bar",
+        },
+      })
+    ).toEqual([undefined]);
+    expect(
+      _whereCacheGet(
+        { where: { X: { var: new Map([["foo", new Map()]]) } } },
+        schema,
+        "X",
+        {
+          where: {
+            field: "var",
+            column: "foo",
+            value: "bar",
+          },
+        }
+      )
+    ).toEqual([undefined]);
+  });
+
+  test("whereCacheGet - summarize query, missing cache values", () => {
+    expect(
+      _whereCacheGet({}, schema, "X", {
+        summarize: {
+          method: "mean",
+          field: "var",
+          column: "foo",
+          values: ["bar"],
+        },
+      })
+    ).toEqual([undefined]);
+    expect(
+      _whereCacheGet(
+        { summarize: { X: { mean: { var: new Map() } } } },
+        schema,
+        "X",
+        {
+          summarize: {
+            method: "mean",
+            field: "var",
+            column: "foo",
+            values: ["bar"],
+          },
+        }
+      )
+    ).toEqual([undefined]);
+  });
+
+  test("whereCacheGet - varied lookups", () => {
+    const whereCache = {
+      where: {
+        X: {
+          var: new Map([
+            [
+              "foo",
+              new Map([
+                ["bar", [0]],
+                ["baz", [1, 2]],
+              ]),
+            ],
+          ]),
+        },
+      },
+      summarize: {
+        X: {
+          mean: {
+            var: new Map([
+              [
+                "foo",
+                new Map([
+                  [sha1("bar"), [0]],
+                  [sha1("baz"), [1, 2]],
+                ]),
+              ],
+            ]),
+          },
+        },
+      },
+    };
+
+    expect(
+      _whereCacheGet(whereCache, schema, "X", {
+        where: {
+          field: "var",
+          column: "foo",
+          value: "bar",
+        },
+      })
+    ).toEqual([0]);
+    expect(
+      _whereCacheGet(whereCache, schema, "X", {
+        summarize: {
+          method: "mean",
+          field: "var",
+          column: "foo",
+          values: ["bar"],
+        },
+      })
+    ).toEqual([0]);
+    expect(
+      _whereCacheGet(whereCache, schema, "X", {
+        where: {
+          field: "var",
+          column: "foo",
+          value: "baz",
+        },
+      })
+    ).toEqual([1, 2]);
+    expect(
+      _whereCacheGet(whereCache, schema, "X", {
+        summarize: {
+          method: "mean",
+          field: "var",
+          column: "foo",
+          values: ["baz"],
+        },
+      })
+    ).toEqual([1, 2]);
+    expect(_whereCacheGet(whereCache, schema, "Y", {})).toEqual([undefined]);
+    expect(
+      _whereCacheGet(whereCache, schema, "X", {
+        where: {
+          field: "whoknows",
+          column: "whatever",
+          value: "snork",
+        },
+      })
+    ).toEqual([undefined]);
+    expect(
+      _whereCacheGet(whereCache, schema, "X", {
+        where: {
+          field: "var",
+          column: "whatever",
+          value: "snork",
+        },
+      })
+    ).toEqual([undefined]);
+    expect(
+      _whereCacheGet(whereCache, schema, "X", {
+        where: {
+          field: "var",
+          column: "foo",
+          value: "snork",
+        },
+      })
+    ).toEqual([undefined]);
+  });
+
+  test("whereCacheCreate, where query", () => {
+    const query = {
+      where: {
+        field: "queryField",
+        column: "queryColumn",
+        value: "queryValue",
+      },
+    };
+    const wc = _whereCacheCreate(
+      "field",
+      {
+        where: {
+          field: "queryField",
+          column: "queryColumn",
+          value: "queryValue",
+        },
+      },
+      [0, 1, 2]
+    );
+    expect(wc).toBeDefined();
+    expect(wc).toEqual(
+      expect.objectContaining({
+        where: {
+          field: {
+            queryField: expect.any(Map),
+          },
+        },
+      })
+    );
+    expect(wc.where.field.queryField.has("queryColumn")).toEqual(true);
+    expect(wc.where.field.queryField.get("queryColumn")).toBeInstanceOf(Map);
+    expect(
+      wc.where.field.queryField.get("queryColumn").has("queryValue")
+    ).toEqual(true);
+    expect(_whereCacheGet(wc, schema, "field", query)).toEqual([0, 1, 2]);
+  });
+
+  test("whereCacheCreate, summarize query", () => {
+    const query = {
+      summarize: {
+        method: "method",
+        field: "queryField",
+        column: "queryColumn",
+        values: ["queryValue"],
+      },
+    };
+    const wc = _whereCacheCreate("field", query, [0, 1, 2]);
+    expect(_whereCacheGet(wc, schema, "field", query)).toEqual([0, 1, 2]);
+  });
+
+  test("whereCacheCreate, unknown query type", () => {
+    expect(_whereCacheCreate("field", { foobar: true }, [1])).toEqual({});
+  });
+
+  test("whereCacheMerge, where queries", () => {
+    let wc;
+
+    // remember, will mutate dst
+    const src = _whereCacheCreate(
+      "field",
+      { where: { field: "queryField", column: "queryColumn", value: "foo" } },
+      ["foo"]
+    );
+
+    const dst1 = _whereCacheCreate(
+      "field",
+      { where: { field: "queryField", column: "queryColumn", value: "bar" } },
+      ["dst1"]
+    );
+    wc = _whereCacheMerge(dst1, src);
+    expect(
+      _whereCacheGet(wc, schema, "field", {
+        where: {
+          field: "queryField",
+          column: "queryColumn",
+          value: "foo",
+        },
+      })
+    ).toEqual(["foo"]);
+    expect(
+      _whereCacheGet(wc, schema, "field", {
+        where: {
+          field: "queryField",
+          column: "queryColumn",
+          value: "bar",
+        },
+      })
+    ).toEqual(["dst1"]);
+
+    const dst2 = _whereCacheCreate(
+      "field",
+      { where: { field: "queryField", column: "queryColumn", value: "bar" } },
+      ["dst2"]
+    );
+    wc = _whereCacheMerge(dst2, dst1, src);
+    expect(
+      _whereCacheGet(wc, schema, "field", {
+        where: {
+          field: "queryField",
+          column: "queryColumn",
+          value: "foo",
+        },
+      })
+    ).toEqual(["foo"]);
+    expect(
+      _whereCacheGet(wc, schema, "field", {
+        where: {
+          field: "queryField",
+          column: "queryColumn",
+          value: "bar",
+        },
+      })
+    ).toEqual(["dst1"]);
+
+    wc = _whereCacheMerge({}, src);
+    expect(wc).toEqual(src);
+
+    wc = _whereCacheMerge({ where: { field: { queryField: new Map() } } }, src);
+    expect(wc).toEqual(src);
+  });
+
+  test("whereCacheMerge, mixed queries", () => {
+    const wc = _whereCacheMerge(
+      _whereCacheCreate(
+        "field",
+        {
+          where: {
+            field: "queryField",
+            column: "queryColumn",
+            value: "foo",
+          },
+        },
+        ["a"]
+      ),
+      _whereCacheCreate(
+        "field",
+        {
+          summarize: {
+            method: "mean",
+            field: "queryField",
+            column: "queryColumn",
+            values: ["foo", "bar", "baz"],
+          },
+        },
+        ["b"]
+      )
+    );
+
+    expect(
+      _whereCacheGet(wc, schema, "field", {
+        where: {
+          field: "queryField",
+          column: "queryColumn",
+          value: "foo",
+        },
+      })
+    ).toEqual(["a"]);
+
+    expect(
+      _whereCacheGet(wc, schema, "field", {
+        summarize: {
+          method: "mean",
+          field: "queryField",
+          column: "queryColumn",
+          values: ["foo", "bar", "baz"],
+        },
+      })
+    ).toEqual(["b"]);
+
+    expect(
+      _whereCacheGet(wc, schema, "field", {
+        where: {
+          field: "queryField",
+          column: "queryColumn",
+          value: "does-not-exist",
+        },
+      })
+    ).toEqual([undefined]);
+
+    expect(
+      _whereCacheGet(wc, schema, "field", {
+        summarize: {
+          method: "no-such-method",
+          field: "queryField",
+          column: "queryColumn",
+          values: ["does-not-exist"],
+        },
+      })
+    ).toEqual([undefined]);
+  });
+});

client/__tests__/util/centroid.test.js

@@ -0,0 +1,75 @@
+import cloneDeep from "lodash.clonedeep";
+
+import calcCentroid from "../../src/util/centroid";
+import quantile from "../../src/util/quantile";
+import { matrixFBSToDataframe } from "../../src/util/stateManager/matrix";
+import * as REST from "./stateManager/sampleResponses";
+import { indexEntireSchema } from "../../src/util/stateManager/schemaHelpers";
+import { normalizeWritableCategoricalSchema } from "../../src/annoMatrix/normalize";
+
+describe("centroid", () => {
+  let schema;
+  let obsAnnotations;
+  let obsLayout;
+
+  beforeAll(() => {
+    schema = indexEntireSchema(cloneDeep(REST.schema.schema));
+    obsAnnotations = matrixFBSToDataframe(REST.annotationsObs);
+    obsLayout = matrixFBSToDataframe(REST.layoutObs);
+
+    normalizeWritableCategoricalSchema(
+      schema.annotations.obsByName.field3,
+      obsAnnotations.col("field3")
+    );
+  });
+
+  test("field4 (categorical obsAnnotation)", () => {
+    const centroidResult = calcCentroid(
+      schema,
+      "field4",
+      obsAnnotations,
+      { current: "umap", currentDimNames: ["umap_0", "umap_1"] },
+      obsLayout
+    );
+
+    // Check to see that a centroid has been calculated for every categorical value
+    const keysAsArray = Array.from(centroidResult.keys());
+    expect(keysAsArray).toEqual(
+      expect.arrayContaining([83, true, "foo", 2.222222])
+    );
+
+    // This expected result assumes that all cells belong in all categorical values inside of sample response
+    const expectedResult = [
+      quantile([0.5], obsLayout.col("umap_0").asArray())[0],
+      quantile([0.5], obsLayout.col("umap_1").asArray())[0],
+    ];
+
+    centroidResult.forEach((coordinate) => {
+      expect(coordinate).toEqual(expectedResult);
+    });
+  });
+
+  test("field3 (boolean obsAnnotation)", () => {
+    const centroidResult = calcCentroid(
+      schema,
+      "field3",
+      obsAnnotations,
+      { current: "umap", currentDimNames: ["umap_0", "umap_1"] },
+      obsLayout
+    );
+
+    // Check to see that a centroid has been calculated for every categorical value
+    const keysAsArray = Array.from(centroidResult.keys());
+    expect(keysAsArray).toEqual(expect.arrayContaining([false, true]));
+
+    // This expected result assumes that all cells belong in all categorical values inside of sample response
+    const expectedResult = [
+      quantile([0.5], obsLayout.col("umap_0").asArray())[0],
+      quantile([0.5], obsLayout.col("umap_1").asArray())[0],
+    ];
+
+    centroidResult.forEach((coordinate) => {
+      expect(coordinate).toEqual(expectedResult);
+    });
+  });
+});

client/__tests__/util/dataframe/histogram.test.js

@@ -14,7 +14,7 @@ describe("Dataframe column histogram", () => {
       new Map([
         ["n1", new Map([["c1", 1]])],
         ["n2", new Map([["c2", 1]])],
-        ["n3", new Map([["c3", 1]])]
+        ["n3", new Map([["c3", 1]])],
       ])
     );
     // memoized?
@@ -31,7 +31,11 @@ describe("Dataframe column histogram", () => {
 
     const h1 = df.col("value").histogram(3, [0, 2], df.col("name"));
     expect(h1).toMatchObject(
-      new Map([["n1", [1, 0, 0]], ["n2", [0, 1, 0]], ["n3", [0, 0, 1]]])
+      new Map([
+        ["n1", [1, 0, 0]],
+        ["n2", [0, 1, 0]],
+        ["n3", [0, 0, 1]],
+      ])
     );
     // memoized?
     expect(df.col("value").histogram(3, [0, 2], df.col("name"))).toMatchObject(
@@ -48,7 +52,13 @@ describe("Dataframe column histogram", () => {
     );
 
     const h1 = df.col("cat").histogram();
-    expect(h1).toMatchObject(new Map([["c1", 1], ["c2", 1], ["c3", 1]]));
+    expect(h1).toMatchObject(
+      new Map([
+        ["c1", 1],
+        ["c2", 1],
+        ["c3", 1],
+      ])
+    );
     // memoized?
     expect(df.col("value").histogram(3, [0, 2])).toMatchObject(h1);
   });
@@ -78,16 +88,7 @@ describe("Dataframe column histogram", () => {
     expect(df.col(1).histogram(5, [0, 100])).toEqual([5, 1, 0, 0, 2]);
     expect(df.col(0).histogram(2, [0, 10])).toEqual([2, 2]);
     expect(df.col(0).histogram(10, [0, 100])).toEqual([
-      3,
-      2,
-      1,
-      0,
-      0,
-      0,
-      0,
-      0,
-      0,
-      2
+      3, 2, 1, 0, 0, 0, 0, 0, 0, 2,
     ]);
   });
 });

client/__tests__/util/dataframe/summarize.test.js

@@ -13,7 +13,7 @@ describe("Dataframe column summary", () => {
         categorical: true,
         categories: [],
         categoryCounts: new Map(),
-        numCategories: 0
+        numCategories: 0,
       })
     );
   });
@@ -27,7 +27,7 @@ describe("Dataframe column summary", () => {
         [true],
         new Float32Array([39.3]),
         new Int32Array([99]),
-        [1]
+        [1],
       ],
       null,
       new Dataframe.KeyIndex([
@@ -36,7 +36,7 @@ describe("Dataframe column summary", () => {
         "nameBoolean",
         "nameFloat32",
         "nameInt32",
-        "nameCategorical"
+        "nameCategorical",
       ])
     );
 
@@ -45,7 +45,7 @@ describe("Dataframe column summary", () => {
         categorical: true,
         categories: ["n1"],
         categoryCounts: new Map([["n1", 1]]),
-        numCategories: 1
+        numCategories: 1,
       })
     );
     expect(df.icol(1).summarize()).toEqual(
@@ -53,7 +53,7 @@ describe("Dataframe column summary", () => {
         categorical: true,
         categories: ["hi"],
         categoryCounts: new Map([["hi", 1]]),
-        numCategories: 1
+        numCategories: 1,
       })
     );
     expect(df.icol(2).summarize()).toEqual(
@@ -61,7 +61,7 @@ describe("Dataframe column summary", () => {
         categorical: true,
         categories: [true],
         categoryCounts: new Map([[true, 1]]),
-        numCategories: 1
+        numCategories: 1,
       })
     );
     expect(df.icol(3).summarize()).toEqual(
@@ -71,7 +71,7 @@ describe("Dataframe column summary", () => {
         max: float32Conversion(39.3),
         nan: 0,
         ninf: 0,
-        pinf: 0
+        pinf: 0,
       })
     );
     expect(df.icol(4).summarize()).toEqual(
@@ -81,7 +81,7 @@ describe("Dataframe column summary", () => {
         max: 99,
         nan: 0,
         ninf: 0,
-        pinf: 0
+        pinf: 0,
       })
     );
     expect(df.icol(5).summarize()).toEqual(
@@ -89,7 +89,7 @@ describe("Dataframe column summary", () => {
         categorical: true,
         categories: [1],
         categoryCounts: new Map([[1, 1]]),
-        numCategories: 1
+        numCategories: 1,
       })
     );
   });
@@ -103,7 +103,7 @@ describe("Dataframe column summary", () => {
         [false, true, true],
         new Float32Array([39.3, 39.3, 0]),
         new Int32Array([99, 99, 99]),
-        [1, false, "0"]
+        [1, false, "0"],
       ],
       null,
       new Dataframe.KeyIndex([
@@ -112,7 +112,7 @@ describe("Dataframe column summary", () => {
         "nameBoolean",
         "nameFloat32",
         "nameInt32",
-        "nameCategorical"
+        "nameCategorical",
       ])
     );
 
@@ -120,24 +120,34 @@ describe("Dataframe column summary", () => {
       expect.objectContaining({
         categorical: true,
         categories: expect.arrayContaining(["n0", "n1", "n2"]),
-        categoryCounts: new Map([["n0", 1], ["n1", 1], ["n2", 1]]),
-        numCategories: 3
+        categoryCounts: new Map([
+          ["n0", 1],
+          ["n1", 1],
+          ["n2", 1],
+        ]),
+        numCategories: 3,
       })
     );
     expect(df.icol(1).summarize()).toEqual(
       expect.objectContaining({
         categorical: true,
         categories: expect.arrayContaining(["hi", "bye"]),
-        categoryCounts: new Map([["hi", 2], ["bye", 1]]),
-        numCategories: 2
+        categoryCounts: new Map([
+          ["hi", 2],
+          ["bye", 1],
+        ]),
+        numCategories: 2,
       })
     );
     expect(df.icol(2).summarize()).toEqual(
       expect.objectContaining({
         categorical: true,
         categories: expect.arrayContaining([true, false]),
-        categoryCounts: new Map([[true, 2], [false, 1]]),
-        numCategories: 2
+        categoryCounts: new Map([
+          [true, 2],
+          [false, 1],
+        ]),
+        numCategories: 2,
       })
     );
     expect(df.icol(3).summarize()).toEqual(
@@ -147,7 +157,7 @@ describe("Dataframe column summary", () => {
         max: float32Conversion(39.3),
         nan: 0,
         ninf: 0,
-        pinf: 0
+        pinf: 0,
       })
     );
     expect(df.icol(4).summarize()).toEqual(
@@ -157,15 +167,19 @@ describe("Dataframe column summary", () => {
         max: 99,
         nan: 0,
         ninf: 0,
-        pinf: 0
+        pinf: 0,
       })
     );
     expect(df.icol(5).summarize()).toEqual(
       expect.objectContaining({
         categorical: true,
         categories: expect.arrayContaining([1, false, "0"]),
-        categoryCounts: new Map([[1, 1], [false, 1], ["0", 1]]),
-        numCategories: 3
+        categoryCounts: new Map([
+          [1, 1],
+          [false, 1],
+          ["0", 1],
+        ]),
+        numCategories: 3,
       })
     );
   });
@@ -181,10 +195,10 @@ describe("Dataframe column summary", () => {
           39.3,
           Number.NEGATIVE_INFINITY,
           Number.NaN,
-          Number.POSITIVE_INFINITY
+          Number.POSITIVE_INFINITY,
         ]),
         new Int32Array([99, 99, 99, 99]),
-        [1, false, "0", "0"]
+        [1, false, "0", "0"],
       ],
       null,
       new Dataframe.KeyIndex([
@@ -193,7 +207,7 @@ describe("Dataframe column summary", () => {
         "nameBoolean",
         "nameFloat32",
         "nameInt32",
-        "nameCategorical"
+        "nameCategorical",
       ])
     );
 
@@ -201,24 +215,34 @@ describe("Dataframe column summary", () => {
       expect.objectContaining({
         categorical: true,
         categories: expect.arrayContaining(["n0", "n1", "n2"]),
-        categoryCounts: new Map([["n0", 1], ["n1", 1], ["n2", 2]]),
-        numCategories: 3
+        categoryCounts: new Map([
+          ["n0", 1],
+          ["n1", 1],
+          ["n2", 2],
+        ]),
+        numCategories: 3,
       })
     );
     expect(df.icol(1).summarize()).toEqual(
       expect.objectContaining({
         categorical: true,
         categories: expect.arrayContaining(["hi", "bye"]),
-        categoryCounts: new Map([["hi", 2], ["bye", 1]]),
-        numCategories: 2
+        categoryCounts: new Map([
+          ["hi", 2],
+          ["bye", 1],
+        ]),
+        numCategories: 2,
       })
     );
     expect(df.icol(2).summarize()).toEqual(
       expect.objectContaining({
         categorical: true,
         categories: expect.arrayContaining([true, false]),
-        categoryCounts: new Map([[true, 2], [false, 1]]),
-        numCategories: 2
+        categoryCounts: new Map([
+          [true, 2],
+          [false, 1],
+        ]),
+        numCategories: 2,
       })
     );
     expect(df.icol(3).summarize()).toEqual(
@@ -228,7 +252,7 @@ describe("Dataframe column summary", () => {
         max: float32Conversion(39.3),
         nan: 1,
         ninf: 1,
-        pinf: 1
+        pinf: 1,
       })
     );
     expect(df.icol(4).summarize()).toEqual(
@@ -238,15 +262,19 @@ describe("Dataframe column summary", () => {
         max: 99,
         nan: 0,
         ninf: 0,
-        pinf: 0
+        pinf: 0,
       })
     );
     expect(df.icol(5).summarize()).toEqual(
       expect.objectContaining({
         categorical: true,
         categories: expect.arrayContaining([1, false, "0"]),
-        categoryCounts: new Map([[1, 1], [false, 1], ["0", 1]]),
-        numCategories: 3
+        categoryCounts: new Map([
+          [1, 1],
+          [false, 1],
+          ["0", 1],
+        ]),
+        numCategories: 3,
       })
     );
   });

client/__tests__/util/nameCreators.test.js

@@ -6,7 +6,7 @@ import {
   obsAnnoDimensionName,
   diffexpDimensionName,
   userDefinedDimensionName,
-  makeContinuousDimensionName
+  makeContinuousDimensionName,
 } from "../../src/util/nameCreators";
 
 describe("nameCreators", () => {
@@ -14,25 +14,25 @@ describe("nameCreators", () => {
     layoutDimensionName,
     obsAnnoDimensionName,
     diffexpDimensionName,
-    userDefinedDimensionName
+    userDefinedDimensionName,
   ];
 
   test("check for namespace isolation", () => {
     const foo = "foo";
-    nameCreators.forEach(fn => expect(fn(foo)).not.toBe(foo));
+    nameCreators.forEach((fn) => expect(fn(foo)).not.toBe(foo));
 
     const bar = "bar";
-    nameCreators.forEach(fn => expect(fn(foo)).not.toBe(fn(bar)));
+    nameCreators.forEach((fn) => expect(fn(foo)).not.toBe(fn(bar)));
 
-    nameCreators.forEach(fn => {
-      const allOtherFn = nameCreators.filter(elmnt => elmnt !== fn);
-      allOtherFn.forEach(otherFn => expect(fn(foo)).not.toBe(otherFn(foo)));
+    nameCreators.forEach((fn) => {
+      const allOtherFn = nameCreators.filter((elmnt) => elmnt !== fn);
+      allOtherFn.forEach((otherFn) => expect(fn(foo)).not.toBe(otherFn(foo)));
     });
   });
 
   test("check for legal keys", () => {
     /* need namespace creators to return strings only */
-    nameCreators.forEach(fn => expect(fn("X")).toMatch(/X/));
+    nameCreators.forEach((fn) => expect(fn("X")).toMatch(/X/));
   });
 });
 

client/__tests__/util/promiseLimit.test.js

@@ -0,0 +1,96 @@
+import PromiseLimit from "../../src/util/promiseLimit";
+import { range } from "../../src/util/range";
+
+const delay = (t) => new Promise((resolve) => setTimeout(resolve, t));
+
+describe("PromiseLimit", () => {
+  test("simple evaluation, concurrency 1", async () => {
+    const plimit = new PromiseLimit(1);
+    const result = await Promise.all([
+      plimit.add(() => Promise.resolve(1)),
+      plimit.add(() => Promise.resolve(2)),
+      plimit.add(() => Promise.resolve(3)),
+      plimit.add(() => Promise.resolve(4)),
+    ]);
+    expect(result).toEqual([1, 2, 3, 4]);
+  });
+
+  test("simple evaluation, concurrency > 1", async () => {
+    const plimit = new PromiseLimit(100);
+    const result = await Promise.all([
+      plimit.add(() => Promise.resolve(1)),
+      plimit.add(() => Promise.resolve(2)),
+      plimit.add(() => Promise.resolve(3)),
+      plimit.add(() => Promise.resolve(4)),
+    ]);
+    expect(result).toEqual([1, 2, 3, 4]);
+  });
+
+  test("eval in order of insertion", async () => {
+    const plimit = new PromiseLimit(100);
+
+    const result = await Promise.all([
+      plimit.add(() => Promise.resolve(1)),
+      plimit.add(() => Promise.resolve(2)),
+      plimit.add(() => Promise.resolve(3)),
+      plimit.add(() => Promise.resolve(4)),
+    ]);
+
+    expect(result).toEqual([1, 2, 3, 4]);
+  });
+
+  test("obeys concurrency limit", async () => {
+    const plimit = new PromiseLimit(2);
+    let running = 0;
+    let maxRunning = 0;
+
+    const callback = async () => {
+      running += 1;
+      maxRunning = running > maxRunning ? running : maxRunning;
+      await delay(100);
+      running -= 1;
+    };
+
+    await Promise.all(range(10).map((i) => plimit.add(() => callback(i))));
+
+    expect(maxRunning).toEqual(2);
+  });
+
+  test("rejection", async () => {
+    const plimit = new PromiseLimit(2);
+    const result = await Promise.all([
+      plimit.add(() => Promise.resolve("OK")),
+      // eslint-disable-next-line prefer-promise-reject-errors -- unit test
+      plimit.add(() => Promise.reject("not OK")).catch((e) => e),
+      plimit.add(() => Promise.resolve("OK")),
+      plimit
+        .add(() => {
+          throw new Error("not OK");
+        })
+        .catch((e) => e.message),
+    ]);
+    expect(result).toEqual(["OK", "not OK", "OK", "not OK"]);
+  });
+
+  test("priority queue", async () => {
+    const plimit = new PromiseLimit(1);
+
+    let finishOrder = 0;
+    const callback = () => async () => {
+      await delay(100);
+      const result = finishOrder;
+      finishOrder += 1;
+      return result;
+    };
+
+    const result = await Promise.all([
+      plimit.add(callback()),
+      plimit.priorityAdd(4, callback()),
+      plimit.priorityAdd(0, callback()),
+      plimit.priorityAdd(1, callback()),
+      plimit.priorityAdd(-1, callback()),
+    ]);
+
+    expect(result).toEqual([0, 4, 2, 3, 1]);
+  });
+});

client/__tests__/util/quantile.test.js

@@ -1,29 +1,25 @@
 import quantile from "../../src/util/quantile";
 
 describe("quantile", () => {
-	test("single q", () => {
-		const arr = new Float32Array([9, 3, 5, 6, 0]);
-		expect(quantile([1.0], arr)).toMatchObject([9]);
-		expect(quantile([0.9], arr)).toMatchObject([9]);
-		expect(quantile([0.8], arr)).toMatchObject([9]);
-		expect(quantile([0.7], arr)).toMatchObject([6]);
-		expect(quantile([0.6], arr)).toMatchObject([6]);
-		expect(quantile([0.5], arr)).toMatchObject([5]);
-		expect(quantile([0.4], arr)).toMatchObject([5]);
-		expect(quantile([0.3], arr)).toMatchObject([3]);
-		expect(quantile([0.2], arr)).toMatchObject([3]);
-		expect(quantile([0.1], arr)).toMatchObject([0]);
-		expect(quantile([0], arr)).toMatchObject([0]);
-	});
+  test("single q", () => {
+    const arr = new Float32Array([9, 3, 5, 6, 0]);
+    expect(quantile([1.0], arr)).toMatchObject([9]);
+    expect(quantile([0.9], arr)).toMatchObject([9]);
+    expect(quantile([0.8], arr)).toMatchObject([9]);
+    expect(quantile([0.7], arr)).toMatchObject([6]);
+    expect(quantile([0.6], arr)).toMatchObject([6]);
+    expect(quantile([0.5], arr)).toMatchObject([5]);
+    expect(quantile([0.4], arr)).toMatchObject([5]);
+    expect(quantile([0.3], arr)).toMatchObject([3]);
+    expect(quantile([0.2], arr)).toMatchObject([3]);
+    expect(quantile([0.1], arr)).toMatchObject([0]);
+    expect(quantile([0], arr)).toMatchObject([0]);
+  });
 
-	test("multi q", () => {
-		const arr = new Float32Array([9, 3, 5, 6, 0]);
-		expect(quantile([0, 0.25, 0.5, 0.75, 1.0], arr)).toMatchObject([
-			0,
-			3,
-			5,
-			6,
-			9
-		]);
-	});
+  test("multi q", () => {
+    const arr = new Float32Array([9, 3, 5, 6, 0]);
+    expect(quantile([0, 0.25, 0.5, 0.75, 1.0], arr)).toMatchObject([
+      0, 3, 5, 6, 9,
+    ]);
+  });
 });

client/__tests__/util/range.test.js

@@ -1,42 +1,48 @@
-import { range, rangeFill } from "../../src/util/range";
+import { range, rangeFill, linspace } from "../../src/util/range";
 
 describe("range", () => {
-	test("no defaults", () => {
-		expect(range(0, 3, 1)).toMatchObject([0, 1, 2]);
-	});
+  test("no defaults", () => {
+    expect(range(0, 3, 1)).toMatchObject([0, 1, 2]);
+  });
 
-	test("range(stop)", () => {
-		expect(range(3)).toMatchObject([0, 1, 2]);
-		expect(range(0)).toMatchObject([]);
-		expect(range(1)).toMatchObject([0]);
-	});
+  test("range(stop)", () => {
+    expect(range(3)).toMatchObject([0, 1, 2]);
+    expect(range(0)).toMatchObject([]);
+    expect(range(1)).toMatchObject([0]);
+  });
 
-	test("range(start,stop)", () => {
-		expect(range(0, 0)).toMatchObject([]);
-		expect(range(0, 2)).toMatchObject([0, 1]);
-		expect(range(4, 8)).toMatchObject([4, 5, 6, 7]);
-	});
+  test("range(start,stop)", () => {
+    expect(range(0, 0)).toMatchObject([]);
+    expect(range(0, 2)).toMatchObject([0, 1]);
+    expect(range(4, 8)).toMatchObject([4, 5, 6, 7]);
+  });
 
-	test("range(start, stop, step", () => {
-		expect(range(4, 0, -1)).toMatchObject([4, 3, 2, 1]);
-		expect(range(0, 4, 2)).toMatchObject([0, 2]);
-	});
+  test("range(start, stop, step", () => {
+    expect(range(4, 0, -1)).toMatchObject([4, 3, 2, 1]);
+    expect(range(0, 4, 2)).toMatchObject([0, 2]);
+  });
 });
 
 describe("rangefill", () => {
-	test("rangeFill(arr)", () => {
-		expect(rangeFill(new Int32Array(3))).toMatchObject(
-			new Int32Array([0, 1, 2])
-		);
-	});
-	test("rangeFill(arr, start)", () => {
-		expect(rangeFill(new Int32Array(2), 1)).toMatchObject(
-			new Int32Array([1, 2])
-		);
-	});
-	test("rangeFill(arr, start, step)", () => {
-		expect(rangeFill(new Int32Array(3), 2, -1)).toMatchObject(
-			new Int32Array([2, 1, 0])
-		);
-	});
+  test("rangeFill(arr)", () => {
+    expect(rangeFill(new Int32Array(3))).toMatchObject(
+      new Int32Array([0, 1, 2])
+    );
+  });
+  test("rangeFill(arr, start)", () => {
+    expect(rangeFill(new Int32Array(2), 1)).toMatchObject(
+      new Int32Array([1, 2])
+    );
+  });
+  test("rangeFill(arr, start, step)", () => {
+    expect(rangeFill(new Int32Array(3), 2, -1)).toMatchObject(
+      new Int32Array([2, 1, 0])
+    );
+  });
+});
+
+describe("linspace", () => {
+  test("linspace(arr, start, step)", () => {
+    expect(linspace(0.0, 2.0, 5)).toMatchObject([0.0, 0.5, 1.0, 1.5, 2.0]);
+  });
 });

client/__tests__/util/stateManager/colorHelpers.test.js

@@ -0,0 +1,198 @@
+/* eslint-disable no-bitwise -- unsigned right shift better than Math.round */
+
+/*
+test color helpers
+*/
+import {
+  createColorTable,
+  loadUserColorConfig,
+} from "../../../src/util/stateManager/colorHelpers";
+import * as Dataframe from "../../../src/util/dataframe";
+
+describe("categorical color helpers", () => {
+  /*
+  Primary test constraint for categorical colors is that they are ordered/identified
+  by schema order, NOT by value.  Ie,
+
+    scale(schemaIndex) should match rgb[obsOffset]
+  */
+
+  const schema = indexSchema({
+    annotations: {
+      obs: {
+        columns: [
+          {
+            name: "name_0",
+            type: "string",
+            writable: false,
+          },
+          {
+            name: "continuousColumn",
+            type: "float32",
+            writable: false,
+          },
+          {
+            categories: [
+              "CD4 T cells",
+              "CD14+ Monocytes",
+              "B cells",
+              "CD8 T cells",
+              "NK cells",
+              "FCGR3A+ Monocytes",
+              "Dendritic cells",
+              "Megakaryocytes",
+            ],
+            name: "categoricalColumn",
+            type: "categorical",
+            writable: false,
+          },
+        ],
+        index: "name_0",
+      },
+      var: {
+        columns: [
+          {
+            name: "name_0",
+            type: "string",
+            writable: false,
+          },
+        ],
+        index: "name_0",
+      },
+    },
+    dataframe: {
+      nObs: 2638,
+      nVar: 1838,
+      type: "float32",
+    },
+    layout: {},
+  });
+
+  const catColCategories = schema.annotations.obs.columns[2].categories;
+  const obsDataframe = new Dataframe.Dataframe(
+    [schema.dataframe.nObs, 2],
+    [
+      new Float32Array(schema.dataframe.nObs).map(() => Math.random()),
+      new Array(schema.dataframe.nObs)
+        .fill("")
+        .map(
+          () =>
+            catColCategories[(Math.random() * catColCategories.length) >>> 0]
+        ),
+    ],
+    null,
+    new Dataframe.KeyIndex(["continuousColumn", "categoricalColumn"])
+  );
+
+  test("default category order", () => {
+    const ct = createColorTable(
+      "color by categorical metadata",
+      "categoricalColumn",
+      obsDataframe,
+      schema
+    );
+    expect(ct).toBeDefined();
+    const data = obsDataframe.col("categoricalColumn").asArray();
+    const cats = schema.annotations.obsByName.categoricalColumn.categories;
+    for (let i = 0; i < schema.dataframe.nObs; i += 1) {
+      expect(makeScale(ct.rgb[i])).toEqual(ct.scale(cats.indexOf(data[i])));
+    }
+  });
+
+  test("shuffle category order", () => {
+    const schemaClone = indexSchema(JSON.parse(JSON.stringify(schema)));
+    shuffle(schemaClone.annotations.obsByName.categoricalColumn.categories);
+    const ct = createColorTable(
+      "color by categorical metadata",
+      "categoricalColumn",
+      obsDataframe,
+      schemaClone
+    );
+    expect(ct).toBeDefined();
+    const data = obsDataframe.col("categoricalColumn").asArray();
+    const cats = schemaClone.annotations.obsByName.categoricalColumn.categories;
+    for (let i = 0; i < schemaClone.dataframe.nObs; i += 1) {
+      expect(makeScale(ct.rgb[i])).toEqual(ct.scale(cats.indexOf(data[i])));
+    }
+  });
+
+  test("user defined color order", () => {
+    const cats = schema.annotations.obsByName.categoricalColumn.categories;
+    const shuffleCats = shuffle(
+      Array.from(schema.annotations.obsByName.categoricalColumn.categories)
+    );
+    const userDefinedColorTable = {
+      categoricalColumn: shuffleCats.reduce((acc, label) => {
+        acc[label] = randRGBColor();
+        return acc;
+      }, {}),
+    };
+
+    const userColors = loadUserColorConfig(userDefinedColorTable);
+    expect(userColors).toBeDefined();
+
+    const ct = createColorTable(
+      "color by categorical metadata",
+      "categoricalColumn",
+      obsDataframe,
+      schema,
+      userColors
+    );
+    expect(ct).toBeDefined();
+    const data = obsDataframe.col("categoricalColumn").asArray();
+    for (let i = 0; i < schema.dataframe.nObs; i += 1) {
+      expect(makeScale(ct.rgb[i])).toEqual(
+        ct.scale(cats.indexOf(data[i])).toString()
+      );
+    }
+  });
+});
+
+/*
+TODO:
+1. mix up category order in schema to make sure it works with varied order
+2. user defined colors
+*/
+
+function indexSchema(schema) {
+  schema.annotations.obsByName = Object.fromEntries(
+    schema.annotations?.obs?.columns?.map((v) => [v.name, v]) ?? []
+  );
+  schema.annotations.varByName = Object.fromEntries(
+    schema.annotations?.var?.columns?.map((v) => [v.name, v]) ?? []
+  );
+  schema.layout.obsByName = Object.fromEntries(
+    schema.layout?.obs?.map((v) => [v.name, v]) ?? []
+  );
+  schema.layout.varByName = Object.fromEntries(
+    schema.layout?.var?.map((v) => [v.name, v]) ?? []
+  );
+
+  return schema;
+}
+
+function makeScale(rgb) {
+  // make a scale string from a rgb float triple
+  return `rgb(${(rgb[0] * 255) >>> 0}, ${(rgb[1] * 255) >>> 0}, ${
+    (rgb[2] * 256) >>> 0
+  })`;
+}
+
+function shuffle(array) {
+  for (let i = array.length - 1; i > 0; i -= 1) {
+    const j = (Math.random() * (i + 1)) >>> 0;
+    [array[i], array[j]] = [array[j], array[i]];
+  }
+  return array;
+}
+
+function randHexColor() {
+  const hex = ((Math.random() * 255) >>> 0).toString(16);
+  return `0${hex}`.slice(-2);
+}
+
+function randRGBColor() {
+  return `#${randHexColor()}${randHexColor()}${randHexColor()}`;
+}
+
+/* eslint-enable no-bitwise -- unsigned right shift better than Math.round */

client/__tests__/util/stateManager/controlsHelpers.test.js

@@ -0,0 +1,9 @@
+/*
+test controls helpers
+*/
+// TODO #2227 test: improve test coverage on control helper functions
+// (`isSelectableCategoryName()`, `selectableCategoryNames()`, `createCategorySummaryFromDfCol()`, `createCategoricalSelection()`, )
+
+describe("controls helpers", () => {
+  test("placeholder", () => {});
+});

client/__tests__/util/stateManager/fbs.test.js

@@ -3,32 +3,32 @@ test FBS encode/decode API
 */
 import { Dataframe, KeyIndex } from "../../../src/util/dataframe";
 import {
-	decodeMatrixFBS,
-	encodeMatrixFBS
+  decodeMatrixFBS,
+  encodeMatrixFBS,
 } from "../../../src/util/stateManager/matrix";
 
 describe("encode/decode", () => {
-	test("round trip", () => {
-		const columns = [
-			["red", "green", "blue"],
-			new Int32Array(3).fill(0),
-			new Uint32Array(3).fill(1),
-			new Float32Array(3).fill(2)
-		];
+  test("round trip", () => {
+    const columns = [
+      ["red", "green", "blue"],
+      new Int32Array(3).fill(0),
+      new Uint32Array(3).fill(1),
+      new Float32Array(3).fill(2),
+    ];
 
-		const dfNoColIdx = new Dataframe([3, 4], columns);
-		const dfA = decodeMatrixFBS(encodeMatrixFBS(dfNoColIdx));
-		expect([dfA.nRows, dfA.nCols]).toEqual(dfNoColIdx.dims);
-		expect(dfA.colIdx).toBeNull();
-		expect(dfA.rowIdx).toBeNull();
-		expect(dfA.columns).toEqual(columns);
+    const dfNoColIdx = new Dataframe([3, 4], columns);
+    const dfA = decodeMatrixFBS(encodeMatrixFBS(dfNoColIdx));
+    expect([dfA.nRows, dfA.nCols]).toEqual(dfNoColIdx.dims);
+    expect(dfA.colIdx).toBeNull();
+    expect(dfA.rowIdx).toBeNull();
+    expect(dfA.columns).toEqual(columns);
 
-		const colIndex = new KeyIndex(["a", "b", "c", "d"]);
-		const dfWithColIdx = new Dataframe([3, 4], columns, null, colIndex);
-		const dfB = decodeMatrixFBS(encodeMatrixFBS(dfWithColIdx));
-		expect([dfB.nRows, dfB.nCols]).toEqual(dfWithColIdx.dims);
-		expect(dfB.colIdx).toEqual(colIndex.keys());
-		expect(dfB.rowIdx).toBeNull();
-		expect(dfB.columns).toEqual(columns);
-	});
+    const colIndex = new KeyIndex(["a", "b", "c", "d"]);
+    const dfWithColIdx = new Dataframe([3, 4], columns, null, colIndex);
+    const dfB = decodeMatrixFBS(encodeMatrixFBS(dfWithColIdx));
+    expect([dfB.nRows, dfB.nCols]).toEqual(dfWithColIdx.dims);
+    expect(dfB.colIdx).toEqual(colIndex.labels());
+    expect(dfB.rowIdx).toBeNull();
+    expect(dfB.columns).toEqual(columns);
+  });
 });

client/__tests__/util/stateManager/sampleResponses.js

@@ -1,4 +1,7 @@
-/* eslint no-bitwise: "off" */
+import every from "lodash.every";
+import map from "lodash.map";
+import isNumber from "lodash.isnumber";
+import zip from "lodash.zip";
 import _ from "lodash";
 import { flatbuffers } from "flatbuffers";
 import { NetEncoding } from "../../../src/util/stateManager/matrix_generated";
@@ -17,13 +20,13 @@ const aConfigResponse = {
       { method: "POST", path: "/cluster/", available: false },
       { method: "POST", path: "/layout/", available: false },
       { method: "POST", path: "/diffexp/", available: false },
-      { method: "POST", path: "/saveLocal/", available: false }
+      { method: "POST", path: "/saveLocal/", available: false },
     ],
     displayNames: {
       engine: "the little engine that could",
-      dataset: "all your zeros are mine"
-    }
-  }
+      dataset: "all your zeros are mine",
+    },
+  },
 };
 
 const aSchemaResponse = {
@@ -31,7 +34,7 @@ const aSchemaResponse = {
     dataframe: {
       nObs,
       nVar,
-      type: "float32"
+      type: "float32",
     },
     annotations: {
       obs: {
@@ -44,9 +47,9 @@ const aSchemaResponse = {
           {
             name: "field4",
             type: "categorical",
-            categories: field4Categories
-          }
-        ]
+            categories: field4Categories,
+          },
+        ],
       },
       var: {
         index: "name",
@@ -58,46 +61,48 @@ const aSchemaResponse = {
           {
             name: "fieldD",
             type: "categorical",
-            categories: fieldDCategories
-          }
-        ]
-      }
+            categories: fieldDCategories,
+          },
+        ],
+      },
     },
     layout: {
       obs: [{ name: "umap", type: "float32", dims: ["umap_0", "umap_1"] }],
-      var: []
-    }
-  }
+      var: [],
+    },
+  },
 };
 
 const anAnnotationsObsJSONResponse = {
   names: ["name", "field1", "field2", "field3", "field4"],
   data: _()
     .range(nObs)
-    .map(idx => [
+    .map((idx) => [
       idx,
       `obs${idx}`,
       2 * idx,
       idx + 0.0133,
+      // eslint-disable-next-line no-bitwise -- idx & 1 to check for odd numbers
       !!(idx & 1),
-      field4Categories[idx % field4Categories.length]
+      field4Categories[idx % field4Categories.length],
     ])
-    .value()
+    .value(),
 };
 
 const anAnnotationsVarJSONResponse = {
   names: ["fieldA", "fieldB", "fieldC", "fieldD", "name"],
   data: _()
     .range(nVar)
-    .map(idx => [
+    .map((idx) => [
       idx,
       10 * idx,
       idx + 2.90143,
+      // eslint-disable-next-line no-bitwise -- idx & 1 to check for odd numbers
       !!(idx & 1),
       fieldDCategories[idx % fieldDCategories.length],
-      `var${idx}`
+      `var${idx}`,
     ])
-    .value()
+    .value(),
 };
 
 function encodeTypedArray(builder, uType, uData) {
@@ -112,7 +117,7 @@ function encodeTypedArray(builder, uType, uData) {
 function encodeMatrix(columns, colIndex = undefined) {
   /*
   IMPORTANT: this is not a general purpose encoder.  in particular,
-  it doesn't correctly handle all column index types, nor does it 
+  it doesn't correctly handle all column index types, nor does it
   handle all column typedarray types.
 
   encodeMatrixFBS in matrix.py is more general.  This is used only
@@ -120,10 +125,10 @@ function encodeMatrix(columns, colIndex = undefined) {
   */
   const utf8Encoder = new TextEncoder("utf-8");
   const builder = new flatbuffers.Builder(1024);
-  const cols = _.map(columns, carr => {
+  const cols = map(columns, (carr) => {
     let uType;
     let tarr;
-    if (_.every(carr, _.isNumber)) {
+    if (every(carr, isNumber)) {
       uType = NetEncoding.TypedArray.Float32Array;
       tarr = encodeTypedArray(builder, uType, new Float32Array(carr));
     } else {
@@ -166,19 +171,19 @@ function encodeMatrix(columns, colIndex = undefined) {
 }
 
 const anAnnotationsObsFBSResponse = (() => {
-  const columns = _.zip(...anAnnotationsObsJSONResponse.data).slice(1);
+  const columns = zip(...anAnnotationsObsJSONResponse.data).slice(1);
   return encodeMatrix(columns, anAnnotationsObsJSONResponse.names);
 })();
 
 const anAnnotationsVarFBSResponse = (() => {
-  const columns = _.zip(...anAnnotationsVarJSONResponse.data).slice(1);
+  const columns = zip(...anAnnotationsVarJSONResponse.data).slice(1);
   return encodeMatrix(columns, anAnnotationsVarJSONResponse.names);
 })();
 
 const aLayoutFBSResponse = (() => {
   const coords = [
     new Float32Array(nObs).fill(Math.random()),
-    new Float32Array(nObs).fill(Math.random())
+    new Float32Array(nObs).fill(Math.random()),
   ];
   return encodeMatrix(coords, ["umap_0", "umap_1"]);
 })();
@@ -187,8 +192,8 @@ const aDataObsResponse = {
   var: [2, 4, 29],
   obs: _()
     .range(nObs)
-    .map(idx => [idx, Math.random(), Math.random(), Math.random()])
-    .value()
+    .map((idx) => [idx, Math.random(), Math.random(), Math.random()])
+    .value(),
 };
 
 export {
@@ -197,5 +202,5 @@ export {
   anAnnotationsVarFBSResponse as annotationsVar,
   anAnnotationsObsFBSResponse as annotationsObs,
   aSchemaResponse as schema,
-  aConfigResponse as config
+  aConfigResponse as config,
 };

client/__tests__/util/stateManager/universe.test.js

@@ -1,68 +0,0 @@
-import * as Universe from "../../../src/util/stateManager/universe";
-import * as Dataframe from "../../../src/util/dataframe";
-import * as REST from "./sampleResponses";
-
-describe("createUniverseFromResponse", () => {
-  /*
-  test createUniverseFromResponse - this function converts
-  a set of REST 0.2 responses into a "new" Universe.
-
-  createUniverseFromResponse(
-    configResponse,
-    schemaResponse,
-    annotationsObsResponse,
-    annotationsVarResponse,
-    layoutObsResponse
-  ) --> Universe
-
-  where:
-  configResponse: GET /.../config
-  schemaResponse: GET /.../schema
-  annotationsObsResponse: GET /.../annotations/obs
-  annotationsVarResponse: GET /.../annotations/var
-  layoutObsResponse: GET /.../layout/obs
-
-  See spec in docs/REST_API.md.
-  */
-
-  test("create from test data", () => {
-    /*
-    create a universe from sample data nad validate its shape & contents
-    */
-    const { nObs, nVar } = REST.schema.schema.dataframe;
-    const universe = Universe.createUniverseFromResponse(
-      REST.config,
-      REST.schema,
-      REST.annotationsObs,
-      REST.annotationsVar,
-      REST.layoutObs
-    );
-
-    expect(universe).toBeDefined();
-    expect(universe).toMatchObject(
-      expect.objectContaining({
-        nObs,
-        nVar,
-        schema: REST.schema.schema,
-        obsAnnotations: expect.any(Dataframe.Dataframe),
-        varAnnotations: expect.any(Dataframe.Dataframe),
-        obsLayout: expect.any(Dataframe.Dataframe),
-        varData: expect.any(Dataframe.Dataframe)
-      })
-    );
-
-    expect(universe.obsAnnotations.dims).toEqual([
-      nObs,
-      REST.schema.schema.annotations.obs.columns.length
-    ]);
-    expect(universe.obsLayout.dims).toEqual([nObs, 2]);
-    expect(universe.obsLayout.colIndex.keys()).toEqual(
-      universe.schema.layout.obs[0].dims
-    );
-    expect(universe.varAnnotations.dims).toEqual([
-      nVar,
-      REST.schema.schema.annotations.var.columns.length
-    ]);
-    expect(universe.varData.isEmpty()).toBeTruthy();
-  });
-});

client/__tests__/util/stateManager/world.test.js

@@ -1,190 +0,0 @@
-import _ from "lodash";
-import * as Universe from "../../../src/util/stateManager/universe";
-import * as World from "../../../src/util/stateManager/world";
-import * as Dataframe from "../../../src/util/dataframe";
-import Crossfilter from "../../../src/util/typedCrossfilter";
-import { DimTypes } from "../../../src/util/typedCrossfilter/crossfilter";
-import * as REST from "./sampleResponses";
-import {
-  obsAnnoDimensionName,
-  layoutDimensionName
-} from "../../../src/util/nameCreators";
-
-/*
-Helper - creates universe, world, corssfilter and dimensionMap from
-the default REST test response.
-*/
-const defaultBigBang = () => {
-  /* create unverse, world, crossfilter and dimensionMap */
-  /* create universe */
-  const universe = Universe.createUniverseFromResponse(
-    _.cloneDeep(REST.config),
-    _.cloneDeep(REST.schema),
-    _.cloneDeep(REST.annotationsObs),
-    _.cloneDeep(REST.annotationsVar),
-    _.cloneDeep(REST.layoutObs)
-  );
-  /* create world */
-  const world = World.createWorldFromEntireUniverse(universe);
-  /* create crossfilter */
-  const crossfilter = World.createObsDimensions(
-    new Crossfilter(world.obsAnnotations),
-    world,
-    REST.schema.schema.layout.obs[0].dims
-  );
-
-  return {
-    universe,
-    world,
-    crossfilter
-  };
-};
-
-describe("createWorldFromEntireUniverse", () => {
-  test("create from REST sample", () => {
-    const universe = Universe.createUniverseFromResponse(
-      _.cloneDeep(REST.config),
-      _.cloneDeep(REST.schema),
-      _.cloneDeep(REST.annotationsObs),
-      _.cloneDeep(REST.annotationsVar),
-      _.cloneDeep(REST.layoutObs)
-    );
-    expect(universe).toBeDefined();
-
-    const world = World.createWorldFromEntireUniverse(universe);
-    expect(world).toBeDefined();
-
-    expect(world).toMatchObject(
-      expect.objectContaining({
-        nObs: universe.nObs,
-        nVar: universe.nVar,
-        schema: universe.schema,
-        obsAnnotations: expect.any(Dataframe.Dataframe),
-        varAnnotations: expect.any(Dataframe.Dataframe),
-        obsLayout: expect.any(Dataframe.Dataframe),
-        varData: expect.any(Dataframe.Dataframe),
-        clipQuantiles: { min: 0, max: 1 },
-        unclipped: {
-          obsAnnotations: expect.any(Dataframe.Dataframe),
-          varData: expect.any(Dataframe.Dataframe)
-        }
-      })
-    );
-  });
-});
-
-describe("createWorldFromCurrentSelection", () => {
-  test("create from REST sample", () => {
-    const {
-      universe,
-      world: originalWorld,
-      crossfilter: originalCrossfilter
-    } = defaultBigBang();
-
-    /* mock a selection */
-    const crossfilter = originalCrossfilter
-      .select(obsAnnoDimensionName("field1"), { mode: "range", lo: 0, hi: 5 })
-      .select(obsAnnoDimensionName("field3"), {
-        mode: "exact",
-        values: [false]
-      });
-
-    /* create the world from the selection */
-    const world = World.createWorldBySelection(
-      universe,
-      originalWorld,
-      crossfilter
-    );
-    expect(world).toBeDefined();
-    expect(world.nObs).toEqual(crossfilter.countSelected());
-
-    /*
-    calculate expected values and match against result
-    */
-
-    /* matchFilter must match the dimension filters above */
-    const matchFilter = (df, row) => {
-      const field1 = df.at(row, "field1");
-      const field3 = df.at(row, "field3");
-      return field1 >= 0 && field1 < 5 && !field3;
-    };
-    const matchingIndices = _()
-      .range(universe.nObs)
-      .filter(idx => matchFilter(universe.obsAnnotations, idx))
-      .value();
-
-    expect(world).toMatchObject(
-      expect.objectContaining({
-        nObs: matchingIndices.length,
-        nVar: universe.nVar,
-        schema: universe.schema,
-        clipQuantiles: { min: 0, max: 1 },
-        obsAnnotations: expect.any(Dataframe.Dataframe),
-        varAnnotations: expect.any(Dataframe.Dataframe),
-        obsLayout: expect.any(Dataframe.Dataframe),
-        varData: expect.any(Dataframe.Dataframe),
-        unclipped: {
-          obsAnnotations: expect.any(Dataframe.Dataframe),
-          varData: expect.any(Dataframe.Dataframe)
-        }
-      })
-    );
-
-    expect(world.obsAnnotations.rowIndex.keys()).toEqual(
-      new Int32Array(matchingIndices)
-    );
-    expect(world.obsAnnotations.colIndex.keys()).toEqual(
-      universe.obsAnnotations.colIndex.keys()
-    );
-    expect(world.obsLayout.rowIndex.keys()).toEqual(
-      new Int32Array(matchingIndices)
-    );
-    expect(world.obsLayout.colIndex.keys()).toEqual(
-      world.schema.layout.obs[0].dims
-    );
-  });
-});
-
-describe("createObsDimensionMap", () => {
-  test("when universe eq world", () => {
-    /*
-    check for:
-    - creates a dimension for all obsAnnotations, PLUS X/Y layout
-    - check that dimension typing is sane
-    */
-
-    const { crossfilter } = defaultBigBang();
-    const annotationNames = _.map(
-      REST.schema.schema.annotations.obs.columns,
-      c => c.name
-    );
-    const obsIndexColName = REST.schema.schema.annotations.obs.index;
-    const schemaByObsName = _.keyBy(
-      REST.schema.schema.annotations.obs.columns,
-      "name"
-    );
-    expect(crossfilter).toBeDefined();
-    annotationNames.forEach(name => {
-      const dim = crossfilter.dimensions[obsAnnoDimensionName(name)];
-      if (name === obsIndexColName) {
-        expect(dim).toBeUndefined();
-      } else {
-        const { type } = schemaByObsName[name];
-        if (type === "string" || type === "boolean" || type === "categorical") {
-          expect(dim.dim).toBeInstanceOf(DimTypes.enum);
-        } else {
-          expect(dim.dim).toBeInstanceOf(DimTypes.scalar);
-        }
-      }
-    });
-    expect(
-      crossfilter.dimensions[layoutDimensionName("XY")].dim
-    ).toBeInstanceOf(DimTypes.spatial);
-  });
-});
-
-describe("worldEqUniverse", () => {
-  const { universe, world } = defaultBigBang();
-  const result = World.worldEqUniverse(world, universe);
-  expect(result).toBe(true);
-});

client/__tests__/util/typedCrossfilter/bitArray.test.js

@@ -1,5 +1,3 @@
-// jshint esversion: 6
-
 import BitArray from "../../../src/util/typedCrossfilter/bitArray";
 
 const defaultTestLength = 8;
@@ -183,3 +181,24 @@ describe("fillBySelection", () => {
     expect(arr).toEqual(truth);
   });
 });
+
+describe("wide bitarray", () => {
+  test.each([9, 30, 31, 32, 33, 54, 63, 64, 65, 127, 128, 129])(
+    "more than %d dimensions",
+    (ndim) => {
+      /* ensure we move across the uint boundary correctly */
+
+      const ba = new BitArray(defaultTestLength);
+      expect(ba).toBeDefined();
+
+      for (let i = 0; i < ndim; i += 1) {
+        expect(ba.allocDimension()).toEqual(i);
+      }
+      ba.freeDimension(0);
+      expect(ba.allocDimension()).toEqual(0);
+
+      ba.freeDimension(ndim - 1);
+      expect(ba.allocDimension()).toEqual(ndim - 1);
+    }
+  );
+});

client/__tests__/util/typedCrossfilter/crossfilter.test.js

@@ -1,5 +1,5 @@
-import _ from "lodash";
-import { polygonContains } from "d3";
+import filter from "lodash.filter";
+import zip from "lodash.zip";
 
 import Crossfilter from "../../../src/util/typedCrossfilter";
 
@@ -12,7 +12,7 @@ const someData = [
     type: "tab",
     productIDs: ["001"],
     coords: [0, 0],
-    nonFinite: 0.0
+    nonFinite: 0.0,
   },
   {
     date: "2011-11-14T16:20:19Z",
@@ -22,7 +22,7 @@ const someData = [
     type: "tab",
     productIDs: ["001", "005"],
     coords: [0.4, 0.4],
-    nonFinite: Number.NaN
+    nonFinite: Number.NaN,
   },
   {
     date: "2011-11-14T16:28:54Z",
@@ -32,7 +32,7 @@ const someData = [
     type: "visa",
     productIDs: ["004", "005"],
     coords: [0.3, 0.1],
-    nonFinite: Number.POSITIVE_INFINITY
+    nonFinite: Number.POSITIVE_INFINITY,
   },
   {
     date: "2011-11-14T16:30:43Z",
@@ -42,7 +42,7 @@ const someData = [
     type: "tab",
     productIDs: ["001", "002"],
     coords: [0.392, 0.1],
-    nonFinite: Number.NEGATIVE_INFINITY
+    nonFinite: Number.NEGATIVE_INFINITY,
   },
   {
     date: "2011-11-14T16:48:46Z",
@@ -52,7 +52,7 @@ const someData = [
     type: "tab",
     productIDs: ["005"],
     coords: [0.7, 0.0482],
-    nonFinite: 1.0
+    nonFinite: 1.0,
   },
   {
     date: "2011-11-14T16:53:41Z",
@@ -62,7 +62,7 @@ const someData = [
     type: "tab",
     productIDs: ["001", "004", "005"],
     coords: [0.9999, 1.0],
-    nonFinite: Number.NaN
+    nonFinite: Number.NaN,
   },
   {
     date: "2011-11-14T16:54:06Z",
@@ -72,7 +72,7 @@ const someData = [
     type: "cash",
     productIDs: ["001", "002", "003", "004", "005"],
     coords: [0.384, 0.6938],
-    nonFinite: 99.0
+    nonFinite: 99.0,
   },
   {
     date: "2011-11-14T16:58:03Z",
@@ -82,7 +82,7 @@ const someData = [
     type: "tab",
     productIDs: ["001"],
     coords: [0.4822, 0.482],
-    nonFinite: Number.NaN
+    nonFinite: Number.NaN,
   },
   {
     date: "2011-11-14T17:07:21Z",
@@ -92,7 +92,7 @@ const someData = [
     type: "tab",
     productIDs: ["004", "005"],
     coords: [0.2234, 0],
-    nonFinite: Number.NaN
+    nonFinite: Number.NaN,
   },
   {
     date: "2011-11-14T17:22:59Z",
@@ -102,7 +102,7 @@ const someData = [
     type: "tab",
     productIDs: ["001", "002", "004", "005"],
     coords: [0.382, 0.38485],
-    nonFinite: -1
+    nonFinite: -1,
   },
   {
     date: "2011-11-14T17:25:45Z",
@@ -112,7 +112,7 @@ const someData = [
     type: "cash",
     productIDs: ["002"],
     coords: [0.998, 0.8472],
-    nonFinite: 0.0
+    nonFinite: 0.0,
   },
   {
     date: "2011-11-14T17:29:52Z",
@@ -122,8 +122,8 @@ const someData = [
     type: "visa",
     productIDs: ["004"],
     coords: [0.8273, 0.3384],
-    nonFinite: 0.0
-  }
+    nonFinite: 0.0,
+  },
 ];
 
 let payments = null;
@@ -249,16 +249,26 @@ describe("ImmutableTypedCrossfilter", () => {
     test("none", () => {
       expect(p.select("quantity", { mode: "none" }).countSelected()).toEqual(0);
     });
-    test.each([[[]], [[2]], [[2, 1]], [[9, 82]], [[0, 1]]])("exact: %p", v =>
+    test.each([[[]], [[2]], [[2, 1]], [[9, 82]], [[0, 1]]])("exact: %p", (v) =>
       expect(
         p.select("quantity", { mode: "exact", values: v }).countSelected()
-      ).toEqual(_.filter(someData, d => v.includes(d.quantity)).length)
+      ).toEqual(filter(someData, (d) => v.includes(d.quantity)).length)
     );
-    test.each([[0, 1], [1, 2], [0, 99], [99, 100000]])("range %p", (lo, hi) =>
+    test("single value exact", () => {
+      expect(
+        p.select("quantity", { mode: "exact", values: 2 }).countSelected()
+      ).toEqual(filter(someData, (d) => d.quantity === 2).length);
+    });
+    test.each([
+      [0, 1],
+      [1, 2],
+      [0, 99],
+      [99, 100000],
+    ])("range %p", (lo, hi) =>
       expect(
         p.select("quantity", { mode: "range", lo, hi }).countSelected()
       ).toEqual(
-        _.filter(someData, d => d.quantity >= lo && d.quantity < hi).length
+        filter(someData, (d) => d.quantity >= lo && d.quantity < hi).length
       )
     );
     test("bad mode", () => {
@@ -285,12 +295,17 @@ describe("ImmutableTypedCrossfilter", () => {
       [["tab"]],
       [["visa"]],
       [["visa", "tab"]],
-      [["cash", "tab", "visa"]]
-    ])("exact: %p", v =>
+      [["cash", "tab", "visa"]],
+    ])("exact: %p", (v) =>
       expect(
         p.select("type", { mode: "exact", values: v }).countSelected()
-      ).toEqual(_.filter(someData, d => v.includes(d.type)).length)
+      ).toEqual(filter(someData, (d) => v.includes(d.type)).length)
     );
+    test("single value exact", () => {
+      expect(
+        p.select("type", { mode: "exact", values: "tab" }).countSelected()
+      ).toEqual(filter(someData, (d) => d.type === "tab").length);
+    });
     test("range", () => {
       expect(() => p.select("type", { mode: "range", lo: 0, hi: 9 })).toThrow(
         Error
@@ -304,8 +319,8 @@ describe("ImmutableTypedCrossfilter", () => {
   describe("spatial dimension", () => {
     let p;
     beforeEach(() => {
-      const X = someData.map(r => r.coords[0]);
-      const Y = someData.map(r => r.coords[1]);
+      const X = someData.map((r) => r.coords[0]);
+      const Y = someData.map((r) => r.coords[1]);
       p = payments.addDimension("coords", "spatial", X, Y);
     });
 
@@ -317,29 +332,76 @@ describe("ImmutableTypedCrossfilter", () => {
     test("none", () => {
       expect(p.select("coords", { mode: "none" }).countSelected()).toEqual(0);
     });
-    test.each([[0, 0, 1, 1], [0, 0, 0.5, 0.5], [0.5, 0.5, 1, 1]])(
-      "within-rect %d %d %d %d",
-      (minX, minY, maxX, maxY) => {
-        expect(
-          p
-            .select("coords", { mode: "within-rect", minX, minY, maxX, maxY })
-            .allSelected()
-        ).toEqual(
-          _.filter(someData, d => {
-            const [x, y] = d.coords;
-            return minX <= x && x < maxX && minY <= y && y < maxY;
-          })
-        );
-      }
-    );
+    test.each([
+      [0, 0, 1, 1],
+      [0, 0, 0.5, 0.5],
+      [0.5, 0.5, 1, 1],
+    ])("within-rect %d %d %d %d", (minX, minY, maxX, maxY) => {
+      expect(
+        p
+          .select("coords", { mode: "within-rect", minX, minY, maxX, maxY })
+          .allSelected()
+      ).toEqual(
+        filter(someData, (d) => {
+          const [x, y] = d.coords;
+          return minX <= x && x < maxX && minY <= y && y < maxY;
+        })
+      );
+    });
 
     test.each([
-      [[[0, 0], [0, 1], [1, 1], [1, 0]]],
-      [[[0, 0], [0, 0.5], [0.5, 0.5], [0.5, 0]]]
-    ])("within-polygon %p", polygon => {
+      [
+        [
+          [0, 0],
+          [0, 1],
+          [1, 1],
+          [1, 0],
+        ],
+        [
+          true,
+          true,
+          true,
+          true,
+          true,
+          false,
+          true,
+          true,
+          true,
+          true,
+          true,
+          true,
+        ],
+      ],
+      [
+        [
+          [0, 0],
+          [0, 0.5],
+          [0.5, 0.5],
+          [0.5, 0],
+        ],
+        [
+          true,
+          true,
+          true,
+          true,
+          false,
+          false,
+          false,
+          true,
+          true,
+          true,
+          false,
+          false,
+        ],
+      ],
+    ])("within-polygon %p", (polygon, expected) => {
       expect(
         p.select("coords", { mode: "within-polygon", polygon }).allSelected()
-      ).toEqual(_.filter(someData, d => polygonContains(polygon, d.coords)));
+      ).toEqual(
+        zip(someData, expected)
+          .filter((x) => x[1])
+          .map((x) => x[0])
+      );
     });
   });
 
@@ -377,7 +439,7 @@ describe("ImmutableTypedCrossfilter", () => {
         p
           .select("nonFinite", {
             mode: "exact",
-            values: [Number.POSITIVE_INFINITY]
+            values: [Number.POSITIVE_INFINITY],
           })
           .countSelected()
       ).toEqual(1);
@@ -385,7 +447,7 @@ describe("ImmutableTypedCrossfilter", () => {
         p
           .select("nonFinite", {
             mode: "exact",
-            values: [Number.NEGATIVE_INFINITY]
+            values: [Number.NEGATIVE_INFINITY],
           })
           .countSelected()
       ).toEqual(1);
@@ -398,7 +460,7 @@ describe("ImmutableTypedCrossfilter", () => {
         p
           .select("nonFinite", {
             mode: "exact",
-            values: [Number.POSITIVE_INFINITY, 0, 1, 99]
+            values: [Number.POSITIVE_INFINITY, 0, 1, 99],
           })
           .countSelected()
       ).toEqual(6);
@@ -410,7 +472,7 @@ describe("ImmutableTypedCrossfilter", () => {
           .select("nonFinite", {
             mode: "range",
             lo: 0,
-            hi: Number.POSITIVE_INFINITY
+            hi: Number.POSITIVE_INFINITY,
           })
           .countSelected()
       ).toEqual(5);
@@ -419,7 +481,7 @@ describe("ImmutableTypedCrossfilter", () => {
           .select("nonFinite", {
             mode: "range",
             lo: 0,
-            hi: Number.NaN
+            hi: Number.NaN,
           })
           .countSelected()
       ).toEqual(6);
@@ -428,7 +490,7 @@ describe("ImmutableTypedCrossfilter", () => {
           .select("nonFinite", {
             mode: "range",
             lo: Number.NEGATIVE_INFINITY,
-            hi: Number.POSITIVE_INFINITY
+            hi: Number.POSITIVE_INFINITY,
           })
           .countSelected()
       ).toEqual(7);

client/__tests__/util/typedCrossfilter/positiveInterval.test.js

@@ -1,5 +1,3 @@
-// jshint esversion: 6
-
 // const PositiveIntervals = require("../../src/util/typedCrossfilter/positiveIntervals");
 import PositiveIntervals from "../../../src/util/typedCrossfilter/positiveIntervals";
 
@@ -10,15 +8,30 @@ describe("canonicalize", () => {
 
   test("simple, already correct", () => {
     expect(PositiveIntervals.canonicalize([[0, 1]])).toEqual([[0, 1]]);
-    expect(PositiveIntervals.canonicalize([[0, 1], [2, 3]])).toEqual([
+    expect(
+      PositiveIntervals.canonicalize([
+        [0, 1],
+        [2, 3],
+      ])
+    ).toEqual([
       [0, 1],
-      [2, 3]
+      [2, 3],
     ]);
   });
 
   test("non-canonical, need to be canonicalized", () => {
-    expect(PositiveIntervals.canonicalize([[0, 1], [1, 2]])).toEqual([[0, 2]]);
-    expect(PositiveIntervals.canonicalize([[1, 2], [2, 3]])).toEqual([[1, 3]]);
+    expect(
+      PositiveIntervals.canonicalize([
+        [0, 1],
+        [1, 2],
+      ])
+    ).toEqual([[0, 2]]);
+    expect(
+      PositiveIntervals.canonicalize([
+        [1, 2],
+        [2, 3],
+      ])
+    ).toEqual([[1, 3]]);
   });
 });
 
@@ -26,14 +39,30 @@ describe("union", () => {
   test("empty range", () => {
     expect(PositiveIntervals.union([], [])).toEqual([]);
     expect(PositiveIntervals.union([], [[1, 2]])).toEqual([[1, 2]]);
-    expect(PositiveIntervals.union([], [[1, 2], [3, 4]])).toEqual([
+    expect(
+      PositiveIntervals.union(
+        [],
+        [
+          [1, 2],
+          [3, 4],
+        ]
+      )
+    ).toEqual([
       [1, 2],
-      [3, 4]
+      [3, 4],
     ]);
     expect(PositiveIntervals.union([[3, 4]], [])).toEqual([[3, 4]]);
-    expect(PositiveIntervals.union([[1, 2], [3, 4]], [])).toEqual([
+    expect(
+      PositiveIntervals.union(
+        [
+          [1, 2],
+          [3, 4],
+        ],
+        []
+      )
+    ).toEqual([
       [1, 2],
-      [3, 4]
+      [3, 4],
     ]);
     expect(PositiveIntervals.union([[3, 3]], [])).toEqual([[3, 3]]);
     expect(PositiveIntervals.union([], [[3, 3]])).toEqual([[3, 3]]);
@@ -44,17 +73,37 @@ describe("union", () => {
     expect(PositiveIntervals.union([[2, 3]], [[1, 2]])).toEqual([[1, 3]]);
     expect(PositiveIntervals.union([[1, 2]], [[3, 4]])).toEqual([
       [1, 2],
-      [3, 4]
+      [3, 4],
     ]);
     expect(
-      PositiveIntervals.union([[1, 2], [3, 4]], [[6, 7], [19, 40]])
-    ).toEqual([[1, 2], [3, 4], [6, 7], [19, 40]]);
-    expect(PositiveIntervals.union([[1, 4]], [[1, 1], [3, 4]])).toEqual([
-      [1, 4]
+      PositiveIntervals.union(
+        [
+          [1, 2],
+          [3, 4],
+        ],
+        [
+          [6, 7],
+          [19, 40],
+        ]
+      )
+    ).toEqual([
+      [1, 2],
+      [3, 4],
+      [6, 7],
+      [19, 40],
     ]);
+    expect(
+      PositiveIntervals.union(
+        [[1, 4]],
+        [
+          [1, 1],
+          [3, 4],
+        ]
+      )
+    ).toEqual([[1, 4]]);
     expect(PositiveIntervals.union([[3, 3]], [[4, 4]])).toEqual([
       [3, 3],
-      [4, 4]
+      [4, 4],
     ]);
   });
 });
@@ -70,34 +119,43 @@ describe("intersection", () => {
     expect(PositiveIntervals.intersection([[1, 2]], [[2, 3]])).toEqual([]);
     expect(PositiveIntervals.intersection([[2, 3]], [[1, 2]])).toEqual([]);
     expect(PositiveIntervals.intersection([[1, 10]], [[1, 10]])).toEqual([
-      [1, 10]
+      [1, 10],
     ]);
     expect(PositiveIntervals.intersection([[1, 10]], [[2, 8]])).toEqual([
-      [2, 8]
+      [2, 8],
     ]);
     expect(PositiveIntervals.intersection([[2, 8]], [[1, 10]])).toEqual([
-      [2, 8]
+      [2, 8],
     ]);
     expect(PositiveIntervals.intersection([[1, 10]], [[2, 12]])).toEqual([
-      [2, 10]
+      [2, 10],
     ]);
     expect(PositiveIntervals.intersection([[2, 12]], [[1, 10]])).toEqual([
-      [2, 10]
+      [2, 10],
     ]);
     expect(PositiveIntervals.intersection([[1, 10]], [[1, 8]])).toEqual([
-      [1, 8]
+      [1, 8],
     ]);
     expect(PositiveIntervals.intersection([[1, 8]], [[1, 10]])).toEqual([
-      [1, 8]
+      [1, 8],
+    ]);
+    expect(
+      PositiveIntervals.intersection(
+        [[1, 10]],
+        [
+          [1, 2],
+          [6, 9],
+        ]
+      )
+    ).toEqual([
+      [1, 2],
+      [6, 9],
     ]);
-    expect(PositiveIntervals.intersection([[1, 10]], [[1, 2], [6, 9]])).toEqual(
-      [[1, 2], [6, 9]]
-    );
     expect(PositiveIntervals.intersection([[0, 2638]], [[1363, 2638]])).toEqual(
       [[1363, 2638]]
     );
     expect(PositiveIntervals.intersection([[1, 2]], [[1, 2]])).toEqual([
-      [1, 2]
+      [1, 2],
     ]);
   });
 });
@@ -110,32 +168,66 @@ describe("difference", () => {
   });
 
   test("simple", () => {
-    expect(PositiveIntervals.difference([[1, 2], [3, 4]], [])).toEqual([
+    expect(
+      PositiveIntervals.difference(
+        [
+          [1, 2],
+          [3, 4],
+        ],
+        []
+      )
+    ).toEqual([
       [1, 2],
-      [3, 4]
-    ]);
-    expect(PositiveIntervals.difference([[1, 2], [3, 10]], [[5, 10]])).toEqual([
-      [1, 2],
-      [3, 5]
-    ]);
-    expect(PositiveIntervals.difference([[1, 2], [3, 10]], [[0, 5]])).toEqual([
-      [5, 10]
+      [3, 4],
     ]);
     expect(
-      PositiveIntervals.difference([[0, 2638]], [[0, 1363], [2055, 2638]])
+      PositiveIntervals.difference(
+        [
+          [1, 2],
+          [3, 10],
+        ],
+        [[5, 10]]
+      )
+    ).toEqual([
+      [1, 2],
+      [3, 5],
+    ]);
+    expect(
+      PositiveIntervals.difference(
+        [
+          [1, 2],
+          [3, 10],
+        ],
+        [[0, 5]]
+      )
+    ).toEqual([[5, 10]]);
+    expect(
+      PositiveIntervals.difference(
+        [[0, 2638]],
+        [
+          [0, 1363],
+          [2055, 2638],
+        ]
+      )
     ).toEqual([[1363, 2055]]);
     expect(
-      PositiveIntervals.difference([[0, 1363], [2055, 2638]], [[0, 2638]])
+      PositiveIntervals.difference(
+        [
+          [0, 1363],
+          [2055, 2638],
+        ],
+        [[0, 2638]]
+      )
     ).toEqual([]);
     expect(PositiveIntervals.difference([[0, 10]], [[0, 1]])).toEqual([
-      [1, 10]
+      [1, 10],
     ]);
     expect(PositiveIntervals.difference([[0, 10]], [[1, 2]])).toEqual([
       [0, 1],
-      [2, 10]
+      [2, 10],
     ]);
     expect(PositiveIntervals.difference([[0, 10]], [[9, 10]])).toEqual([
-      [0, 9]
+      [0, 9],
     ]);
   });
 });

client/__tests__/util/typedCrossfilter/sort.test.js

@@ -2,9 +2,6 @@ import {
   sortArray,
   sortIndex,
   lowerBound,
-  upperBound,
-  lowerBoundIndirect,
-  upperBoundIndirect
 } from "../../../src/util/typedCrossfilter/sort";
 
 /*
@@ -40,7 +37,7 @@ describe("sortArray", () => {
       ["a", "b", "0", "1"],
       [0, "a", true, null, undefined, 3.1415],
       fillRand(new Array(1000)),
-      ["a", NaN, null, pInf]
+      ["a", NaN, null, pInf],
     ].map((val, idx) =>
       test(`JS vals ${idx}`, () => {
         expect(sortArray(val)).toMatchObject(val.sort());
@@ -49,7 +46,7 @@ describe("sortArray", () => {
   });
 
   describe("finite numbers", () => {
-    [Array, Float32Array, Uint32Array, Int32Array, Float64Array].map(Type =>
+    [Array, Float32Array, Uint32Array, Int32Array, Float64Array].map((Type) =>
       test(Type.name, () => {
         expect(sortArray(Type.from([6, 5, 4, 3, 2, 1, 0]))).toMatchObject(
           Type.from([0, 1, 2, 3, 4, 5, 6])
@@ -68,7 +65,7 @@ describe("sortArray", () => {
   });
 
   describe("non-finite numbers", () => {
-    test("inifinity", () => {
+    test("infinity", () => {
       expect(sortArray(new Float32Array([pInf, nInf, 0, 1, 2]))).toMatchObject(
         new Float32Array([nInf, 0, 1, 2, pInf])
       );
@@ -131,7 +128,7 @@ describe("sortArray", () => {
 
 describe("sortIndex", () => {
   describe("finite numbers", () => {
-    [Array, Float32Array, Uint32Array, Int32Array, Float64Array].map(Type =>
+    [Array, Float32Array, Uint32Array, Int32Array, Float64Array].map((Type) =>
       test(Type.name, () => {
         const source1 = Type.from([6, 5, 4, 3, 2, 1, 0]);
         const index1 = fillRange(new Uint32Array(source1.length));

client/__tests__/util/typedCrossfilter/util.test.js

@@ -1,6 +1,6 @@
 import {
   sliceByIndex,
-  makeSortIndex
+  makeSortIndex,
 } from "../../../src/util/typedCrossfilter/util";
 import { rangeFill as fillRange } from "../../../src/util/range";
 

client/configuration/babel/babel.dev.js

@@ -2,15 +2,24 @@ module.exports = {
   babelrc: false,
   cacheDirectory: true,
   presets: [
-    ["modern-browsers", { loose: true, modules: false }],
-    "@babel/preset-react"
+    [
+      "@babel/preset-env",
+      {
+        useBuiltIns: "entry",
+        corejs: 3,
+        modules: false,
+      },
+    ],
+    "@babel/preset-react",
   ],
   plugins: [
     "@babel/plugin-proposal-function-bind",
     ["@babel/plugin-proposal-decorators", { legacy: true }],
     ["@babel/plugin-proposal-class-properties", { loose: true }],
+    ["@babel/plugin-transform-private-methods", { loose: true }],
+    ["@babel/plugin-transform-private-property-in-object", { loose: true }],
     "@babel/plugin-proposal-export-namespace-from",
     "@babel/plugin-proposal-optional-chaining",
-    "@babel/plugin-proposal-nullish-coalescing-operator"
-  ]
+    "@babel/plugin-proposal-nullish-coalescing-operator",
+  ],
 };

client/configuration/babel/babel.prod.js

@@ -1,17 +1,26 @@
 module.exports = {
   babelrc: false,
   presets: [
-    ["modern-browsers", { loose: true, modules: false }],
-    "@babel/preset-react"
+    [
+      "@babel/preset-env",
+      {
+        useBuiltIns: "entry",
+        corejs: 3,
+        modules: false,
+      },
+    ],
+    "@babel/preset-react",
   ],
   plugins: [
     "@babel/plugin-proposal-function-bind",
     ["@babel/plugin-proposal-decorators", { legacy: true }],
     ["@babel/plugin-proposal-class-properties", { loose: true }],
+    ["@babel/plugin-transform-private-methods", { loose: true }],
+    ["@babel/plugin-transform-private-property-in-object", { loose: true }],
     "@babel/plugin-proposal-export-namespace-from",
     "@babel/plugin-transform-react-constant-elements",
     "@babel/plugin-transform-runtime",
     "@babel/plugin-proposal-optional-chaining",
-    "@babel/plugin-proposal-nullish-coalescing-operator"
-  ]
+    "@babel/plugin-proposal-nullish-coalescing-operator",
+  ],
 };

client/configuration/eslint/eslint.js

@@ -1,18 +1,48 @@
 module.exports = {
   root: true,
-  parser: "babel-eslint",
-  extends: ["airbnb", "prettier", "prettier/react"],
+
+  extends: [
+    "airbnb",
+    "plugin:eslint-comments/recommended",
+    "plugin:@blueprintjs/recommended",
+    "plugin:compat/recommended",
+    "plugin:jsx-a11y/recommended",
+    // (thuang) disable eslint formatting rules, so prettier can do its job
+    // Do not use `plugin:prettier/recommended` per doc below:
+    // https://prettier.io/docs/en/integrating-with-linters.html
+    "prettier",
+  ],
+  settings: {
+    // AbortController is not supported in iOS Safari 10.3, Chrome 61
+    // Headers is not supported in iOS Safari 10.3
+    polyfills: ["Headers", "AbortController"],
+  },
   env: { browser: true, commonjs: true, es6: true },
-  globals: { expect: true },
+  globals: {
+    expect: true,
+    jest: true,
+    jestPuppeteer: true,
+    it: true,
+    page: true,
+    browser: true,
+    context: true,
+    beforeEach: true,
+  },
+  parser: "@babel/eslint-parser",
   parserOptions: {
     ecmaVersion: 2017,
     sourceType: "module",
     ecmaFeatures: {
       jsx: true,
-      generators: true
-    }
+      generators: true,
+    },
+    babelOptions: {
+      configFile: "./configuration/babel/babel.prod.js",
+    },
   },
   rules: {
+    "react/jsx-no-target-blank": "off",
+    "eslint-comments/require-description": ["error"],
     "no-magic-numbers": "off",
     "no-nested-ternary": "off",
     "func-style": "off",
@@ -21,13 +51,7 @@ module.exports = {
     "react/jsx-filename-extension": "off",
     "comma-dangle": "off",
     "no-underscore-dangle": "off",
-    quotes: ["error", "double"],
     "implicit-arrow-linebreak": "off",
-    "operator-linebreak": [
-      "error",
-      "after",
-      { overrides: { "?": "before", ":": "before" } }
-    ],
     "no-console": "off",
     "spaced-comment": ["error", "always", { exceptions: ["*"] }],
     "no-param-reassign": "off",
@@ -35,13 +59,26 @@ module.exports = {
     "react/prop-types": [0],
     "space-before-function-paren": "off",
     "function-paren-newline": "off",
-    "prefer-destructuring": ["error", { object: true, array: false }]
+    "prefer-destructuring": ["error", { object: true, array: false }],
+    "import/prefer-default-export": "off",
+    "no-restricted-syntax": [
+      "error",
+      "ForInStatement",
+      "LabeledStatement",
+      "WithStatement",
+    ],
+    "import/no-extraneous-dependencies": [
+      "error",
+      {
+        devDependencies: true,
+      },
+    ],
   },
   overrides: [
     {
       files: ["**/*.test.js"],
       env: {
-        jest: true // now **/*.test.js files' env has both es6 *and* jest
+        jest: true, // now **/*.test.js files' env has both es6 *and* jest
       },
       // Can't extend in overrides: https://github.com/eslint/eslint/issues/8813
       // "extends": ["plugin:jest/recommended"]
@@ -51,8 +88,8 @@ module.exports = {
         "jest/no-focused-tests": "error",
         "jest/no-identical-title": "error",
         "jest/prefer-to-have-length": "warn",
-        "jest/valid-expect": "error"
-      }
-    }
-  ]
+        "jest/valid-expect": "error",
+      },
+    },
+  ],
 };

client/configuration/lint-staged/lint-staged.config.js

@@ -0,0 +1,4 @@
+module.exports = {
+  "*.js": "eslint --fix",
+  "**/*": "prettier --write --ignore-unknown",
+};

client/configuration/webpack/cspHashPlugin.js

@@ -0,0 +1,53 @@
+/* eslint-disable import/no-extraneous-dependencies -- this file is a devDependency*/
+const cheerio = require("cheerio");
+const crypto = require("crypto");
+const HtmlWebpackPlugin = require("html-webpack-plugin");
+
+const digest = (str) => {
+  const hash = crypto.createHash("sha256").update(str, "utf8").digest("base64");
+  return `sha256-${hash}`;
+};
+class CspHashPlugin {
+  constructor(opts) {
+    this.opts = { ...opts };
+  }
+
+  apply(compiler) {
+    compiler.hooks.compilation.tap("CspHashPlugin", (compilation) => {
+      HtmlWebpackPlugin.getHooks(compilation).beforeEmit.tapAsync(
+        "CspHashPlugin",
+        (data, cb) => {
+          const { filename } = this.opts;
+
+          const $ = cheerio.load(data.html, { decodeEntities: false });
+
+          if (filename) {
+            const results = {};
+            results["script-hashes"] = $("script:not([src]):not([no-csp-hash])")
+              .map((i, elmt) => digest($(elmt).html()))
+              .get();
+
+            const json = JSON.stringify(results);
+            compilation.assets[filename] = {
+              source: () => json,
+              size: () => json.length,
+            };
+          }
+
+          // Remove no-csp-hash attributes. Cheerio does not parse Jinja templates
+          // correctly, so we brute force this with a regular expression.
+          data.html = data.html.replace(
+            /(<script .*)no-csp-hash(.*>)/,
+            (match, p1, p2) => [p1, p2].join("")
+          );
+
+          // Tell webpack to move on
+          cb(null, data);
+        }
+      );
+    });
+  }
+}
+
+module.exports = CspHashPlugin;
+/* eslint-enable import/no-extraneous-dependencies -- enable*/

client/configuration/webpack/obsoleteHTMLTemplate.html

@@ -0,0 +1,75 @@
+<script>
+  /** Reflect ANY changes to this script in the script hash in `server/eb/app.py` **/
+  var root = document.getElementById("root");
+  root.remove();
+  var portals = document.getElementsByClassName("bp3-portal");
+  for (var i = 0; i < portals.length; i += 1) {
+    portals[i].remove();
+  }
+</script>
+<div
+  style="
+    display: flex;
+    flex-direction: column;
+    width: 100vw;
+    height: 100vh;
+    text-align: center;
+    justify-content: center;
+    align-items: center;
+    background: #8080801a;
+    font-family: 'Roboto Condensed, sans serif';
+  "
+>
+  <img
+    src="https://cellxgene.cziscience.com/s3/cellxgene/static/images/cellxgene-logo.png"
+    style="width: 320px;"
+  />
+  <div
+    style="
+      margin-top: 16px;
+      background: white;
+      width: 40vw;
+      border-radius: 4px;
+      padding: 24px 64px;
+      -webkit-box-shadow: 0px 0px 3px 2px rgba(0, 0, 0, 0.38);
+      -moz-box-shadow: 0px 0px 3px 2px rgba(0, 0, 0, 0.38);
+      box-shadow: 0px 0px 3px 2px rgba(0, 0, 0, 0.38);
+      max-width: 550px;
+    "
+  >
+    <div style="margin-bottom: 0; font-weight: bolder; font-size: 1.2em;">
+      Unsupported Browser
+    </div>
+    <div style="margin-top: 0;">
+      cellxgene is currently supported on the following browsers
+    </div>
+    <div
+      style="display: flex; justify-content: space-around; margin-top: 16px;"
+    >
+      <a
+        href="https://www.google.com/chrome/?hl=en%22"
+        aria-label="Download Google Chrome"
+      >
+        <img
+          src="https://cellxgene.cziscience.com/s3/cellxgene/static/images/chrome.png"
+          style="width: 80px; height: 80px;"
+        />
+        <div>Chrome &gt; 60</div>
+      </a>
+      <a href="https://www.mozilla.com/firefox/" aria-label="Download Firefox">
+        <img
+          src="https://cellxgene.cziscience.com/s3/cellxgene/static/images/firefox.png"
+          style="width: 80px; height: 80px;"
+        />
+        <div>Firefox ≥ 60</div>
+      </a>
+      <a href="//www.microsoft.com/edge" aria-label="Download Edge">
+        <img
+          src="https://cellxgene.cziscience.com/s3/cellxgene/static/images/edge.png"
+          style="width: 80px; height: 80px;"
+        />
+        <div>Edge ≥ 79</div>
+      </a>
+    </div>
+  </div>
+</div>

client/configuration/webpack/webpack.config.dev.js

@@ -1,86 +1,66 @@
-// jshint esversion: 6
 const path = require("path");
 const webpack = require("webpack");
 const HtmlWebpackPlugin = require("html-webpack-plugin");
+const MiniCssExtractPlugin = require("mini-css-extract-plugin");
 
-const src = path.resolve("src");
-const nodeModules = path.resolve("node_modules");
+const { merge } = require("webpack-merge");
 
+const sharedConfig = require("./webpack.config.shared");
 const babelOptions = require("../babel/babel.dev");
 
-module.exports = {
+const fonts = path.resolve("src/fonts");
+const images = path.resolve("src/images");
+const nodeModules = path.resolve("node_modules");
+
+const devConfig = {
   mode: "development",
   devtool: "eval",
-  entry: ["./src/index"],
   output: {
-    path: path.resolve("build"),
     pathinfo: true,
     filename: "static/js/bundle.js",
-    publicPath: "/"
   },
   module: {
     rules: [
       {
-        test: /\.js$/,
-        include: src,
+        test: /\.jsx?$/,
         loader: "babel-loader",
-        options: babelOptions
+        options: babelOptions,
       },
       {
-        test: /\.css$/,
-        include: src,
-        exclude: [path.resolve(src, "index.css")],
-        loader: [
-          {
-            loader: "style-loader"
-          },
-          {
-            loader: "css-loader",
-            options: {
-              modules: {
-                localIdentName: "[name]__[local]___[hash:base64:5]"
-              }
-            }
-          }
-        ]
-      },
-      {
-        test: /index\.css$/,
-        include: [path.resolve(src, "index.css")],
-        loader: [
-          {
-            loader: "style-loader"
-          },
-          {
-            loader: "css-loader"
-          }
-        ]
-      },
-
-      { test: /\.json$/, include: [src, nodeModules], loader: "json-loader" },
-      {
-        test: /\.(jpg|png|gif|eot|svg|ttf|woff|woff2)(\?.*)?$/,
-        include: [src, nodeModules],
+        test: /\.(jpg|png|gif|eot|svg|ttf|woff|woff2|otf)$/i,
         loader: "file-loader",
-        query: { name: "static/media/[name].[ext]" }
+        include: [nodeModules, fonts, images],
+        options: {
+          name: "static/assets/[name].[ext]",
+          // (thuang): This is needed to make sure @font url path is '/static/assets/'
+          publicPath: "..",
+        },
       },
-      {
-        test: /\.(mp4|webm)(\?.*)?$/,
-        include: [src, nodeModules],
-        loader: "url-loader",
-        query: { limit: 10000, name: "static/media/[name].[ext]" }
-      }
-    ]
+    ],
   },
   plugins: [
     new HtmlWebpackPlugin({
       inject: true,
       template: path.resolve("index.html"),
-      favicon: path.resolve("favicon.png")
+    }),
+    new MiniCssExtractPlugin({
+      filename: "static/[name].css",
     }),
     new webpack.NoEmitOnErrorsPlugin(),
     new webpack.DefinePlugin({
-      __REACT_DEVTOOLS_GLOBAL_HOOK__: "({ isDisabled: true })"
-    })
-  ]
+      __REACT_DEVTOOLS_GLOBAL_HOOK__: "({ isDisabled: true })",
+    }),
+    new webpack.DefinePlugin({
+      // webpack 5 no longer polyfills NodeJS modules, so fake the one we need
+      "process.env": JSON.stringify({
+        NODE_ENV: process.env.NODE_ENV || "development",
+        CXG_SERVER_PORT: process.env.CXG_SERVER_PORT || "5005",
+      }),
+    }),
+  ],
+  infrastructureLogging: {
+    level: "warn",
+  },
 };
+
+module.exports = merge(sharedConfig, devConfig);

client/configuration/webpack/webpack.config.prod.js

@@ -1,120 +1,87 @@
-// jshint esversion: 6
 const path = require("path");
+const webpack = require("webpack");
 const HtmlWebpackPlugin = require("html-webpack-plugin");
+const { CleanWebpackPlugin } = require("clean-webpack-plugin");
+const TerserJSPlugin = require("terser-webpack-plugin");
+const CssMinimizerPlugin = require("css-minimizer-webpack-plugin");
 const MiniCssExtractPlugin = require("mini-css-extract-plugin");
-const SWPrecacheWebpackPlugin = require("sw-precache-webpack-plugin");
-const HtmlWebpackInlineSourcePlugin = require("html-webpack-inline-source-plugin");
 
-const src = path.resolve("src");
-const nodeModules = path.resolve("node_modules");
+const { merge } = require("webpack-merge");
 
 const babelOptions = require("../babel/babel.prod");
 
-const publicPath = "";
+const CspHashPlugin = require("./cspHashPlugin");
+const sharedConfig = require("./webpack.config.shared");
 
-module.exports = {
+const fonts = path.resolve("src/fonts");
+const images = path.resolve("src/images");
+const nodeModules = path.resolve("node_modules");
+
+const prodConfig = {
   mode: "production",
   bail: true,
   cache: false,
-  devtool: "cheap-source-map",
-  entry: ["./src/index.js"],
   output: {
-    path: path.resolve("build"),
-    filename: "static/js/[name].[chunkhash:8].js",
-    publicPath
+    filename: "static/[name]-[contenthash].js",
   },
+  optimization: {
+    minimize: true,
+    minimizer: [
+      new TerserJSPlugin({}),
+      new CssMinimizerPlugin({
+        minify: CssMinimizerPlugin.cleanCssMinify,
+      }),
+    ],
+  },
+  devtool: "source-map",
   module: {
     rules: [
       {
-        test: /\.js$/,
-        include: src,
+        test: /\.jsx?$/,
         loader: "babel-loader",
-        options: babelOptions
+        options: babelOptions,
       },
       {
-        test: /\.css$/,
-        include: src,
-        exclude: [path.resolve(src, "index.css")],
-        use: [
-          MiniCssExtractPlugin.loader,
-          {
-            loader: "css-loader",
-            options: {
-              modules: {
-                localIdentName: "[name]__[local]___[hash:base64:5]"
-              },
-              importLoaders: 1
-            }
-          }
-        ]
-      },
-      {
-        test: /index\.css$/,
-        include: [path.resolve(src, "index.css")],
-        use: [
-          MiniCssExtractPlugin.loader,
-          {
-            loader: "css-loader",
-            options: {
-              importLoaders: 1
-            }
-          }
-        ]
-      },
-      {
-        test: /\.json$/,
-        include: [src, nodeModules],
-        loader: "json-loader",
-        exclude: /manifest.json$/
-      },
-      {
-        test: /\.(jpg|png|gif|eot|svg|ttf|woff|woff2)(\?.*)?$/,
-        include: [src, nodeModules],
+        test: /\.(jpg|png|gif|eot|svg|ttf|woff|woff2|otf)$/i,
         loader: "file-loader",
+        include: [nodeModules, fonts, images],
         options: {
-          name: "[name].[hash:8].[ext]",
-          outputPath: "static/media/"
-        }
+          name: "static/assets/[name]-[contenthash].[ext]",
+          // (thuang): This is needed to make sure @font url path is '../static/assets/'
+          publicPath: "..",
+        },
       },
-      {
-        test: /\.(mp4|webm)(\?.*)?$/,
-        include: [src, nodeModules],
-        loader: "url-loader",
-        query: { limit: 10000, name: "static/media/[name].[hash:8].[ext]" }
-      }
-    ]
+    ],
   },
   plugins: [
     new HtmlWebpackPlugin({
-      inject: "body",
       filename: "index.html",
       template: path.resolve("index_template.html"),
-      favicon: path.resolve("favicon.png"),
-      inlineSource: ".(js|css)$",
-      minify: {
-        removeComments: true,
-        collapseWhitespace: true,
-        removeRedundantAttributes: true,
-        useShortDoctype: true,
-        removeEmptyAttributes: true,
-        removeStyleLinkTypeAttributes: true,
-        keepClosingSlash: true,
-        minifyJS: true,
-        minifyCSS: true,
-        minifyURLs: true
-      }
+      decodeEntities: false,
+      minify: false,
+    }),
+    new CleanWebpackPlugin({
+      verbose: true,
+      protectWebpackAssets: false,
+      cleanAfterEveryBuildPatterns: ["main.js", "main.css"],
     }),
-    new HtmlWebpackInlineSourcePlugin(),
     new MiniCssExtractPlugin({
-      filename: "static/css/[name].[contenthash:8].css"
+      filename: "static/[name]-[contenthash].css",
+    }),
+    new CspHashPlugin({
+      filename: "csp-hashes.json",
+    }),
+    new webpack.DefinePlugin({
+      // webpack 5 no longer polyfills NodeJS modules, so fake the one we need
+      "process.env": JSON.stringify({
+        NODE_ENV: "production",
+      }),
     }),
-    new SWPrecacheWebpackPlugin({
-      cacheId: "cellxgene",
-      filename: "service-worker.js"
-    })
   ],
   performance: {
     maxEntrypointSize: 2000000,
-    maxAssetSize: 2000000
-  }
+    maxAssetSize: 2000000,
+  },
 };
+
+module.exports = merge(sharedConfig, prodConfig);

client/configuration/webpack/webpack.config.shared.js

@@ -0,0 +1,69 @@
+const path = require("path");
+const fs = require("fs");
+const MiniCssExtractPlugin = require("mini-css-extract-plugin");
+const ObsoleteWebpackPlugin = require("webpack-obsolete-plugin");
+
+const src = path.resolve("src");
+const nodeModules = path.resolve("node_modules");
+
+const publicPath = "";
+
+const rawObsoleteHTMLTemplate = fs.readFileSync(
+  `${__dirname}/obsoleteHTMLTemplate.html`,
+  "utf8"
+);
+
+const obsoleteHTMLTemplate = rawObsoleteHTMLTemplate.replace(/'/g, '"');
+
+module.exports = {
+  entry: [
+    "core-js",
+    "regenerator-runtime/runtime",
+    "whatwg-fetch",
+    "abort-controller/polyfill",
+    "./src/index",
+  ],
+  output: {
+    path: path.resolve("build"),
+    publicPath,
+  },
+  module: {
+    rules: [
+      {
+        test: /\.css$/,
+        include: src,
+        exclude: [path.resolve(src, "index.css")],
+        use: [
+          MiniCssExtractPlugin.loader,
+          {
+            loader: "css-loader",
+            options: {
+              modules: {
+                localIdentName: "[name]__[local]___[contenthash:base64:5]",
+              },
+              importLoaders: 1,
+            },
+          },
+        ],
+      },
+      {
+        test: /index\.css$/,
+        include: [path.resolve(src, "index.css")],
+        use: [MiniCssExtractPlugin.loader, "css-loader"],
+      },
+      {
+        test: /\.json$/,
+        include: [src, nodeModules],
+        loader: "json-loader",
+        exclude: /manifest.json$/,
+      },
+    ],
+  },
+  plugins: [
+    new ObsoleteWebpackPlugin({
+      name: "obsolete",
+      template: obsoleteHTMLTemplate,
+      promptOnNonTargetBrowser: false,
+    }),
+  ],
+};

client/index.html

@@ -1,13 +1,27 @@
 <!DOCTYPE html>
 <html lang="en">
   <head>
-    <meta charset="utf-8">
-    <meta name="viewport" content="width=device-width, initial-scale=1">
-    <title>cell&times;gene</title>
-    <link href="https://fonts.googleapis.com/css?family=Roboto+Condensed:400,400i,700" rel="stylesheet">
+    <meta charset="utf-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1" />
+    <title>CELL&times;GENE | Annotate</title>
     <style>
-      html, body, p, h1, h2, h3, h4, h5, h6, span, button, input, label, text, div {
-        font-family: 'Roboto Condensed','Helvetica Neue','Helvetica','Arial',sans-serif;
+      html,
+      body,
+      p,
+      h1,
+      h2,
+      h3,
+      h4,
+      h5,
+      h6,
+      span,
+      button,
+      input,
+      label,
+      text,
+      div {
+        font-family: "Roboto Condensed", "Helvetica Neue", "Helvetica", "Arial",
+          sans-serif;
         font-size: 14px;
       }
       body {
@@ -21,9 +35,12 @@
     </style>
   </head>
   <body>
-    <noscript>If you're seeing this message, that means <strong>JavaScript has been disabled on your browser</strong>, please <strong>enable JS</strong> to make this app work.</noscript>
+    <noscript
+      >If you're seeing this message, that means
+      <strong>JavaScript has been disabled on your browser</strong>, please
+      <strong>enable JS</strong> to make this app work.</noscript
+    >
 
     <div id="root"></div>
-
   </body>
 </html>

client/index_template.html

@@ -1,42 +1,65 @@
 <!DOCTYPE html>
 <html lang="en">
-<head>
-    <meta charset="utf-8">
-    <meta name="viewport" content="width=device-width, initial-scale=1">
-    <title>cell&times;gene</title>
-    <link href="https://fonts.googleapis.com/css?family=Roboto+Condensed:400,400i,700" rel="stylesheet">
+  <head>
+    <meta charset="utf-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1" />
+    <title>CELL&times;GENE | Annotate</title>
     <style>
-        html, body, p, h1, h2, h3, h4, h5, h6, span, button, input, label, text, div {
-            font-family: 'Roboto Condensed', 'Helvetica Neue', 'Helvetica', 'Arial', sans-serif;
-            font-size: 14px;
-        }
+      html,
+      body,
+      p,
+      h1,
+      h2,
+      h3,
+      h4,
+      h5,
+      h6,
+      span,
+      button,
+      input,
+      label,
+      text,
+      div {
+        font-family: "Roboto Condensed", "Helvetica Neue", "Helvetica", "Arial",
+          sans-serif;
+        font-size: 14px;
+      }
 
-        body {
-            margin: 0;
-            padding: 0;
-        }
+      body {
+        margin: 0;
+        padding: 0;
+      }
 
-        * {
-            box-sizing: border-box;
-        }
+      * {
+        box-sizing: border-box;
+      }
     </style>
+  </head>
+  <body>
+    <script type="text/javascript">
+      window.CELLXGENE = {};
+      window.CELLXGENE.API = {
+        prefix: `${location.origin}${location.pathname}api/`,
+        version: "v0.2/",
+      };
+    </script>
+    <noscript
+      >If you're seeing this message, that means
+      <strong>JavaScript has been disabled on your browser</strong>, please
+      <strong>enable JS</strong> to make this app work.
+    </noscript>
 
-</head>
-<body>
-<script type="text/javascript">
-  window.CELLXGENE = {};
-  window.CELLXGENE.API = {
-    prefix: window.location.href + "api/",
-    version: "v0.2/"
-  };
-</script>
-<noscript>If you're seeing this message, that means <strong>JavaScript has been disabled on your browser</strong>,
-    please <strong>enable JS</strong> to make this app work.
-</noscript>
+    <div id="root"></div>
+    {% for script in SCRIPTS -%}
+    <script type="text/javascript"
+      {{ ('integrity="%s"' % script.integrity) | safe if script.integrity }}
+      {{ ('crossorigin="%s"' % script.crossorigin) | safe if script.crossorigin }}
+      src="{{script.src | safe}}"
+    ></script>
+    {%- endfor %}
 
-<div id="root"></div>
-{% for script in SCRIPTS %}
-<script type="text/javascript" src="{{script | safe}}"></script>
-{% endfor %}
-</body>
+    {% for ils in INLINE_SCRIPTS %}
+    <script type="text/javascript" no-csp-hash>{% include ils %}</script>
+    {% endfor %}
+  </body>
 </html>

client/jest-puppeteer.config.js

@@ -0,0 +1,47 @@
+/**
+ * `client/jest-puppeteer.config.js` is for configuring Puppeteer's launch config options
+ * `client/__tests__/e2e/puppeteer.setup.js` is for configuring `jest`, `browser`,
+ * and `page` objects
+ */
+
+const ENV_DEFAULT = require("../environment.default.json");
+
+const jestEnv = process.env.JEST_ENV || ENV_DEFAULT.JEST_ENV;
+const isHeadful =
+  process.env.HEADFUL === "true" || process.env.HEADLESS === "false";
+
+const DEFAULT_LAUNCH_CONFIG = {
+  headless: !isHeadful,
+  args: ["--ignore-certificate-errors", "--ignore-ssl-errors"],
+  ignoreHTTPSErrors: true,
+  timeout: 90000,
+  defaultViewport: {
+    width: 1280,
+    height: 960,
+  },
+};
+
+const LAUNCH_CONFIG_BY_ENV = {
+  [ENV_DEFAULT.DEBUG]: {
+    ...DEFAULT_LAUNCH_CONFIG,
+    headless: false,
+    slowMo: 100,
+    devtools: true,
+    defaultViewport: {
+      width: DEFAULT_LAUNCH_CONFIG.defaultViewport.width,
+      height: DEFAULT_LAUNCH_CONFIG.defaultViewport.height + 560,
+    },
+  },
+  [ENV_DEFAULT.DEV]: {
+    ...DEFAULT_LAUNCH_CONFIG,
+    headless: false,
+    slowMo: 5,
+  },
+};
+
+const launchConfig = LAUNCH_CONFIG_BY_ENV[jestEnv] || DEFAULT_LAUNCH_CONFIG;
+
+module.exports = {
+  browserContext: "incognito",
+  launch: launchConfig,
+};

client/package.json

@@ -1,26 +1,25 @@
 {
   "name": "cellxgene",
-  "version": "0.13.0",
+  "version": "1.3.0",
   "license": "MIT",
   "description": "cellxgene is a web application for the interactive exploration of single cell sequence data.",
   "repository": "https://github.com/chanzuckerberg/cellxgene",
   "scripts": {
-    "backend-dev": "python3.6 -m venv cellxgene && source cellxgene/bin/activate && yes | pip uninstall cellxgene || true && pip install -e .. && cellxgene launch ",
-    "backend-dev-anno": "python3.6 -m venv cellxgene && source cellxgene/bin/activate && yes | pip uninstall cellxgene || true && pip install -e .. && cellxgene launch --experimental-annotations ",
-    "build": "npm run clean && webpack --config configuration/webpack/webpack.config.prod.js",
+    "build": "npm run clean && webpack --config",
     "clean": "rimraf build",
-    "dev": "npm run clean && webpack --config configuration/webpack/webpack.config.dev.js",
-    "e2e": "node node_modules/jest/bin/jest.js --verbose false --config __tests__/e2e/e2eJestConfig.json e2e/e2e.test.js",
-    "lint": "eslint src",
-    "smoke-test": "start-server-and-test start-server-for-test :5000 e2e",
-    "start": "node server/development.js",
-    "start-server-for-test": "cellxgene launch -p 5000 ../example-dataset/pbmc3k.h5ad",
-    "test": "node node_modules/jest/bin/jest.js",
-    "unit-test": "node node_modules/jest/bin/jest.js --testPathIgnorePatterns e2e"
+    "dev": "npm run build -- configuration/webpack/webpack.config.dev.js",
+    "e2e": "jest --config __tests__/e2e/e2eJestConfig.json e2e/e2e.test.js",
+    "e2e-annotations": "jest --config __tests__/e2e/e2eJestConfig.json e2e/e2eAnnotations.test.js",
+    "fmt": "eslint --fix src __tests__",
+    "lint": "eslint --fix src __tests__",
+    "prod": "npm run build -- configuration/webpack/webpack.config.prod.js",
+    "test": "jest --testPathIgnorePatterns e2e",
+    "prepare": "cd .. && husky install client/.husky"
   },
   "engineStrict": true,
   "engines": {
-    "npm": ">=3.0.0"
+    "npm": ">=9.6.7",
+    "node": "^18.17.0"
   },
   "eslintConfig": {
     "extends": "./configuration/eslint/eslint.js"
@@ -31,89 +30,130 @@
   "resolutions": {
     "eslint-scope": "3.7.1"
   },
+  "browserslist": [
+    "Chrome > 60",
+    "iOS >= 10.3",
+    "Firefox >= 60",
+    "Edge >= 79",
+    "not Explorer > 0",
+    "not Safari > 0"
+  ],
   "dependencies": {
-    "@blueprintjs/core": "^3.20.0",
-    "@blueprintjs/icons": "^3.12.0",
-    "@blueprintjs/select": "^3.11.2",
+    "@babel/eslint-parser": "^7.15.0",
+    "@blueprintjs/core": "^3.44.0",
+    "@blueprintjs/icons": "^3.19.0",
+    "@blueprintjs/popover2": "^0.11.2",
+    "@blueprintjs/select": "^3.16.0",
+    "abort-controller": "^3.0.0",
+    "core-js": "^3.16.3",
     "d3": "^4.10.0",
     "d3-scale-chromatic": "^1.5.0",
     "flatbuffers": "^1.11.0",
     "fuzzysort": "^1.1.4",
-    "gl-mat4": "^1.1.4",
-    "gl-matrix": "^3.1.0",
+    "gl-mat4": "^1.2.0",
+    "gl-matrix": "^3.3.0",
     "gl-vec3": "^1.1.3",
     "is-number": "^7.0.0",
-    "lodash": "^4.17.15",
+    "lodash.clonedeep": "^4.5.0",
+    "lodash.difference": "^4.5.0",
+    "lodash.foreach": "^4.5.0",
+    "lodash.pull": "^4.1.0",
+    "lodash.sortby": "^4.7.0",
+    "lodash.uniq": "^4.5.0",
     "memoize-one": "^5.1.1",
-    "react": "^16.11.0",
-    "react-autocomplete": "^1.7.2",
-    "react-dom": "^16.11.0",
-    "react-flip-toolkit": "7.0.6",
-    "react-helmet": "^5.2.1",
-    "react-icons": "^3.8.0",
-    "react-redux": "^7.1.3",
-    "redux": "^4.0.4",
-    "redux-thunk": "^2.2.0",
-    "regl": "^1.3.13"
+    "pako": "^2.0.3",
+    "react": "^17.0.2",
+    "react-async": "^10.0.1",
+    "react-dom": "^17.0.2",
+    "react-flip-toolkit": "^7.0.12",
+    "react-helmet": "^6.1.0",
+    "react-icons": "^4.2.0",
+    "react-redux": "^7.2.0",
+    "redux": "^4.0.5",
+    "redux-thunk": "^2.3.0",
+    "regenerator-runtime": "^0.13.7",
+    "regl": "^2.1.0",
+    "sha1": "^1.1.1",
+    "tinyqueue": "^2.0.3",
+    "whatwg-fetch": "^3.2.0"
   },
   "devDependencies": {
-    "@babel/core": "^7.7.2",
-    "@babel/plugin-proposal-class-properties": "^7.7.0",
-    "@babel/plugin-proposal-decorators": "^7.7.0",
-    "@babel/plugin-proposal-export-namespace-from": "^7.5.2",
-    "@babel/plugin-proposal-function-bind": "^7.2.0",
-    "@babel/plugin-proposal-nullish-coalescing-operator": "^7.4.4",
-    "@babel/plugin-proposal-optional-chaining": "^7.6.0",
-    "@babel/plugin-transform-react-constant-elements": "^7.6.3",
-    "@babel/plugin-transform-runtime": "^7.6.2",
-    "@babel/preset-env": "^7.7.1",
-    "@babel/preset-react": "^7.7.0",
-    "@babel/register": "^7.7.0",
-    "@babel/runtime": "^7.7.2",
-    "babel-eslint": "^10.0.3",
-    "babel-jest": "^24.9.0",
-    "babel-loader": "^8.0.6",
-    "babel-preset-modern-browsers": "^14.0.0",
-    "chalk": "^3.0.0",
-    "connect-history-api-fallback": "^1.6.0",
-    "copy-webpack-plugin": "^5.0.5",
-    "css-loader": "^3.2.0",
-    "eslint": "^6.6.0",
-    "eslint-config-airbnb": "^18.0.1",
-    "eslint-config-prettier": "^6.5.0",
-    "eslint-loader": "^3.0.2",
+    "@babel/core": "^7.25.2",
+    "@babel/plugin-proposal-class-properties": "^7.10.4",
+    "@babel/plugin-proposal-decorators": "^7.13.15",
+    "@babel/plugin-proposal-export-namespace-from": "^7.10.4",
+    "@babel/plugin-proposal-function-bind": "^7.10.5",
+    "@babel/plugin-proposal-nullish-coalescing-operator": "^7.10.4",
+    "@babel/plugin-proposal-optional-chaining": "^7.10.4",
+    "@babel/plugin-transform-private-property-in-object": "^7.22.11",
+    "@babel/plugin-transform-react-constant-elements": "^7.13.13",
+    "@babel/plugin-transform-runtime": "^7.13.15",
+    "@babel/preset-env": "^7.22.20",
+    "@babel/preset-react": "^7.13.13",
+    "@babel/register": "^7.13.16",
+    "@babel/runtime": "^7.13.16",
+    "@blueprintjs/eslint-plugin": "^0.3.0",
+    "@sentry/webpack-plugin": "^1.15.0",
+    "babel-jest": "^26.1.0",
+    "babel-loader": "^8.1.0",
+    "babel-preset-modern-browsers": "^15.0.2",
+    "chalk": "^4.1.0",
+    "cheerio": "^1.0.0-rc.6",
+    "clean-css": "^5.1.2",
+    "clean-webpack-plugin": "^4.0.0-alpha.0",
+    "codecov": "^3.7.1",
+    "css-loader": "^5.2.4",
+    "css-minimizer-webpack-plugin": "^4.0.0",
+    "eslint": "^7.24.0",
+    "eslint-config-airbnb": "^18.2.0",
+    "eslint-config-prettier": "^8.2.0",
+    "eslint-plugin-compat": "^4.2.0",
+    "eslint-plugin-eslint-comments": "^3.2.0",
     "eslint-plugin-filenames": "^1.3.2",
-    "eslint-plugin-import": "^2.18.2",
-    "eslint-plugin-jest": "^23.0.4",
-    "eslint-plugin-jsx-a11y": "^6.2.3",
-    "eslint-plugin-react": "^7.16.0",
+    "eslint-plugin-import": "^2.24.2",
+    "eslint-plugin-jest": "^24.3.5",
+    "eslint-plugin-jsx-a11y": "^6.3.1",
+    "eslint-plugin-react": "^7.23.2",
+    "eslint-plugin-react-hooks": "^4.0.8",
+    "expect-puppeteer": "^5.0.0",
     "express": "^4.17.1",
-    "file-loader": "^4.2.0",
-    "html-webpack-inline-source-plugin": "0.0.10",
-    "html-webpack-plugin": "^3.2.0",
-    "jest": "^24.9.0",
-    "jest-puppeteer": "^4.3.0",
-    "json-loader": "^0.5.4",
-    "mini-css-extract-plugin": "^0.8.0",
-    "puppeteer": "^2.0.0",
-    "rimraf": "^3.0.0",
-    "serve-favicon": "^2.3.0",
-    "start-server-and-test": "^1.10.6",
-    "style-loader": "^1.0.0",
-    "sw-precache-webpack-plugin": "^0.11.5",
-    "url-loader": "^2.2.0",
-    "webpack": "^4.41.2",
-    "webpack-cli": "^3.3.10",
-    "webpack-dev-middleware": "^3.7.2"
+    "file-loader": "^6.0.0",
+    "html-webpack-plugin": "^5.3.1",
+    "husky": "^7.0.2",
+    "jest": "^27.0.6",
+    "jest-circus": "^27.0.6",
+    "jest-environment-puppeteer": "^5.0.1",
+    "jest-fetch-mock": "^3.0.3",
+    "jest-puppeteer": "^6.2.0",
+    "json-loader": "^0.5.7",
+    "lint-staged": "^10.2.11",
+    "lodash": "^4.17.21",
+    "lodash.every": "^4.6.0",
+    "lodash.filter": "^4.6.0",
+    "lodash.isnumber": "^3.0.3",
+    "lodash.map": "^4.6.0",
+    "lodash.zip": "^4.2.0",
+    "mini-css-extract-plugin": "^1.5.0",
+    "prettier": "^2.0.5",
+    "puppeteer": "^10.4.0",
+    "rimraf": "^3.0.2",
+    "serve-favicon": "^2.5.0",
+    "terser-webpack-plugin": "^5.1.1",
+    "webpack": "^5.94.0",
+    "webpack-cli": "^4.6.0",
+    "webpack-dev-middleware": "^4.1.0",
+    "webpack-merge": "^5.0.9",
+    "webpack-obsolete-plugin": "^1.0.5"
   },
   "jest": {
     "testMatch": [
       "**/__tests__/**/?(*.)(spec|test).js?(x)"
     ],
-    "testURL": "http://localhost/",
     "setupFiles": [
       "./__tests__/setupMissingGlobals.js"
-    ]
+    ],
+    "coverageDirectory": "./coverage/",
+    "collectCoverage": true
   },
   "babel": {
     "env": {
@@ -136,6 +176,18 @@
               "loose": true
             }
           ],
+          [
+            "@babel/plugin-transform-private-methods",
+            {
+              "loose": true
+            }
+          ],
+          [
+            "@babel/plugin-transform-private-property-in-object",
+            {
+              "loose": true
+            }
+          ],
           "@babel/plugin-proposal-export-namespace-from",
           "@babel/plugin-transform-react-constant-elements",
           "@babel/plugin-transform-runtime",

client/server/development.js

@@ -1,49 +1,53 @@
-/* eslint-disable */
-// jshint esversion: 6
-var path = require("path");
-var historyApiFallback = require("connect-history-api-fallback");
-var chalk = require("chalk");
-var express = require("express");
-var favicon = require("serve-favicon");
-var webpack = require("webpack");
-var config = require("../configuration/webpack/webpack.config.dev");
-var utils = require("./utils");
+const chalk = require("chalk");
+const express = require("express");
+const favicon = require("serve-favicon");
+const webpack = require("webpack");
+const devMiddleware = require("webpack-dev-middleware");
+const config = require("../configuration/webpack/webpack.config.dev");
+const utils = require("./utils");
 
 process.env.NODE_ENV = "development";
 
-var PORT = process.env.PORT || 3000;
+const CLIENT_PORT = process.env.CXG_CLIENT_PORT;
+const { CXG_SERVER_PORT } = process.env;
+
+const API = {
+  prefix: `http://localhost:${CXG_SERVER_PORT}/`,
+};
 
 // Set up compiler
-var compiler = webpack(config);
+const compiler = webpack(config);
 
-compiler.plugin("invalid", () => {
+compiler.hooks.invalid.tap("invalid", () => {
   utils.clearConsole();
   console.log("Compiling...");
 });
 
-compiler.plugin("done", stats => {
-  utils.formatStats(stats, PORT);
+compiler.hooks.done.tap("done", (stats) => {
+  utils.formatStats(stats, CLIENT_PORT);
 });
 
 // Launch server
-var app = express();
-
-app.use(historyApiFallback({ verbose: false }));
+const app = express();
 
 app.use(
-  require("webpack-dev-middleware")(compiler, {
-    logLevel: "warn",
-    publicPath: config.output.publicPath
+  devMiddleware(compiler, {
+    publicPath: config.output.publicPath,
+    index: true,
   })
 );
 
 app.use(favicon("./favicon.png"));
 
-app.get("*", (req, res) => {
-  res.sendFile(path.resolve("index.html"));
+app.get("/logout", async (req, res) => {
+  try {
+    res.redirect(`${API.prefix}logout?dataset=http://localhost:${CLIENT_PORT}`);
+  } catch (err) {
+    console.error(err);
+  }
 });
 
-app.listen(PORT, err => {
+app.listen(CLIENT_PORT, (err) => {
   if (err) {
     console.log(err);
     return;

client/server/utils.js

@@ -8,7 +8,9 @@ function isLikelyASyntaxError(message) {
   return message.indexOf(friendlySyntaxErrorLabel) !== -1;
 }
 
-function formatMessage(message) {
+function formatMessage(messageObject) {
+  let { message, details } = messageObject;
+  if (details) message = message + ": " + details;
   return message
     .replace("Module build failed: SyntaxError:", friendlySyntaxErrorLabel)
     .replace(
@@ -36,10 +38,10 @@ var formatStats = (stats, port) => {
 
   var json = stats.toJson();
   var formattedErrors = json.errors.map(
-    message => "Error in " + formatMessage(message)
+    (message) => "Error in " + formatMessage(message)
   );
   var formattedWarnings = json.warnings.map(
-    message => "Warning in " + formatMessage(message)
+    (message) => "Warning in " + formatMessage(message)
   );
 
   if (hasErrors) {
@@ -48,7 +50,7 @@ var formatStats = (stats, port) => {
     if (formattedErrors.some(isLikelyASyntaxError)) {
       formattedErrors = formattedErrors.filter(isLikelyASyntaxError);
     }
-    formattedErrors.forEach(message => {
+    formattedErrors.forEach((message) => {
       console.log(message);
       console.log();
     });
@@ -58,7 +60,7 @@ var formatStats = (stats, port) => {
   if (hasWarnings) {
     console.log(chalk.yellow("Compiled with warnings."));
     console.log();
-    formattedWarnings.forEach(message => {
+    formattedWarnings.forEach((message) => {
       console.log(message);
       console.log();
     });

client/src/actions/annotation.js

@@ -0,0 +1,455 @@
+/*
+Action creators for user annotation
+*/
+import difference from "lodash.difference";
+import pako from "pako";
+import * as globals from "../globals";
+import { MatrixFBS, AnnotationsHelpers } from "../util/stateManager";
+
+const { isUserAnnotation } = AnnotationsHelpers;
+
+export const annotationCreateCategoryAction =
+  (newCategoryName, categoryToDuplicate) => async (dispatch, getState) => {
+    /*
+  Add a new user-created category to the obs annotations.
+
+  Arguments:
+    newCategoryName - string name for the category.
+    categoryToDuplicate - obs category to use for initial values, or null.
+  */
+    const { annoMatrix: prevAnnoMatrix, obsCrossfilter: prevObsCrossfilter } =
+      getState();
+    if (!prevAnnoMatrix || !prevObsCrossfilter) return;
+    const { schema } = prevAnnoMatrix;
+
+    /* name must be a string,  non-zero length */
+    if (typeof newCategoryName !== "string" || newCategoryName.length === 0)
+      throw new Error("user annotations require string name");
+
+    /* ensure the name isn't already in use! */
+    if (schema.annotations.obsByName[newCategoryName])
+      throw new Error("name collision on annotation category create");
+
+    let initialValue;
+    let newSchema;
+    let ctor;
+    if (categoryToDuplicate) {
+      /* if we are duplicating a category, retrieve it */
+      const catDupSchema = schema.annotations.obsByName[categoryToDuplicate];
+      const catDupType = catDupSchema?.type;
+      if (catDupType !== "string" && catDupType !== "categorical")
+        throw new Error(
+          "categoryToDuplicate does not exist or has invalid type"
+        );
+
+      const catToDupDf = await prevAnnoMatrix
+        .base()
+        .fetch("obs", categoryToDuplicate);
+      const col = catToDupDf.col(categoryToDuplicate);
+      initialValue = col.asArray();
+      const { categories } = col.summarizeCategorical();
+      // all user-created annotations must have the unassigned category
+      if (!categories.includes(globals.unassignedCategoryLabel)) {
+        categories.push(globals.unassignedCategoryLabel);
+      }
+      ctor = initialValue.constructor;
+      newSchema = {
+        ...catDupSchema,
+        name: newCategoryName,
+        categories,
+        writable: true,
+      };
+    } else {
+      /* else assign to the standard default value */
+      initialValue = globals.unassignedCategoryLabel;
+      ctor = Array;
+      newSchema = {
+        name: newCategoryName,
+        type: "categorical",
+        categories: [globals.unassignedCategoryLabel],
+        writable: true,
+      };
+    }
+
+    const obsCrossfilter = prevObsCrossfilter.addObsColumn(
+      newSchema,
+      ctor,
+      initialValue
+    );
+
+    dispatch({
+      type: "annotation: create category",
+      data: newCategoryName,
+      categoryToDuplicate,
+      annoMatrix: obsCrossfilter.annoMatrix,
+      obsCrossfilter,
+    });
+  };
+
+export const annotationRenameCategoryAction =
+  (oldCategoryName, newCategoryName) => (dispatch, getState) => {
+    /*
+  Rename a user-created annotation category
+  */
+    const { annoMatrix: prevAnnoMatrix, obsCrossfilter: prevObsCrossfilter } =
+      getState();
+    if (!prevAnnoMatrix || !prevObsCrossfilter) return;
+    if (!isUserAnnotation(prevAnnoMatrix, oldCategoryName))
+      throw new Error("not a user annotation");
+
+    /* name must be a string,  non-zero length */
+    if (typeof newCategoryName !== "string" || newCategoryName.length === 0)
+      throw new Error("user annotations require string name");
+
+    if (oldCategoryName === newCategoryName) return;
+
+    const obsCrossfilter = prevObsCrossfilter.renameObsColumn(
+      oldCategoryName,
+      newCategoryName
+    );
+
+    dispatch({
+      type: "annotation: category edited",
+      annoMatrix: obsCrossfilter.annoMatrix,
+      obsCrossfilter,
+      metadataField: oldCategoryName,
+      newCategoryText: newCategoryName,
+      data: newCategoryName,
+    });
+  };
+
+export const annotationDeleteCategoryAction =
+  (categoryName) => (dispatch, getState) => {
+    /*
+  Delete a user-created category
+  */
+    const { annoMatrix: prevAnnoMatrix, obsCrossfilter: prevObsCrossfilter } =
+      getState();
+    if (!prevAnnoMatrix || !prevObsCrossfilter) return;
+    if (!isUserAnnotation(prevAnnoMatrix, categoryName))
+      throw new Error("not a user annotation");
+
+    const obsCrossfilter = prevObsCrossfilter.dropObsColumn(categoryName);
+    dispatch({
+      type: "annotation: delete category",
+      annoMatrix: obsCrossfilter.annoMatrix,
+      obsCrossfilter,
+      metadataField: categoryName,
+    });
+  };
+
+export const annotationCreateLabelInCategory =
+  (categoryName, labelName, assignSelected) => async (dispatch, getState) => {
+    /*
+  Add a new label to a user-defined category.  If assignSelected is true, assign
+  the label to all currently selected cells.
+  */
+    const { annoMatrix: prevAnnoMatrix, obsCrossfilter: prevObsCrossfilter } =
+      getState();
+    if (!prevAnnoMatrix || !prevObsCrossfilter) return;
+    if (!isUserAnnotation(prevAnnoMatrix, categoryName))
+      throw new Error("not a user annotation");
+
+    let obsCrossfilter = prevObsCrossfilter.addObsAnnoCategory(
+      categoryName,
+      labelName
+    );
+    if (assignSelected) {
+      obsCrossfilter = await obsCrossfilter.setObsColumnValues(
+        categoryName,
+        prevObsCrossfilter.allSelectedLabels(),
+        labelName
+      );
+    }
+
+    dispatch({
+      type: "annotation: add new label to category",
+      annoMatrix: obsCrossfilter.annoMatrix,
+      obsCrossfilter,
+      metadataField: categoryName,
+      newLabelText: labelName,
+      assignSelectedCells: assignSelected,
+    });
+  };
+
+export const annotationDeleteLabelFromCategory =
+  (categoryName, labelName) => async (dispatch, getState) => {
+    /*
+  delete a label from a user-defined category
+  */
+    const { annoMatrix: prevAnnoMatrix, obsCrossfilter: prevObsCrossfilter } =
+      getState();
+    if (!prevAnnoMatrix || !prevObsCrossfilter) return;
+    if (!isUserAnnotation(prevAnnoMatrix, categoryName))
+      throw new Error("not a user annotation");
+
+    const obsCrossfilter = await prevObsCrossfilter.removeObsAnnoCategory(
+      categoryName,
+      labelName,
+      globals.unassignedCategoryLabel
+    );
+
+    dispatch({
+      type: "annotation: delete label",
+      metadataField: categoryName,
+      label: labelName,
+      annoMatrix: obsCrossfilter.annoMatrix,
+      obsCrossfilter,
+    });
+  };
+
+export const annotationRenameLabelInCategory =
+  (categoryName, oldLabelName, newLabelName) => async (dispatch, getState) => {
+    /*
+  label name change
+  */
+    const { annoMatrix: prevAnnoMatrix, obsCrossfilter: prevObsCrossfilter } =
+      getState();
+    if (!prevAnnoMatrix || !prevObsCrossfilter) return;
+    if (!isUserAnnotation(prevAnnoMatrix, categoryName))
+      throw new Error("not a user annotation");
+
+    let obsCrossfilter = await prevObsCrossfilter.resetObsColumnValues(
+      categoryName,
+      oldLabelName,
+      newLabelName
+    );
+    obsCrossfilter = await obsCrossfilter.removeObsAnnoCategory(
+      categoryName,
+      oldLabelName,
+      globals.unassignedCategoryLabel
+    );
+
+    dispatch({
+      type: "annotation: label edited",
+      editedLabel: newLabelName,
+      metadataField: categoryName,
+      label: oldLabelName,
+      annoMatrix: obsCrossfilter.annoMatrix,
+      obsCrossfilter,
+    });
+  };
+
+export const annotationLabelCurrentSelection =
+  (categoryName, labelName) => async (dispatch, getState) => {
+    /*
+  set the label on all currently selected
+  */
+    const { annoMatrix: prevAnnoMatrix, obsCrossfilter: prevObsCrossfilter } =
+      getState();
+    if (!prevAnnoMatrix || !prevObsCrossfilter) return;
+    if (!isUserAnnotation(prevAnnoMatrix, categoryName))
+      throw new Error("not a user annotation");
+
+    const obsCrossfilter = await prevObsCrossfilter.setObsColumnValues(
+      categoryName,
+      prevObsCrossfilter.allSelectedLabels(),
+      labelName
+    );
+
+    dispatch({
+      type: "annotation: label current cell selection",
+      metadataField: categoryName,
+      label: labelName,
+      obsCrossfilter,
+      annoMatrix: obsCrossfilter.annoMatrix,
+    });
+  };
+
+function writableAnnotations(annoMatrix) {
+  return annoMatrix.schema.annotations.obs.columns
+    .filter((s) => s.writable)
+    .map((s) => s.name);
+}
+
+export const needToSaveObsAnnotations = (annoMatrix, lastSavedAnnoMatrix) => {
+  /*
+  Return true if there are LIKELY user-defined annotation modifications between the two
+  annoMatrices.  Technically not an action creator, but intimately intertwined
+  with the save process.
+
+  Two conditions will trigger a need to save:
+    * the collection of user-defined columns have changed
+    * the contents of the user-defined columns have change
+  */
+
+  annoMatrix = annoMatrix.base();
+
+  // if the annoMatrix hasn't changed, we are guaranteed no changes to the matrix schema or contents.
+  if (annoMatrix === lastSavedAnnoMatrix) return false;
+
+  // if the schema has changed, we need to save
+  const currentWritable = writableAnnotations(annoMatrix);
+  if (difference(currentWritable, writableAnnotations(lastSavedAnnoMatrix))) {
+    return true;
+  }
+
+  // no schema changes; check for change in contents
+  return currentWritable.some(
+    (col) => annoMatrix.col(col) !== lastSavedAnnoMatrix.col(col)
+  );
+};
+
+export const saveObsAnnotationsAction = () => async (dispatch, getState) => {
+  /*
+  Save the user-created obs annotations IF any have changed.
+  */
+  const state = getState();
+  const { annotations, autosave } = state;
+  const { dataCollectionNameIsReadOnly, dataCollectionName } = annotations;
+  const { lastSavedAnnoMatrix, saveInProgress } = autosave;
+
+  const annoMatrix = state.annoMatrix.base();
+
+  if (saveInProgress || annoMatrix === lastSavedAnnoMatrix) return;
+  if (!needToSaveObsAnnotations(annoMatrix, lastSavedAnnoMatrix)) {
+    dispatch({
+      type: "writable obs annotations - save complete",
+      lastSavedAnnoMatrix: annoMatrix,
+    });
+    return;
+  }
+
+  /*
+  Else, we really do need to save
+  */
+
+  dispatch({
+    type: "writable obs annotations - save started",
+  });
+
+  const df = await annoMatrix.fetch("obs", writableAnnotations(annoMatrix));
+  const matrix = MatrixFBS.encodeMatrixFBS(df);
+  const compressedMatrix = pako.deflate(matrix);
+  try {
+    const queryString =
+      !dataCollectionNameIsReadOnly && !!dataCollectionName
+        ? `?annotation-collection-name=${encodeURIComponent(
+            dataCollectionName
+          )}`
+        : "";
+    const res = await fetch(
+      `${globals.API.prefix}${globals.API.version}annotations/obs${queryString}`,
+      {
+        method: "PUT",
+        body: compressedMatrix,
+        headers: new Headers({
+          "Content-Type": "application/octet-stream",
+        }),
+        credentials: "include",
+      }
+    );
+    if (res.ok) {
+      dispatch({
+        type: "writable obs annotations - save complete",
+        lastSavedAnnoMatrix: annoMatrix,
+      });
+    } else {
+      dispatch({
+        type: "writable obs annotations - save error",
+        message: `HTTP error ${res.status} - ${res.statusText}`,
+        res,
+      });
+    }
+  } catch (error) {
+    dispatch({
+      type: "writable obs annotations - save error",
+      message: error.toString(),
+      error,
+    });
+  }
+};
+
+export const saveGenesetsAction = () => async (dispatch, getState) => {
+  const state = getState();
+
+  // bail if gene sets not available, or in readonly mode.
+  const { config } = state;
+  const { lastTid, genesets } = state.genesets;
+
+  const genesetsAreAvailable =
+    config?.parameters?.annotations_genesets ?? false;
+  const genesetsReadonly =
+    config?.parameters?.annotations_genesets_readonly ?? true;
+  if (!genesetsAreAvailable || genesetsReadonly) {
+    // our non-save was completed!
+    return dispatch({
+      type: "autosave: genesets complete",
+      lastSavedGenesets: genesets,
+    });
+  }
+
+  dispatch({
+    type: "autosave: genesets started",
+  });
+
+  /* Create the JSON OTA data structure */
+  const tid = (lastTid ?? 0) + 1;
+  const genesetsOTA = [];
+  for (const [name, gs] of genesets) {
+    const genes = [];
+    for (const g of gs.genes.values()) {
+      genes.push({
+        gene_symbol: g.geneSymbol,
+        gene_description: g.geneDescription,
+      });
+    }
+    genesetsOTA.push({
+      geneset_name: name,
+      geneset_description: gs.genesetDescription,
+      genes,
+    });
+  }
+  const ota = {
+    tid,
+    genesets: genesetsOTA,
+  };
+
+  /* Save to server */
+  try {
+    const { dataCollectionNameIsReadOnly, dataCollectionName } =
+      state.annotations;
+    const queryString =
+      !dataCollectionNameIsReadOnly && !!dataCollectionName
+        ? `?annotation-collection-name=${encodeURIComponent(
+            dataCollectionName
+          )}`
+        : "";
+
+    const res = await fetch(
+      `${globals.API.prefix}${globals.API.version}genesets${queryString}`,
+      {
+        method: "PUT",
+        headers: new Headers({
+          Accept: "application/json",
+          "Content-Type": "application/json",
+        }),
+        body: JSON.stringify(ota),
+        credentials: "include",
+      }
+    );
+    if (!res.ok) {
+      return dispatch({
+        type: "autosave: genesets error",
+        message: `HTTP error ${res.status} - ${res.statusText}`,
+        res,
+      });
+    }
+    return Promise.all([
+      dispatch({
+        type: "autosave: genesets complete",
+        lastSavedGenesets: genesets,
+      }),
+      dispatch({
+        type: "geneset: set tid",
+        tid,
+      }),
+    ]);
+  } catch (error) {
+    return dispatch({
+      type: "autosave: genesets error",
+      message: error.toString(),
+      error,
+    });
+  }
+};

client/src/actions/embedding.js

@@ -0,0 +1,47 @@
+/*
+action creators related to embeddings choice
+*/
+
+import { AnnoMatrixObsCrossfilter } from "../annoMatrix";
+import { _setEmbeddingSubset } from "../util/stateManager/viewStackHelpers";
+
+export async function _switchEmbedding(
+  prevAnnoMatrix,
+  prevCrossfilter,
+  newEmbeddingName
+) {
+  /*
+  DRY helper used by embedding action creators
+  */
+  const base = prevAnnoMatrix.base();
+  const embeddingDf = await base.fetch("emb", newEmbeddingName);
+  const annoMatrix = _setEmbeddingSubset(prevAnnoMatrix, embeddingDf);
+  const obsCrossfilter = await new AnnoMatrixObsCrossfilter(
+    annoMatrix,
+    prevCrossfilter.obsCrossfilter
+  ).select("emb", newEmbeddingName, {
+    mode: "all",
+  });
+  return [annoMatrix, obsCrossfilter];
+}
+
+export const layoutChoiceAction =
+  (newLayoutChoice) => async (dispatch, getState) => {
+    /*
+  On layout choice, make sure we have selected all on the previous layout, AND the new
+  layout.
+  */
+    const { annoMatrix: prevAnnoMatrix, obsCrossfilter: prevCrossfilter } =
+      getState();
+    const [annoMatrix, obsCrossfilter] = await _switchEmbedding(
+      prevAnnoMatrix,
+      prevCrossfilter,
+      newLayoutChoice
+    );
+    dispatch({
+      type: "set layout choice",
+      layoutChoice: newLayoutChoice,
+      obsCrossfilter,
+      annoMatrix,
+    });
+  };

client/src/actions/geneset.js

@@ -0,0 +1,149 @@
+import { postUserErrorToast } from "../components/framework/toasters";
+/*
+Action creators for gene sets
+
+Primarily used to keep the crossfilter and underlying data in sync with the UI.
+
+The behavior manifest in these action creators:
+
+    Delete a gene set, will
+      * drop index & clear selection state on the gene set summary
+      * drop index & clear selection state of each gene in the geneset
+
+    Delete a gene from a gene set, will:
+      * drop index & clear selection state on the gene set summary
+      * drop index & clear selection state on the gene
+
+    Add a gene to a gene set, will:
+      * drop index & clear selection state on the gene set summary
+      * will NOT touch the selection state for the gene
+
+Note that crossfilter indices are lazy created, as needed.
+*/
+
+export const genesetDelete = (genesetName) => (dispatch, getState) => {
+  const state = getState();
+  const { genesets } = state;
+  const gs = genesets?.genesets?.get(genesetName) ?? {};
+  const geneSymbols = Array.from(gs.genes.keys());
+  const obsCrossfilter = dropGeneset(dispatch, state, genesetName, geneSymbols);
+  if (genesetName === state.colors.colorAccessor) {
+    dispatch({
+      type: "reset colorscale",
+    });
+  }
+  dispatch({
+    type: "geneset: delete",
+    genesetName,
+    obsCrossfilter,
+    annoMatrix: obsCrossfilter.annoMatrix,
+  });
+};
+
+export const genesetAddGenes =
+  (genesetName, genes) => async (dispatch, getState) => {
+    const state = getState();
+    const { obsCrossfilter: prevObsCrossfilter, annoMatrix } = state;
+    const { schema } = annoMatrix;
+    const varIndex = schema.annotations.var.index;
+    const df = await annoMatrix.fetch("var", varIndex);
+    const geneNames = df.col(varIndex).asArray();
+    genes = genes.reduce((acc, gene) => {
+      if (geneNames.indexOf(gene.geneSymbol) === -1) {
+        postUserErrorToast(
+          `${gene.geneSymbol} doesn't appear to be a valid gene name.`
+        );
+      } else acc.push(gene);
+      return acc;
+    }, []);
+
+    const obsCrossfilter = dropGenesetSummaryDimension(
+      prevObsCrossfilter,
+      state,
+      genesetName
+    );
+    dispatch({
+      type: "continuous metadata histogram cancel",
+      continuousNamespace: { isGeneSetSummary: true },
+      selection: genesetName,
+    });
+    return dispatch({
+      type: "geneset: add genes",
+      genesetName,
+      genes,
+      obsCrossfilter,
+      annoMatrix: obsCrossfilter.annoMatrix,
+    });
+  };
+
+export const genesetDeleteGenes =
+  (genesetName, geneSymbols) => (dispatch, getState) => {
+    const state = getState();
+    const obsCrossfilter = dropGeneset(
+      dispatch,
+      state,
+      genesetName,
+      geneSymbols
+    );
+    return dispatch({
+      type: "geneset: delete genes",
+      genesetName,
+      geneSymbols,
+      obsCrossfilter,
+      annoMatrix: obsCrossfilter.annoMatrix,
+    });
+  };
+
+/*
+Private
+*/
+
+function dropGenesetSummaryDimension(obsCrossfilter, state, genesetName) {
+  const { annoMatrix, genesets } = state;
+  const varIndex = annoMatrix.schema.annotations?.var?.index;
+  const gs = genesets?.genesets?.get(genesetName) ?? {};
+  const genes = Array.from(gs.genes.keys());
+  const query = {
+    summarize: {
+      method: "mean",
+      field: "var",
+      column: varIndex,
+      values: genes,
+    },
+  };
+  return obsCrossfilter.dropDimension("X", query);
+}
+
+function dropGeneDimension(obsCrossfilter, state, gene) {
+  const { annoMatrix } = state;
+  const varIndex = annoMatrix.schema.annotations?.var?.index;
+  const query = {
+    where: {
+      field: "var",
+      column: varIndex,
+      value: gene,
+    },
+  };
+  return obsCrossfilter.dropDimension("X", query);
+}
+
+function dropGeneset(dispatch, state, genesetName, geneSymbols) {
+  const { obsCrossfilter: prevObsCrossfilter } = state;
+  const obsCrossfilter = geneSymbols.reduce(
+    (crossfilter, gene) => dropGeneDimension(crossfilter, state, gene),
+    dropGenesetSummaryDimension(prevObsCrossfilter, state, genesetName)
+  );
+  dispatch({
+    type: "continuous metadata histogram cancel",
+    continuousNamespace: { isGeneSetSummary: true },
+    selection: genesetName,
+  });
+  geneSymbols.forEach((g) =>
+    dispatch({
+      type: "continuous metadata histogram cancel",
+      continuousNamespace: { isUserDefined: true },
+      selection: g,
+    })
+  );
+  return obsCrossfilter;
+}

client/src/actions/index.js

@@ -1,216 +1,141 @@
-// jshint esversion: 6
-import _ from "lodash";
 import * as globals from "../globals";
-import { Universe, MatrixFBS } from "../util/stateManager";
+import { AnnoMatrixLoader, AnnoMatrixObsCrossfilter } from "../annoMatrix";
 import {
   catchErrorsWrap,
   doJsonRequest,
-  doBinaryRequest,
-  dispatchNetworkErrorMessageToUser
+  dispatchNetworkErrorMessageToUser,
 } from "../util/actionHelpers";
+import { loadUserColorConfig } from "../util/stateManager/colorHelpers";
+import * as selnActions from "./selection";
+import * as annoActions from "./annotation";
+import * as viewActions from "./viewStack";
+import * as embActions from "./embedding";
+import * as genesetActions from "./geneset";
+
+function setGlobalConfig(config) {
+  /**
+   * Set any global run-time config not _exclusively_ managed by the config reducer.
+   * This should only set fields defined in globals.globalConfig.
+   */
+  globals.globalConfig.maxCategoricalOptionsToDisplay =
+    config?.parameters?.["max-category-items"] ??
+    globals.globalConfig.maxCategoricalOptionsToDisplay;
+}
 
 /*
-Bootstrap application with the initial data loading.
-  * /config - application configuration
-  * /schema - schema of dataframe
-  * /annotations - all metadata annotation
-  * /layout - all default layout
+return promise fetching user-configured colors
+*/
+async function userColorsFetchAndLoad(dispatch) {
+  return fetchJson("colors").then((response) =>
+    dispatch({
+      type: "universe: user color load success",
+      userColors: loadUserColorConfig(response),
+    })
+  );
+}
+
+async function schemaFetch() {
+  return fetchJson("schema");
+}
+
+async function configFetch(dispatch) {
+  return fetchJson("config").then((response) => {
+    const config = { ...globals.configDefaults, ...response.config };
+
+    setGlobalConfig(config);
+
+    dispatch({
+      type: "configuration load complete",
+      config,
+    });
+    return config;
+  });
+}
+
+async function genesetsFetch(dispatch, config) {
+  /* request genesets ONLY if the backend supports the feature */
+  const defaultResponse = {
+    genesets: [],
+    tid: 0,
+  };
+  if (config?.parameters?.annotations_genesets ?? false) {
+    fetchJson("genesets").then((response) => {
+      dispatch({
+        type: "geneset: initial load",
+        data: response ?? defaultResponse,
+      });
+    });
+  } else {
+    dispatch({
+      type: "geneset: initial load",
+      data: defaultResponse,
+    });
+  }
+}
+
+function prefetchEmbeddings(annoMatrix) {
+  /*
+  prefetch requests for all embeddings
+  */
+  const { schema } = annoMatrix;
+  const available = schema.layout.obs.map((v) => v.name);
+  available.forEach((embName) => annoMatrix.prefetch("emb", embName));
+}
+
+/*
+Application bootstrap
 */
 const doInitialDataLoad = () =>
-  catchErrorsWrap(async dispatch => {
+  catchErrorsWrap(async (dispatch) => {
     dispatch({ type: "initial data load start" });
 
     try {
-      /*
-      Step 1 - config & schema, all JSON
-      */
-      const requestJson = ["config", "schema"]
-        .map(r => `${globals.API.prefix}${globals.API.version}${r}`)
-        .map(url => doJsonRequest(url));
-      const stepOneResults = await Promise.all(requestJson);
-      /* set config defaults */
-      const config = { ...globals.configDefaults, ...stepOneResults[0].config };
-      const schema = stepOneResults[1];
+      const [config, schema] = await Promise.all([
+        configFetch(dispatch),
+        schemaFetch(dispatch),
+        userColorsFetchAndLoad(dispatch),
+      ]);
 
-      /*
-      Step 2 - dataframes, all binary.  NOTE: uses results of step 1.
-      */
-      /* only load names for var annotations, if possible*/
-      const varIndexName = schema?.schema?.annotations?.var?.index;
-      const varAnnotationsQuery = varIndexName
-        ? `?annotation-name=${encodeURIComponent(varIndexName)}`
-        : "";
-      const varAnnotationsURL = `annotations/var${varAnnotationsQuery}`;
-      const requestBinary = ["annotations/obs", varAnnotationsURL, "layout/obs"]
-        .map(r => `${globals.API.prefix}${globals.API.version}${r}`)
-        .map(url => doBinaryRequest(url));
-      const stepTwoResults = await Promise.all(requestBinary);
-      const [obsAnno, varAnno, obsLayout] = [...stepTwoResults];
+      genesetsFetch(dispatch, config);
 
-      const universe = Universe.createUniverseFromResponse(
-        config,
-        schema,
-        obsAnno,
-        varAnno,
-        obsLayout
-      );
+      const baseDataUrl = `${globals.API.prefix}${globals.API.version}`;
+      const annoMatrix = new AnnoMatrixLoader(baseDataUrl, schema.schema);
+      const obsCrossfilter = new AnnoMatrixObsCrossfilter(annoMatrix);
+      prefetchEmbeddings(annoMatrix);
 
       dispatch({
-        type: "configuration load complete",
-        config
-      });
-      dispatch({
-        type: "initial data load complete (universe exists)",
-        universe
+        type: "annoMatrix: init complete",
+        annoMatrix,
+        obsCrossfilter,
       });
+      dispatch({ type: "initial data load complete" });
+
+      const defaultEmbedding = config?.parameters?.default_embedding;
+      const layoutSchema = schema?.schema?.layout?.obs ?? [];
+      if (
+        defaultEmbedding &&
+        layoutSchema.some((s) => s.name === defaultEmbedding)
+      ) {
+        dispatch(embActions.layoutChoiceAction(defaultEmbedding));
+      }
     } catch (error) {
       dispatch({ type: "initial data load error", error });
     }
   }, true);
 
-/*
-Set the view (world) to current selection.   Placeholder for an async action
-which also does re-layout.
-*/
-const regraph = () => (dispatch, getState) => {
-  const { universe, world, crossfilter } = getState();
-  dispatch({
-    type: "set World to current selection",
-    universe,
-    world,
-    crossfilter
-  });
-};
-
-// Throws
-const dispatchExpressionErrors = (dispatch, res) => {
-  const msg = `Unexpected HTTP response while fetching expression data ${res.status}, ${res.statusText}`;
-  dispatchNetworkErrorMessageToUser(msg);
-  throw new Error(msg);
-};
-
-/*
-Fetch expression vectors for each gene in genes.   This is NOT an action
-function, but rather a helper to be called from an action helper that
-needs expression data.
-
-Transparently utilizes cached data if it is already present.
-*/
-async function _doRequestExpressionData(dispatch, getState, genes) {
-  const state = getState();
-  const { universe } = state;
-  const varIndexName = universe.schema.annotations.var.index;
-
-  /* helper for this function only */
-  const fetchData = async geneNames => {
-    const res = await fetch(
-      `${globals.API.prefix}${globals.API.version}data/var`,
-      {
-        method: "PUT",
-        body: JSON.stringify({
-          filter: {
-            var: {
-              annotation_value: [{ name: varIndexName, values: geneNames }]
-            }
-          }
-        }),
-        headers: new Headers({
-          accept: "application/octet-stream",
-          "Content-Type": "application/json"
-        }),
-        credentials: "include"
-      }
-    );
-
-    if (
-      !res.ok ||
-      res.headers.get("Content-Type") !== "application/octet-stream"
-    ) {
-      // WILL throw
-      return dispatchExpressionErrors(dispatch, res);
-    }
-
-    const data = await res.arrayBuffer();
-    return Universe.convertDataFBStoObject(universe, data);
-  };
-
-  /* preload data already in cache */
-  let expressionData = _.transform(
-    genes,
-    (expData, g) => {
-      const data = universe.varData.col(g);
-      if (data) {
-        expData[g] = data.asArray();
-      }
-    },
-    {}
-  ); // --> { gene: data }
-  /* make a list of genes for which we do not have data */
-  const genesToFetch = _.filter(genes, g => expressionData[g] === undefined);
-
-  dispatch({ type: "expression load start" });
-
-  /* Fetch data for any genes not in cache */
-  if (genesToFetch.length) {
-    try {
-      const newExpressionData = await fetchData(genesToFetch);
-      expressionData = {
-        ...expressionData,
-        ...newExpressionData
-      };
-    } catch (error) {
-      dispatch({ type: "expression load error", error });
-      throw error; // rethrow
-    }
-  }
-
-  dispatch({ type: "expression load success", expressionData });
-  return expressionData;
-}
-
 function requestSingleGeneExpressionCountsForColoringPOST(gene) {
-  return async (dispatch, getState) => {
-    dispatch({ type: "get single gene expression for coloring started" });
-    try {
-      await _doRequestExpressionData(dispatch, getState, [gene]);
-      const { world } = getState();
-      dispatch({
-        type: "color by expression",
-        gene,
-        data: {
-          [gene]: world.varData.col(gene).asArray()
-        }
-      });
-    } catch (error) {
-      dispatch({
-        type: "get single gene expression for coloring error",
-        error
-      });
-    }
+  return {
+    type: "color by expression",
+    gene,
   };
 }
 
-const requestUserDefinedGene = gene => async (dispatch, getState) => {
-  dispatch({ type: "request user defined gene started" });
-  try {
-    await await _doRequestExpressionData(dispatch, getState, [gene]);
-    const { world } = getState();
-
-    /* then send the success case action through */
-    return dispatch({
-      type: "request user defined gene success",
-      data: {
-        genes: [gene],
-        expression: world.varData.col(gene).asArray()
-      }
-    });
-  } catch (error) {
-    return dispatch({
-      type: "request user defined gene error",
-      error
-    });
-  }
-};
+const requestUserDefinedGene = (gene) => ({
+  type: "request user defined gene success",
+  data: {
+    genes: [gene],
+  },
+});
 
 const dispatchDiffExpErrors = (dispatch, response) => {
   switch (response.status) {
@@ -229,174 +154,122 @@ const dispatchDiffExpErrors = (dispatch, response) => {
       dispatchNetworkErrorMessageToUser(msg);
       dispatch({
         type: "request differential expression error",
-        error: new Error(msg)
+        error: new Error(msg),
       });
     }
   }
 };
 
-const requestDifferentialExpression = (set1, set2, num_genes = 10) => async (
-  dispatch,
-  getState
-) => {
-  dispatch({ type: "request differential expression started" });
-  try {
-    /*
+const requestDifferentialExpression =
+  (set1, set2, num_genes = 50) =>
+  async (dispatch, getState) => {
+    dispatch({ type: "request differential expression started" });
+    try {
+      /*
     Steps:
     1. get the most differentially expressed genes
     2. get expression data for each
     */
-    const state = getState();
-    const { universe } = state;
-    const varIndexName = universe.schema.annotations.var.index;
+      const { annoMatrix } = getState();
+      const varIndexName = annoMatrix.schema.annotations.var.index;
 
-    // Legal values are null, Array or TypedArray.  Null is initial state.
-    if (!set1) set1 = [];
-    if (!set2) set2 = [];
+      // Legal values are null, Array or TypedArray.  Null is initial state.
+      if (!set1) set1 = [];
+      if (!set2) set2 = [];
 
-    // These lines ensure that we convert any TypedArray to an Array.
-    // This is necessary because JSON.stringify() does some very strange
-    // things with TypedArrays (they are marshalled to JSON objects, rather
-    // than being marshalled as a JSON array).
-    set1 = Array.isArray(set1) ? set1 : Array.from(set1);
-    set2 = Array.isArray(set2) ? set2 : Array.from(set2);
+      // These lines ensure that we convert any TypedArray to an Array.
+      // This is necessary because JSON.stringify() does some very strange
+      // things with TypedArrays (they are marshalled to JSON objects, rather
+      // than being marshalled as a JSON array).
+      set1 = Array.isArray(set1) ? set1 : Array.from(set1);
+      set2 = Array.isArray(set2) ? set2 : Array.from(set2);
 
-    const res = await fetch(
-      `${globals.API.prefix}${globals.API.version}diffexp/obs`,
-      {
-        method: "POST",
-        headers: new Headers({
-          Accept: "application/json",
-          "Content-Type": "application/json"
-        }),
-        body: JSON.stringify({
-          mode: "topN",
-          count: num_genes,
-          set1: { filter: { obs: { index: set1 } } },
-          set2: { filter: { obs: { index: set2 } } }
-        }),
-        credentials: "include"
+      const res = await fetch(
+        `${globals.API.prefix}${globals.API.version}diffexp/obs`,
+        {
+          method: "POST",
+          headers: new Headers({
+            Accept: "application/json",
+            "Content-Type": "application/json",
+          }),
+          body: JSON.stringify({
+            mode: "topN",
+            count: num_genes,
+            set1: { filter: { obs: { index: set1 } } },
+            set2: { filter: { obs: { index: set2 } } },
+          }),
+          credentials: "include",
+        }
+      );
+
+      if (!res.ok || res.headers.get("Content-Type") !== "application/json") {
+        return dispatchDiffExpErrors(dispatch, res);
       }
-    );
 
-    if (!res.ok || res.headers.get("Content-Type") !== "application/json") {
-      return dispatchDiffExpErrors(dispatch, res);
-    }
-
-    const data = await res.json();
-    // result is [ [varIdx, ...], ... ]
-    const topNGenes = _.map(data, r =>
-      universe.varAnnotations.at(r[0], varIndexName)
-    );
-
-    /*
-    Kick off secondary action to fetch all of the expression data for the
-    topN expressed genes.
-    */
-    await _doRequestExpressionData(dispatch, getState, topNGenes);
-
-    /* then send the success case action through */
-    return dispatch({
-      type: "request differential expression success",
-      data
-    });
-  } catch (error) {
-    return dispatch({
-      type: "request differential expression error",
-      error
-    });
-  }
-};
-
-const resetInterface = () => (dispatch, getState) => {
-  const { universe } = getState();
-
-  dispatch({
-    type: "user reset start"
-  });
-  dispatch({
-    type: "clear all user defined genes"
-  });
-  dispatch({
-    type: "clear differential expression"
-  });
-  dispatch({
-    type: "reset colorscale"
-  });
-  dispatch({
-    type: "clear scatterplot"
-  });
-  dispatch({
-    type: "reset World to eq Universe",
-    universe
-  });
-  dispatch({
-    type: "increment graph render counter"
-  });
-  dispatch({
-    type: "user reset end"
-  });
-};
-
-const saveObsAnnotations = () => async (dispatch, getState) => {
-  const { universe, annotations } = getState();
-  const { obsAnnotations, schema } = universe;
-  const { dataCollectionNameIsReadOnly, dataCollectionName } = annotations;
-
-  dispatch({
-    type: "writable obs annotations - save started"
-  });
-
-  const writableAnnotations = schema.annotations.obs.columns
-    .filter(s => s.writable)
-    .map(s => s.name);
-  const df = obsAnnotations.subset(null, writableAnnotations);
-  const matrix = MatrixFBS.encodeMatrixFBS(df);
-  try {
-    const queryString =
-      !dataCollectionNameIsReadOnly && !!dataCollectionName
-        ? `?annotation-collection-name=${encodeURIComponent(
-            dataCollectionName
-          )}`
-        : "";
-    const res = await fetch(
-      `${globals.API.prefix}${globals.API.version}annotations/obs${queryString}`,
-      {
-        method: "PUT",
-        body: matrix,
-        headers: new Headers({
-          "Content-Type": "application/octet-stream"
-        }),
-        credentials: "include"
+      const response = await res.json();
+      const varIndex = await annoMatrix.fetch("var", varIndexName);
+      const diffexpLists = { negative: [], positive: [] };
+      for (const polarity of Object.keys(diffexpLists)) {
+        diffexpLists[polarity] = response[polarity].map((v) => [
+          varIndex.at(v[0], varIndexName),
+          ...v.slice(1),
+        ]);
       }
-    );
-    if (res.ok) {
-      dispatch({
-        type: "writable obs annotations - save complete",
-        obsAnnotations
+
+      /* then send the success case action through */
+      return dispatch({
+        type: "request differential expression success",
+        data: diffexpLists,
       });
-    } else {
-      dispatch({
-        type: "writable obs annotations - save error",
-        message: `HTTP error ${res.status} - ${res.statusText}`,
-        res
+    } catch (error) {
+      return dispatch({
+        type: "request differential expression error",
+        error,
       });
     }
-  } catch (error) {
-    dispatch({
-      type: "writable obs annotations - save error",
-      message: error.toString(),
-      error
-    });
-  }
-};
+  };
+
+function fetchJson(pathAndQuery) {
+  return doJsonRequest(
+    `${globals.API.prefix}${globals.API.version}${pathAndQuery}`
+  );
+}
 
 export default {
-  regraph,
-  resetInterface,
-  requestSingleGeneExpressionCountsForColoringPOST,
-  requestDifferentialExpression,
-  requestUserDefinedGene,
   doInitialDataLoad,
-  saveObsAnnotations
+  requestDifferentialExpression,
+  requestSingleGeneExpressionCountsForColoringPOST,
+  requestUserDefinedGene,
+  selectContinuousMetadataAction: selnActions.selectContinuousMetadataAction,
+  selectCategoricalMetadataAction: selnActions.selectCategoricalMetadataAction,
+  selectCategoricalAllMetadataAction:
+    selnActions.selectCategoricalAllMetadataAction,
+  graphBrushStartAction: selnActions.graphBrushStartAction,
+  graphBrushChangeAction: selnActions.graphBrushChangeAction,
+  graphBrushDeselectAction: selnActions.graphBrushDeselectAction,
+  graphBrushCancelAction: selnActions.graphBrushCancelAction,
+  graphBrushEndAction: selnActions.graphBrushEndAction,
+  graphLassoStartAction: selnActions.graphLassoStartAction,
+  graphLassoEndAction: selnActions.graphLassoEndAction,
+  graphLassoCancelAction: selnActions.graphLassoCancelAction,
+  graphLassoDeselectAction: selnActions.graphLassoDeselectAction,
+  clipAction: viewActions.clipAction,
+  subsetAction: viewActions.subsetAction,
+  resetSubsetAction: viewActions.resetSubsetAction,
+  annotationCreateCategoryAction: annoActions.annotationCreateCategoryAction,
+  annotationRenameCategoryAction: annoActions.annotationRenameCategoryAction,
+  annotationDeleteCategoryAction: annoActions.annotationDeleteCategoryAction,
+  annotationCreateLabelInCategory: annoActions.annotationCreateLabelInCategory,
+  annotationDeleteLabelFromCategory:
+    annoActions.annotationDeleteLabelFromCategory,
+  annotationRenameLabelInCategory: annoActions.annotationRenameLabelInCategory,
+  annotationLabelCurrentSelection: annoActions.annotationLabelCurrentSelection,
+  saveObsAnnotationsAction: annoActions.saveObsAnnotationsAction,
+  saveGenesetsAction: annoActions.saveGenesetsAction,
+  needToSaveObsAnnotations: annoActions.needToSaveObsAnnotations,
+  layoutChoiceAction: embActions.layoutChoiceAction,
+  setCellSetFromSelection: selnActions.setCellSetFromSelection,
+  genesetDelete: genesetActions.genesetDelete,
+  genesetAddGenes: genesetActions.genesetAddGenes,
+  genesetDeleteGenes: genesetActions.genesetDeleteGenes,
 };

client/src/actions/selection.js

@@ -0,0 +1,192 @@
+/*
+Action creators for selection 
+*/
+export const selectContinuousMetadataAction =
+  (type, query, range, oldProps = {}) =>
+  async (dispatch, getState) => {
+    const { obsCrossfilter: prevObsCrossfilter } = getState();
+
+    const selection = range
+      ? {
+          mode: "range",
+          lo: range[0],
+          hi: range[1],
+          inclusive: true, // [lo, hi] incluisve selection
+        }
+      : { mode: "all" };
+
+    const obsCrossfilter = await prevObsCrossfilter.select(...query, selection);
+
+    dispatch({
+      type,
+      obsCrossfilter,
+      range,
+      ...oldProps,
+    });
+  };
+
+export const selectCategoricalMetadataAction =
+  (
+    type, // action type
+    metadataField, // annotation category name
+    labels,
+    label, // the label being selected/deselected
+    isSelected, // bool
+    oldProps = {}
+  ) =>
+  async (dispatch, getState) => {
+    const { obsCrossfilter: prevObsCrossfilter, categoricalSelection } =
+      getState();
+
+    const labelSelectionState = new Map(categoricalSelection[metadataField]);
+    labels.forEach(
+      (l) => labelSelectionState.has(l) || labelSelectionState.set(l, true)
+    );
+    labelSelectionState.set(label, isSelected);
+
+    const values = Array.from(labelSelectionState.keys()).filter((k) =>
+      labelSelectionState.get(k)
+    );
+    const selection = {
+      mode: "exact",
+      values,
+    };
+    const obsCrossfilter = await prevObsCrossfilter.select(
+      "obs",
+      metadataField,
+      selection
+    );
+
+    dispatch({
+      type,
+      obsCrossfilter,
+      metadataField,
+      labelSelectionState,
+      ...oldProps,
+    });
+  };
+
+export const selectCategoricalAllMetadataAction =
+  (
+    type, // action type
+    metadataField, // annotation category name
+    labels,
+    isSelected, // bool, select all or none
+    oldProps = {}
+  ) =>
+  async (dispatch, getState) => {
+    const { obsCrossfilter: prevObsCrossfilter, categoricalSelection } =
+      getState();
+
+    const labelSelectionState = new Map(categoricalSelection[metadataField]);
+    labels.forEach((label) => labelSelectionState.set(label, isSelected));
+
+    const selection = { mode: isSelected ? "all" : "none" };
+    const obsCrossfilter = await prevObsCrossfilter.select(
+      "obs",
+      metadataField,
+      selection
+    );
+
+    dispatch({
+      type,
+      obsCrossfilter,
+      metadataField,
+      labelSelectionState,
+      ...oldProps,
+    });
+  };
+
+/**
+ ** Graph selection-related actions
+ **/
+
+export const graphBrushStartAction = () =>
+  /* no change to crossfilter until a change fires */
+  ({ type: "graph brush start" });
+
+const _graphBrushWithinRectAction =
+  (type, embName, brushCoords) => async (dispatch, getState) => {
+    const { obsCrossfilter: prevObsCrossfilter } = getState();
+
+    const selection = { mode: "within-rect", ...brushCoords };
+    const obsCrossfilter = await prevObsCrossfilter.select(
+      "emb",
+      embName,
+      selection
+    );
+
+    dispatch({
+      type,
+      obsCrossfilter,
+      brushCoords,
+    });
+  };
+
+const _graphAllAction = (type, embName) => async (dispatch, getState) => {
+  const { obsCrossfilter: prevObsCrossfilter } = getState();
+
+  const obsCrossfilter = await prevObsCrossfilter.select("emb", embName, {
+    mode: "all",
+  });
+
+  dispatch({
+    type,
+    obsCrossfilter,
+  });
+};
+
+export const graphBrushChangeAction = (embName, brushCoords) =>
+  _graphBrushWithinRectAction("graph brush change", embName, brushCoords);
+
+export const graphBrushEndAction = (embName, brushCoords) =>
+  _graphBrushWithinRectAction("graph brush end", embName, brushCoords);
+
+export const graphBrushCancelAction = (embName) =>
+  _graphAllAction("graph brush cancel", embName);
+export const graphBrushDeselectAction = (embName) =>
+  _graphAllAction("graph brush deselect", embName);
+
+export const graphLassoStartAction = () =>
+  /* no change to crossfilter until a change fires */
+  ({ type: "graph lasso start" });
+
+export const graphLassoCancelAction = (embName) =>
+  _graphAllAction("graph lasso cancel", embName);
+
+export const graphLassoDeselectAction = (embName) =>
+  _graphAllAction("graph lasso cancel", embName);
+
+export const graphLassoEndAction =
+  (embName, polygon) => async (dispatch, getState) => {
+    const { obsCrossfilter: prevObsCrossfilter } = getState();
+
+    const selection = {
+      mode: "within-polygon",
+      polygon,
+    };
+    const obsCrossfilter = await prevObsCrossfilter.select(
+      "emb",
+      embName,
+      selection
+    );
+
+    dispatch({
+      type: "graph lasso end",
+      obsCrossfilter,
+      polygon,
+    });
+  };
+
+/*
+Differential expression set selection
+*/
+export const setCellSetFromSelection = (cellSetId) => (dispatch, getState) => {
+  const { obsCrossfilter } = getState();
+  const selected = obsCrossfilter.allSelectedLabels();
+
+  dispatch({
+    type: `store current cell selection as differential set ${cellSetId}`,
+    data: selected.length > 0 ? selected : null,
+  });
+};

client/src/actions/viewStack.js

@@ -0,0 +1,74 @@
+/*
+The following actions manage the view stack for annoMatrix.
+
+Conventions used and assumed elsewhere in the code base:
+  * there will be zero or one clip view, and it will be the TOP view always.
+  * there will be zero or more subset views
+
+In other words, in our current use, we do not stack multiple clip views but we do
+stack multiple subsets.
+
+If these conventions change, code elsewhere (eg. menubar/clip.js) will need to
+change as well.
+*/
+import { AnnoMatrixObsCrossfilter } from "../annoMatrix";
+import {
+  _clipAnnoMatrix,
+  _userSubsetAnnoMatrix,
+  _userResetSubsetAnnoMatrix,
+} from "../util/stateManager/viewStackHelpers";
+
+export const clipAction = (min, max) => (dispatch, getState) => {
+  /*
+  apply a clip to the current annoMatrix.  By convention, the clip
+  view is ALWAYS the top view.
+  */
+  const { annoMatrix: prevAnnoMatrix } = getState();
+  const annoMatrix = _clipAnnoMatrix(prevAnnoMatrix, min, max);
+  const obsCrossfilter = new AnnoMatrixObsCrossfilter(annoMatrix);
+  dispatch({
+    type: "set clip quantiles",
+    clipQuantiles: { min, max },
+    annoMatrix,
+    obsCrossfilter,
+  });
+};
+
+export const subsetAction = () => (dispatch, getState) => {
+  /*
+  Subset the annoMatrix to the current crossfilter selection by pushing a 
+  subset view.
+
+  By convention, a clip view is ALWAYS the top view, so if present, pop
+  off and re-apply
+  */
+  const { annoMatrix: prevAnnoMatrix, obsCrossfilter: prevObsCrossfilter } =
+    getState();
+  const annoMatrix = _userSubsetAnnoMatrix(
+    prevAnnoMatrix,
+    prevObsCrossfilter.allSelectedMask()
+  );
+  const obsCrossfilter = new AnnoMatrixObsCrossfilter(annoMatrix);
+  dispatch({
+    type: "subset to selection",
+    annoMatrix,
+    obsCrossfilter,
+  });
+};
+
+export const resetSubsetAction = () => (dispatch, getState) => {
+  /*
+  Reset the annoMatrix to all data.  Because we may have multiple views
+  stacked, we pop them all.  By convention, any clip transformation will
+  be the top of the stack, and must be preserved.
+  */
+
+  const { annoMatrix: prevAnnoMatrix } = getState();
+  const annoMatrix = _userResetSubsetAnnoMatrix(prevAnnoMatrix);
+  const obsCrossfilter = new AnnoMatrixObsCrossfilter(annoMatrix);
+  dispatch({
+    type: "reset subset",
+    annoMatrix,
+    obsCrossfilter,
+  });
+};