chore: CCIE-4984 conform to open sourcing guidelines

Trademark

.bumpversion.cfg

@@ -1,5 +1,16 @@
 [bumpversion]
-current_version = 0.16.0
+current_version = 1.3.0
+commit = True
+parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(?:-(?P<prerel>rc)\.(?P<prerelversion>\d+))?
+serialize = 
+	{major}.{minor}.{patch}-{prerel}.{prerelversion}
+	{major}.{minor}.{patch}
+
+[bumpversion:part:prerel]
+optional_value = release
+values = 
+	rc
+	release
 
 [bumpversion:file:setup.py]
 search = version="{current_version}"

.github/ISSUE_TEMPLATE/tech-issue.md

@@ -0,0 +1,23 @@
+---
+name: Tech Issue
+about: Engineering-specific technical work that is not product-specific. Engineering team "owns" these issues.
+title: ""
+labels: tech
+assignees: ""
+---
+
+## Motivation
+
+Why is this work important to engineers?
+
+## Definition of Done
+
+What should the end result look like? What will have been changed?
+
+## Tasks
+
+Detail the specific tasks that can be used to accomplish the desired changes.
+If detailed steps cannot be provided at this time, please file a [Tech Proposal](https://docs.google.com/document/d/1o2vuvl-kXwRJN1nBoPzJS_MAQgDGYnjmPZWa4qRDi-I/edit#heading=h.7dvzhm7gqc3v) instead.
+
+- [ ]
+- [ ]

.github/workflows/close-stale-prs.yml

@@ -0,0 +1,23 @@
+name: Close inactive pull requests
+on:
+  schedule:
+    - cron: "30 1 * * *"
+
+jobs:
+  close-issues:
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write
+      pull-requests: write
+    steps:
+      - uses: actions/stale@v5
+        with:
+          days-before-issue-stale: -1 # Do not mark any issues as stale
+          days-before-pr-stale: 14
+          days-before-pr-close: 3
+          stale-pr-message: "This PR has not seen any activity in the past 2 weeks; if no one comments or reviews it in the next 3 days, this PR will be closed."
+          close-pr-message: "This PR was closed because it has been inactive for 17 days, 3 days since being marked as stale. Please re-open if you still need this to be addressed."
+          stale-pr-label: "stale"
+          close-pr-label: "autoclosed"
+          exempt-draft-pr: true
+          repo-token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/codeql-analysis.yml

@@ -0,0 +1,67 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL Scan"
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [ main ]
+  schedule:
+    - cron: '0 8 * * *'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'javascript', 'python' ]
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ]
+        # Learn more:
+        # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v1
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+        # queries: ./path/to/local/query, your-org/your-repo/queries@main
+
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v1
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 https://git.io/JvXDl
+
+    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
+    #    and modify them (or add more) to build your code if your project
+    #    uses a compiled language
+
+    #- run: |
+    #   make bootstrap
+    #   make release
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v1

.github/workflows/compatibility_tests.yml

@@ -2,7 +2,7 @@ name: Compatibility Tests
 
 on:
   schedule:
-    - cron: '0 8 7 * 2'
+    - cron: "0 8 7 * 2"
   push:
     branches:
       - main
@@ -14,105 +14,99 @@ jobs:
   docker-build:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
       - name: Build docker image
         run: docker build .
 
-  cellxgene-main-with-python-and-anndata-versions:
-    name: python versions x anndata versions
-    runs-on: ubuntu-latest
+  matrix-compatibility-test:
+    name: cxg:${{ matrix.cellxgene_build }} os:${{ matrix.os }} py:${{ matrix.python-version }} anndata:${{ matrix.anndata_version || 'latest' }}
+    runs-on: ${{ matrix.os }}
     strategy:
+      fail-fast: false
       matrix:
-        python-version: [3.6, 3.7, 3.8]
-        anndata-version: [0.6.22.post1, 0.7.1]
-        test-suite: [smoke-test, smoke-test-annotations]
+        os: [ubuntu-latest, macos-latest, macos-13]
+        python-version: ["3.10", "3.11", "3.12"]
+        cellxgene_build: [main, latest]
+        # add anndata pinned version test for subset of matrix configurations,
+        # in order to reduce matrix cross-product explosion
+        include:
+          - python-version: 3.12
+            cellxgene_build: latest
+            # TODO: dynamically use the literal version in requirements.txt,
+            # to avoid having to update this in manually in the future
+            # TODO: Do not bother running this if anndata latest version
+            # matches this pinned version, to avoid a redundant test
+            anndata_version: "==0.10.9"
     steps:
-    - uses: actions/checkout@v2
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v1
-      with:
-        python-version: ${{ matrix.python-version }}
-    - name: Install dependencies
-      run: |
-        # 1. only install the dev requirements on top of what is in the cellxgene pip package
-        sudo apt-get update && sudo apt-get install -y libhdf5-serial-dev
-        sed -i 's/-r requirements.txt//' server/requirements-dev.txt
-        pip install -r server/requirements-dev.txt
-        # 2. install cellxgene
-        make pydist install-dist
-        # 3. install anndata
-        pip install anndata==${{ matrix.anndata-version }}
-        # workaround for anndata 0.6.22.post1 bug
-        [[ "0.6.22.post1" = "${{ matrix.anndata-version }}" ]] && pip install h5py==2.9.0 || true
-    - name: Tests
-      run: make unit-test ${{ matrix.test-suite }}
-
-  cellxgene-release-with-anndata-master:
-    name: cellxgene release with anndata master
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        test-suite: [smoke-test, smoke-test-annotations]
-    steps:
-      - uses: actions/checkout@v2
-      - name: Set up Python 3.7
-        uses: actions/setup-python@v1
+      - uses: actions/checkout@v4
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
         with:
-          python-version: 3.7
-      - name: Checkout
-        uses: actions/checkout@v2
+          python-version: ${{ matrix.python-version }}
+      - name: Cache env vars
+        run: echo "PIP_CACHE=`python -m pip cache dir`" >> $GITHUB_ENV
+      - name: Cache env vars (MacOS)
+        if: startsWith(matrix.os, 'macos')
+        run: echo "BREW_CACHE=`brew --cache`" >> $GITHUB_ENV
+      # FIXME: Only working for Linux
+      - name: Python cache
+        uses: actions/cache@v4
         with:
-          path: cellxgene
-      - name: Checkout tools repo
-        uses: actions/checkout@v2
+          path: ${{ env.PIP_CACHE }}
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+      - name: Node cache
+        uses: actions/cache@v4
         with:
-          repository: theislab/anndata
-          path: anndata
-      - name: Install dependencies
+          path: ~/.npm
+          key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-node-
+      - name: Brew cache (MacOS)
+        if: startsWith(matrix.os, 'macos')
+        uses: actions/cache@v4
+        with:
+          path: ${{ env.BREW_CACHE }}
+          key: ${{ runner.os }}-brew-
+      - name: Install dependencies (Ubuntu Linux)
+        if: startsWith(matrix.os, 'ubuntu')
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y libhdf5-serial-dev
+      - name: Install dependencies (MacOS)
+        if: startsWith(matrix.os, 'macos')
+        run: brew install hdf5
+      - name: Install cellxgene from `main` branch
+        if: matrix.cellxgene_build == 'main'
         run: |
-          cd cellxgene
-          # 1. only install the dev requirements on top of what is in the cellxgene pip package
-          make dev-env-client
-          sed -i 's/-r requirements.txt//' server/requirements-dev.txt
           pip install -r server/requirements-dev.txt
-          # 2. install cellxgene
-          pip install --upgrade cellxgene
-          # 3. install anndata
-          cd ../anndata && pip install -e .
-      - name: Tests
-        run: cd cellxgene && make unit-test ${{ matrix.test-suite }}
-
-  cellxgene-main-with-anndata-master:
-    name: cellxgene main with anndata master
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        test-suite: [smoke-test, smoke-test-annotations]
-    steps:
-      - uses: actions/checkout@v2
-      - name: Set up Python 3.7
-        uses: actions/setup-python@v1
-        with:
-          python-version: 3.7
-      - name: Checkout
-        uses: actions/checkout@v2
-        with:
-          path: cellxgene
-      - name: Checkout tools repo
-        uses: actions/checkout@v2
-        with:
-          repository: theislab/anndata
-          path: anndata
-      - name: Install dependencies
+          make pydist install-dist
+      - name: Install cellxgene from latest release (pypi.org)
+        if: matrix.cellxgene_build == 'latest'
         run: |
-          cd cellxgene
-          sed -i -E 's/^anndata[>=]=[0-9]+.[0-9]+.[0-9]+$/anndata/g' server/requirements.txt
-          make pydist install-dist dev-env
-          cd ../anndata
-          pip install -e .
-      - name: Tests
-        run: cd cellxgene && make unit-test ${{ matrix.test-suite }}
+          pip install --upgrade cellxgene
+          # install the additional dev requirements on top of what is in the
+          # cellxgene pip package, which are needed for testing, but otherwise
+          # keep same pip pkg versions as in the cxg release
+          sed -i'' -e 's/-r requirements.txt//' server/requirements-dev.txt
+          pip install -r server/requirements-dev.txt
+          pip install --force-reinstall numpy==2.0.1 numba>=0.60.0 pandas flatbuffers==2.0.7
+      - name: Install anndata version per matrix variable
+        run: pip install anndata${{ matrix.anndata_version }}
+      - name: Install node
+        run: make dev-env-client
+      # Run different types of test separately, to facilitate troubleshooting
+      - name: Unit Tests - client
+        run: make unit-test-client
+      - name: Unit Tests - server
+        run: make unit-test-server
+      - name: Smoke Tests
+        run: make smoke-test
+    # FIXME: Fails intermittently. See https://app.zenhub.com/workspaces/single-cell-5e2a191dad828d52cc78b028/issues/chanzuckerberg/cellxgene/2415
+    # - name: Smoke Tests with Annotations
+    #   run: make smoke-test-annotations

.github/workflows/deploy.yml

@@ -1,13 +0,0 @@
-name: Deploy via single cell infra repo
-
-on:
-  push:
-    branches: main
-
-jobs:
-  deploy:
-    runs-on: ubuntu-latest
-    steps:
-      - name: repository dispatch
-        run: |
-          curl -XPOST -u czi-sci-single-cell-eng:${{secrets.SCI_GITHUB_TOKEN}} -H "Accept: application/vnd.github.everest-preview+json"  -H "Content-Type: application/json" https://api.github.com/repos/chanzuckerberg/single-cell-infra/dispatches --data '{"event_type": "cellxgene-hook"}'

.github/workflows/lint-pr-commit-message.yml

@@ -0,0 +1,19 @@
+
+name: "Lint PR commit message"
+
+on:
+  pull_request_target:
+    types:
+      - opened
+      - edited
+      - synchronize
+
+jobs:
+  main:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: amannn/action-semantic-pull-request@v3.4.1
+        with:
+          validateSingleCommit: true
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/push_tests.yml

@@ -14,15 +14,15 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - run: |
           git fetch --depth=1 origin +${{github.base_ref}}
-      - name: Set up Python 3.7
-        uses: actions/setup-python@v1
+      - name: Set up Python 3.12
+        uses: actions/setup-python@v5
         with:
-          python-version: 3.7
+          python-version: 3.12
       - name: Node cache
-        uses: actions/cache@v1
+        uses: actions/cache@v4
         with:
           path: ~/.npm
           key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
@@ -31,33 +31,36 @@ jobs:
       - name: Install dependencies
         run: |
           pip install flake8
+          pip install black
           cd client
           npm install
-      - name: Lint with flake8
+      - name: Format with black and lint with flake8
         run: |
           make lint-server
       - name: Lint src with eslint
         working-directory: ./client
         run: |
-          make lint
+          npx eslint src __tests__
 
   unit-test:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - name: Set up Python 3.7
-        uses: actions/setup-python@v1
+      - uses: actions/checkout@v4
+      - name: Set up Python 3.12 (pyenv) # pyenv needed for mlflow in cli annotate tests
+        uses: gabrielfalcao/pyenv-action@v9
         with:
-          python-version: 3.7
+          default: 3.12
+          command: pip install -U pip # upgrade pip after installing python
+      - run: pip install virtualenv # virtualenv needed for mlflow in cli annotate tests
       - name: Python cache
-        uses: actions/cache@v1
+        uses: actions/cache@v4
         with:
           path: ~/.cache/pip
           key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}
           restore-keys: |
             ${{ runner.os }}-pip-
       - name: Node cache
-        uses: actions/cache@v1
+        uses: actions/cache@v4
         with:
           path: ~/.npm
           key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
@@ -67,66 +70,69 @@ jobs:
         run: make pydist install-dist dev-env-server
       - name: Unit tests
         run: |
-          make unit-test
-          bash <(curl -s https://codecov.io/bash) -y .codecov.yml -k server -cF backend,python,unitTest
+          make unit-test-server unit-test-client
+          bash <(curl -s https://codecov.io/bash) -y .codecov.yml -k server -cF server,python,unitTest
           cd client && ./node_modules/codecov/bin/codecov --yml=../.codecov.yml --root=../ --gcov-root=../ -C -F frontend,javascript,unitTest
 
   smoke-tests:
     runs-on: macos-latest
     timeout-minutes: 20
     steps:
-      - uses: actions/checkout@v2
-      - name: Set up Python 3.7
-        uses: actions/setup-python@v1
+      - uses: actions/checkout@v4
+      - name: Set up Python 3.12
+        uses: actions/setup-python@v5
         with:
-          python-version: 3.7
+          python-version: 3.12
       - name: Python cache
-        uses: actions/cache@v1
+        uses: actions/cache@v4
         with:
           path: ~/.cache/pip
           key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}
           restore-keys: |
             ${{ runner.os }}-pip-
       - name: Node cache
-        uses: actions/cache@v1
+        uses: actions/cache@v4
         with:
           path: ~/.npm
           key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
           restore-keys: |
             ${{ runner.os }}-node-
       - name: Install dependencies
-        run: make pydist install-dist
+        run: |
+          pip install setuptools
+          make pydist install-dist
       - name: Smoke tests (without annotations feature)
         run: |
           cd client && make smoke-test
           ./node_modules/codecov/bin/codecov --yml=../.codecov.yml --root=../ --gcov-root=../ -C -F frontend,javascript,smokeTest
 
-  smoke-tests-annotations:
-    runs-on: ubuntu-latest
-    timeout-minutes: 20
-    steps:
-      - uses: actions/checkout@v2
-      - name: Set up Python 3.7
-        uses: actions/setup-python@v1
-        with:
-          python-version: 3.7
-      - name: Python cache
-        uses: actions/cache@v1
-        with:
-          path: ~/.cache/pip
-          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}
-          restore-keys: |
-            ${{ runner.os }}-pip-
-      - name: Node cache
-        uses: actions/cache@v1
-        with:
-          path: ~/.npm
-          key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
-          restore-keys: |
-            ${{ runner.os }}-node-
-      - name: Install dependencies
-        run: make pydist install-dist
-      - name: Smoke tests (with annotations feature)
-        run: |
-          cd client && make smoke-test-annotations
-          ./node_modules/codecov/bin/codecov --yml=../.codecov.yml --root=../ --gcov-root=../ -C -F frontend,javascript,smokeTestAnnotations
+  # TODO: reinstate: https://github.com/chanzuckerberg/cellxgene/issues/2544
+  # smoke-tests-annotations:
+  #   runs-on: ubuntu-latest
+  #   timeout-minutes: 20
+  #   steps:
+  #     - uses: actions/checkout@v2
+  #     - name: Set up Python 3.9
+  #       uses: actions/setup-python@v4
+  #       with:
+  #         python-version: 3.9
+  #     - name: Python cache
+  #       uses: actions/cache@v1
+  #       with:
+  #         path: ~/.cache/pip
+  #         key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}
+  #         restore-keys: |
+  #           ${{ runner.os }}-pip-
+  #     - name: Node cache
+  #       uses: actions/cache@v1
+  #       with:
+  #         path: ~/.npm
+  #         key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
+  #         restore-keys: |
+  #           ${{ runner.os }}-node-
+  #     - name: Install dependencies
+  #       run: make pydist install-dist
+  #     - name: Smoke tests (with annotations feature)
+  #       run: |
+  #         cd client && make smoke-test-annotations
+  #         ./node_modules/codecov/bin/codecov --yml=../.codecov.yml --root=../ --gcov-root=../ -C -F frontend,javascript,smokeTestAnnotations

.gitignore

@@ -15,7 +15,7 @@ dist/
 *.egg-info
 
 # Environments
-venv/
+venv*/
 cellxgene/
 
 # client build
@@ -54,3 +54,6 @@ client/.eslintcache
 
 # E2E Testing
 ignoreE2E*
+
+# annotate subcmd
+.models_cache

CONTRIBUTING.md

@@ -1,3 +1,3 @@
 We warmly welcome contributions from the community!
 
-Whether you want to contribute ideas, requests, documentation, or code, you can get started by visiting our [contribution guide](https://chanzuckerberg.github.io/cellxgene/posts/contribute).
+Whether you want to contribute ideas, requests, documentation, or code, you can get started by visiting our [contribution guide](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/contribute.md).

Dockerfile

@@ -5,6 +5,7 @@ ENV LANG=C.UTF-8
 
 RUN apt-get update && \
     apt-get install -y build-essential libxml2-dev python3-dev python3-pip zlib1g-dev python3-requests python3-aiohttp && \
+    python3 -m pip install --upgrade pip && \
     pip3 install cellxgene
 
 ENTRYPOINT ["cellxgene"]

LICENSE

@@ -0,0 +1,20 @@
+The MIT License (MIT)
+
+Copyright (c) 2017-2023 Chan Zuckerberg Initiative
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

LICENSE.txt

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2013
+Copyright (c) 2017-2023 Chan Zuckerberg Initiative
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of
 this software and associated documentation files (the "Software"), to deal in
@@ -17,4 +17,4 @@ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
 FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
 COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
 IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
-CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

MANIFEST.in

@@ -3,3 +3,6 @@ recursive-include server/common/web/static *
 
 include server/requirements.txt
 include server/requirements-prepare.txt
+include server/requirements-annotate.txt
+include server/converters/schema/hgnc_complete_set.txt.gz
+include server/converters/schema/schema_definitions/*

Makefile

@@ -16,8 +16,13 @@ clean: clean-lite clean-server clean-client
 clean-lite:
 	rm -rf $(CLEANFILES)
 
-clean-%:
-	cd $(*) && $(MAKE) clean
+.PHONY: clean-client
+clean-client:
+	cd client && $(MAKE) clean
+
+.PHONY: clean-server
+clean-server:
+	cd server && $(MAKE) clean
 
 
 # BUILDING PACKAGE
@@ -28,20 +33,20 @@ build-client:
 
 .PHONY: build
 build: clean build-client
-	git ls-files server/ | grep -v 'server/test/' | cpio -pdm $(BUILDDIR)
+	git ls-files server/ | cpio -pdm $(BUILDDIR)
 	cp -r client/build/  $(CLIENTBUILD)
 	$(call copy_client_assets,$(CLIENTBUILD),$(SERVERBUILD))
 	cp MANIFEST.in README.md setup.cfg setup.py $(BUILDDIR)
 
 # If you are actively developing in the server folder use this, dirties the source tree
 .PHONY: build-for-server-dev
-build-for-server-dev: clean-server build-client
-	$(call copy_client_assets,client/build,server)
+build-for-server-dev: clean-server build-client copy-client-assets
 
 .PHONY: copy-client-assets
 copy-client-assets:
 	$(call copy_client_assets,client/build,server)
 
+
 # TESTING
 .PHONY: test
 test: unit-test smoke-test
@@ -49,8 +54,22 @@ test: unit-test smoke-test
 .PHONY: unit-test
 unit-test: unit-test-server unit-test-client
 
-unit-test-%:
-	cd $(*) && $(MAKE) unit-test
+.PHONY: test-server
+test-server: unit-test-server smoke-test
+
+.PHONY: unit-test-client
+unit-test-client:
+	cd client && $(MAKE) unit-test
+
+.PHONY: unit-test-server
+unit-test-server:
+	PYTHONWARNINGS=ignore:ResourceWarning coverage run \
+		--source=server \
+		--omit=.coverage,venv \
+		-m unittest discover \
+		--start-directory test/unit \
+		--verbose; test_result=$$?; \
+	exit $$test_result \
 
 .PHONY: smoke-test
 smoke-test:
@@ -60,14 +79,9 @@ smoke-test:
 smoke-test-annotations:
 	cd client && $(MAKE) smoke-test-annotations
 
-.PHONY: test-db
-test-db:
-	cd server && $(MAKE) test-db
-
-
 # FORMATTING CODE
 
-.PHOHY: fmt
+.PHONY: fmt
 fmt: fmt-client fmt-py
 
 .PHONY: fmt-client
@@ -81,9 +95,10 @@ fmt-py:
 .PHONY: lint
 lint: lint-server lint-client
 
+
 .PHONY: lint-server
-lint-server:
-	flake8 server
+lint-server: fmt-py
+	flake8 server --per-file-ignores='test/fixtures/dataset_config_outline.py:F821 test/fixtures/server_config_outline.py:F821 test/performance/scale_test_annotations.py:E501'
 
 .PHONY: lint-client
 lint-client:
@@ -96,31 +111,46 @@ pydist: build
 	cd $(BUILDDIR); python setup.py sdist -d ../dist
 	@echo "done"
 
-
 # RELEASE HELPERS
 
-# create new version to commit to main
-.PHONY: release-stage-1
-release-stage-1: dev-env bump clean-lite gen-package-lock
+# Set PART=[major, minor, patch] as param to make bump.
+# This will create a release candidate. (i.e. 0.16.1 -> 0.16.2-rc.0 for a patch bump)
+.PHONY: bump-version
+bump-version:
+	bumpversion --config-file .bumpversion.cfg $(PART)
+
+# Create new version to commit to main
+.PHONY: create-release-candidate
+create-release-candidate: bump-version clean-lite gen-package-lock
 	@echo "Version bumped part:$(PART) and client built. Ready to commit and push"
 
-# build dist and release to dev pypi
-.PHONY: release-stage-2
-release-stage-2: dev-env pydist twine
+# Bump the release candidate version if needed (i.e. the previous release candidate had errors).
+.PHONY: recreate-release-candidate
+recreate-release-candidate: bump-release-candidate clean-lite gen-package-lock
+	@echo "Version bumped part:$(PART) and client built. Ready to commit and push"
+
+# Build dist and release to Test PyPI
+.PHONY: release-candidate-to-test-pypi
+release-candidate-to-test-pypi: pydist twine
 	@echo "Dist built and uploaded to test.pypi.org"
 	@echo "Test the install:"
 	@echo "    make install-release-test"
-	@echo "Then upload to Pypi prod:"
-	@echo "    make twine-prod"
 
-.PHONY: release-stage-final
-release-stage-final: twine-prod
+# Build final dist (gets rid of the rc tag) and release final candidate to TestPyPI
+.PHONY: release-final-to-test-pypi
+release-final-to-test-pypi: bump-release clean-lite gen-package-lock pydist twine
+	@echo "Final release dist built and uploaded to test.pypi.org"
+	@echo "Test the install:"
+	@echo "    make install-release-test"
+
+.PHONY: release-final
+release-final: twine-prod
 	@echo "Release uploaded to pypi.org"
 
 # DANGER: releases directly to prod
-# use this if you accidently burned a test release version number,
+# use this if you accidentally burned a test release version number,
 .PHONY: release-directly-to-prod
-release-directly-to-prod: dev-env pydist twine-prod
+release-directly-to-prod: pydist twine-prod
 	@echo "Dist built and uploaded to pypi.org"
 	@echo "Test the install:"
 	@echo "    make install-release"
@@ -136,10 +166,15 @@ dev-env-client:
 dev-env-server:
 	pip install -r server/requirements-dev.txt
 
-# give PART=[major, minor, part] as param to make bump
-.PHONY: bump
-bump:
-	bumpversion --config-file .bumpversion.cfg $(PART)
+# Increments the release candidate version (i.e. 0.16.2-rc.1 -> 0.16.2-rc.2)
+.PHONY: bump-release-candidate
+bump-release-candidate:
+	bumpversion --config-file .bumpversion.cfg prerelversion --allow-dirty
+
+# Finalizes the release candidate by removing the release candidate tag (i.e. 0.16.2-rc.2 -> 0.16.2).
+.PHONY: bump-release
+bump-release:
+	bumpversion --config-file .bumpversion.cfg prerel --allow-dirty
 
 .PHONY: twine
 twine:
@@ -165,7 +200,7 @@ install-dev: uninstall
 # install from test.pypi to test your release
 .PHONY: install-release-test
 install-release-test: uninstall
-	pip install --no-cache-dir --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple cellxgene
+	pip install --no-cache-dir --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple cellxgene==$(VERSION)
 	@echo "Installed cellxgene from test.pypi.org, now run and smoke test"
 
 # install from pypi to test your release

PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,11 @@
+#### Reviewers
+**Functional:** 
+
+**Readability:** 
+
+---
+
+## Changes
+- add
+- remove
+- modify

README.md

@@ -7,27 +7,27 @@ _an interactive explorer for single-cell transcriptomics data_
 [![Compatibility Tests](https://github.com/chanzuckerberg/cellxgene/workflows/Compatibility%20Tests/badge.svg)](https://github.com/chanzuckerberg/cellxgene/actions?query=workflow%3A%22Compatibility+Tests%22)
 ![Code Coverage](https://codecov.io/gh/chanzuckerberg/cellxgene/branch/main/graph/badge.svg)
 
-cellxgene (pronounced "cell-by-gene") is an interactive data explorer for single-cell transcriptomics datasets, such as those coming from the [Human Cell Atlas](https://humancellatlas.org). Leveraging modern web development techniques to enable fast visualizations of at least 1 million cells, we hope to enable biologists and computational researchers to explore their data.
+CZ CELLxGENE Annotate (pronounced "cell-by-gene") is an interactive data explorer for single-cell datasets, such as those coming from the [Human Cell Atlas](https://humancellatlas.org). Leveraging modern web development techniques to enable fast visualizations of at least 1 million cells, we hope to enable biologists and computational researchers to explore their data.
 
-Whether you need to visualize one thousand cells or one million, cellxgene helps you gain insight into your single-cell data.
+Whether you need to visualize one thousand cells or one million, CELLxGENE Annotate helps you gain insight into your single-cell data.
 
 <img src="https://github.com/chanzuckerberg/cellxgene/raw/main/docs/images/crossfilter.gif" width="350" height="200" hspace="30"><img src="https://github.com/chanzuckerberg/cellxgene/raw/main/docs/images/category-breakdown.gif" width="350" height="200" hspace="30">
 
 # Getting started
 
-### The comprehensive guide to cellxgene
+### The comprehensive guide to CZ CELLxGENE Annotate
 
-[The cellxgene documentation is your one-stop-shop for information about cellxgene](https://chanzuckerberg.github.io/cellxgene/)! You may be particularly interested in:
+[The CZ CELLxGENE Annotate documentation is your one-stop-shop for information about CELLxGENE Annotate](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/README.md)! You may be particularly interested in:
 
-- Seeing [what cellxgene can do](https://chanzuckerberg.github.io/cellxgene/posts/gallery)
-- Learning more about cellxgene [installation](https://chanzuckerberg.github.io/cellxgene/posts/install) and [usage](https://chanzuckerberg.github.io/cellxgene/posts/launch)
-- [Preparing your own data](https://chanzuckerberg.github.io/cellxgene/posts/prepare) for use in cellxgene
-- Checking out [our roadmap](https://chanzuckerberg.github.io/cellxgene/posts/roadmap) for future development
-- [Contributing](https://chanzuckerberg.github.io/cellxgene/posts/contribute) to cellxgene
+- Seeing [what Annotate can do](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/explore-data/explorer-tutorials.md)
+- Learning more about Annotate [installation](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/desktop/install.md) and [usage](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/desktop/quick-start.md#quick-start-1)
+- [Preparing your own data](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/desktop/data-reqs.md) for use in Annotate
+- Checking out [our roadmap](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/roadmap.md) for future development
+- [Contributing](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/contribute.md) to Annotate
 
 ### Quick start
 
-To install cellxgene you need Python 3.6+. We recommend [installing cellxgene into a conda or virtual environment.](https://chanzuckerberg.github.io/cellxgene/posts/install)
+To install CELLxGENE Annotate you need Python 3.10+. We recommend [installing Annotate into a conda or virtual environment.](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/desktop/install.md)
 
 Install the package.
 
@@ -35,69 +35,67 @@ Install the package.
 pip install cellxgene
 ```
 
-Launch cellxgene with an example [anndata](https://anndata.readthedocs.io/en/latest/) file
+Launch Annotate with an example [anndata](https://anndata.readthedocs.io/en/latest/) file
 
 ```bash
 cellxgene launch https://cellxgene-example-data.czi.technology/pbmc3k.h5ad
 ```
 
-To explore more datasets already formatted for cellxgene, check out the [Demo data](https://chanzuckerberg.github.io/cellxgene/posts/demo-data) or
-see [Preparing your data](https://chanzuckerberg.github.io/cellxgene/posts/prepare) to learn more about formatting your own
-data for cellxgene.
+To explore more datasets already formatted for Annotate, check out the [Demo data](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/desktop/quick-start.md#example-datasets) or
+see [Preparing your data](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/desktop/data-reqs.md) to learn more about formatting your own
+data for CELLxGENE Annotate.
 
 ### Supported browsers
 
-cellxgene currently supports the following browsers:
+CELLxGENE Annotate currently supports the following browsers:
 
 - Google Chrome 61+
 - Edge 15+
 - Firefox 60+
-- Safari 10.1+
 
 Please [file an issue](https://github.com/chanzuckerberg/cellxgene/issues/new/choose) if you would like us to add support for an unsupported browser.
 
 ### Finding help
 
 We'd love to hear from you!
-For questions, suggestions, or accolades, [join the `#cellxgene-users` channel on the CZI Science Slack](https://join-cellxgene-users.herokuapp.com/) and say "hi!".
+For questions, suggestions, or accolades, join the `#cellxgene-users` channel on the [CZI Science Community Slack](https://czi.co/science-slack) and say "hi!".
 
 For any errors, [report bugs on Github](https://github.com/chanzuckerberg/cellxgene/issues).
 
-# Developing with cellxgene
+# Developing with CZ CELLxGENE Annotate
 
 ### Contributing
 
-We warmly welcome contributions from the community! Please see our [contributing guide](https://chanzuckerberg.github.io/cellxgene/posts/contribute) and don't hesitate to open an issue or send a pull request to improve cellxgene.
+We warmly welcome contributions from the community! Please see our [contributing guide](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/contribute.md) and don't hesitate to open an issue or send a pull request to improve CELLxGENE Annotate. Please see the [dev_docs](https://github.com/chanzuckerberg/cellxgene/tree/main/dev_docs) for pull request suggestions, unit test details, local documentation preview, and other development specifics.
 
 This project adheres to the Contributor Covenant [code of conduct](https://github.com/chanzuckerberg/.github/blob/master/CODE_OF_CONDUCT.md). By participating, you are expected to uphold this code. Please report unacceptable behavior to opensource@chanzuckerberg.com.
 
 ### Reuse
 
-This project was started with the sole goal of empowering the scientific community to explore and understand their data. As such, we encourage other scientific tool builders in academia or industry to adopt the patterns, tools, and code from this project, and reach out to us with ideas or questions. All code is freely available for reuse under the [MIT license](https://opensource.org/licenses/MIT).
+This project was started with the sole goal of empowering the scientific community to explore and understand their data.
+As such, we encourage other scientific tool builders in academia or industry to adopt the patterns, tools, and code from
+this project. All code is freely available for reuse under the [MIT license](https://opensource.org/licenses/MIT).
+
+Before extending CELLxGENE Annotate, we encourage you to reach out to us with ideas or questions. It might be possible that an
+extension could be directly contributed, which would make it available for a wider audience, or that it's on our
+[roadmap](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/roadmap.md) and under active development.
+
+See the [CELLxGENE extensions](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/community-extensions.md) section of our documentation for examples of community use and CELLxGENE extensions.
+
+### Trademarks
+
+CZ CELLXGENE, CZ CELLXGENE DISCOVER, and CZ CELLXGENE ANNOTATE are trademarks of the Chan Zuckerberg Initiative. All rights reserved.
+
+Use, reuse, modification, and re-distribution of the source code in this repository is subject to the terms of the applicable open source [license](LICENSE.txt). However, that license does not grant permission to use the trademarks without separate, express permission from the Chan Zuckerberg Initiative.
+
 
 ### Security
 
 If you believe you have found a security issue, we would appreciate notification. Please send email to <security@chanzuckerberg.com>.
 
-# About
+# Inspiration
 
-### Core team
-
-The current core team:
-
-- Colin Megill, frontend & product design
-- Bruce Martin, software engineer
-- Sidney Bell, computational biologist
-- Lia Prins, designer
-- Severiano Badajoz, software engineer
-
-We would also like to gratefully acknowledge contributions from past core team members:
-
-- Charlotte Weaver, software engineer
-
-### Inspiration
-
-We've been heavily inspired by several other related single-cell visualization projects, including the [UCSC Cell Browswer](http://cells.ucsc.edu/), [Cytoscape](http://www.cytoscape.org/), [Xena](https://xena.ucsc.edu/), [ASAP](https://asap.epfl.ch/), [Gene Pattern](http://genepattern-notebook.org/), and many others. We hope to explore collaborations where useful as this community works together on improving interactive visualization for single-cell data.
+We've been heavily inspired by several other related single-cell visualization projects, including the [UCSC Cell Browser](http://cells.ucsc.edu/), [Cytoscape](http://www.cytoscape.org/), [Xena](https://xena.ucsc.edu/), [ASAP](https://asap.epfl.ch/), [GenePattern](http://genepattern-notebook.org/), and many others. We hope to explore collaborations where useful as this community works together on improving interactive visualization for single-cell data.
 
 We were inspired by Mike Bostock and the [crossfilter](https://github.com/crossfilter) team for the design of our filtering implementation.
 

SECURITY.md

@@ -0,0 +1,2 @@
+## Reporting Security Issues
+If you believe you have found a security issue, please responsibly disclose by contacting us at [security@chanzuckerberg.com](mailto:security@chanzuckerberg.com).

client/.husky/pre-commit

@@ -0,0 +1,5 @@
+#!/bin/sh
+. "$(dirname "$0")/_/husky.sh"
+
+cd client
+npx --no-install lint-staged --config "./configuration/lint-staged/lint-staged.config.js"

client/.nvmrc

@@ -0,0 +1 @@
+18.17.0

client/Makefile

@@ -1,7 +1,13 @@
 include ../common.mk
 
-ANNOTATIONS := $(if $(ANNOTATIONS),$(ANNOTATIONS),../server/test/fixtures/pbmc3k-annotations.csv)
+ANNOTATIONS := $(if $(ANNOTATIONS),$(ANNOTATIONS),../test/fixtures/pbmc3k-annotations.csv)
+GENE_SETS := $(if $(GENE_SETS),$(GENE_SETS),../test/fixtures/pbmc3k-genesets.csv)
 ANNOTATIONS_FILENAME := $(shell basename $(ANNOTATIONS))
+GENE_SETS_FILENAME := $(shell basename $(GENE_SETS))
+
+CXG_CONFIG := $(if $(CXG_CONFIG),$(CXG_CONFIG),./__tests__/e2e/test_config.yaml)
+
+
 
 # Packaging
 .PHONY: clean
@@ -31,7 +37,7 @@ start-frontend:
 .PHONY: smoke-test
 smoke-test:
 	start_server_and_test \
-		'CXG_OPTIONS="--disable-annotations" $(MAKE) start-server' \
+		'CXG_OPTIONS="--config-file $(CXG_CONFIG)" $(MAKE) start-server' \
 		$(CXG_SERVER_PORT) \
 		'CXG_URL_BASE="http://localhost:$(CXG_SERVER_PORT)" npm run e2e -- --verbose false'
 
@@ -40,8 +46,9 @@ smoke-test:
 smoke-test-annotations:
 	$(eval TMP_DIR := $(shell  mktemp -d /tmp/cellxgene_XXXXXX))
 	cp $(ANNOTATIONS) $(TMP_DIR)/ && \
+	cp $(GENE_SETS) $(TMP_DIR)/ && \
 	start_server_and_test \
-		'CXG_OPTIONS="--annotations-file $(TMP_DIR)/$(ANNOTATIONS_FILENAME)" $(MAKE) start-server' \
+		'CXG_OPTIONS="--annotations-file $(TMP_DIR)/$(ANNOTATIONS_FILENAME) --gene-sets-file $(TMP_DIR)/$(GENE_SETS_FILENAME)" $(MAKE) start-server' \
 		$(CXG_SERVER_PORT) \
 		'CXG_URL_BASE="http://localhost:$(CXG_SERVER_PORT)" npm run e2e-annotations -- --verbose false'
 	rm -rf $(TMP_DIR)

client/__tests__/e2e/__snapshots__/e2e.test.js.snap

@@ -1,5 +1,5 @@
 // Jest Snapshot v1, https://goo.gl/fbAQLP
 
-exports[`did launch page launched 1`] = `"<span style=\\"width: 185px; display: flex; overflow: hidden; justify-content: flex-start;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">pbm</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">c3k</span><span style=\\"position: absolute; right: 0px; color: inherit;\\">c3k</span></span></span>"`;
+exports[`did launch page launched 1`] = `"<span style=\\"max-width: 155px; display: flex; overflow: hidden; justify-content: flex-start; width: 100%; padding: 0px;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">pbm</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">c3k</span><span style=\\"position: absolute; right: 0px; color: inherit;\\">c3k</span></span></span>"`;
 
-exports[`metadata loads categories and values from dataset appear 1`] = `"<div style=\\"display: flex; justify-content: space-between; align-items: baseline;\\"><div style=\\"display: flex; justify-content: flex-start; align-items: flex-start;\\"><label class=\\"bp3-control bp3-checkbox\\" for=\\"category-select-louvain\\"><input id=\\"category-select-louvain\\" data-testclass=\\"category-select\\" data-testid=\\"louvain:category-select\\" type=\\"checkbox\\" checked=\\"\\"><span class=\\"bp3-control-indicator\\"></span></label><span role=\\"menuitem\\" tabindex=\\"0\\" data-testclass=\\"category-expand\\" data-testid=\\"louvain:category-expand\\" style=\\"cursor: pointer;\\"><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\"><span data-testid=\\"louvain:category-label\\" aria-label=\\"louvain\\" class=\\"\\" tabindex=\\"0\\" style=\\"max-width: 265px;\\"><span style=\\"max-width: 265px; display: flex; overflow: hidden; justify-content: flex-start;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">lou</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">vain</span><span style=\\"position: absolute; right: 0px; color: inherit;\\">vain</span></span></span></span></span></span><svg stroke=\\"currentColor\\" fill=\\"currentColor\\" stroke-width=\\"0\\" viewBox=\\"0 0 320 512\\" data-testclass=\\"category-expand-is-not-expanded\\" height=\\"1em\\" width=\\"1em\\" xmlns=\\"http://www.w3.org/2000/svg\\" style=\\"font-size: 10px; margin-left: 5px;\\"><path d=\\"M285.476 272.971L91.132 467.314c-9.373 9.373-24.569 9.373-33.941 0l-22.667-22.667c-9.357-9.357-9.375-24.522-.04-33.901L188.505 256 34.484 101.255c-9.335-9.379-9.317-24.544.04-33.901l22.667-22.667c9.373-9.373 24.569-9.373 33.941 0L285.475 239.03c9.373 9.372 9.373 24.568.001 33.941z\\"></path></svg></span></div><div><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\"><a role=\\"button\\" data-testclass=\\"colorby\\" data-testid=\\"colorby-louvain\\" class=\\"bp3-button\\" tabindex=\\"0\\"><span icon=\\"tint\\" class=\\"bp3-icon bp3-icon-tint\\"><svg data-icon=\\"tint\\" width=\\"16\\" height=\\"16\\" viewBox=\\"0 0 16 16\\"><desc>tint</desc><path d=\\"M7.88 1s-4.9 6.28-4.9 8.9c.01 2.82 2.34 5.1 4.99 5.1 2.65-.01 5.03-2.3 5.03-5.13C12.99 7.17 7.88 1 7.88 1z\\" fill-rule=\\"evenodd\\"></path></svg></span></a></span></span></div></div><div style=\\"margin-left: 26px;\\"></div><div></div>"`;
+exports[`metadata loads categories and values from dataset appear 1`] = `"<div style=\\"display: flex; justify-content: space-between; align-items: baseline;\\"><div style=\\"display: flex; justify-content: flex-start; align-items: flex-start;\\"><label class=\\"bp3-control bp3-checkbox\\" for=\\"category-select-louvain\\"><input id=\\"category-select-louvain\\" data-testclass=\\"category-select\\" data-testid=\\"louvain:category-select\\" type=\\"checkbox\\" checked=\\"\\"><span class=\\"bp3-control-indicator\\"></span></label><span role=\\"menuitem\\" tabindex=\\"0\\" data-testclass=\\"category-expand\\" data-testid=\\"louvain:category-expand\\" style=\\"cursor: pointer;\\"><span aria-haspopup=\\"true\\" class=\\"bp3-popover2-target\\"><span data-testid=\\"louvain:category-label\\" tabindex=\\"-1\\" aria-label=\\"louvain\\" class=\\"\\" style=\\"max-width: 265px;\\"><span style=\\"max-width: 265px; display: flex; overflow: hidden; justify-content: flex-start; width: 100%; padding: 0px;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">lou</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">vain</span><span style=\\"position: absolute; right: 0px; color: inherit;\\">vain</span></span></span></span></span><svg stroke=\\"currentColor\\" fill=\\"currentColor\\" stroke-width=\\"0\\" viewBox=\\"0 0 320 512\\" data-testclass=\\"category-expand-is-not-expanded\\" height=\\"1em\\" width=\\"1em\\" xmlns=\\"http://www.w3.org/2000/svg\\" style=\\"font-size: 10px; margin-left: 5px;\\"><path d=\\"M285.476 272.971L91.132 467.314c-9.373 9.373-24.569 9.373-33.941 0l-22.667-22.667c-9.357-9.357-9.375-24.522-.04-33.901L188.505 256 34.484 101.255c-9.335-9.379-9.317-24.544.04-33.901l22.667-22.667c9.373-9.373 24.569-9.373 33.941 0L285.475 239.03c9.373 9.372 9.373 24.568.001 33.941z\\"></path></svg></span></div><div><span class=\\"bp3-popover-wrapper\\"><span aria-haspopup=\\"true\\" class=\\"bp3-popover-target\\"><a role=\\"button\\" data-testclass=\\"colorby\\" data-testid=\\"colorby-louvain\\" class=\\"bp3-button\\" tabindex=\\"0\\"><span icon=\\"tint\\" aria-hidden=\\"true\\" tabindex=\\"0\\" class=\\"bp3-icon bp3-icon-tint\\"><svg data-icon=\\"tint\\" width=\\"16\\" height=\\"16\\" viewBox=\\"0 0 16 16\\"><path d=\\"M7.88 1s-4.9 6.28-4.9 8.9c.01 2.82 2.34 5.1 4.99 5.1 2.65-.01 5.03-2.3 5.03-5.13C12.99 7.17 7.88 1 7.88 1z\\" fill-rule=\\"evenodd\\"></path></svg></span></a></span></span></div></div><div style=\\"margin-left: 26px;\\"></div>"`;

client/__tests__/e2e/cellxgeneActions.js

@@ -58,11 +58,9 @@ export async function getAllHistograms(testclass, testIds) {
   const allHistograms = await getAllByClass(testclass);
 
   const testIDs = await Promise.all(
-    allHistograms.map((hist) => {
-      return page.evaluate((elem) => {
-        return elem.dataset.testid;
-      }, hist);
-    })
+    allHistograms.map((hist) =>
+      page.evaluate((elem) => elem.dataset.testid, hist)
+    )
   );
 
   return testIDs.map((id) => id.replace(/^histogram-/, ""));
@@ -173,6 +171,137 @@ export async function createCategory(categoryName) {
   await clickOn("submit-category");
 }
 
+/*
+
+  GENESET
+
+*/
+
+export async function colorByGeneset(genesetName) {
+  await clickOn(`${genesetName}:colorby-entire-geneset`);
+}
+
+export async function colorByGene(gene) {
+  await clickOn(`colorby-${gene}`);
+}
+
+export async function assertColorLegendLabel(label) {
+  const handle = await waitByID("continuous_legend_color_by_label");
+
+  const result = await handle.evaluate((node) =>
+    node.getAttribute("aria-label")
+  );
+
+  return expect(result).toBe(label);
+}
+
+export async function expandGeneset(genesetName) {
+  const expand = await waitByID(`${genesetName}:geneset-expand`);
+  const notExpanded = await expand.$(
+    "[data-testclass='geneset-expand-is-not-expanded']"
+  );
+  if (notExpanded) await clickOn(`${genesetName}:geneset-expand`);
+}
+
+export async function createGeneset(genesetName) {
+  await clickOnUntil("open-create-geneset-dialog", async () => {
+    await expect(page).toMatchElement(getTestId("create-geneset-input"));
+  });
+
+  await typeInto("create-geneset-input", genesetName);
+  await clickOn("submit-geneset");
+  await waitByClass("autosave-complete");
+}
+
+export async function editGenesetName(genesetName, editText) {
+  const editButton = `${genesetName}:edit-genesetName-mode`;
+  const submitButton = `${genesetName}:submit-geneset`;
+  await clickOnUntil(`${genesetName}:see-actions`, async () => {
+    await expect(page).toMatchElement(getTestId(editButton));
+  });
+  await clickOn(editButton);
+  await typeInto("rename-geneset-modal", editText);
+  await clickOn(submitButton);
+}
+
+export async function deleteGeneset(genesetName) {
+  const targetId = `${genesetName}:delete-geneset`;
+
+  await clickOnUntil(`${genesetName}:see-actions`, async () => {
+    await expect(page).toMatchElement(getTestId(targetId));
+  });
+
+  await clickOn(targetId);
+
+  await assertGenesetDoesNotExist(genesetName);
+  await waitByClass("autosave-complete");
+}
+
+export async function assertGenesetDoesNotExist(genesetName) {
+  const result = await isElementPresent(
+    getTestId(`${genesetName}:geneset-name`)
+  );
+  await expect(result).toBe(false);
+}
+
+export async function assertGenesetExists(genesetName) {
+  const handle = await waitByID(`${genesetName}:geneset-name`);
+
+  const result = await handle.evaluate((node) =>
+    node.getAttribute("aria-label")
+  );
+
+  return expect(result).toBe(genesetName);
+}
+
+/*
+
+  GENE
+
+*/
+
+export async function addGeneToSet(genesetName, geneToAddToSet) {
+  const submitButton = `${genesetName}:submit-gene`;
+
+  await clickOn(`${genesetName}:add-new-gene-to-geneset`);
+  await typeInto("add-genes", geneToAddToSet);
+  await clickOn(submitButton);
+}
+
+export async function removeGene(geneSymbol) {
+  const targetId = `delete-from-geneset:${geneSymbol}`;
+
+  await clickOn(targetId);
+
+  await waitByClass("autosave-complete");
+}
+
+export async function assertGeneExistsInGeneset(geneSymbol) {
+  const handle = await waitByID(`${geneSymbol}:gene-label`);
+
+  const result = await handle.evaluate((node) =>
+    node.getAttribute("aria-label")
+  );
+
+  return expect(result).toBe(geneSymbol);
+}
+
+export async function assertGeneDoesNotExist(geneSymbol) {
+  const result = await isElementPresent(getTestId(`${geneSymbol}:gene-label`));
+
+  await expect(result).toBe(false);
+}
+
+export async function expandGene(geneSymbol) {
+  await clickOn(`maximize-${geneSymbol}`);
+}
+
+/*
+
+  CATEGORY
+
+*/
+
 export async function duplicateCategory(categoryName) {
   await clickOn("open-annotation-dialog");
 
@@ -235,7 +364,7 @@ export async function createLabel(categoryName, labelName) {
    * 4. You will see `123` is persisted in the input box
    * 5. Expected behavior is to get an empty input box
    */
-  await page.waitFor(500);
+  await page.waitForTimeout(500);
 
   await clickOn(`${categoryName}:see-actions`);
 
@@ -312,4 +441,5 @@ export async function assertCategoryDoesNotExist(categoryName) {
 
   await expect(result).toBe(false);
 }
+
 /* eslint-enable no-await-in-loop -- await in loop is needed to emulate sequential user actions */

client/__tests__/e2e/config.js

@@ -6,3 +6,5 @@ export const appUrlBase =
 export const DATASET = "pbmc3k";
 export const isDev = jestEnv === ENV_DEFAULT.DEV;
 export const isDebug = jestEnv === ENV_DEFAULT.DEBUG;
+export const TEST_EMAIL = "user@example.com";
+export const TEST_PASSWORD = process.env.TEST_ACCOUNT_PASS ?? "";

client/__tests__/e2e/diffexpGeneSets.js

@@ -0,0 +1,105 @@
+export const diffexpPop1Genes = [
+  "CD79A",
+  "HLA-DRB1",
+  "HLA-DQA1",
+  "HLA-DPB1",
+  "HLA-DQB1",
+  "HLA-DPA1",
+  "MS4A1",
+  "LTB",
+  "CD79B",
+  "CD37",
+  "HLA-DMA",
+  "TCL1A",
+  "LINC00926",
+  "HLA-DMB",
+  "HVCN1",
+  "EAF2",
+  "FCRLA",
+  "IRF8",
+  "PKIG",
+  "P2RX5",
+  "HLA-DOB",
+  "SPIB",
+  "BLNK",
+  "SWAP70",
+  "PNOC",
+  "CD19",
+  "SMIM14",
+  "CD72",
+  "KIAA0125",
+  "IGLL5",
+  "ARHGAP24",
+  "COTL1",
+  "C16orf74",
+  "BTK",
+  "SNX29P2",
+  "ADAM28",
+  "FCGR2B",
+  "PLD4",
+  "PPP1R14A",
+  "MZB1",
+  "KIAA0040",
+  "PHACTR1",
+  "FCRL2",
+  "RIC3",
+  "P2RY10",
+  "SCPEP1",
+  "DRAM2",
+  "RP5-887A10.1",
+  "CD82",
+  "GPX1",
+];
+
+export const diffexpPop2Genes = [
+  "NKG7",
+  "GZMB",
+  "CTSW",
+  "PRF1",
+  "GNLY",
+  "GZMA",
+  "CST7",
+  "FGFBP2",
+  "SRGN",
+  "CD247",
+  "FCGR3A",
+  "TYROBP",
+  "FCER1G",
+  "ID2",
+  "SPON2",
+  "CCL4",
+  "CCL5",
+  "GZMH",
+  "GIMAP7",
+  "CLIC3",
+  "HOPX",
+  "XCL2",
+  "LGALS1",
+  "IGFBP7",
+  "AKR1C3",
+  "IL32",
+  "EFHD2",
+  "PRSS23",
+  "TTC38",
+  "ZAP70",
+  "S1PR5",
+  "SAMD3",
+  "GIMAP4",
+  "CCL3",
+  "ABI3",
+  "XCL1",
+  "S100A6",
+  "UBB",
+  "GPR56",
+  "PDIA3",
+  "S100A11",
+  "APOBEC3G",
+  "HAVCR2",
+  "PLEKHF1",
+  "LITAF",
+  "ARPC5L",
+  "PTGDR",
+  "PRMT2",
+  "GSTP1",
+  "FCRL6",
+];

client/__tests__/e2e/e2e.test.js

@@ -15,22 +15,16 @@ import {
   getOneElementInnerHTML,
   getTestId,
   goToPage,
-  typeInto,
   waitByID,
 } from "./puppeteerUtils";
 
 import {
-  addGeneToSearch,
-  bulkAddGenes,
   calcDragCoordinates,
   clip,
   drag,
   getAllCategoriesAndCounts,
-  getAllHistograms,
   getCellSetCount,
-  runDiffExp,
   selectCategory,
-  subset,
 } from "./cellxgeneActions";
 
 const data = datasets[DATASET];
@@ -146,47 +140,6 @@ describe("cell selection", () => {
   });
 });
 
-describe("gene entry", () => {
-  test("search for single gene", async () => {
-    await goToPage(appUrlBase);
-    await addGeneToSearch(data.genes.search);
-  });
-
-  test("bulk add genes", async () => {
-    await goToPage(appUrlBase);
-
-    const testGenes = data.genes.bulkadd;
-
-    await bulkAddGenes(testGenes);
-    const allHistograms = await getAllHistograms(
-      "histogram-user-gene",
-      testGenes
-    );
-
-    expect(allHistograms).toEqual(expect.arrayContaining(testGenes));
-    expect(allHistograms).toHaveLength(testGenes.length);
-  });
-});
-
-describe("differential expression", () => {
-  test("selects cells, saves them and performs diffexp", async () => {
-    await goToPage(appUrlBase);
-
-    await runDiffExp(data.diffexp.cellset1, data.diffexp.cellset2);
-
-    const allHistograms = await getAllHistograms(
-      "histogram-diffexp",
-      data.diffexp["gene-results"]
-    );
-
-    expect(allHistograms).toEqual(
-      expect.arrayContaining(data.diffexp["gene-results"])
-    );
-
-    expect(allHistograms).toHaveLength(data.diffexp["gene-results"].length);
-  });
-});
-
 describe("subset", () => {
   test("subset - cell count matches", async () => {
     await goToPage(appUrlBase);
@@ -233,94 +186,6 @@ describe("subset", () => {
     const cellCount = await getCellSetCount(1);
     expect(cellCount).toBe(data.subset.lasso.count);
   });
-
-  test("undo selection appends the top diff exp genes to user defined genes", async () => {
-    await goToPage(appUrlBase);
-
-    const userDefinedGenes = data.genes.bulkadd;
-    const diffExpGenes = data.diffexp["gene-results"];
-
-    await bulkAddGenes(userDefinedGenes);
-    const userDefinedHistograms = await getAllHistograms(
-      "histogram-user-gene",
-      userDefinedGenes
-    );
-
-    expect(userDefinedHistograms).toEqual(
-      expect.arrayContaining(userDefinedGenes)
-    );
-
-    await subset({ x1: 0.15, y1: 0.1, x2: 0.98, y2: 0.98 });
-    await runDiffExp(data.diffexp.cellset1, data.diffexp.cellset2);
-
-    const diffExpHistograms = await getAllHistograms(
-      "histogram-diffexp",
-      diffExpGenes
-    );
-
-    expect(diffExpHistograms).toEqual(expect.arrayContaining(diffExpGenes));
-
-    await clickOn("reset-subset-button");
-    const expected = [].concat(userDefinedGenes, diffExpGenes);
-    const userDefinedHistogramsAfterSubset = await getAllHistograms(
-      "histogram-user-gene",
-      expected
-    );
-
-    expect(userDefinedHistogramsAfterSubset).toEqual(
-      expect.arrayContaining(expected)
-    );
-  });
-
-  test("subset selection appends the top diff exp genes to user defined genes", async () => {
-    await goToPage(appUrlBase);
-
-    const userDefinedGenes = data.genes.bulkadd;
-    const diffExpGenes = data.diffexp["gene-results"];
-
-    await bulkAddGenes(userDefinedGenes);
-    const userDefinedHistograms = await getAllHistograms(
-      "histogram-user-gene",
-      userDefinedGenes
-    );
-
-    expect(userDefinedHistograms).toEqual(
-      expect.arrayContaining(userDefinedGenes)
-    );
-
-    await subset({ x1: 0.15, y1: 0.1, x2: 0.98, y2: 0.98 });
-    await runDiffExp(data.diffexp.cellset1, data.diffexp.cellset2);
-
-    const diffExpHistograms = await getAllHistograms(
-      "histogram-diffexp",
-      diffExpGenes
-    );
-
-    expect(diffExpHistograms).toEqual(expect.arrayContaining(diffExpGenes));
-
-    await subset({ x1: 0.16, y1: 0.11, x2: 0.97, y2: 0.97 });
-
-    const expected = [].concat(userDefinedGenes, diffExpGenes);
-    const userDefinedHistogramsAfterSubset = await getAllHistograms(
-      "histogram-user-gene",
-      expected
-    );
-
-    expect(userDefinedHistogramsAfterSubset).toEqual(
-      expect.arrayContaining(expected)
-    );
-  });
-});
-
-describe("scatter plot", () => {
-  test("scatter plot appears", async () => {
-    await goToPage(appUrlBase);
-
-    await bulkAddGenes(Object.values(data.scatter.genes));
-    await clickOn(`plot-x-${data.scatter.genes.x}`);
-    await clickOn(`plot-y-${data.scatter.genes.y}`);
-    await waitByID("scatterplot");
-  });
 });
 
 describe("clipping", () => {
@@ -337,30 +202,6 @@ describe("clipping", () => {
     const cellCount = await getCellSetCount(1);
     expect(cellCount).toBe(data.clip.count);
   });
-
-  test("clip gene", async () => {
-    await goToPage(appUrlBase);
-
-    await typeInto("gene-search", data.clip.gene);
-    await page.keyboard.press("Enter");
-
-    await page.waitForSelector(`[data-testid='histogram-${data.clip.gene}']`);
-
-    await clip(data.clip.min, data.clip.max);
-
-    const histBrushableAreaId = `histogram-${data.clip.gene}-plot-brushable-area`;
-
-    const coords = await calcDragCoordinates(
-      histBrushableAreaId,
-      data.clip["coordinates-as-percent"]
-    );
-
-    await drag(histBrushableAreaId, coords.start, coords.end);
-
-    const cellCount = await getCellSetCount(1);
-
-    expect(cellCount).toBe(data.clip["gene-cell-count"]);
-  });
 });
 
 // interact with UI elements just that they do not break
@@ -378,17 +219,6 @@ describe("ui elements don't error", () => {
     }
   });
 
-  test("color by for gene", async () => {
-    await goToPage(appUrlBase);
-
-    await typeInto("gene-search", data.genes.search);
-    await page.keyboard.press("Enter");
-    await page.waitForSelector(
-      `[data-testid='histogram-${data.genes.search}']`
-    );
-    await clickOn(`colorby-${data.genes.search}`);
-  });
-
   test("pan and zoom", async () => {
     await goToPage(appUrlBase);
 

client/__tests__/e2e/e2eAnnotations.test.js

@@ -12,6 +12,8 @@ import {
   getTestId,
   getTestClass,
   getAllByClass,
+  clickOnUntil,
+  getOneElementInnerHTML,
 } from "./puppeteerUtils";
 
 import {
@@ -27,6 +29,21 @@ import {
   renameLabel,
   subset,
   duplicateCategory,
+  createGeneset,
+  deleteGeneset,
+  assertGenesetExists,
+  assertGenesetDoesNotExist,
+  getCellSetCount,
+  expandGeneset,
+  editGenesetName,
+  addGeneToSet,
+  assertGeneExistsInGeneset,
+  removeGene,
+  assertGeneDoesNotExist,
+  expandGene,
+  colorByGeneset,
+  assertColorLegendLabel,
+  colorByGene,
 } from "./cellxgeneActions";
 
 const data = datasets[DATASET];
@@ -34,12 +51,45 @@ const data = datasets[DATASET];
 const perTestCategoryName = "TEST-CATEGORY";
 const perTestLabelName = "TEST-LABEL";
 
+// geneset CRUD
+const genesetToDeleteName = "geneset_to_delete";
+const preExistingGenesetName = "fifth_dataset";
+const meanExpressionBrushGenesetName = "second_gene_set";
+const meanExpressionBrushCellsSelected = "557";
+const subsetMeanExpressionBrushCellsSelected = "452";
+
+// initial text, the text we type in, the result
+const editableGenesetName = "geneset_to_edit";
+const editText = "_111";
+const newGenesetName = "geneset_to_edit_111";
+
+// add gene to set
+const geneToAddToSet = "RER1";
+const setToAddGeneTo = "fill_this_geneset";
+
+// remove gene from set
+const geneToRemove = "SIK1";
+const setToRemoveFrom = "empty_this_geneset";
+
+// brush a gene
+const geneToBrushAndColorBy = "SIK1";
+const brushThisGeneGeneset = "brush_this_gene";
+const geneBrushedCellCount = "109";
+const subsetGeneBrushedCellCount = "96";
+
+const genesetDescriptionID =
+  "geneset-description-tooltip-fourth_gene_set: fourth description";
+const genesetDescriptionString = "fourth_gene_set: fourth description";
+const genesetToCheckForDescription = "fourth_gene_set";
+
 async function setup(config) {
   await goToPage(appUrlBase);
 
-  // setup the test fixtures
-  await createCategory(perTestCategoryName);
-  await createLabel(perTestCategoryName, perTestLabelName);
+  if (config.categoricalAnno) {
+    // setup the test fixtures
+    await createCategory(perTestCategoryName);
+    await createLabel(perTestCategoryName, perTestLabelName);
+  }
 
   if (config.withSubset) {
     await subset({ x1: 0.1, y1: 0.1, x2: 0.8, y2: 0.8 });
@@ -51,6 +101,200 @@ async function setup(config) {
 describe.each([
   { withSubset: true, tag: "subset" },
   { withSubset: false, tag: "whole" },
+])("geneSET crud operations and interactions", (config) => {
+  test("genesets load from csv", async () => {
+    await setup(config);
+
+    await assertGenesetExists(preExistingGenesetName);
+  });
+  test("brush on geneset mean", async () => {
+    await setup(config);
+
+    await expandGeneset(meanExpressionBrushGenesetName);
+
+    const histBrushableAreaId = `histogram-${meanExpressionBrushGenesetName}-plot-brushable-area`;
+
+    const coords = await calcDragCoordinates(histBrushableAreaId, {
+      x1: 0.25,
+      y1: 0.5,
+      x2: 0.55,
+      y2: 0.5,
+    });
+
+    await drag(histBrushableAreaId, coords.start, coords.end);
+
+    const cellCount = await getCellSetCount(1);
+    if (config.withSubset) {
+      expect(cellCount).toBe(subsetMeanExpressionBrushCellsSelected);
+    } else {
+      expect(cellCount).toBe(meanExpressionBrushCellsSelected);
+    }
+  });
+  test("color by mean expression", async () => {
+    await setup(config);
+
+    await colorByGeneset(meanExpressionBrushGenesetName);
+    await assertColorLegendLabel(meanExpressionBrushGenesetName);
+  });
+  test("diffexp", async () => {
+    if (config.withSubset) return;
+
+    await setup(config);
+
+    // set the two cell sets to b cells vs nk cells
+    await expandCategory(`louvain`);
+    await clickOn(`louvain:category-select`);
+    await clickOn(`categorical-value-select-louvain-B cells`);
+    await clickOn(`cellset-button-1`);
+    await clickOn(`categorical-value-select-louvain-B cells`);
+    await clickOn(`categorical-value-select-louvain-NK cells`);
+    await clickOn(`cellset-button-2`);
+
+    // run diffexp
+    await clickOn(`diffexp-button`);
+    await waitByClass("pop-1-geneset-expand");
+    await expect(page).toClick(getTestClass("pop-1-geneset-expand"));
+
+    await page.waitForFunction(
+      (selector) => !document.querySelector(selector),
+      {},
+      getTestClass("gene-loading-spinner")
+    );
+
+    let genesHTML = await getOneElementInnerHTML(
+      getTestClass("gene-set-genes")
+    );
+
+    expect(genesHTML).toMatchSnapshot();
+
+    await expect(page).toClick(getTestClass("pop-1-geneset-expand"));
+    await expect(page).toClick(getTestClass("pop-2-geneset-expand"));
+
+    await page.waitForFunction(
+      (selector) => !document.querySelector(selector),
+      {},
+      getTestClass("gene-loading-spinner")
+    );
+
+    genesHTML = await getOneElementInnerHTML(getTestClass("gene-set-genes"));
+
+    expect(genesHTML).toMatchSnapshot();
+  });
+  test("create a new geneset and undo/redo", async () => {
+    if (config.withSubset) return;
+
+    await setup(config);
+
+    const genesetName = `test-geneset-foo-123`;
+    await assertGenesetDoesNotExist(genesetName);
+    await createGeneset(genesetName);
+    /* note: as of June 2021, the aria label is in the truncate component which clones the element */
+    await assertGenesetExists(genesetName);
+    await clickOn("undo");
+    await assertGenesetDoesNotExist(genesetName);
+    await clickOn("redo");
+    await assertGenesetExists(genesetName);
+  });
+  test("edit geneset name and undo/redo", async () => {
+    await setup(config);
+
+    await editGenesetName(editableGenesetName, editText);
+    await assertGenesetExists(newGenesetName);
+    await clickOn("undo");
+    await assertGenesetExists(editableGenesetName);
+    await clickOn("redo");
+    await assertGenesetExists(newGenesetName);
+  });
+  test("delete a geneset and undo/redo", async () => {
+    if (config.withSubset) return;
+
+    await setup(config);
+
+    await deleteGeneset(genesetToDeleteName);
+    await clickOn("undo");
+    await assertGenesetExists(genesetToDeleteName);
+    await clickOn("redo");
+    await assertGenesetDoesNotExist(genesetToDeleteName);
+  });
+  test("geneset description", async () => {
+    if (config.withSubset) return;
+
+    await setup(config);
+
+    await clickOnUntil(
+      `${genesetToCheckForDescription}:geneset-expand`,
+      async () => {
+        expect(page).toMatchElement(getTestId(genesetDescriptionID), {
+          text: genesetDescriptionString,
+        });
+      }
+    );
+  });
+});
+
+describe.each([
+  { withSubset: true, tag: "subset" },
+  { withSubset: false, tag: "whole" },
+])("GENE crud operations and interactions", (config) => {
+  test("add a gene to geneset and undo/redo", async () => {
+    await setup(config);
+
+    await addGeneToSet(setToAddGeneTo, geneToAddToSet);
+    await expandGeneset(setToAddGeneTo);
+    await assertGeneExistsInGeneset(geneToAddToSet);
+    await clickOn("undo");
+    await assertGeneDoesNotExist(geneToAddToSet);
+    await clickOn("redo");
+    await assertGeneExistsInGeneset(geneToAddToSet);
+  });
+  test("expand gene and brush", async () => {
+    await setup(config);
+
+    await expandGeneset(brushThisGeneGeneset);
+    await expandGene(geneToBrushAndColorBy);
+    const histBrushableAreaId = `histogram-${geneToBrushAndColorBy}-plot-brushable-area`;
+
+    const coords = await calcDragCoordinates(histBrushableAreaId, {
+      x1: 0.25,
+      y1: 0.5,
+      x2: 0.55,
+      y2: 0.5,
+    });
+    await drag(histBrushableAreaId, coords.start, coords.end);
+    const cellCount = await getCellSetCount(1);
+    if (config.withSubset) {
+      expect(cellCount).toBe(subsetGeneBrushedCellCount);
+    } else {
+      expect(cellCount).toBe(geneBrushedCellCount);
+    }
+  });
+  test("color by gene in geneset", async () => {
+    await setup(config);
+
+    await expandGeneset(meanExpressionBrushGenesetName);
+
+    await colorByGene(geneToBrushAndColorBy);
+    await assertColorLegendLabel(geneToBrushAndColorBy);
+  });
+  test("delete gene from geneset and undo/redo", async () => {
+    // We've already deleted the gene
+    if (config.withSubset) return;
+
+    await setup(config);
+
+    await expandGeneset(setToRemoveFrom);
+    await removeGene(geneToRemove);
+    await assertGeneDoesNotExist(geneToRemove);
+    await clickOn("undo");
+    await assertGeneExistsInGeneset(geneToRemove);
+    await clickOn("redo");
+    await assertGeneDoesNotExist(geneToRemove);
+  });
+});
+
+describe.each([
+  { withSubset: true, tag: "subset", categoricalAnno: true },
+  { withSubset: false, tag: "whole", categoricalAnno: true },
 ])("annotations", (config) => {
   test("create a category", async () => {
     await setup(config);
@@ -287,11 +531,7 @@ describe.each([
     const labels = await getAllByClass("categorical-row");
 
     const result = await Promise.all(
-      labels.map((label) => {
-        return page.evaluate((element) => {
-          return element.outerHTML;
-        }, label);
-      })
+      labels.map((label) => page.evaluate((element) => element.outerHTML, label))
     );
 
     expect(result).toMatchSnapshot();

client/__tests__/e2e/e2eJestConfig.json

@@ -4,7 +4,7 @@
   "testMatch": ["**/__tests__/**/?(*.)(spec|test).js?(x)"],
   "setupFiles": ["../setupMissingGlobals.js"],
   "setupFilesAfterEnv": ["expect-puppeteer", "./puppeteer.setup.js"],
-  "globalSetup": "jest-environment-puppeteer/setup",
+  "globalSetup": "../globalSetup.js",
   "globalTeardown": "jest-environment-puppeteer/teardown",
   "testEnvironment": "./screenshot_env.js"
 }

client/__tests__/e2e/puppeteer.setup.js

@@ -13,11 +13,13 @@ import * as ENV_DEFAULT from "../../../environment.default.json";
 // a test can take more time to finish, so we don't want
 // jest to shut off the test too soon
 jest.setTimeout(2 * 60 * 1000);
-setDefaultOptions({ timeout: 20 * 1000 });
+setDefaultOptions({ timeout: 60 * 1000 });
 
 jest.retryTimes(ENV_DEFAULT.RETRY_ATTEMPTS);
 
-(async () => {
+beforeEach(async () => {
+  await jestPuppeteer.resetBrowser();
+
   const userAgent = await browser.userAgent();
   await page.setUserAgent(`${userAgent}bot`);
 
@@ -53,6 +55,4 @@ jest.retryTimes(ENV_DEFAULT.RETRY_ATTEMPTS);
       }
     }
   });
-})().catch((error) => {
-  console.error("puppeteer.setup.js error", error);
 });

client/__tests__/e2e/puppeteerUtils.js

@@ -12,7 +12,7 @@ export async function waitByID(testId, props = {}) {
 }
 
 export async function waitByClass(testClass, props = {}) {
-  await page.waitForSelector(`[data-testclass='${testClass}']`, props);
+  return page.waitForSelector(`[data-testclass='${testClass}']`, props);
 }
 
 export async function waitForAllByIds(testIds) {
@@ -32,7 +32,7 @@ export async function typeInto(testId, text) {
   const selector = getTestId(testId);
   // type ahead can be annoying if you don't pause before you type
   await page.click(selector);
-  await page.waitFor(200);
+  await page.waitForTimeout(200);
   await page.type(selector, text);
 }
 
@@ -42,7 +42,7 @@ export async function clearInputAndTypeInto(testId, text) {
   // only works for text without special characters
   // type ahead can be annoying if you don't pause before you type
   await page.click(selector);
-  await page.waitFor(200);
+  await page.waitForTimeout(200);
   // select all
   await page.click(selector, { clickCount: 3 });
   await page.keyboard.press("Backspace");
@@ -72,7 +72,7 @@ export async function clickOnUntil(testId, assert) {
     } catch (error) {
       retry += 1;
 
-      await page.waitFor(WAIT_FOR_MS);
+      await page.waitForTimeout(WAIT_FOR_MS);
     }
   }
 
@@ -100,12 +100,6 @@ export async function getElementCoordinates(testId) {
   });
 }
 
-async function clickTermsOfService() {
-  if (!(await isElementPresent(getTestId("tos-cookies-accept")))) return;
-
-  await clickOn("tos-cookies-accept");
-}
-
 async function nameNewAnnotation() {
   if (await isElementPresent(getTestId("annotation-dialog"))) {
     await typeInto("new-annotation-name", "ignoreE2E");
@@ -122,7 +116,6 @@ export async function goToPage(url) {
   });
 
   await nameNewAnnotation();
-  await clickTermsOfService();
 }
 
 export async function isElementPresent(selector, options) {

client/__tests__/e2e/test_config.yaml

@@ -0,0 +1,19 @@
+server:
+  app:
+    force_https: true
+    port: 5005
+
+dataset:
+  presentation:
+    max_categories: 1000
+    custom_colors: true
+
+  user_annotations:
+    enable: false
+    type: local_file_csv
+    local_file_csv:
+      directory: null
+      file: null
+
+  embeddings:
+    names: []

client/__tests__/globalSetup.js

@@ -0,0 +1,5 @@
+const { setup } = require("jest-environment-puppeteer");
+
+module.exports = async () => {
+  await setup();
+};

client/__tests__/reducers/genesets.test.js

@@ -0,0 +1,524 @@
+import genesetsReducer from "../../src/reducers/genesets";
+
+describe("initial reducer state", () => {
+  test("some other action", () => {
+    expect(genesetsReducer(undefined, { type: "foo" })).toMatchObject({
+      initialized: false,
+      lastTid: undefined,
+      genesets: new Map(),
+    });
+  });
+});
+
+describe("geneset: initial load", () => {
+  test("missing JSON response", () => {
+    expect(() =>
+      genesetsReducer(undefined, {
+        type: "geneset: initial load",
+      })
+    ).toThrow("missing or malformed JSON response");
+  });
+
+  test("empty geneset", () => {
+    expect(
+      genesetsReducer(undefined, {
+        type: "geneset: initial load",
+        data: {
+          tid: 0,
+          genesets: [],
+        },
+      })
+    ).toMatchObject({
+      initialized: true,
+      lastTid: 0,
+      genesets: new Map(),
+    });
+  });
+
+  test("non-empty geneset", () => {
+    expect(
+      genesetsReducer(undefined, {
+        type: "geneset: initial load",
+        data: {
+          tid: 99,
+          genesets: [
+            {
+              geneset_name: "G1",
+              genes: [{ gene_symbol: "F5" }],
+            },
+            {
+              geneset_name: "G2",
+              geneset_description: "G2 desc",
+              genes: [{ gene_symbol: "F6" }],
+            },
+            {
+              geneset_name: "G3",
+              geneset_description: "G3 desc",
+              genes: [{ gene_symbol: "F7", gene_description: "gene desc" }],
+            },
+          ],
+        },
+      })
+    ).toMatchObject({
+      initialized: true,
+      lastTid: 99,
+      genesets: new Map([
+        [
+          "G1",
+          {
+            genesetName: "G1",
+            genesetDescription: "",
+            genes: new Map([["F5", { geneSymbol: "F5", geneDescription: "" }]]),
+          },
+        ],
+        [
+          "G2",
+          {
+            genesetName: "G2",
+            genesetDescription: "G2 desc",
+            genes: new Map([["F6", { geneSymbol: "F6", geneDescription: "" }]]),
+          },
+        ],
+        [
+          "G3",
+          {
+            genesetName: "G3",
+            genesetDescription: "G3 desc",
+            genes: new Map([
+              ["F7", { geneSymbol: "F7", geneDescription: "gene desc" }],
+            ]),
+          },
+        ],
+      ]),
+    });
+  });
+});
+
+describe("geneset: create", () => {
+  const initialState = genesetsReducer(undefined, {
+    type: "geneset: initial load",
+    data: {
+      tid: 0,
+      genesets: [],
+    },
+  });
+
+  test("simple create", () => {
+    expect(
+      genesetsReducer(initialState, {
+        type: "geneset: create",
+        genesetName: "a geneset",
+        genesetDescription: "",
+      })
+    ).toMatchObject({
+      ...initialState,
+      genesets: new Map([
+        [
+          "a geneset",
+          {
+            genesetName: "a geneset",
+            genesetDescription: "",
+            genes: new Map(),
+          },
+        ],
+      ]),
+    });
+  });
+
+  test("error - duplicate name", () => {
+    expect(() => {
+      genesetsReducer(
+        genesetsReducer(initialState, {
+          type: "geneset: create",
+          genesetName: "foo",
+          genesetDescription: "foo",
+        }),
+        {
+          type: "geneset: create",
+          genesetName: "foo",
+          genesetDescription: "bar",
+        }
+      );
+    }).toThrow("name already defined");
+  });
+
+  test("error - missing required action values", () => {
+    expect(() => {
+      genesetsReducer(initialState, {
+        type: "geneset: create",
+        genesetDescription: "foo",
+      });
+    }).toThrow();
+    expect(() => {
+      genesetsReducer(initialState, {
+        type: "geneset: create",
+        genesetName: "foo",
+      });
+    }).toThrow("name or description unspecified");
+  });
+});
+
+describe("geneset: delete", () => {
+  const initialState = genesetsReducer(undefined, {
+    type: "geneset: initial load",
+    data: {
+      tid: 0,
+      genesets: [],
+    },
+  });
+
+  test("simple delete", () => {
+    expect(
+      genesetsReducer(
+        genesetsReducer(initialState, {
+          type: "geneset: create",
+          genesetName: "foo",
+          genesetDescription: "foo",
+        }),
+        {
+          type: "geneset: delete",
+          genesetName: "foo",
+        }
+      )
+    ).toMatchObject({
+      initialized: true,
+      lastTid: 0,
+      genesets: new Map(),
+    });
+  });
+
+  test("error - missing name", () => {
+    expect(() => {
+      genesetsReducer(initialState, {
+        type: "geneset: delete",
+        genesetName: "foo",
+      });
+    }).toThrow("name does not exist");
+  });
+});
+
+describe("geneset: update", () => {
+  const initialState = genesetsReducer(undefined, {
+    type: "geneset: initial load",
+    data: {
+      tid: 0,
+      genesets: [],
+    },
+  });
+
+  test("simple update", () => {
+    expect(
+      genesetsReducer(
+        genesetsReducer(
+          genesetsReducer(initialState, {
+            type: "geneset: create",
+            genesetName: "foo1",
+            genesetDescription: "foo1",
+          }),
+          {
+            type: "geneset: create",
+            genesetName: "foo2",
+            genesetDescription: "foo2",
+          }
+        ),
+        {
+          type: "geneset: update",
+          genesetName: "foo1",
+          update: {
+            genesetName: "bar",
+            genesetDescription: "bar",
+          },
+        }
+      )
+    ).toMatchObject({
+      initialized: true,
+      lastTid: 0,
+      genesets: new Map([
+        [
+          "bar",
+          { genesetName: "bar", genesetDescription: "bar", genes: new Map() },
+        ],
+        [
+          "foo2",
+          { genesetName: "foo2", genesetDescription: "foo2", genes: new Map() },
+        ],
+      ]),
+    });
+  });
+
+  test("error - unknown name", () => {
+    expect(() => {
+      genesetsReducer(initialState, {
+        type: "geneset: update",
+        genesetName: "foo",
+        update: {
+          genesetName: "foo",
+          genesetDescription: "bar",
+        },
+      });
+    }).toThrow("name unspecified or does not exist");
+  });
+
+  test("error - duplicate name", () => {
+    expect(() => {
+      genesetsReducer(
+        genesetsReducer(initialState, {
+          type: "geneset: create",
+          genesetName: "foo",
+          genesetDescription: "foo",
+        }),
+        {
+          type: "geneset: update",
+          genesetName: "foo",
+          update: {
+            genesetName: "foo",
+            genesetDescription: "foo",
+          },
+        }
+      );
+    }).toThrow("update specified existing name and description");
+  });
+});
+
+describe("geneset: add genes", () => {
+  const initialState = genesetsReducer(
+    genesetsReducer(undefined, {
+      type: "geneset: initial load",
+      data: {
+        tid: 0,
+        genesets: [],
+      },
+    }),
+    {
+      type: "geneset: create",
+      genesetName: "test",
+      genesetDescription: "",
+    }
+  );
+
+  test("add a gene", () => {
+    expect(
+      genesetsReducer(initialState, {
+        type: "geneset: add genes",
+        genesetName: "test",
+        genes: [{ geneSymbol: "F5" }],
+      })
+    ).toMatchObject({
+      ...initialState,
+      genesets: new Map([
+        [
+          "test",
+          {
+            genesetName: "test",
+            genesetDescription: "",
+            genes: new Map([["F5", { geneSymbol: "F5", geneDescription: "" }]]),
+          },
+        ],
+      ]),
+    });
+
+    expect(
+      genesetsReducer(initialState, {
+        type: "geneset: add genes",
+        genesetName: "test",
+        genes: [
+          { geneSymbol: "F5", geneDescription: "desc" },
+          { geneSymbol: "SET1", geneDescription: "" },
+        ],
+      })
+    ).toMatchObject({
+      ...initialState,
+      genesets: new Map([
+        [
+          "test",
+          {
+            genesetName: "test",
+            genesetDescription: "",
+            genes: new Map([
+              ["F5", { geneSymbol: "F5", geneDescription: "desc" }],
+              ["SET1", { geneSymbol: "SET1", geneDescription: "" }],
+            ]),
+          },
+        ],
+      ]),
+    });
+  });
+
+  test("no such geneset error", () => {
+    expect(() => {
+      genesetsReducer(initialState, {
+        type: "geneset: add genes",
+        genesetName: "mumble",
+        genes: [],
+      });
+    }).toThrow("geneset name does not exist");
+  });
+});
+
+describe("geneset: delete genes", () => {
+  const initialState = genesetsReducer(
+    genesetsReducer(
+      genesetsReducer(undefined, {
+        type: "geneset: initial load",
+        data: {
+          tid: 0,
+          genesets: [],
+        },
+      }),
+      {
+        type: "geneset: create",
+        genesetName: "test",
+        genesetDescription: "",
+      }
+    ),
+    {
+      type: "geneset: add genes",
+      genesetName: "test",
+      genes: [{ geneSymbol: "F5" }],
+    }
+  );
+
+  test("simple", () => {
+    expect(
+      genesetsReducer(initialState, {
+        type: "geneset: delete genes",
+        genesetName: "test",
+        geneSymbols: ["F5"],
+      })
+    ).toMatchObject({
+      ...initialState,
+      genesets: new Map([
+        [
+          "test",
+          {
+            genesetName: "test",
+            genesetDescription: "",
+            genes: new Map(),
+          },
+        ],
+      ]),
+    });
+  });
+
+  test("no such geneset error", () => {
+    expect(() => {
+      genesetsReducer(initialState, {
+        type: "geneset: delete genes",
+        genesetName: "mumble",
+        geneSymbols: [],
+      });
+    }).toThrow("name does not exist");
+  });
+});
+
+describe("geneset: set gene description", () => {
+  const initialState = genesetsReducer(
+    genesetsReducer(
+      genesetsReducer(undefined, {
+        type: "geneset: initial load",
+        data: {
+          tid: 0,
+          genesets: [],
+        },
+      }),
+      {
+        type: "geneset: create",
+        genesetName: "test",
+        genesetDescription: "",
+      }
+    ),
+    {
+      type: "geneset: add genes",
+      genesetName: "test",
+      genes: [{ geneSymbol: "F5" }],
+    }
+  );
+
+  test("simple set", () => {
+    expect(
+      genesetsReducer(initialState, {
+        type: "geneset: set gene description",
+        genesetName: "test",
+        update: {
+          geneSymbol: "F5",
+          geneDescription: "mumble",
+        },
+      })
+    ).toMatchObject({
+      ...initialState,
+      genesets: new Map([
+        [
+          "test",
+          {
+            genesetName: "test",
+            genesetDescription: "",
+            genes: new Map([
+              ["F5", { geneSymbol: "F5", geneDescription: "mumble" }],
+            ]),
+          },
+        ],
+      ]),
+    });
+  });
+
+  test("no such geneset error", () => {
+    expect(() => {
+      genesetsReducer(initialState, {
+        type: "geneset: set gene description",
+        genesetName: "does not exist",
+        update: {
+          geneSymbol: "F5",
+          geneDescription: "mumble",
+        },
+      });
+    }).toThrow("geneset name does not exist");
+  });
+
+  test("no such gene error", () => {
+    expect(() => {
+      genesetsReducer(initialState, {
+        type: "geneset: set gene description",
+        genesetName: "test",
+        update: {
+          geneSymbol: "NO SUCH GENE",
+          geneDescription: "mumble",
+        },
+      });
+    }).toThrow("no such gene");
+  });
+});
+
+describe("geneset: set tid", () => {
+  test("simple set", () => {
+    expect(
+      genesetsReducer(undefined, {
+        type: "geneset: set tid",
+        tid: 1,
+      })
+    ).toMatchObject({ lastTid: 1 });
+  });
+
+  test("not a number error", () => {
+    expect(() => {
+      genesetsReducer(
+        { lastTid: 1 },
+        {
+          type: "geneset: set tid",
+          tid: "0",
+        }
+      );
+    }).toThrow("must be a positive integer");
+  });
+
+  test("decrement error", () => {
+    expect(() => {
+      genesetsReducer(
+        { lastTid: 1 },
+        {
+          type: "geneset: set tid",
+          tid: 0,
+        }
+      );
+    }).toThrow("may not be decremented");
+  });
+});

client/__tests__/reducers/genesetsUI.test.js

@@ -0,0 +1,75 @@
+import genesetsUIReducer from "../../src/reducers/genesetsUI";
+
+// Format: GeneSetsUI(state,action)
+
+const initialState = {
+  createGenesetModeActive: false,
+  isEditingGenesetName: false,
+  isAddingGenesToGeneset: false,
+};
+
+/* initial */
+describe("geneset UI states", () => {
+  test("initial state, some other action", () => {
+    expect(
+      genesetsUIReducer(undefined, {
+        type: "foo",
+      })
+    ).toMatchObject(initialState);
+  });
+  test("geneset: activate add new geneset mode", () => {
+    expect(
+      genesetsUIReducer(initialState, {
+        type: "geneset: activate add new geneset mode",
+      })
+    ).toMatchObject({
+      createGenesetModeActive: true,
+      isEditingGenesetName: false,
+      isAddingGenesToGeneset: false,
+    });
+  });
+  test("geneset: disable create geneset mode", () => {
+    expect(
+      genesetsUIReducer(undefined, { isEditingGenesetName: false })
+    ).toMatchObject(initialState);
+  });
+
+  test("activate add new genes mode", () => {
+    expect(
+      genesetsUIReducer(undefined, {
+        type: "geneset: activate add new genes mode",
+        geneset: "a geneset name",
+      })
+    ).toMatchObject({
+      createGenesetModeActive: false,
+      isEditingGenesetName: false,
+      isAddingGenesToGeneset: "a geneset name",
+    });
+  });
+  test("disable create geneset mode", () => {
+    expect(
+      genesetsUIReducer(undefined, {
+        type: "geneset: disable create geneset mode",
+      })
+    ).toMatchObject(initialState);
+  });
+  test("activate rename geneset mode", () => {
+    expect(
+      genesetsUIReducer(undefined, {
+        type: "geneset: activate rename geneset mode",
+        data: "a geneset name",
+      })
+    ).toMatchObject({
+      createGenesetModeActive: false,
+      isEditingGenesetName: "a geneset name",
+      isAddingGenesToGeneset: false,
+    });
+  });
+  test("disable rename geneset mode", () => {
+    expect(
+      genesetsUIReducer(undefined, {
+        type: "geneset: disable rename geneset mode",
+      })
+    ).toMatchObject(initialState);
+  });
+});

client/__tests__/reducers/undoable.test.js

@@ -23,9 +23,7 @@ describe("create", () => {
 describe("undo", () => {
   test("expected state modifications", () => {
     const initialState = { a: 0, b: 1000 };
-    const reducer = (state) => {
-      return { a: state.a + 1, b: state.b + 1 };
-    };
+    const reducer = (state) => ({ a: state.a + 1, b: state.b + 1 });
     const undoableReducer = undoable(reducer, ["a"]);
 
     const s1 = undoableReducer(initialState, { type: "test" });
@@ -43,9 +41,7 @@ describe("undo", () => {
 
 describe("redo", () => {
   const initialState = { a: 0, b: 1000 };
-  const reducer = (state) => {
-    return { a: state.a + 1, b: state.b + 1 };
-  };
+  const reducer = (state) => ({ a: state.a + 1, b: state.b + 1 });
   let UR;
 
   beforeEach(() => {

client/__tests__/util/actionHelpers.test.js

@@ -11,18 +11,14 @@ describe("rangeEncodeIndices", () => {
 
   test("sorted flag", () => {
     expect(rangeEncodeIndices([1, 9, 432], 10, true)).toMatchObject([
-      1,
-      9,
-      432,
+      1, 9, 432,
     ]);
     expect(rangeEncodeIndices([1, 9, 432], 10, false)).toMatchObject([
-      1,
-      9,
-      432,
+      1, 9, 432,
     ]);
-    expect(
-      rangeEncodeIndices([0, 1, 2, 3, 9, 10, 432], 2, true)
-    ).toMatchObject([[0, 3], [9, 10], 432]);
+    expect(rangeEncodeIndices([0, 1, 2, 3, 9, 10, 432], 2, true)).toMatchObject(
+      [[0, 3], [9, 10], 432]
+    );
     expect(
       rangeEncodeIndices([0, 1, 2, 3, 9, 10, 432], 2, false)
     ).toMatchObject([[0, 3], [9, 10], 432]);

client/__tests__/util/annoMatrix/annoMatrix.test.js

@@ -85,9 +85,11 @@ describe("AnnoMatrix", () => {
       fetch.once(serverMocks.responder);
       await expect(
         annoMatrix.fetch("X", {
-          field: "var",
-          column: annoMatrix.schema.annotations.var.index,
-          value: "TYMP",
+          where: {
+            field: "var",
+            column: annoMatrix.schema.annotations.var.index,
+            value: "TYMP",
+          },
         })
       ).resolves.toBeInstanceOf(Dataframe);
 
@@ -103,14 +105,18 @@ describe("AnnoMatrix", () => {
       await expect(
         annoMatrix.fetch("X", [
           {
-            field: "var",
-            column: varIndex,
-            value: "SUMO3",
+            where: {
+              field: "var",
+              column: varIndex,
+              value: "SUMO3",
+            },
           },
           {
-            field: "var",
-            column: varIndex,
-            value: "TYMP",
+            where: {
+              field: "var",
+              column: varIndex,
+              value: "TYMP",
+            },
           },
         ])
       ).resolves.toBeInstanceOf(Dataframe);

client/__tests__/util/annoMatrix/crossfilter.test.js

@@ -170,9 +170,11 @@ describe("AnnoMatrixCrossfilter", () => {
       const xfltr = await crossfilter.select(
         "X",
         {
-          field: "var",
-          column: varIndex,
-          value: "TYMP",
+          where: {
+            field: "var",
+            column: varIndex,
+            value: "TYMP",
+          },
         },
         {
           mode: "range",
@@ -186,9 +188,11 @@ describe("AnnoMatrixCrossfilter", () => {
       expect(xfltr.countSelected()).toEqual(501);
 
       const df = await annoMatrix.fetch("X", {
-        field: "var",
-        column: varIndex,
-        value: "TYMP",
+        where: {
+          field: "var",
+          column: varIndex,
+          value: "TYMP",
+        },
       });
       const values = df.icol(0).asArray();
       const selected = xfltr.allSelectedMask();

client/__tests__/util/annoMatrix/serverMocks/index.js

@@ -1,11 +1,4 @@
 export const baseDataURL = "https://a.fake.url/api/v0.2";
 
-window.CELLXGENE = {
-  API: {
-    prefix: baseDataURL,
-    version: "v0.2/",
-  },
-};
-
 export { schema } from "./schema";
 export * from "./routes";

client/__tests__/util/annoMatrix/whereCache.test.js

@@ -1,3 +1,4 @@
+import sha1 from "sha1";
 import {
   _whereCacheGet,
   _whereCacheCreate,
@@ -7,37 +8,82 @@ import {
 const schema = {};
 
 describe("whereCache", () => {
-  test("whereCacheGet - missing cache values", () => {
+  test("whereCacheGet - where query, missing cache values", () => {
     expect(
       _whereCacheGet({}, schema, "X", {
-        field: "var",
-        column: "foo",
-        value: "bar",
+        where: {
+          field: "var",
+          column: "foo",
+          value: "bar",
+        },
       })
     ).toEqual([undefined]);
     expect(
-      _whereCacheGet({ X: {} }, schema, "X", {
-        field: "var",
-        column: "foo",
-        value: "bar",
+      _whereCacheGet({}, schema, "X", {
+        summarize: {
+          field: "var",
+          column: "foo",
+          values: ["bar"],
+        },
       })
     ).toEqual([undefined]);
     expect(
-      _whereCacheGet({ X: { var: new Map() } }, schema, "X", {
-        field: "var",
-        column: "foo",
-        value: "bar",
+      _whereCacheGet({ where: { X: {} } }, schema, "X", {
+        where: {
+          field: "var",
+          column: "foo",
+          value: "bar",
+        },
+      })
+    ).toEqual([undefined]);
+    expect(
+      _whereCacheGet({ where: { X: { var: new Map() } } }, schema, "X", {
+        where: {
+          field: "var",
+          column: "foo",
+          value: "bar",
+        },
       })
     ).toEqual([undefined]);
     expect(
       _whereCacheGet(
-        { X: { var: new Map([["foo", new Map()]]) } },
+        { where: { X: { var: new Map([["foo", new Map()]]) } } },
         schema,
         "X",
         {
+          where: {
+            field: "var",
+            column: "foo",
+            value: "bar",
+          },
+        }
+      )
+    ).toEqual([undefined]);
+  });
+
+  test("whereCacheGet - summarize query, missing cache values", () => {
+    expect(
+      _whereCacheGet({}, schema, "X", {
+        summarize: {
+          method: "mean",
           field: "var",
           column: "foo",
-          value: "bar",
+          values: ["bar"],
+        },
+      })
+    ).toEqual([undefined]);
+    expect(
+      _whereCacheGet(
+        { summarize: { X: { mean: { var: new Map() } } } },
+        schema,
+        "X",
+        {
+          summarize: {
+            method: "mean",
+            field: "var",
+            column: "foo",
+            values: ["bar"],
+          },
         }
       )
     ).toEqual([undefined]);
@@ -45,140 +91,292 @@ describe("whereCache", () => {
 
   test("whereCacheGet - varied lookups", () => {
     const whereCache = {
-      X: {
-        var: new Map([
-          [
-            "foo",
-            new Map([
-              ["bar", [0]],
-              ["baz", [1, 2]],
+      where: {
+        X: {
+          var: new Map([
+            [
+              "foo",
+              new Map([
+                ["bar", [0]],
+                ["baz", [1, 2]],
+              ]),
+            ],
+          ]),
+        },
+      },
+      summarize: {
+        X: {
+          mean: {
+            var: new Map([
+              [
+                "foo",
+                new Map([
+                  [sha1("bar"), [0]],
+                  [sha1("baz"), [1, 2]],
+                ]),
+              ],
             ]),
-          ],
-        ]),
+          },
+        },
       },
     };
 
     expect(
       _whereCacheGet(whereCache, schema, "X", {
-        field: "var",
-        column: "foo",
-        value: "bar",
+        where: {
+          field: "var",
+          column: "foo",
+          value: "bar",
+        },
       })
     ).toEqual([0]);
     expect(
       _whereCacheGet(whereCache, schema, "X", {
-        field: "var",
-        column: "foo",
-        value: "baz",
+        summarize: {
+          method: "mean",
+          field: "var",
+          column: "foo",
+          values: ["bar"],
+        },
+      })
+    ).toEqual([0]);
+    expect(
+      _whereCacheGet(whereCache, schema, "X", {
+        where: {
+          field: "var",
+          column: "foo",
+          value: "baz",
+        },
+      })
+    ).toEqual([1, 2]);
+    expect(
+      _whereCacheGet(whereCache, schema, "X", {
+        summarize: {
+          method: "mean",
+          field: "var",
+          column: "foo",
+          values: ["baz"],
+        },
       })
     ).toEqual([1, 2]);
     expect(_whereCacheGet(whereCache, schema, "Y", {})).toEqual([undefined]);
     expect(
       _whereCacheGet(whereCache, schema, "X", {
-        field: "whoknows",
-        column: "whatever",
-        value: "snork",
+        where: {
+          field: "whoknows",
+          column: "whatever",
+          value: "snork",
+        },
       })
     ).toEqual([undefined]);
     expect(
       _whereCacheGet(whereCache, schema, "X", {
-        field: "var",
-        column: "whatever",
-        value: "snork",
+        where: {
+          field: "var",
+          column: "whatever",
+          value: "snork",
+        },
       })
     ).toEqual([undefined]);
     expect(
       _whereCacheGet(whereCache, schema, "X", {
-        field: "var",
-        column: "foo",
-        value: "snork",
+        where: {
+          field: "var",
+          column: "foo",
+          value: "snork",
+        },
       })
     ).toEqual([undefined]);
   });
 
-  test("whereCacheCreate", () => {
+  test("whereCacheCreate, where query", () => {
     const query = {
-      field: "queryField",
-      column: "queryColumn",
-      value: "queryValue",
+      where: {
+        field: "queryField",
+        column: "queryColumn",
+        value: "queryValue",
+      },
     };
     const wc = _whereCacheCreate(
       "field",
-      { field: "queryField", column: "queryColumn", value: "queryValue" },
+      {
+        where: {
+          field: "queryField",
+          column: "queryColumn",
+          value: "queryValue",
+        },
+      },
       [0, 1, 2]
     );
     expect(wc).toBeDefined();
     expect(wc).toEqual(
       expect.objectContaining({
-        field: {
-          queryField: expect.any(Map),
+        where: {
+          field: {
+            queryField: expect.any(Map),
+          },
         },
       })
     );
-    expect(wc.field.queryField.has("queryColumn")).toEqual(true);
-    expect(wc.field.queryField.get("queryColumn")).toBeInstanceOf(Map);
-    expect(wc.field.queryField.get("queryColumn").has("queryValue")).toEqual(
-      true
-    );
+    expect(wc.where.field.queryField.has("queryColumn")).toEqual(true);
+    expect(wc.where.field.queryField.get("queryColumn")).toBeInstanceOf(Map);
+    expect(
+      wc.where.field.queryField.get("queryColumn").has("queryValue")
+    ).toEqual(true);
     expect(_whereCacheGet(wc, schema, "field", query)).toEqual([0, 1, 2]);
   });
 
-  test("whereCacheMerge", () => {
+  test("whereCacheCreate, summarize query", () => {
+    const query = {
+      summarize: {
+        method: "method",
+        field: "queryField",
+        column: "queryColumn",
+        values: ["queryValue"],
+      },
+    };
+    const wc = _whereCacheCreate("field", query, [0, 1, 2]);
+    expect(_whereCacheGet(wc, schema, "field", query)).toEqual([0, 1, 2]);
+  });
+
+  test("whereCacheCreate, unknown query type", () => {
+    expect(_whereCacheCreate("field", { foobar: true }, [1])).toEqual({});
+  });
+
+  test("whereCacheMerge, where queries", () => {
     let wc;
 
     // remember, will mutate dst
     const src = _whereCacheCreate(
       "field",
-      { field: "queryField", column: "queryColumn", value: "foo" },
+      { where: { field: "queryField", column: "queryColumn", value: "foo" } },
       ["foo"]
     );
 
     const dst1 = _whereCacheCreate(
       "field",
-      { field: "queryField", column: "queryColumn", value: "bar" },
+      { where: { field: "queryField", column: "queryColumn", value: "bar" } },
       ["dst1"]
     );
     wc = _whereCacheMerge(dst1, src);
     expect(
       _whereCacheGet(wc, schema, "field", {
-        field: "queryField",
-        column: "queryColumn",
-        value: "foo",
+        where: {
+          field: "queryField",
+          column: "queryColumn",
+          value: "foo",
+        },
       })
     ).toEqual(["foo"]);
     expect(
       _whereCacheGet(wc, schema, "field", {
-        field: "queryField",
-        column: "queryColumn",
-        value: "bar",
+        where: {
+          field: "queryField",
+          column: "queryColumn",
+          value: "bar",
+        },
       })
     ).toEqual(["dst1"]);
 
     const dst2 = _whereCacheCreate(
       "field",
-      { field: "queryField", column: "queryColumn", value: "bar" },
+      { where: { field: "queryField", column: "queryColumn", value: "bar" } },
       ["dst2"]
     );
     wc = _whereCacheMerge(dst2, dst1, src);
     expect(
       _whereCacheGet(wc, schema, "field", {
-        field: "queryField",
-        column: "queryColumn",
-        value: "foo",
+        where: {
+          field: "queryField",
+          column: "queryColumn",
+          value: "foo",
+        },
       })
     ).toEqual(["foo"]);
     expect(
       _whereCacheGet(wc, schema, "field", {
-        field: "queryField",
-        column: "queryColumn",
-        value: "bar",
+        where: {
+          field: "queryField",
+          column: "queryColumn",
+          value: "bar",
+        },
       })
     ).toEqual(["dst1"]);
 
     wc = _whereCacheMerge({}, src);
     expect(wc).toEqual(src);
 
-    wc = _whereCacheMerge({ field: { queryField: new Map() } }, src);
+    wc = _whereCacheMerge({ where: { field: { queryField: new Map() } } }, src);
     expect(wc).toEqual(src);
   });
+
+  test("whereCacheMerge, mixed queries", () => {
+    const wc = _whereCacheMerge(
+      _whereCacheCreate(
+        "field",
+        {
+          where: {
+            field: "queryField",
+            column: "queryColumn",
+            value: "foo",
+          },
+        },
+        ["a"]
+      ),
+      _whereCacheCreate(
+        "field",
+        {
+          summarize: {
+            method: "mean",
+            field: "queryField",
+            column: "queryColumn",
+            values: ["foo", "bar", "baz"],
+          },
+        },
+        ["b"]
+      )
+    );
+
+    expect(
+      _whereCacheGet(wc, schema, "field", {
+        where: {
+          field: "queryField",
+          column: "queryColumn",
+          value: "foo",
+        },
+      })
+    ).toEqual(["a"]);
+
+    expect(
+      _whereCacheGet(wc, schema, "field", {
+        summarize: {
+          method: "mean",
+          field: "queryField",
+          column: "queryColumn",
+          values: ["foo", "bar", "baz"],
+        },
+      })
+    ).toEqual(["b"]);
+
+    expect(
+      _whereCacheGet(wc, schema, "field", {
+        where: {
+          field: "queryField",
+          column: "queryColumn",
+          value: "does-not-exist",
+        },
+      })
+    ).toEqual([undefined]);
+
+    expect(
+      _whereCacheGet(wc, schema, "field", {
+        summarize: {
+          method: "no-such-method",
+          field: "queryField",
+          column: "queryColumn",
+          values: ["does-not-exist"],
+        },
+      })
+    ).toEqual([undefined]);
+  });
 });

client/__tests__/util/centroid.test.js

@@ -1,11 +1,11 @@
-import _ from "lodash";
+import cloneDeep from "lodash.clonedeep";
 
 import calcCentroid from "../../src/util/centroid";
 import quantile from "../../src/util/quantile";
 import { matrixFBSToDataframe } from "../../src/util/stateManager/matrix";
 import * as REST from "./stateManager/sampleResponses";
 import { indexEntireSchema } from "../../src/util/stateManager/schemaHelpers";
-import { _normalizeCategoricalSchema } from "../../src/annoMatrix/schema";
+import { normalizeWritableCategoricalSchema } from "../../src/annoMatrix/normalize";
 
 describe("centroid", () => {
   let schema;
@@ -13,11 +13,11 @@ describe("centroid", () => {
   let obsLayout;
 
   beforeAll(() => {
-    schema = indexEntireSchema(_.cloneDeep(REST.schema.schema));
+    schema = indexEntireSchema(cloneDeep(REST.schema.schema));
     obsAnnotations = matrixFBSToDataframe(REST.annotationsObs);
     obsLayout = matrixFBSToDataframe(REST.layoutObs);
 
-    _normalizeCategoricalSchema(
+    normalizeWritableCategoricalSchema(
       schema.annotations.obsByName.field3,
       obsAnnotations.col("field3")
     );

client/__tests__/util/dataframe/dataframe.test.js

@@ -793,9 +793,7 @@ describe("dataframe factories", () => {
         [3, 3],
         [new Array(3).fill(0), new Array(3).fill(0), new Array(3).fill(0)]
       );
-      const dfB = dfA.mapColumns(() => {
-        return new Array(3).fill(1);
-      });
+      const dfB = dfA.mapColumns(() => new Array(3).fill(1));
       expect(dfA).not.toBe(dfB);
       expect(dfB.iat(0, 0)).toEqual(1);
       expect(dfB.iat(0, 1)).toEqual(1);

client/__tests__/util/dataframe/histogram.test.js

@@ -88,16 +88,7 @@ describe("Dataframe column histogram", () => {
     expect(df.col(1).histogram(5, [0, 100])).toEqual([5, 1, 0, 0, 2]);
     expect(df.col(0).histogram(2, [0, 10])).toEqual([2, 2]);
     expect(df.col(0).histogram(10, [0, 100])).toEqual([
-      3,
-      2,
-      1,
-      0,
-      0,
-      0,
-      0,
-      0,
-      0,
-      2,
+      3, 2, 1, 0, 0, 0, 0, 0, 0, 2,
     ]);
   });
 });

client/__tests__/util/quantile.test.js

@@ -19,11 +19,7 @@ describe("quantile", () => {
   test("multi q", () => {
     const arr = new Float32Array([9, 3, 5, 6, 0]);
     expect(quantile([0, 0.25, 0.5, 0.75, 1.0], arr)).toMatchObject([
-      0,
-      3,
-      5,
-      6,
-      9,
+      0, 3, 5, 6, 9,
     ]);
   });
 });

client/__tests__/util/stateManager/colorHelpers.test.js

@@ -0,0 +1,198 @@
+/* eslint-disable no-bitwise -- unsigned right shift better than Math.round */
+
+/*
+test color helpers
+*/
+import {
+  createColorTable,
+  loadUserColorConfig,
+} from "../../../src/util/stateManager/colorHelpers";
+import * as Dataframe from "../../../src/util/dataframe";
+
+describe("categorical color helpers", () => {
+  /*
+  Primary test constraint for categorical colors is that they are ordered/identified
+  by schema order, NOT by value.  Ie,
+
+    scale(schemaIndex) should match rgb[obsOffset]
+  */
+
+  const schema = indexSchema({
+    annotations: {
+      obs: {
+        columns: [
+          {
+            name: "name_0",
+            type: "string",
+            writable: false,
+          },
+          {
+            name: "continuousColumn",
+            type: "float32",
+            writable: false,
+          },
+          {
+            categories: [
+              "CD4 T cells",
+              "CD14+ Monocytes",
+              "B cells",
+              "CD8 T cells",
+              "NK cells",
+              "FCGR3A+ Monocytes",
+              "Dendritic cells",
+              "Megakaryocytes",
+            ],
+            name: "categoricalColumn",
+            type: "categorical",
+            writable: false,
+          },
+        ],
+        index: "name_0",
+      },
+      var: {
+        columns: [
+          {
+            name: "name_0",
+            type: "string",
+            writable: false,
+          },
+        ],
+        index: "name_0",
+      },
+    },
+    dataframe: {
+      nObs: 2638,
+      nVar: 1838,
+      type: "float32",
+    },
+    layout: {},
+  });
+
+  const catColCategories = schema.annotations.obs.columns[2].categories;
+  const obsDataframe = new Dataframe.Dataframe(
+    [schema.dataframe.nObs, 2],
+    [
+      new Float32Array(schema.dataframe.nObs).map(() => Math.random()),
+      new Array(schema.dataframe.nObs)
+        .fill("")
+        .map(
+          () =>
+            catColCategories[(Math.random() * catColCategories.length) >>> 0]
+        ),
+    ],
+    null,
+    new Dataframe.KeyIndex(["continuousColumn", "categoricalColumn"])
+  );
+
+  test("default category order", () => {
+    const ct = createColorTable(
+      "color by categorical metadata",
+      "categoricalColumn",
+      obsDataframe,
+      schema
+    );
+    expect(ct).toBeDefined();
+    const data = obsDataframe.col("categoricalColumn").asArray();
+    const cats = schema.annotations.obsByName.categoricalColumn.categories;
+    for (let i = 0; i < schema.dataframe.nObs; i += 1) {
+      expect(makeScale(ct.rgb[i])).toEqual(ct.scale(cats.indexOf(data[i])));
+    }
+  });
+
+  test("shuffle category order", () => {
+    const schemaClone = indexSchema(JSON.parse(JSON.stringify(schema)));
+    shuffle(schemaClone.annotations.obsByName.categoricalColumn.categories);
+    const ct = createColorTable(
+      "color by categorical metadata",
+      "categoricalColumn",
+      obsDataframe,
+      schemaClone
+    );
+    expect(ct).toBeDefined();
+    const data = obsDataframe.col("categoricalColumn").asArray();
+    const cats = schemaClone.annotations.obsByName.categoricalColumn.categories;
+    for (let i = 0; i < schemaClone.dataframe.nObs; i += 1) {
+      expect(makeScale(ct.rgb[i])).toEqual(ct.scale(cats.indexOf(data[i])));
+    }
+  });
+
+  test("user defined color order", () => {
+    const cats = schema.annotations.obsByName.categoricalColumn.categories;
+    const shuffleCats = shuffle(
+      Array.from(schema.annotations.obsByName.categoricalColumn.categories)
+    );
+    const userDefinedColorTable = {
+      categoricalColumn: shuffleCats.reduce((acc, label) => {
+        acc[label] = randRGBColor();
+        return acc;
+      }, {}),
+    };
+
+    const userColors = loadUserColorConfig(userDefinedColorTable);
+    expect(userColors).toBeDefined();
+
+    const ct = createColorTable(
+      "color by categorical metadata",
+      "categoricalColumn",
+      obsDataframe,
+      schema,
+      userColors
+    );
+    expect(ct).toBeDefined();
+    const data = obsDataframe.col("categoricalColumn").asArray();
+    for (let i = 0; i < schema.dataframe.nObs; i += 1) {
+      expect(makeScale(ct.rgb[i])).toEqual(
+        ct.scale(cats.indexOf(data[i])).toString()
+      );
+    }
+  });
+});
+
+/*
+TODO:
+1. mix up category order in schema to make sure it works with varied order
+2. user defined colors
+*/
+
+function indexSchema(schema) {
+  schema.annotations.obsByName = Object.fromEntries(
+    schema.annotations?.obs?.columns?.map((v) => [v.name, v]) ?? []
+  );
+  schema.annotations.varByName = Object.fromEntries(
+    schema.annotations?.var?.columns?.map((v) => [v.name, v]) ?? []
+  );
+  schema.layout.obsByName = Object.fromEntries(
+    schema.layout?.obs?.map((v) => [v.name, v]) ?? []
+  );
+  schema.layout.varByName = Object.fromEntries(
+    schema.layout?.var?.map((v) => [v.name, v]) ?? []
+  );
+
+  return schema;
+}
+
+function makeScale(rgb) {
+  // make a scale string from a rgb float triple
+  return `rgb(${(rgb[0] * 255) >>> 0}, ${(rgb[1] * 255) >>> 0}, ${
+    (rgb[2] * 256) >>> 0
+  })`;
+}
+
+function shuffle(array) {
+  for (let i = array.length - 1; i > 0; i -= 1) {
+    const j = (Math.random() * (i + 1)) >>> 0;
+    [array[i], array[j]] = [array[j], array[i]];
+  }
+  return array;
+}
+
+function randHexColor() {
+  const hex = ((Math.random() * 255) >>> 0).toString(16);
+  return `0${hex}`.slice(-2);
+}
+
+function randRGBColor() {
+  return `#${randHexColor()}${randHexColor()}${randHexColor()}`;
+}
+
+/* eslint-enable no-bitwise -- unsigned right shift better than Math.round */

client/__tests__/util/stateManager/controlsHelpers.test.js

@@ -1,38 +1,9 @@
 /*
 test controls helpers
 */
-import { subsetAndResetGeneLists } from "../../../src/util/stateManager/controlsHelpers";
-import * as globals from "../../../src/globals";
+// TODO #2227 test: improve test coverage on control helper functions
+// (`isSelectableCategoryName()`, `selectableCategoryNames()`, `createCategorySummaryFromDfCol()`, `createCategoricalSelection()`, )
 
 describe("controls helpers", () => {
-  test("subsetAndResetGeneLists", () => {
-    const geneList = [];
-    const genRandGene = () => Math.random().toString(36).substring(2, 6);
-
-    // build a unique set of genes
-    for (let i = 0; i < 150; i += 1) {
-      let randGene = genRandGene();
-      while (geneList.includes(randGene)) randGene = genRandGene();
-      geneList.push(randGene);
-    }
-    // insert duplicates
-    geneList[0] = "dupl";
-    geneList[20] = "dupl";
-    const state = {
-      userDefinedGenes: geneList.slice(0, 20),
-      diffexpGenes: geneList.slice(20),
-    };
-    const [newUserDefinedGenes, newDiffExpGenes] = subsetAndResetGeneLists(
-      state
-    );
-    const expectedNewUserDefinedGenes = [
-      ...geneList.slice(0, 20),
-      ...geneList.slice(21),
-    ].slice(0, globals.maxGenes);
-    expect(globals.maxUserDefinedGenes).toBeLessThan(globals.maxGenes);
-    expect(geneList.length).toBeGreaterThan(globals.maxGenes);
-    expect(newUserDefinedGenes).toHaveLength(globals.maxGenes);
-    expect(newUserDefinedGenes).toStrictEqual(expectedNewUserDefinedGenes);
-    expect(newDiffExpGenes).toStrictEqual([]);
-  });
+  test("placeholder", () => {});
 });

client/__tests__/util/stateManager/sampleResponses.js

@@ -1,3 +1,7 @@
+import every from "lodash.every";
+import map from "lodash.map";
+import isNumber from "lodash.isnumber";
+import zip from "lodash.zip";
 import _ from "lodash";
 import { flatbuffers } from "flatbuffers";
 import { NetEncoding } from "../../../src/util/stateManager/matrix_generated";
@@ -121,10 +125,10 @@ function encodeMatrix(columns, colIndex = undefined) {
   */
   const utf8Encoder = new TextEncoder("utf-8");
   const builder = new flatbuffers.Builder(1024);
-  const cols = _.map(columns, (carr) => {
+  const cols = map(columns, (carr) => {
     let uType;
     let tarr;
-    if (_.every(carr, _.isNumber)) {
+    if (every(carr, isNumber)) {
       uType = NetEncoding.TypedArray.Float32Array;
       tarr = encodeTypedArray(builder, uType, new Float32Array(carr));
     } else {
@@ -167,12 +171,12 @@ function encodeMatrix(columns, colIndex = undefined) {
 }
 
 const anAnnotationsObsFBSResponse = (() => {
-  const columns = _.zip(...anAnnotationsObsJSONResponse.data).slice(1);
+  const columns = zip(...anAnnotationsObsJSONResponse.data).slice(1);
   return encodeMatrix(columns, anAnnotationsObsJSONResponse.names);
 })();
 
 const anAnnotationsVarFBSResponse = (() => {
-  const columns = _.zip(...anAnnotationsVarJSONResponse.data).slice(1);
+  const columns = zip(...anAnnotationsVarJSONResponse.data).slice(1);
   return encodeMatrix(columns, anAnnotationsVarJSONResponse.names);
 })();
 

client/__tests__/util/typedCrossfilter/bitArray.test.js

@@ -1,5 +1,3 @@
-// jshint esversion: 6
-
 import BitArray from "../../../src/util/typedCrossfilter/bitArray";
 
 const defaultTestLength = 8;

client/__tests__/util/typedCrossfilter/crossfilter.test.js

@@ -1,4 +1,5 @@
-import _ from "lodash";
+import filter from "lodash.filter";
+import zip from "lodash.zip";
 
 import Crossfilter from "../../../src/util/typedCrossfilter";
 
@@ -251,12 +252,12 @@ describe("ImmutableTypedCrossfilter", () => {
     test.each([[[]], [[2]], [[2, 1]], [[9, 82]], [[0, 1]]])("exact: %p", (v) =>
       expect(
         p.select("quantity", { mode: "exact", values: v }).countSelected()
-      ).toEqual(_.filter(someData, (d) => v.includes(d.quantity)).length)
+      ).toEqual(filter(someData, (d) => v.includes(d.quantity)).length)
     );
     test("single value exact", () => {
       expect(
         p.select("quantity", { mode: "exact", values: 2 }).countSelected()
-      ).toEqual(_.filter(someData, (d) => d.quantity === 2).length);
+      ).toEqual(filter(someData, (d) => d.quantity === 2).length);
     });
     test.each([
       [0, 1],
@@ -267,7 +268,7 @@ describe("ImmutableTypedCrossfilter", () => {
       expect(
         p.select("quantity", { mode: "range", lo, hi }).countSelected()
       ).toEqual(
-        _.filter(someData, (d) => d.quantity >= lo && d.quantity < hi).length
+        filter(someData, (d) => d.quantity >= lo && d.quantity < hi).length
       )
     );
     test("bad mode", () => {
@@ -298,12 +299,12 @@ describe("ImmutableTypedCrossfilter", () => {
     ])("exact: %p", (v) =>
       expect(
         p.select("type", { mode: "exact", values: v }).countSelected()
-      ).toEqual(_.filter(someData, (d) => v.includes(d.type)).length)
+      ).toEqual(filter(someData, (d) => v.includes(d.type)).length)
     );
     test("single value exact", () => {
       expect(
         p.select("type", { mode: "exact", values: "tab" }).countSelected()
-      ).toEqual(_.filter(someData, (d) => d.type === "tab").length);
+      ).toEqual(filter(someData, (d) => d.type === "tab").length);
     });
     test("range", () => {
       expect(() => p.select("type", { mode: "range", lo: 0, hi: 9 })).toThrow(
@@ -341,7 +342,7 @@ describe("ImmutableTypedCrossfilter", () => {
           .select("coords", { mode: "within-rect", minX, minY, maxX, maxY })
           .allSelected()
       ).toEqual(
-        _.filter(someData, (d) => {
+        filter(someData, (d) => {
           const [x, y] = d.coords;
           return minX <= x && x < maxX && minY <= y && y < maxY;
         })
@@ -397,7 +398,7 @@ describe("ImmutableTypedCrossfilter", () => {
       expect(
         p.select("coords", { mode: "within-polygon", polygon }).allSelected()
       ).toEqual(
-        _.zip(someData, expected)
+        zip(someData, expected)
           .filter((x) => x[1])
           .map((x) => x[0])
       );

client/__tests__/util/typedCrossfilter/positiveInterval.test.js

@@ -1,5 +1,3 @@
-// jshint esversion: 6
-
 // const PositiveIntervals = require("../../src/util/typedCrossfilter/positiveIntervals");
 import PositiveIntervals from "../../../src/util/typedCrossfilter/positiveIntervals";
 
@@ -153,9 +151,9 @@ describe("intersection", () => {
       [1, 2],
       [6, 9],
     ]);
-    expect(
-      PositiveIntervals.intersection([[0, 2638]], [[1363, 2638]])
-    ).toEqual([[1363, 2638]]);
+    expect(PositiveIntervals.intersection([[0, 2638]], [[1363, 2638]])).toEqual(
+      [[1363, 2638]]
+    );
     expect(PositiveIntervals.intersection([[1, 2]], [[1, 2]])).toEqual([
       [1, 2],
     ]);

client/configuration/babel/babel.dev.js

@@ -16,6 +16,8 @@ module.exports = {
     "@babel/plugin-proposal-function-bind",
     ["@babel/plugin-proposal-decorators", { legacy: true }],
     ["@babel/plugin-proposal-class-properties", { loose: true }],
+    ["@babel/plugin-transform-private-methods", { loose: true }],
+    ["@babel/plugin-transform-private-property-in-object", { loose: true }],
     "@babel/plugin-proposal-export-namespace-from",
     "@babel/plugin-proposal-optional-chaining",
     "@babel/plugin-proposal-nullish-coalescing-operator",

client/configuration/babel/babel.prod.js

@@ -15,6 +15,8 @@ module.exports = {
     "@babel/plugin-proposal-function-bind",
     ["@babel/plugin-proposal-decorators", { legacy: true }],
     ["@babel/plugin-proposal-class-properties", { loose: true }],
+    ["@babel/plugin-transform-private-methods", { loose: true }],
+    ["@babel/plugin-transform-private-property-in-object", { loose: true }],
     "@babel/plugin-proposal-export-namespace-from",
     "@babel/plugin-transform-react-constant-elements",
     "@babel/plugin-transform-runtime",

client/configuration/eslint/eslint.js

@@ -1,23 +1,21 @@
 module.exports = {
   root: true,
-  parser: "babel-eslint",
+
   extends: [
     "airbnb",
     "plugin:eslint-comments/recommended",
+    "plugin:@blueprintjs/recommended",
     "plugin:compat/recommended",
-    "plugin:prettier/recommended",
-    "prettier/react",
+    "plugin:jsx-a11y/recommended",
+    // (thuang) disable eslint formatting rules, so prettier can do its job
+    // Do not use `plugin:prettier/recommended` per doc below:
+    // https://prettier.io/docs/en/integrating-with-linters.html
+    "prettier",
   ],
   settings: {
-    polyfills: [
-      "TextDecoder",
-      "TextEncoder",
-      "fetch",
-      "Request",
-      "Response",
-      "Headers",
-      "AbortController",
-    ],
+    // AbortController is not supported in iOS Safari 10.3, Chrome 61
+    // Headers is not supported in iOS Safari 10.3
+    polyfills: ["Headers", "AbortController"],
   },
   env: { browser: true, commonjs: true, es6: true },
   globals: {
@@ -30,6 +28,7 @@ module.exports = {
     context: true,
     beforeEach: true,
   },
+  parser: "@babel/eslint-parser",
   parserOptions: {
     ecmaVersion: 2017,
     sourceType: "module",
@@ -37,8 +36,12 @@ module.exports = {
       jsx: true,
       generators: true,
     },
+    babelOptions: {
+      configFile: "./configuration/babel/babel.prod.js",
+    },
   },
   rules: {
+    "react/jsx-no-target-blank": "off",
     "eslint-comments/require-description": ["error"],
     "no-magic-numbers": "off",
     "no-nested-ternary": "off",
@@ -64,6 +67,12 @@ module.exports = {
       "LabeledStatement",
       "WithStatement",
     ],
+    "import/no-extraneous-dependencies": [
+      "error",
+      {
+        devDependencies: true,
+      },
+    ],
   },
   overrides: [
     {

client/configuration/lint-staged/lint-staged.config.js

@@ -1,3 +1,4 @@
 module.exports = {
   "*.js": "eslint --fix",
+  "**/*": "prettier --write --ignore-unknown",
 };

client/configuration/webpack/obsoleteHTMLTemplate.html

@@ -56,13 +56,6 @@
         />
         <div>Chrome &gt; 60</div>
       </a>
-      <a href="https://www.apple.com/safari/" aria-label="Download Safari">
-        <img
-          src="https://cellxgene.cziscience.com/s3/cellxgene/static/images/safari.png"
-          style="width: 80px; height: 80px;"
-        />
-        <div>Safari ≥ 10.1</div>
-      </a>
       <a href="https://www.mozilla.com/firefox/" aria-label="Download Firefox">
         <img
           src="https://cellxgene.cziscience.com/s3/cellxgene/static/images/firefox.png"
@@ -75,7 +68,7 @@
           src="https://cellxgene.cziscience.com/s3/cellxgene/static/images/edge.png"
           style="width: 80px; height: 80px;"
         />
-        <div>Edge ≥ 15</div>
+        <div>Edge ≥ 79</div>
       </a>
     </div>
   </div>

client/configuration/webpack/webpack.config.dev.js

@@ -1,8 +1,6 @@
 const path = require("path");
 const webpack = require("webpack");
 const HtmlWebpackPlugin = require("html-webpack-plugin");
-const FaviconsWebpackPlugin = require("favicons-webpack-plugin");
-const ScriptExtHtmlWebpackPlugin = require("script-ext-html-webpack-plugin");
 const MiniCssExtractPlugin = require("mini-css-extract-plugin");
 
 const { merge } = require("webpack-merge");
@@ -11,6 +9,7 @@ const sharedConfig = require("./webpack.config.shared");
 const babelOptions = require("../babel/babel.dev");
 
 const fonts = path.resolve("src/fonts");
+const images = path.resolve("src/images");
 const nodeModules = path.resolve("node_modules");
 
 const devConfig = {
@@ -30,8 +29,12 @@ const devConfig = {
       {
         test: /\.(jpg|png|gif|eot|svg|ttf|woff|woff2|otf)$/i,
         loader: "file-loader",
-        include: [nodeModules, fonts],
-        query: { name: "static/assets/[name].[ext]" },
+        include: [nodeModules, fonts, images],
+        options: {
+          name: "static/assets/[name].[ext]",
+          // (thuang): This is needed to make sure @font url path is '/static/assets/'
+          publicPath: "..",
+        },
       },
     ],
   },
@@ -40,21 +43,6 @@ const devConfig = {
       inject: true,
       template: path.resolve("index.html"),
     }),
-    new FaviconsWebpackPlugin({
-      logo: "./favicon.png",
-      prefix: "static/img/",
-      favicons: {
-        icons: {
-          android: false,
-          appleIcon: false,
-          appleStartup: false,
-          coast: false,
-          firefox: false,
-          windows: false,
-          yandex: false,
-        },
-      },
-    }),
     new MiniCssExtractPlugin({
       filename: "static/[name].css",
     }),
@@ -63,14 +51,16 @@ const devConfig = {
       __REACT_DEVTOOLS_GLOBAL_HOOK__: "({ isDisabled: true })",
     }),
     new webpack.DefinePlugin({
-      "process.env.CXG_SERVER_PORT": JSON.stringify(
-        process.env.CXG_SERVER_PORT
-      ),
-    }),
-    new ScriptExtHtmlWebpackPlugin({
-      async: "obsolete",
+      // webpack 5 no longer polyfills NodeJS modules, so fake the one we need
+      "process.env": JSON.stringify({
+        NODE_ENV: process.env.NODE_ENV || "development",
+        CXG_SERVER_PORT: process.env.CXG_SERVER_PORT || "5005",
+      }),
     }),
   ],
+  infrastructureLogging: {
+    level: "warn",
+  },
 };
 
 module.exports = merge(sharedConfig, devConfig);

client/configuration/webpack/webpack.config.prod.js

@@ -1,10 +1,9 @@
 const path = require("path");
+const webpack = require("webpack");
 const HtmlWebpackPlugin = require("html-webpack-plugin");
 const { CleanWebpackPlugin } = require("clean-webpack-plugin");
 const TerserJSPlugin = require("terser-webpack-plugin");
-const CleanCss = require("clean-css");
-const OptimizeCSSAssetsPlugin = require("optimize-css-assets-webpack-plugin");
-const FaviconsWebpackPlugin = require("favicons-webpack-plugin");
+const CssMinimizerPlugin = require("css-minimizer-webpack-plugin");
 const MiniCssExtractPlugin = require("mini-css-extract-plugin");
 
 const { merge } = require("webpack-merge");
@@ -15,6 +14,7 @@ const CspHashPlugin = require("./cspHashPlugin");
 const sharedConfig = require("./webpack.config.shared");
 
 const fonts = path.resolve("src/fonts");
+const images = path.resolve("src/images");
 const nodeModules = path.resolve("node_modules");
 
 const prodConfig = {
@@ -28,8 +28,8 @@ const prodConfig = {
     minimize: true,
     minimizer: [
       new TerserJSPlugin({}),
-      new OptimizeCSSAssetsPlugin({
-        cssProcessor: CleanCss,
+      new CssMinimizerPlugin({
+        minify: CssMinimizerPlugin.cleanCssMinify,
       }),
     ],
   },
@@ -44,8 +44,12 @@ const prodConfig = {
       {
         test: /\.(jpg|png|gif|eot|svg|ttf|woff|woff2|otf)$/i,
         loader: "file-loader",
-        include: [nodeModules, fonts],
-        query: { name: "static/assets/[name]-[contenthash].[ext]" },
+        include: [nodeModules, fonts, images],
+        options: {
+          name: "static/assets/[name]-[contenthash].[ext]",
+          // (thuang): This is needed to make sure @font url path is '../static/assets/'
+          publicPath: "..",
+        },
       },
     ],
   },
@@ -61,27 +65,18 @@ const prodConfig = {
       protectWebpackAssets: false,
       cleanAfterEveryBuildPatterns: ["main.js", "main.css"],
     }),
-    new FaviconsWebpackPlugin({
-      logo: "./favicon.png",
-      prefix: "static/assets/",
-      favicons: {
-        icons: {
-          android: false,
-          appleIcon: false,
-          appleStartup: false,
-          coast: false,
-          firefox: false,
-          windows: false,
-          yandex: false,
-        },
-      },
-    }),
     new MiniCssExtractPlugin({
       filename: "static/[name]-[contenthash].css",
     }),
     new CspHashPlugin({
       filename: "csp-hashes.json",
     }),
+    new webpack.DefinePlugin({
+      // webpack 5 no longer polyfills NodeJS modules, so fake the one we need
+      "process.env": JSON.stringify({
+        NODE_ENV: "production",
+      }),
+    }),
   ],
   performance: {
     maxEntrypointSize: 2000000,

client/configuration/webpack/webpack.config.shared.js

@@ -1,13 +1,12 @@
 const path = require("path");
 const fs = require("fs");
 const MiniCssExtractPlugin = require("mini-css-extract-plugin");
-const ObsoleteWebpackPlugin = require("obsolete-webpack-plugin");
-const ScriptExtHtmlWebpackPlugin = require("script-ext-html-webpack-plugin");
+const ObsoleteWebpackPlugin = require("webpack-obsolete-plugin");
 
 const src = path.resolve("src");
 const nodeModules = path.resolve("node_modules");
 
-const publicPath = "/";
+const publicPath = "";
 
 const rawObsoleteHTMLTemplate = fs.readFileSync(
   `${__dirname}/obsoleteHTMLTemplate.html`,
@@ -20,7 +19,6 @@ module.exports = {
   entry: [
     "core-js",
     "regenerator-runtime/runtime",
-    "fastestsmallesttextencoderdecoder",
     "whatwg-fetch",
     "abort-controller/polyfill",
     "./src/index",
@@ -41,7 +39,7 @@ module.exports = {
             loader: "css-loader",
             options: {
               modules: {
-                localIdentName: "[name]__[local]___[hash:base64:5]",
+                localIdentName: "[name]__[local]___[contenthash:base64:5]",
               },
               importLoaders: 1,
             },
@@ -67,8 +65,5 @@ module.exports = {
       template: obsoleteHTMLTemplate,
       promptOnNonTargetBrowser: false,
     }),
-    new ScriptExtHtmlWebpackPlugin({
-      async: "obsolete",
-    }),
   ],
 };

client/index.html

@@ -3,7 +3,7 @@
   <head>
     <meta charset="utf-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1" />
-    <title>cell&times;gene</title>
+    <title>CELL&times;GENE | Annotate</title>
     <style>
       html,
       body,

client/index_template.html

@@ -3,7 +3,7 @@
   <head>
     <meta charset="utf-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1" />
-    <title>cell&times;gene</title>
+    <title>CELL&times;GENE | Annotate</title>
     <style>
       html,
       body,

client/jest-puppeteer.config.js

@@ -14,6 +14,7 @@ const DEFAULT_LAUNCH_CONFIG = {
   headless: !isHeadful,
   args: ["--ignore-certificate-errors", "--ignore-ssl-errors"],
   ignoreHTTPSErrors: true,
+  timeout: 90000,
   defaultViewport: {
     width: 1280,
     height: 960,

client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cellxgene",
-  "version": "0.16.0",
+  "version": "1.3.0",
   "license": "MIT",
   "description": "cellxgene is a web application for the interactive exploration of single cell sequence data.",
   "repository": "https://github.com/chanzuckerberg/cellxgene",
@@ -10,15 +10,16 @@
     "dev": "npm run build -- configuration/webpack/webpack.config.dev.js",
     "e2e": "jest --config __tests__/e2e/e2eJestConfig.json e2e/e2e.test.js",
     "e2e-annotations": "jest --config __tests__/e2e/e2eJestConfig.json e2e/e2eAnnotations.test.js",
-    "e2e-prod": "CXG_URL_BASE='https://cellxgene.cziscience.com/d/pbmc3k.cxg/' jest --config __tests__/e2e/e2eJestConfig.json e2e/e2e.test.js",
     "fmt": "eslint --fix src __tests__",
     "lint": "eslint --fix src __tests__",
     "prod": "npm run build -- configuration/webpack/webpack.config.prod.js",
-    "test": "jest --testPathIgnorePatterns e2e"
+    "test": "jest --testPathIgnorePatterns e2e",
+    "prepare": "cd .. && husky install client/.husky"
   },
   "engineStrict": true,
   "engines": {
-    "npm": ">=3.0.0"
+    "npm": ">=9.6.7",
+    "node": "^18.17.0"
   },
   "eslintConfig": {
     "extends": "./configuration/eslint/eslint.js"
@@ -31,111 +32,118 @@
   },
   "browserslist": [
     "Chrome > 60",
-    "Safari >= 10.1",
     "iOS >= 10.3",
     "Firefox >= 60",
-    "Edge >= 15",
-    "not Explorer > 0"
+    "Edge >= 79",
+    "not Explorer > 0",
+    "not Safari > 0"
   ],
   "dependencies": {
-    "@blueprintjs/core": "^3.30.0",
+    "@babel/eslint-parser": "^7.15.0",
+    "@blueprintjs/core": "^3.44.0",
     "@blueprintjs/icons": "^3.19.0",
-    "@blueprintjs/select": "^3.13.5",
+    "@blueprintjs/popover2": "^0.11.2",
+    "@blueprintjs/select": "^3.16.0",
     "abort-controller": "^3.0.0",
-    "core-js": "^3.6.5",
+    "core-js": "^3.16.3",
     "d3": "^4.10.0",
     "d3-scale-chromatic": "^1.5.0",
-    "fastestsmallesttextencoderdecoder": "^1.0.22",
     "flatbuffers": "^1.11.0",
     "fuzzysort": "^1.1.4",
     "gl-mat4": "^1.2.0",
     "gl-matrix": "^3.3.0",
     "gl-vec3": "^1.1.3",
     "is-number": "^7.0.0",
-    "lodash": "^4.17.19",
+    "lodash.clonedeep": "^4.5.0",
+    "lodash.difference": "^4.5.0",
+    "lodash.foreach": "^4.5.0",
+    "lodash.pull": "^4.1.0",
+    "lodash.sortby": "^4.7.0",
+    "lodash.uniq": "^4.5.0",
     "memoize-one": "^5.1.1",
-    "react": "^16.13.1",
+    "pako": "^2.0.3",
+    "react": "^17.0.2",
     "react-async": "^10.0.1",
-    "react-dom": "^16.13.1",
+    "react-dom": "^17.0.2",
     "react-flip-toolkit": "^7.0.12",
-    "react-helmet": "^5.2.1",
-    "react-icons": "^3.10.0",
+    "react-helmet": "^6.1.0",
+    "react-icons": "^4.2.0",
     "react-redux": "^7.2.0",
     "redux": "^4.0.5",
     "redux-thunk": "^2.3.0",
     "regenerator-runtime": "^0.13.7",
-    "regl": "^1.6.1",
-    "script-ext-html-webpack-plugin": "^2.1.4",
+    "regl": "^2.1.0",
+    "sha1": "^1.1.1",
     "tinyqueue": "^2.0.3",
-    "webpack-merge": "^5.0.9",
     "whatwg-fetch": "^3.2.0"
   },
   "devDependencies": {
-    "@babel/core": "^7.10.5",
+    "@babel/core": "^7.25.2",
     "@babel/plugin-proposal-class-properties": "^7.10.4",
-    "@babel/plugin-proposal-decorators": "^7.10.5",
+    "@babel/plugin-proposal-decorators": "^7.13.15",
     "@babel/plugin-proposal-export-namespace-from": "^7.10.4",
     "@babel/plugin-proposal-function-bind": "^7.10.5",
     "@babel/plugin-proposal-nullish-coalescing-operator": "^7.10.4",
     "@babel/plugin-proposal-optional-chaining": "^7.10.4",
-    "@babel/plugin-transform-react-constant-elements": "^7.10.4",
-    "@babel/plugin-transform-runtime": "^7.10.5",
-    "@babel/preset-env": "^7.10.4",
-    "@babel/preset-react": "^7.10.4",
-    "@babel/register": "^7.10.5",
-    "@babel/runtime": "^7.10.5",
-    "@sentry/webpack-plugin": "^1.12.0",
-    "babel-eslint": "^10.1.0",
+    "@babel/plugin-transform-private-property-in-object": "^7.22.11",
+    "@babel/plugin-transform-react-constant-elements": "^7.13.13",
+    "@babel/plugin-transform-runtime": "^7.13.15",
+    "@babel/preset-env": "^7.22.20",
+    "@babel/preset-react": "^7.13.13",
+    "@babel/register": "^7.13.16",
+    "@babel/runtime": "^7.13.16",
+    "@blueprintjs/eslint-plugin": "^0.3.0",
+    "@sentry/webpack-plugin": "^1.15.0",
     "babel-jest": "^26.1.0",
     "babel-loader": "^8.1.0",
-    "babel-preset-modern-browsers": "^14.2.1",
+    "babel-preset-modern-browsers": "^15.0.2",
     "chalk": "^4.1.0",
-    "cheerio": "^1.0.0-rc.3",
-    "clean-css": "^4.2.3",
-    "clean-webpack-plugin": "^3.0.0",
+    "cheerio": "^1.0.0-rc.6",
+    "clean-css": "^5.1.2",
+    "clean-webpack-plugin": "^4.0.0-alpha.0",
     "codecov": "^3.7.1",
-    "connect-history-api-fallback": "^1.6.0",
-    "css-loader": "^3.6.0",
-    "eslint": "^7.4.0",
+    "css-loader": "^5.2.4",
+    "css-minimizer-webpack-plugin": "^4.0.0",
+    "eslint": "^7.24.0",
     "eslint-config-airbnb": "^18.2.0",
-    "eslint-config-prettier": "^6.11.0",
-    "eslint-loader": "^3.0.4",
-    "eslint-plugin-compat": "^3.8.0",
+    "eslint-config-prettier": "^8.2.0",
+    "eslint-plugin-compat": "^4.2.0",
     "eslint-plugin-eslint-comments": "^3.2.0",
     "eslint-plugin-filenames": "^1.3.2",
-    "eslint-plugin-import": "^2.22.0",
-    "eslint-plugin-jest": "^23.18.0",
+    "eslint-plugin-import": "^2.24.2",
+    "eslint-plugin-jest": "^24.3.5",
     "eslint-plugin-jsx-a11y": "^6.3.1",
-    "eslint-plugin-prettier": "^3.1.4",
-    "eslint-plugin-react": "^7.20.3",
+    "eslint-plugin-react": "^7.23.2",
     "eslint-plugin-react-hooks": "^4.0.8",
-    "expect-puppeteer": "^4.4.0",
+    "expect-puppeteer": "^5.0.0",
     "express": "^4.17.1",
-    "favicons-webpack-plugin": "^3.0.1",
     "file-loader": "^6.0.0",
-    "html-webpack-plugin": "^4.3.0",
-    "husky": "^4.2.5",
-    "jest": "^26.1.0",
-    "jest-circus": "^26.1.0",
-    "jest-environment-puppeteer": "^4.4.0",
+    "html-webpack-plugin": "^5.3.1",
+    "husky": "^7.0.2",
+    "jest": "^27.0.6",
+    "jest-circus": "^27.0.6",
+    "jest-environment-puppeteer": "^5.0.1",
     "jest-fetch-mock": "^3.0.3",
-    "jest-puppeteer": "^4.4.0",
+    "jest-puppeteer": "^6.2.0",
     "json-loader": "^0.5.7",
     "lint-staged": "^10.2.11",
-    "mini-css-extract-plugin": "^0.9.0",
-    "obsolete-webpack-plugin": "^0.5.6",
-    "optimize-css-assets-webpack-plugin": "^5.0.3",
+    "lodash": "^4.17.21",
+    "lodash.every": "^4.6.0",
+    "lodash.filter": "^4.6.0",
+    "lodash.isnumber": "^3.0.3",
+    "lodash.map": "^4.6.0",
+    "lodash.zip": "^4.2.0",
+    "mini-css-extract-plugin": "^1.5.0",
     "prettier": "^2.0.5",
-    "puppeteer": "^3.3.0",
+    "puppeteer": "^10.4.0",
     "rimraf": "^3.0.2",
     "serve-favicon": "^2.5.0",
-    "style-loader": "^1.2.1",
-    "sw-precache-webpack-plugin": "^1.0.0",
-    "terser-webpack-plugin": "^3.0.7",
-    "url-loader": "^4.1.0",
-    "webpack": "^4.43.0",
-    "webpack-cli": "^3.3.12",
-    "webpack-dev-middleware": "^3.7.2"
+    "terser-webpack-plugin": "^5.1.1",
+    "webpack": "^5.94.0",
+    "webpack-cli": "^4.6.0",
+    "webpack-dev-middleware": "^4.1.0",
+    "webpack-merge": "^5.0.9",
+    "webpack-obsolete-plugin": "^1.0.5"
   },
   "jest": {
     "testMatch": [
@@ -168,6 +176,18 @@
               "loose": true
             }
           ],
+          [
+            "@babel/plugin-transform-private-methods",
+            {
+              "loose": true
+            }
+          ],
+          [
+            "@babel/plugin-transform-private-property-in-object",
+            {
+              "loose": true
+            }
+          ],
           "@babel/plugin-proposal-export-namespace-from",
           "@babel/plugin-transform-react-constant-elements",
           "@babel/plugin-transform-runtime",
@@ -176,10 +196,5 @@
         ]
       }
     }
-  },
-  "husky": {
-    "hooks": {
-      "pre-commit": "lint-staged --config \"./configuration/lint-staged/lint-staged.config.js\""
-    }
   }
 }

client/server/development.js

@@ -1,46 +1,50 @@
-/* eslint-disable */
-// jshint esversion: 6
-var path = require("path");
-var historyApiFallback = require("connect-history-api-fallback");
-var chalk = require("chalk");
-var express = require("express");
-var favicon = require("serve-favicon");
-var webpack = require("webpack");
-var config = require("../configuration/webpack/webpack.config.dev");
-var utils = require("./utils");
+const chalk = require("chalk");
+const express = require("express");
+const favicon = require("serve-favicon");
+const webpack = require("webpack");
+const devMiddleware = require("webpack-dev-middleware");
+const config = require("../configuration/webpack/webpack.config.dev");
+const utils = require("./utils");
 
 process.env.NODE_ENV = "development";
 
 const CLIENT_PORT = process.env.CXG_CLIENT_PORT;
+const { CXG_SERVER_PORT } = process.env;
+
+const API = {
+  prefix: `http://localhost:${CXG_SERVER_PORT}/`,
+};
 
 // Set up compiler
-var compiler = webpack(config);
+const compiler = webpack(config);
 
-compiler.plugin("invalid", () => {
+compiler.hooks.invalid.tap("invalid", () => {
   utils.clearConsole();
   console.log("Compiling...");
 });
 
-compiler.plugin("done", (stats) => {
+compiler.hooks.done.tap("done", (stats) => {
   utils.formatStats(stats, CLIENT_PORT);
 });
 
 // Launch server
-var app = express();
-
-app.use(historyApiFallback({ verbose: false }));
+const app = express();
 
 app.use(
-  require("webpack-dev-middleware")(compiler, {
-    logLevel: "warn",
+  devMiddleware(compiler, {
     publicPath: config.output.publicPath,
+    index: true,
   })
 );
 
 app.use(favicon("./favicon.png"));
 
-app.get("*", (req, res) => {
-  res.sendFile(path.resolve("index.html"));
+app.get("/logout", async (req, res) => {
+  try {
+    res.redirect(`${API.prefix}logout?dataset=http://localhost:${CLIENT_PORT}`);
+  } catch (err) {
+    console.error(err);
+  }
 });
 
 app.listen(CLIENT_PORT, (err) => {

client/server/utils.js

@@ -8,7 +8,9 @@ function isLikelyASyntaxError(message) {
   return message.indexOf(friendlySyntaxErrorLabel) !== -1;
 }
 
-function formatMessage(message) {
+function formatMessage(messageObject) {
+  let { message, details } = messageObject;
+  if (details) message = message + ": " + details;
   return message
     .replace("Module build failed: SyntaxError:", friendlySyntaxErrorLabel)
     .replace(

client/src/actions/annotation.js

@@ -1,287 +1,260 @@
 /*
 Action creators for user annotation
 */
-import _ from "lodash";
+import difference from "lodash.difference";
+import pako from "pako";
 import * as globals from "../globals";
 import { MatrixFBS, AnnotationsHelpers } from "../util/stateManager";
 
 const { isUserAnnotation } = AnnotationsHelpers;
 
-export const annotationCreateCategoryAction = (
-  newCategoryName,
-  categoryToDuplicate
-) => async (dispatch, getState) => {
-  /*
+export const annotationCreateCategoryAction =
+  (newCategoryName, categoryToDuplicate) => async (dispatch, getState) => {
+    /*
   Add a new user-created category to the obs annotations.
 
   Arguments:
     newCategoryName - string name for the category.
     categoryToDuplicate - obs category to use for initial values, or null.
   */
-  const {
-    annoMatrix: prevAnnoMatrix,
-    obsCrossfilter: prevObsCrossfilter,
-  } = getState();
-  if (!prevAnnoMatrix || !prevObsCrossfilter) return;
-  const { schema } = prevAnnoMatrix;
+    const { annoMatrix: prevAnnoMatrix, obsCrossfilter: prevObsCrossfilter } =
+      getState();
+    if (!prevAnnoMatrix || !prevObsCrossfilter) return;
+    const { schema } = prevAnnoMatrix;
 
-  /* name must be a string,  non-zero length */
-  if (typeof newCategoryName !== "string" || newCategoryName.length === 0)
-    throw new Error("user annotations require string name");
+    /* name must be a string,  non-zero length */
+    if (typeof newCategoryName !== "string" || newCategoryName.length === 0)
+      throw new Error("user annotations require string name");
 
-  /* ensure the name isn't already in use! */
-  if (schema.annotations.obsByName[newCategoryName])
-    throw new Error("name collision on annotation category create");
+    /* ensure the name isn't already in use! */
+    if (schema.annotations.obsByName[newCategoryName])
+      throw new Error("name collision on annotation category create");
 
-  let initialValue;
-  let newSchema;
-  let ctor;
-  if (categoryToDuplicate) {
-    /* if we are duplicating a category, retrieve it */
-    const catDupSchema = schema.annotations.obsByName[categoryToDuplicate];
-    const catDupType = catDupSchema?.type;
-    if (catDupType !== "string" && catDupType !== "categorical")
-      throw new Error("categoryToDuplicate does not exist or has invalid type");
+    let initialValue;
+    let newSchema;
+    let ctor;
+    if (categoryToDuplicate) {
+      /* if we are duplicating a category, retrieve it */
+      const catDupSchema = schema.annotations.obsByName[categoryToDuplicate];
+      const catDupType = catDupSchema?.type;
+      if (catDupType !== "string" && catDupType !== "categorical")
+        throw new Error(
+          "categoryToDuplicate does not exist or has invalid type"
+        );
 
-    const catToDupDf = await prevAnnoMatrix
-      .base()
-      .fetch("obs", categoryToDuplicate);
-    const col = catToDupDf.col(categoryToDuplicate);
-    initialValue = col.asArray();
-    const { categories } = col.summarizeCategorical();
-    // all user-created annotations must have the unassigned category
-    if (!categories.includes(globals.unassignedCategoryLabel)) {
-      categories.push(globals.unassignedCategoryLabel);
+      const catToDupDf = await prevAnnoMatrix
+        .base()
+        .fetch("obs", categoryToDuplicate);
+      const col = catToDupDf.col(categoryToDuplicate);
+      initialValue = col.asArray();
+      const { categories } = col.summarizeCategorical();
+      // all user-created annotations must have the unassigned category
+      if (!categories.includes(globals.unassignedCategoryLabel)) {
+        categories.push(globals.unassignedCategoryLabel);
+      }
+      ctor = initialValue.constructor;
+      newSchema = {
+        ...catDupSchema,
+        name: newCategoryName,
+        categories,
+        writable: true,
+      };
+    } else {
+      /* else assign to the standard default value */
+      initialValue = globals.unassignedCategoryLabel;
+      ctor = Array;
+      newSchema = {
+        name: newCategoryName,
+        type: "categorical",
+        categories: [globals.unassignedCategoryLabel],
+        writable: true,
+      };
     }
-    ctor = initialValue.constructor;
-    newSchema = {
-      ...catDupSchema,
-      name: newCategoryName,
-      categories,
-      writable: true,
-    };
-  } else {
-    /* else assign to the standard default value */
-    initialValue = globals.unassignedCategoryLabel;
-    ctor = Array;
-    newSchema = {
-      name: newCategoryName,
-      type: "categorical",
-      categories: [globals.unassignedCategoryLabel],
-      writable: true,
-    };
-  }
 
-  const obsCrossfilter = prevObsCrossfilter.addObsColumn(
-    newSchema,
-    ctor,
-    initialValue
-  );
+    const obsCrossfilter = prevObsCrossfilter.addObsColumn(
+      newSchema,
+      ctor,
+      initialValue
+    );
 
-  dispatch({
-    type: "annotation: create category",
-    data: newCategoryName,
-    categoryToDuplicate,
-    annoMatrix: obsCrossfilter.annoMatrix,
-    obsCrossfilter,
-  });
-};
+    dispatch({
+      type: "annotation: create category",
+      data: newCategoryName,
+      categoryToDuplicate,
+      annoMatrix: obsCrossfilter.annoMatrix,
+      obsCrossfilter,
+    });
+  };
 
-export const annotationRenameCategoryAction = (
-  oldCategoryName,
-  newCategoryName
-) => (dispatch, getState) => {
-  /*
+export const annotationRenameCategoryAction =
+  (oldCategoryName, newCategoryName) => (dispatch, getState) => {
+    /*
   Rename a user-created annotation category
   */
-  const {
-    annoMatrix: prevAnnoMatrix,
-    obsCrossfilter: prevObsCrossfilter,
-  } = getState();
-  if (!prevAnnoMatrix || !prevObsCrossfilter) return;
-  if (!isUserAnnotation(prevAnnoMatrix, oldCategoryName))
-    throw new Error("not a user annotation");
+    const { annoMatrix: prevAnnoMatrix, obsCrossfilter: prevObsCrossfilter } =
+      getState();
+    if (!prevAnnoMatrix || !prevObsCrossfilter) return;
+    if (!isUserAnnotation(prevAnnoMatrix, oldCategoryName))
+      throw new Error("not a user annotation");
 
-  /* name must be a string,  non-zero length */
-  if (typeof newCategoryName !== "string" || newCategoryName.length === 0)
-    throw new Error("user annotations require string name");
+    /* name must be a string,  non-zero length */
+    if (typeof newCategoryName !== "string" || newCategoryName.length === 0)
+      throw new Error("user annotations require string name");
 
-  if (oldCategoryName === newCategoryName) return;
+    if (oldCategoryName === newCategoryName) return;
 
-  const obsCrossfilter = prevObsCrossfilter.renameObsColumn(
-    oldCategoryName,
-    newCategoryName
-  );
+    const obsCrossfilter = prevObsCrossfilter.renameObsColumn(
+      oldCategoryName,
+      newCategoryName
+    );
 
-  dispatch({
-    type: "annotation: category edited",
-    annoMatrix: obsCrossfilter.annoMatrix,
-    obsCrossfilter,
-    metadataField: oldCategoryName,
-    newCategoryText: newCategoryName,
-    data: newCategoryName,
-  });
-};
+    dispatch({
+      type: "annotation: category edited",
+      annoMatrix: obsCrossfilter.annoMatrix,
+      obsCrossfilter,
+      metadataField: oldCategoryName,
+      newCategoryText: newCategoryName,
+      data: newCategoryName,
+    });
+  };
 
-export const annotationDeleteCategoryAction = (categoryName) => (
-  dispatch,
-  getState
-) => {
-  /*
+export const annotationDeleteCategoryAction =
+  (categoryName) => (dispatch, getState) => {
+    /*
   Delete a user-created category
   */
-  const {
-    annoMatrix: prevAnnoMatrix,
-    obsCrossfilter: prevObsCrossfilter,
-  } = getState();
-  if (!prevAnnoMatrix || !prevObsCrossfilter) return;
-  if (!isUserAnnotation(prevAnnoMatrix, categoryName))
-    throw new Error("not a user annotation");
+    const { annoMatrix: prevAnnoMatrix, obsCrossfilter: prevObsCrossfilter } =
+      getState();
+    if (!prevAnnoMatrix || !prevObsCrossfilter) return;
+    if (!isUserAnnotation(prevAnnoMatrix, categoryName))
+      throw new Error("not a user annotation");
 
-  const obsCrossfilter = prevObsCrossfilter.dropObsColumn(categoryName);
-  dispatch({
-    type: "annotation: delete category",
-    annoMatrix: obsCrossfilter.annoMatrix,
-    obsCrossfilter,
-    metadataField: categoryName,
-  });
-};
+    const obsCrossfilter = prevObsCrossfilter.dropObsColumn(categoryName);
+    dispatch({
+      type: "annotation: delete category",
+      annoMatrix: obsCrossfilter.annoMatrix,
+      obsCrossfilter,
+      metadataField: categoryName,
+    });
+  };
 
-export const annotationCreateLabelInCategory = (
-  categoryName,
-  labelName,
-  assignSelected
-) => async (dispatch, getState) => {
-  /*
-  Add a new label to a user-defined category.  If assignSelected is true, assign 
+export const annotationCreateLabelInCategory =
+  (categoryName, labelName, assignSelected) => async (dispatch, getState) => {
+    /*
+  Add a new label to a user-defined category.  If assignSelected is true, assign
   the label to all currently selected cells.
   */
-  const {
-    annoMatrix: prevAnnoMatrix,
-    obsCrossfilter: prevObsCrossfilter,
-  } = getState();
-  if (!prevAnnoMatrix || !prevObsCrossfilter) return;
-  if (!isUserAnnotation(prevAnnoMatrix, categoryName))
-    throw new Error("not a user annotation");
+    const { annoMatrix: prevAnnoMatrix, obsCrossfilter: prevObsCrossfilter } =
+      getState();
+    if (!prevAnnoMatrix || !prevObsCrossfilter) return;
+    if (!isUserAnnotation(prevAnnoMatrix, categoryName))
+      throw new Error("not a user annotation");
 
-  let obsCrossfilter = prevObsCrossfilter.addObsAnnoCategory(
-    categoryName,
-    labelName
-  );
-  if (assignSelected) {
-    obsCrossfilter = await obsCrossfilter.setObsColumnValues(
+    let obsCrossfilter = prevObsCrossfilter.addObsAnnoCategory(
+      categoryName,
+      labelName
+    );
+    if (assignSelected) {
+      obsCrossfilter = await obsCrossfilter.setObsColumnValues(
+        categoryName,
+        prevObsCrossfilter.allSelectedLabels(),
+        labelName
+      );
+    }
+
+    dispatch({
+      type: "annotation: add new label to category",
+      annoMatrix: obsCrossfilter.annoMatrix,
+      obsCrossfilter,
+      metadataField: categoryName,
+      newLabelText: labelName,
+      assignSelectedCells: assignSelected,
+    });
+  };
+
+export const annotationDeleteLabelFromCategory =
+  (categoryName, labelName) => async (dispatch, getState) => {
+    /*
+  delete a label from a user-defined category
+  */
+    const { annoMatrix: prevAnnoMatrix, obsCrossfilter: prevObsCrossfilter } =
+      getState();
+    if (!prevAnnoMatrix || !prevObsCrossfilter) return;
+    if (!isUserAnnotation(prevAnnoMatrix, categoryName))
+      throw new Error("not a user annotation");
+
+    const obsCrossfilter = await prevObsCrossfilter.removeObsAnnoCategory(
+      categoryName,
+      labelName,
+      globals.unassignedCategoryLabel
+    );
+
+    dispatch({
+      type: "annotation: delete label",
+      metadataField: categoryName,
+      label: labelName,
+      annoMatrix: obsCrossfilter.annoMatrix,
+      obsCrossfilter,
+    });
+  };
+
+export const annotationRenameLabelInCategory =
+  (categoryName, oldLabelName, newLabelName) => async (dispatch, getState) => {
+    /*
+  label name change
+  */
+    const { annoMatrix: prevAnnoMatrix, obsCrossfilter: prevObsCrossfilter } =
+      getState();
+    if (!prevAnnoMatrix || !prevObsCrossfilter) return;
+    if (!isUserAnnotation(prevAnnoMatrix, categoryName))
+      throw new Error("not a user annotation");
+
+    let obsCrossfilter = await prevObsCrossfilter.resetObsColumnValues(
+      categoryName,
+      oldLabelName,
+      newLabelName
+    );
+    obsCrossfilter = await obsCrossfilter.removeObsAnnoCategory(
+      categoryName,
+      oldLabelName,
+      globals.unassignedCategoryLabel
+    );
+
+    dispatch({
+      type: "annotation: label edited",
+      editedLabel: newLabelName,
+      metadataField: categoryName,
+      label: oldLabelName,
+      annoMatrix: obsCrossfilter.annoMatrix,
+      obsCrossfilter,
+    });
+  };
+
+export const annotationLabelCurrentSelection =
+  (categoryName, labelName) => async (dispatch, getState) => {
+    /*
+  set the label on all currently selected
+  */
+    const { annoMatrix: prevAnnoMatrix, obsCrossfilter: prevObsCrossfilter } =
+      getState();
+    if (!prevAnnoMatrix || !prevObsCrossfilter) return;
+    if (!isUserAnnotation(prevAnnoMatrix, categoryName))
+      throw new Error("not a user annotation");
+
+    const obsCrossfilter = await prevObsCrossfilter.setObsColumnValues(
       categoryName,
       prevObsCrossfilter.allSelectedLabels(),
       labelName
     );
-  }
 
-  dispatch({
-    type: "annotation: add new label to category",
-    annoMatrix: obsCrossfilter.annoMatrix,
-    obsCrossfilter,
-    metadataField: categoryName,
-    newLabelText: labelName,
-    assignSelectedCells: assignSelected,
-  });
-};
-
-export const annotationDeleteLabelFromCategory = (
-  categoryName,
-  labelName
-) => async (dispatch, getState) => {
-  /*
-  delete a label from a user-defined category
-  */
-  const {
-    annoMatrix: prevAnnoMatrix,
-    obsCrossfilter: prevObsCrossfilter,
-  } = getState();
-  if (!prevAnnoMatrix || !prevObsCrossfilter) return;
-  if (!isUserAnnotation(prevAnnoMatrix, categoryName))
-    throw new Error("not a user annotation");
-
-  const obsCrossfilter = await prevObsCrossfilter.removeObsAnnoCategory(
-    categoryName,
-    labelName,
-    globals.unassignedCategoryLabel
-  );
-
-  dispatch({
-    type: "annotation: delete label",
-    metadataField: categoryName,
-    label: labelName,
-    annoMatrix: obsCrossfilter.annoMatrix,
-    obsCrossfilter,
-  });
-};
-
-export const annotationRenameLabelInCategory = (
-  categoryName,
-  oldLabelName,
-  newLabelName
-) => async (dispatch, getState) => {
-  /*
-  label name change
-  */
-  const {
-    annoMatrix: prevAnnoMatrix,
-    obsCrossfilter: prevObsCrossfilter,
-  } = getState();
-  if (!prevAnnoMatrix || !prevObsCrossfilter) return;
-  if (!isUserAnnotation(prevAnnoMatrix, categoryName))
-    throw new Error("not a user annotation");
-
-  let obsCrossfilter = await prevObsCrossfilter.resetObsColumnValues(
-    categoryName,
-    oldLabelName,
-    newLabelName
-  );
-  obsCrossfilter = await obsCrossfilter.removeObsAnnoCategory(
-    categoryName,
-    oldLabelName,
-    globals.unassignedCategoryLabel
-  );
-
-  dispatch({
-    type: "annotation: label edited",
-    editedLabel: newLabelName,
-    metadataField: categoryName,
-    label: oldLabelName,
-    annoMatrix: obsCrossfilter.annoMatrix,
-    obsCrossfilter,
-  });
-};
-
-export const annotationLabelCurrentSelection = (
-  categoryName,
-  labelName
-) => async (dispatch, getState) => {
-  /*
-  set the label on all currently selected
-  */
-  const {
-    annoMatrix: prevAnnoMatrix,
-    obsCrossfilter: prevObsCrossfilter,
-  } = getState();
-  if (!prevAnnoMatrix || !prevObsCrossfilter) return;
-  if (!isUserAnnotation(prevAnnoMatrix, categoryName))
-    throw new Error("not a user annotation");
-
-  const obsCrossfilter = await prevObsCrossfilter.setObsColumnValues(
-    categoryName,
-    prevObsCrossfilter.allSelectedLabels(),
-    labelName
-  );
-
-  dispatch({
-    type: "annotation: label current cell selection",
-    metadataField: categoryName,
-    label: labelName,
-    obsCrossfilter,
-    annoMatrix: obsCrossfilter.annoMatrix,
-  });
-};
+    dispatch({
+      type: "annotation: label current cell selection",
+      metadataField: categoryName,
+      label: labelName,
+      obsCrossfilter,
+      annoMatrix: obsCrossfilter.annoMatrix,
+    });
+  };
 
 function writableAnnotations(annoMatrix) {
   return annoMatrix.schema.annotations.obs.columns
@@ -307,7 +280,7 @@ export const needToSaveObsAnnotations = (annoMatrix, lastSavedAnnoMatrix) => {
 
   // if the schema has changed, we need to save
   const currentWritable = writableAnnotations(annoMatrix);
-  if (_.difference(currentWritable, writableAnnotations(lastSavedAnnoMatrix))) {
+  if (difference(currentWritable, writableAnnotations(lastSavedAnnoMatrix))) {
     return true;
   }
 
@@ -347,6 +320,7 @@ export const saveObsAnnotationsAction = () => async (dispatch, getState) => {
 
   const df = await annoMatrix.fetch("obs", writableAnnotations(annoMatrix));
   const matrix = MatrixFBS.encodeMatrixFBS(df);
+  const compressedMatrix = pako.deflate(matrix);
   try {
     const queryString =
       !dataCollectionNameIsReadOnly && !!dataCollectionName
@@ -358,7 +332,7 @@ export const saveObsAnnotationsAction = () => async (dispatch, getState) => {
       `${globals.API.prefix}${globals.API.version}annotations/obs${queryString}`,
       {
         method: "PUT",
-        body: matrix,
+        body: compressedMatrix,
         headers: new Headers({
           "Content-Type": "application/octet-stream",
         }),
@@ -385,3 +359,97 @@ export const saveObsAnnotationsAction = () => async (dispatch, getState) => {
     });
   }
 };
+
+export const saveGenesetsAction = () => async (dispatch, getState) => {
+  const state = getState();
+
+  // bail if gene sets not available, or in readonly mode.
+  const { config } = state;
+  const { lastTid, genesets } = state.genesets;
+
+  const genesetsAreAvailable =
+    config?.parameters?.annotations_genesets ?? false;
+  const genesetsReadonly =
+    config?.parameters?.annotations_genesets_readonly ?? true;
+  if (!genesetsAreAvailable || genesetsReadonly) {
+    // our non-save was completed!
+    return dispatch({
+      type: "autosave: genesets complete",
+      lastSavedGenesets: genesets,
+    });
+  }
+
+  dispatch({
+    type: "autosave: genesets started",
+  });
+
+  /* Create the JSON OTA data structure */
+  const tid = (lastTid ?? 0) + 1;
+  const genesetsOTA = [];
+  for (const [name, gs] of genesets) {
+    const genes = [];
+    for (const g of gs.genes.values()) {
+      genes.push({
+        gene_symbol: g.geneSymbol,
+        gene_description: g.geneDescription,
+      });
+    }
+    genesetsOTA.push({
+      geneset_name: name,
+      geneset_description: gs.genesetDescription,
+      genes,
+    });
+  }
+  const ota = {
+    tid,
+    genesets: genesetsOTA,
+  };
+
+  /* Save to server */
+  try {
+    const { dataCollectionNameIsReadOnly, dataCollectionName } =
+      state.annotations;
+    const queryString =
+      !dataCollectionNameIsReadOnly && !!dataCollectionName
+        ? `?annotation-collection-name=${encodeURIComponent(
+            dataCollectionName
+          )}`
+        : "";
+
+    const res = await fetch(
+      `${globals.API.prefix}${globals.API.version}genesets${queryString}`,
+      {
+        method: "PUT",
+        headers: new Headers({
+          Accept: "application/json",
+          "Content-Type": "application/json",
+        }),
+        body: JSON.stringify(ota),
+        credentials: "include",
+      }
+    );
+    if (!res.ok) {
+      return dispatch({
+        type: "autosave: genesets error",
+        message: `HTTP error ${res.status} - ${res.statusText}`,
+        res,
+      });
+    }
+    return Promise.all([
+      dispatch({
+        type: "autosave: genesets complete",
+        lastSavedGenesets: genesets,
+      }),
+      dispatch({
+        type: "geneset: set tid",
+        tid,
+      }),
+    ]);
+  } catch (error) {
+    return dispatch({
+      type: "autosave: genesets error",
+      message: error.toString(),
+      error,
+    });
+  }
+};

client/src/actions/embedding.js

@@ -5,40 +5,43 @@ action creators related to embeddings choice
 import { AnnoMatrixObsCrossfilter } from "../annoMatrix";
 import { _setEmbeddingSubset } from "../util/stateManager/viewStackHelpers";
 
-export async function _switchEmbedding(prevAnnoMatrix, newEmbeddingName) {
+export async function _switchEmbedding(
+  prevAnnoMatrix,
+  prevCrossfilter,
+  newEmbeddingName
+) {
   /*
-  DRY helper used by this and reembedding action creators
+  DRY helper used by embedding action creators
   */
   const base = prevAnnoMatrix.base();
   const embeddingDf = await base.fetch("emb", newEmbeddingName);
   const annoMatrix = _setEmbeddingSubset(prevAnnoMatrix, embeddingDf);
-  const obsCrossfilter = await new AnnoMatrixObsCrossfilter(annoMatrix).select(
-    "emb",
-    newEmbeddingName,
-    {
-      mode: "all",
-    }
-  );
+  const obsCrossfilter = await new AnnoMatrixObsCrossfilter(
+    annoMatrix,
+    prevCrossfilter.obsCrossfilter
+  ).select("emb", newEmbeddingName, {
+    mode: "all",
+  });
   return [annoMatrix, obsCrossfilter];
 }
 
-export const layoutChoiceAction = (newLayoutChoice) => async (
-  dispatch,
-  getState
-) => {
-  /*
+export const layoutChoiceAction =
+  (newLayoutChoice) => async (dispatch, getState) => {
+    /*
   On layout choice, make sure we have selected all on the previous layout, AND the new
   layout.
   */
-  const { annoMatrix: prevAnnoMatrix } = getState();
-  const [annoMatrix, obsCrossfilter] = await _switchEmbedding(
-    prevAnnoMatrix,
-    newLayoutChoice
-  );
-  dispatch({
-    type: "set layout choice",
-    layoutChoice: newLayoutChoice,
-    obsCrossfilter,
-    annoMatrix,
-  });
-};
+    const { annoMatrix: prevAnnoMatrix, obsCrossfilter: prevCrossfilter } =
+      getState();
+    const [annoMatrix, obsCrossfilter] = await _switchEmbedding(
+      prevAnnoMatrix,
+      prevCrossfilter,
+      newLayoutChoice
+    );
+    dispatch({
+      type: "set layout choice",
+      layoutChoice: newLayoutChoice,
+      obsCrossfilter,
+      annoMatrix,
+    });
+  };

client/src/actions/geneset.js

@@ -0,0 +1,149 @@
+import { postUserErrorToast } from "../components/framework/toasters";
+/*
+Action creators for gene sets
+
+Primarily used to keep the crossfilter and underlying data in sync with the UI.
+
+The behavior manifest in these action creators:
+
+    Delete a gene set, will
+      * drop index & clear selection state on the gene set summary
+      * drop index & clear selection state of each gene in the geneset
+
+    Delete a gene from a gene set, will:
+      * drop index & clear selection state on the gene set summary
+      * drop index & clear selection state on the gene
+
+    Add a gene to a gene set, will:
+      * drop index & clear selection state on the gene set summary
+      * will NOT touch the selection state for the gene
+
+Note that crossfilter indices are lazy created, as needed.
+*/
+
+export const genesetDelete = (genesetName) => (dispatch, getState) => {
+  const state = getState();
+  const { genesets } = state;
+  const gs = genesets?.genesets?.get(genesetName) ?? {};
+  const geneSymbols = Array.from(gs.genes.keys());
+  const obsCrossfilter = dropGeneset(dispatch, state, genesetName, geneSymbols);
+  if (genesetName === state.colors.colorAccessor) {
+    dispatch({
+      type: "reset colorscale",
+    });
+  }
+  dispatch({
+    type: "geneset: delete",
+    genesetName,
+    obsCrossfilter,
+    annoMatrix: obsCrossfilter.annoMatrix,
+  });
+};
+
+export const genesetAddGenes =
+  (genesetName, genes) => async (dispatch, getState) => {
+    const state = getState();
+    const { obsCrossfilter: prevObsCrossfilter, annoMatrix } = state;
+    const { schema } = annoMatrix;
+    const varIndex = schema.annotations.var.index;
+    const df = await annoMatrix.fetch("var", varIndex);
+    const geneNames = df.col(varIndex).asArray();
+    genes = genes.reduce((acc, gene) => {
+      if (geneNames.indexOf(gene.geneSymbol) === -1) {
+        postUserErrorToast(
+          `${gene.geneSymbol} doesn't appear to be a valid gene name.`
+        );
+      } else acc.push(gene);
+      return acc;
+    }, []);
+
+    const obsCrossfilter = dropGenesetSummaryDimension(
+      prevObsCrossfilter,
+      state,
+      genesetName
+    );
+    dispatch({
+      type: "continuous metadata histogram cancel",
+      continuousNamespace: { isGeneSetSummary: true },
+      selection: genesetName,
+    });
+    return dispatch({
+      type: "geneset: add genes",
+      genesetName,
+      genes,
+      obsCrossfilter,
+      annoMatrix: obsCrossfilter.annoMatrix,
+    });
+  };
+
+export const genesetDeleteGenes =
+  (genesetName, geneSymbols) => (dispatch, getState) => {
+    const state = getState();
+    const obsCrossfilter = dropGeneset(
+      dispatch,
+      state,
+      genesetName,
+      geneSymbols
+    );
+    return dispatch({
+      type: "geneset: delete genes",
+      genesetName,
+      geneSymbols,
+      obsCrossfilter,
+      annoMatrix: obsCrossfilter.annoMatrix,
+    });
+  };
+
+/*
+Private
+*/
+
+function dropGenesetSummaryDimension(obsCrossfilter, state, genesetName) {
+  const { annoMatrix, genesets } = state;
+  const varIndex = annoMatrix.schema.annotations?.var?.index;
+  const gs = genesets?.genesets?.get(genesetName) ?? {};
+  const genes = Array.from(gs.genes.keys());
+  const query = {
+    summarize: {
+      method: "mean",
+      field: "var",
+      column: varIndex,
+      values: genes,
+    },
+  };
+  return obsCrossfilter.dropDimension("X", query);
+}
+
+function dropGeneDimension(obsCrossfilter, state, gene) {
+  const { annoMatrix } = state;
+  const varIndex = annoMatrix.schema.annotations?.var?.index;
+  const query = {
+    where: {
+      field: "var",
+      column: varIndex,
+      value: gene,
+    },
+  };
+  return obsCrossfilter.dropDimension("X", query);
+}
+
+function dropGeneset(dispatch, state, genesetName, geneSymbols) {
+  const { obsCrossfilter: prevObsCrossfilter } = state;
+  const obsCrossfilter = geneSymbols.reduce(
+    (crossfilter, gene) => dropGeneDimension(crossfilter, state, gene),
+    dropGenesetSummaryDimension(prevObsCrossfilter, state, genesetName)
+  );
+  dispatch({
+    type: "continuous metadata histogram cancel",
+    continuousNamespace: { isGeneSetSummary: true },
+    selection: genesetName,
+  });
+  geneSymbols.forEach((g) =>
+    dispatch({
+      type: "continuous metadata histogram cancel",
+      continuousNamespace: { isUserDefined: true },
+      selection: g,
+    })
+  );
+  return obsCrossfilter;
+}

client/src/actions/index.js

@@ -5,14 +5,22 @@ import {
   doJsonRequest,
   dispatchNetworkErrorMessageToUser,
 } from "../util/actionHelpers";
-import {
-  requestReembed /* , reembedResetWorldToUniverse -- disabled temporarily, TODO issue #1606 */,
-} from "./reembed";
 import { loadUserColorConfig } from "../util/stateManager/colorHelpers";
 import * as selnActions from "./selection";
 import * as annoActions from "./annotation";
 import * as viewActions from "./viewStack";
 import * as embActions from "./embedding";
+import * as genesetActions from "./geneset";
+
+function setGlobalConfig(config) {
+  /**
+   * Set any global run-time config not _exclusively_ managed by the config reducer.
+   * This should only set fields defined in globals.globalConfig.
+   */
+  globals.globalConfig.maxCategoricalOptionsToDisplay =
+    config?.parameters?.["max-category-items"] ??
+    globals.globalConfig.maxCategoricalOptionsToDisplay;
+}
 
 /*
 return promise fetching user-configured colors
@@ -33,6 +41,9 @@ async function schemaFetch() {
 async function configFetch(dispatch) {
   return fetchJson("config").then((response) => {
     const config = { ...globals.configDefaults, ...response.config };
+
+    setGlobalConfig(config);
+
     dispatch({
       type: "configuration load complete",
       config,
@@ -41,6 +52,27 @@ async function configFetch(dispatch) {
   });
 }
 
+async function genesetsFetch(dispatch, config) {
+  /* request genesets ONLY if the backend supports the feature */
+  const defaultResponse = {
+    genesets: [],
+    tid: 0,
+  };
+  if (config?.parameters?.annotations_genesets ?? false) {
+    fetchJson("genesets").then((response) => {
+      dispatch({
+        type: "geneset: initial load",
+        data: response ?? defaultResponse,
+      });
+    });
+  } else {
+    dispatch({
+      type: "geneset: initial load",
+      data: defaultResponse,
+    });
+  }
+}
+
 function prefetchEmbeddings(annoMatrix) {
   /*
   prefetch requests for all embeddings
@@ -64,6 +96,8 @@ const doInitialDataLoad = () =>
         userColorsFetchAndLoad(dispatch),
       ]);
 
+      genesetsFetch(dispatch, config);
+
       const baseDataUrl = `${globals.API.prefix}${globals.API.version}`;
       const annoMatrix = new AnnoMatrixLoader(baseDataUrl, schema.schema);
       const obsCrossfilter = new AnnoMatrixObsCrossfilter(annoMatrix);
@@ -76,7 +110,7 @@ const doInitialDataLoad = () =>
       });
       dispatch({ type: "initial data load complete" });
 
-      const defaultEmbedding = config?.parameters?.["default_embedding"];
+      const defaultEmbedding = config?.parameters?.default_embedding;
       const layoutSchema = schema?.schema?.layout?.obs ?? [];
       if (
         defaultEmbedding &&
@@ -126,72 +160,74 @@ const dispatchDiffExpErrors = (dispatch, response) => {
   }
 };
 
-const requestDifferentialExpression = (set1, set2, num_genes = 10) => async (
-  dispatch,
-  getState
-) => {
-  dispatch({ type: "request differential expression started" });
-  try {
-    /*
+const requestDifferentialExpression =
+  (set1, set2, num_genes = 50) =>
+  async (dispatch, getState) => {
+    dispatch({ type: "request differential expression started" });
+    try {
+      /*
     Steps:
     1. get the most differentially expressed genes
     2. get expression data for each
     */
-    const { annoMatrix } = getState();
-    const varIndexName = annoMatrix.schema.annotations.var.index;
+      const { annoMatrix } = getState();
+      const varIndexName = annoMatrix.schema.annotations.var.index;
 
-    // Legal values are null, Array or TypedArray.  Null is initial state.
-    if (!set1) set1 = [];
-    if (!set2) set2 = [];
+      // Legal values are null, Array or TypedArray.  Null is initial state.
+      if (!set1) set1 = [];
+      if (!set2) set2 = [];
 
-    // These lines ensure that we convert any TypedArray to an Array.
-    // This is necessary because JSON.stringify() does some very strange
-    // things with TypedArrays (they are marshalled to JSON objects, rather
-    // than being marshalled as a JSON array).
-    set1 = Array.isArray(set1) ? set1 : Array.from(set1);
-    set2 = Array.isArray(set2) ? set2 : Array.from(set2);
+      // These lines ensure that we convert any TypedArray to an Array.
+      // This is necessary because JSON.stringify() does some very strange
+      // things with TypedArrays (they are marshalled to JSON objects, rather
+      // than being marshalled as a JSON array).
+      set1 = Array.isArray(set1) ? set1 : Array.from(set1);
+      set2 = Array.isArray(set2) ? set2 : Array.from(set2);
 
-    const res = await fetch(
-      `${globals.API.prefix}${globals.API.version}diffexp/obs`,
-      {
-        method: "POST",
-        headers: new Headers({
-          Accept: "application/json",
-          "Content-Type": "application/json",
-        }),
-        body: JSON.stringify({
-          mode: "topN",
-          count: num_genes,
-          set1: { filter: { obs: { index: set1 } } },
-          set2: { filter: { obs: { index: set2 } } },
-        }),
-        credentials: "include",
+      const res = await fetch(
+        `${globals.API.prefix}${globals.API.version}diffexp/obs`,
+        {
+          method: "POST",
+          headers: new Headers({
+            Accept: "application/json",
+            "Content-Type": "application/json",
+          }),
+          body: JSON.stringify({
+            mode: "topN",
+            count: num_genes,
+            set1: { filter: { obs: { index: set1 } } },
+            set2: { filter: { obs: { index: set2 } } },
+          }),
+          credentials: "include",
+        }
+      );
+
+      if (!res.ok || res.headers.get("Content-Type") !== "application/json") {
+        return dispatchDiffExpErrors(dispatch, res);
       }
-    );
 
-    if (!res.ok || res.headers.get("Content-Type") !== "application/json") {
-      return dispatchDiffExpErrors(dispatch, res);
+      const response = await res.json();
+      const varIndex = await annoMatrix.fetch("var", varIndexName);
+      const diffexpLists = { negative: [], positive: [] };
+      for (const polarity of Object.keys(diffexpLists)) {
+        diffexpLists[polarity] = response[polarity].map((v) => [
+          varIndex.at(v[0], varIndexName),
+          ...v.slice(1),
+        ]);
+      }
+
+      /* then send the success case action through */
+      return dispatch({
+        type: "request differential expression success",
+        data: diffexpLists,
+      });
+    } catch (error) {
+      return dispatch({
+        type: "request differential expression error",
+        error,
+      });
     }
-
-    const response = await res.json();
-    const varIndex = await annoMatrix.fetch("var", varIndexName);
-    const data = response.map((v) => [
-      varIndex.at(v[0], varIndexName),
-      ...v.slice(1),
-    ]);
-
-    /* then send the success case action through */
-    return dispatch({
-      type: "request differential expression success",
-      data,
-    });
-  } catch (error) {
-    return dispatch({
-      type: "request differential expression error",
-      error,
-    });
-  }
-};
+  };
 
 function fetchJson(pathAndQuery) {
   return doJsonRequest(
@@ -204,7 +240,6 @@ export default {
   requestDifferentialExpression,
   requestSingleGeneExpressionCountsForColoringPOST,
   requestUserDefinedGene,
-  requestReembed,
   selectContinuousMetadataAction: selnActions.selectContinuousMetadataAction,
   selectCategoricalMetadataAction: selnActions.selectCategoricalMetadataAction,
   selectCategoricalAllMetadataAction:
@@ -230,7 +265,11 @@ export default {
   annotationRenameLabelInCategory: annoActions.annotationRenameLabelInCategory,
   annotationLabelCurrentSelection: annoActions.annotationLabelCurrentSelection,
   saveObsAnnotationsAction: annoActions.saveObsAnnotationsAction,
+  saveGenesetsAction: annoActions.saveGenesetsAction,
   needToSaveObsAnnotations: annoActions.needToSaveObsAnnotations,
   layoutChoiceAction: embActions.layoutChoiceAction,
   setCellSetFromSelection: selnActions.setCellSetFromSelection,
+  genesetDelete: genesetActions.genesetDelete,
+  genesetAddGenes: genesetActions.genesetAddGenes,
+  genesetDeleteGenes: genesetActions.genesetDeleteGenes,
 };

client/src/actions/reembed.js

@@ -1,108 +0,0 @@
-import { API } from "../globals";
-import {
-  postNetworkErrorToast,
-  postAsyncSuccessToast,
-  postAsyncFailureToast,
-} from "../components/framework/toasters";
-import { _switchEmbedding } from "./embedding";
-
-function abortableFetch(request, opts, timeout = 0) {
-  const controller = new AbortController();
-  const { signal } = controller;
-
-  return {
-    abort: () => controller.abort(),
-    isAborted: () => signal.aborted,
-    ready: () => {
-      if (timeout) {
-        setTimeout(() => controller.abort(), timeout);
-      }
-      return fetch(request, { ...opts, signal });
-    },
-  };
-}
-
-async function doReembedFetch(dispatch, getState) {
-  const state = getState();
-  let cells = state.annoMatrix.rowIndex.labels();
-
-  // These lines ensure that we convert any TypedArray to an Array.
-  // This is necessary because JSON.stringify() does some very strange
-  // things with TypedArrays (they are marshalled to JSON objects, rather
-  // than being marshalled as a JSON array).
-  cells = Array.isArray(cells) ? cells : Array.from(cells);
-
-  const af = abortableFetch(
-    `${API.prefix}${API.version}layout/obs`,
-    {
-      method: "PUT",
-      headers: new Headers({
-        Accept: "application/octet-stream",
-        "Content-Type": "application/json",
-      }),
-      body: JSON.stringify({
-        method: "umap",
-        filter: { obs: { index: cells } },
-      }),
-      credentials: "include",
-    },
-    60000 // 1 minute timeout
-  );
-  dispatch({
-    type: "reembed: request start",
-    abortableFetch: af,
-  });
-  const res = await af.ready();
-
-  if (res.ok && res.headers.get("Content-Type").includes("application/json")) {
-    return res;
-  }
-
-  // else an error
-  let msg = `Unexpected HTTP response ${res.status}, ${res.statusText}`;
-  const body = await res.text();
-  if (body && body.length > 0) {
-    msg = `${msg} -- ${body}`;
-  }
-  throw new Error(msg);
-}
-
-/*
-functions below are dispatch-able
-*/
-export function requestReembed() {
-  return async (dispatch, getState) => {
-    try {
-      const res = await doReembedFetch(dispatch, getState);
-      const schema = await res.json();
-      dispatch({
-        type: "reembed: request completed",
-      });
-
-      const { annoMatrix: prevAnnoMatrix } = getState();
-      const base = prevAnnoMatrix.base().addEmbedding(schema);
-      const [annoMatrix, obsCrossfilter] = await _switchEmbedding(
-        base,
-        schema.name
-      );
-      dispatch({
-        type: "reembed: add reembedding",
-        schema,
-        annoMatrix,
-        obsCrossfilter,
-      });
-
-      postAsyncSuccessToast("Re-embedding has completed.");
-    } catch (error) {
-      dispatch({
-        type: "reembed: request aborted",
-      });
-      if (error.name === "AbortError") {
-        postAsyncFailureToast("Re-embedding calculation was aborted.");
-      } else {
-        postNetworkErrorToast(`Re-embedding: ${error.message}`);
-      }
-      console.log("Reembed exception:", error, error.name, error.message);
-    }
-  };
-}

client/src/actions/selection.js

@@ -1,104 +1,101 @@
 /*
 Action creators for selection 
 */
-export const selectContinuousMetadataAction = (
-  type,
-  query,
-  range,
-  oldProps = {}
-) => async (dispatch, getState) => {
-  const { obsCrossfilter: prevObsCrossfilter } = getState();
+export const selectContinuousMetadataAction =
+  (type, query, range, oldProps = {}) =>
+  async (dispatch, getState) => {
+    const { obsCrossfilter: prevObsCrossfilter } = getState();
 
-  const selection = range
-    ? {
-        mode: "range",
-        lo: range[0],
-        hi: range[1],
-        inclusive: true, // [lo, hi] incluisve selection
-      }
-    : { mode: "all" };
+    const selection = range
+      ? {
+          mode: "range",
+          lo: range[0],
+          hi: range[1],
+          inclusive: true, // [lo, hi] incluisve selection
+        }
+      : { mode: "all" };
 
-  const obsCrossfilter = await prevObsCrossfilter.select(...query, selection);
+    const obsCrossfilter = await prevObsCrossfilter.select(...query, selection);
 
-  dispatch({
-    type,
-    obsCrossfilter,
-    range,
-    ...oldProps,
-  });
-};
-
-export const selectCategoricalMetadataAction = (
-  type, // action type
-  metadataField, // annotation category name
-  labels,
-  label, // the label being selected/deselected
-  isSelected, // bool
-  oldProps = {}
-) => async (dispatch, getState) => {
-  const {
-    obsCrossfilter: prevObsCrossfilter,
-    categoricalSelection,
-  } = getState();
-
-  const labelSelectionState = new Map(categoricalSelection[metadataField]);
-  labels.forEach(
-    (l) => labelSelectionState.has(l) || labelSelectionState.set(l, true)
-  );
-  labelSelectionState.set(label, isSelected);
-
-  const values = Array.from(labelSelectionState.keys()).filter((k) =>
-    labelSelectionState.get(k)
-  );
-  const selection = {
-    mode: "exact",
-    values,
+    dispatch({
+      type,
+      obsCrossfilter,
+      range,
+      ...oldProps,
+    });
   };
-  const obsCrossfilter = await prevObsCrossfilter.select(
-    "obs",
-    metadataField,
-    selection
-  );
 
-  dispatch({
-    type,
-    obsCrossfilter,
-    metadataField,
-    labelSelectionState,
-    ...oldProps,
-  });
-};
+export const selectCategoricalMetadataAction =
+  (
+    type, // action type
+    metadataField, // annotation category name
+    labels,
+    label, // the label being selected/deselected
+    isSelected, // bool
+    oldProps = {}
+  ) =>
+  async (dispatch, getState) => {
+    const { obsCrossfilter: prevObsCrossfilter, categoricalSelection } =
+      getState();
 
-export const selectCategoricalAllMetadataAction = (
-  type, // action type
-  metadataField, // annotation category name
-  labels,
-  isSelected, // bool, select all or none
-  oldProps = {}
-) => async (dispatch, getState) => {
-  const {
-    obsCrossfilter: prevObsCrossfilter,
-    categoricalSelection,
-  } = getState();
+    const labelSelectionState = new Map(categoricalSelection[metadataField]);
+    labels.forEach(
+      (l) => labelSelectionState.has(l) || labelSelectionState.set(l, true)
+    );
+    labelSelectionState.set(label, isSelected);
 
-  const labelSelectionState = new Map(categoricalSelection[metadataField]);
-  labels.forEach((label) => labelSelectionState.set(label, isSelected));
+    const values = Array.from(labelSelectionState.keys()).filter((k) =>
+      labelSelectionState.get(k)
+    );
+    const selection = {
+      mode: "exact",
+      values,
+    };
+    const obsCrossfilter = await prevObsCrossfilter.select(
+      "obs",
+      metadataField,
+      selection
+    );
 
-  const selection = { mode: isSelected ? "all" : "none" };
-  const obsCrossfilter = await prevObsCrossfilter.select(
-    "obs",
-    metadataField,
-    selection
-  );
+    dispatch({
+      type,
+      obsCrossfilter,
+      metadataField,
+      labelSelectionState,
+      ...oldProps,
+    });
+  };
 
-  dispatch({
-    type,
-    obsCrossfilter,
-    metadataField,
-    labelSelectionState,
-    ...oldProps,
-  });
-};
+export const selectCategoricalAllMetadataAction =
+  (
+    type, // action type
+    metadataField, // annotation category name
+    labels,
+    isSelected, // bool, select all or none
+    oldProps = {}
+  ) =>
+  async (dispatch, getState) => {
+    const { obsCrossfilter: prevObsCrossfilter, categoricalSelection } =
+      getState();
+
+    const labelSelectionState = new Map(categoricalSelection[metadataField]);
+    labels.forEach((label) => labelSelectionState.set(label, isSelected));
+
+    const selection = { mode: isSelected ? "all" : "none" };
+    const obsCrossfilter = await prevObsCrossfilter.select(
+      "obs",
+      metadataField,
+      selection
+    );
+
+    dispatch({
+      type,
+      obsCrossfilter,
+      metadataField,
+      labelSelectionState,
+      ...oldProps,
+    });
+  };
 
 /**
  ** Graph selection-related actions
@@ -108,25 +105,23 @@ export const graphBrushStartAction = () =>
   /* no change to crossfilter until a change fires */
   ({ type: "graph brush start" });
 
-const _graphBrushWithinRectAction = (type, embName, brushCoords) => async (
-  dispatch,
-  getState
-) => {
-  const { obsCrossfilter: prevObsCrossfilter } = getState();
+const _graphBrushWithinRectAction =
+  (type, embName, brushCoords) => async (dispatch, getState) => {
+    const { obsCrossfilter: prevObsCrossfilter } = getState();
 
-  const selection = { mode: "within-rect", ...brushCoords };
-  const obsCrossfilter = await prevObsCrossfilter.select(
-    "emb",
-    embName,
-    selection
-  );
+    const selection = { mode: "within-rect", ...brushCoords };
+    const obsCrossfilter = await prevObsCrossfilter.select(
+      "emb",
+      embName,
+      selection
+    );
 
-  dispatch({
-    type,
-    obsCrossfilter,
-    brushCoords,
-  });
-};
+    dispatch({
+      type,
+      obsCrossfilter,
+      brushCoords,
+    });
+  };
 
 const _graphAllAction = (type, embName) => async (dispatch, getState) => {
   const { obsCrossfilter: prevObsCrossfilter } = getState();
@@ -162,28 +157,26 @@ export const graphLassoCancelAction = (embName) =>
 export const graphLassoDeselectAction = (embName) =>
   _graphAllAction("graph lasso cancel", embName);
 
-export const graphLassoEndAction = (embName, polygon) => async (
-  dispatch,
-  getState
-) => {
-  const { obsCrossfilter: prevObsCrossfilter } = getState();
+export const graphLassoEndAction =
+  (embName, polygon) => async (dispatch, getState) => {
+    const { obsCrossfilter: prevObsCrossfilter } = getState();
 
-  const selection = {
-    mode: "within-polygon",
-    polygon,
+    const selection = {
+      mode: "within-polygon",
+      polygon,
+    };
+    const obsCrossfilter = await prevObsCrossfilter.select(
+      "emb",
+      embName,
+      selection
+    );
+
+    dispatch({
+      type: "graph lasso end",
+      obsCrossfilter,
+      polygon,
+    });
   };
-  const obsCrossfilter = await prevObsCrossfilter.select(
-    "emb",
-    embName,
-    selection
-  );
-
-  dispatch({
-    type: "graph lasso end",
-    obsCrossfilter,
-    polygon,
-  });
-};
 
 /*
 Differential expression set selection

client/src/actions/viewStack.js

@@ -42,10 +42,8 @@ export const subsetAction = () => (dispatch, getState) => {
   By convention, a clip view is ALWAYS the top view, so if present, pop
   off and re-apply
   */
-  const {
-    annoMatrix: prevAnnoMatrix,
-    obsCrossfilter: prevObsCrossfilter,
-  } = getState();
+  const { annoMatrix: prevAnnoMatrix, obsCrossfilter: prevObsCrossfilter } =
+    getState();
   const annoMatrix = _userSubsetAnnoMatrix(
     prevAnnoMatrix,
     prevObsCrossfilter.allSelectedMask()

client/src/annoMatrix/annoMatrix.js

@@ -12,6 +12,7 @@ import {
 import { indexEntireSchema } from "../util/stateManager/schemaHelpers";
 import { _whereCacheGet, _whereCacheMerge } from "./whereCache";
 import _shallowClone from "./clone";
+import { _queryValidate, _queryCacheKey } from "./query";
 
 const _dataframeCache = dataframeMemo(128);
 
@@ -181,7 +182,7 @@ export default class AnnoMatrix {
 		Returns a Promise for the query result, which will resolve to a dataframe.
 
 		Field must be one of the matrix fields: 'obs', 'var', 'X', 'emb'.  Value
-		represents the underlying object upon which the query is occuring.
+		represents the underlying object upon which the query is occurring.
 
 		Query is one of:
 			* a string, representing a single column name from the field, eg,
@@ -197,12 +198,6 @@ export default class AnnoMatrix {
 		field/column, similar to a join.  Currently only supported on the var
 		dimension, allowing query of X columns by var value (eg, gene name)
 
-    The query filter	is a single value filter:
-			{ "field name": [
-				{name: "column name", values: [ list of values ]}
-			]}
-		One and only one value filter is allowed in a value query.
-
 		Examples:
 
     1. Fetch the "n_genes" column the "obs":
@@ -220,7 +215,9 @@ export default class AnnoMatrix {
        value "TYMP" in the var index.
 
 			fetch("X", { 
-				where: {field: "var", column: this.schema.annotations.var.index, value: "TYMP"}
+				where: {
+          field: "var", column: this.schema.annotations.var.index, value: "TYMP"
+        }
 			})
 
       In AnnData & Pandas DataFrame API, this is equivalent to:
@@ -410,6 +407,14 @@ export default class AnnoMatrix {
     _subclassResponsibility();
   }
 
+  getCacheKeys(field, query) {
+    /*
+    Return cache keys for columns associated with this query.  May return
+    [unknown] if no keys are known (ie, nothing is or was cached).
+    */
+    return _whereCacheGet(this._whereCache, this.schema, field, query);
+  }
+
   /**
    ** Private interfaces below.
    **/
@@ -427,6 +432,7 @@ export default class AnnoMatrix {
   async _fetch(field, q) {
     if (!AnnoMatrix.fields().includes(field)) return undefined;
     const queries = Array.isArray(q) ? q : [q];
+    queries.forEach(_queryValidate);
 
     /* find cached columns we need, and GC the rest */
     const cachedColumns = this._resolveCachedQueries(field, queries);
@@ -507,7 +513,7 @@ export default class AnnoMatrix {
     * obs, var and emb do not grow without bounds, and are needed constantly 
       for rendering.  
       a) There is no upside to GC'ing these in the base (loader)
-      b) The undo/redo cache can hold a large number in views, which is worht GC'ing
+      b) The undo/redo cache can hold a large number in views, which is worth GC'ing
     * X is often much larger than memory, and the UI allows add/del from
       this.  Most of the GC potential is here in both the base and views.
 
@@ -522,7 +528,7 @@ export default class AnnoMatrix {
   as much of the cache is pinned by that data structure.
   */
   _gcField(field, isHot, pinnedColumns) {
-    const maxColumns = isHot ? 256 : 10; // maybe to aggessive?
+    const maxColumns = isHot ? 256 : 10; // maybe to aggressive?
 
     const cache = this._cache[field];
     if (cache.colIndex.size() < maxColumns) return; // trivial rejection
@@ -590,23 +596,18 @@ export default class AnnoMatrix {
     called each time a query is performed, allowing the gc to update any bookkeeping
     information.  Currently, this is just a simple last-fetched timestamp, stored
     in a Map.
-
-    Map objects preserve order of insertion. This is leveraged as a cheap way to
-    do LRU, by removing and re-inserting keys.  IMPORTANT: the cleanup code assumes
-    the map insertion order is least-recently-used first.
     */
     const cols = dataframe.colIndex.labels();
     const { _gcInfo } = this;
     const now = Date.now();
     cols.forEach((c) => {
-      // gcInfo.delete(c);
       _gcInfo.set(_columnCacheKey(field, c), now);
     });
   }
 
   /**
-  Cloning sublcass protocol - we rely in cloning to preserve immutable
-  symantics while not causing races or other side effects in internal
+  Cloning subclass protocol - we rely in cloning to preserve immutable
+  semantics while not causing races or other side effects in internal
   cache management.
 
   Subclasses must override _cloneDeeper() if they have state which requires 
@@ -639,15 +640,6 @@ export default class AnnoMatrix {
 /*
 private utility functions below
 */
-
-function _queryCacheKey(field, query) {
-  if (typeof query === "object") {
-    const { field: queryField, column: queryColumn, value: queryValue } = query;
-    return `${field}/${queryField}/${queryColumn}/${queryValue}`;
-  }
-  return `${field}/${query}`;
-}
-
 function _columnCacheKey(field, column) {
   return `${field}/${column}`;
 }

client/src/annoMatrix/crossfilter.js

@@ -123,6 +123,24 @@ export default class AnnoMatrixObsCrossfilter {
     return new AnnoMatrixObsCrossfilter(annoMatrix, this.obsCrossfilter);
   }
 
+  /**
+   * Drop the crossfilter dimension. Do not change the annoMatrix. Useful when we
+   * want to stop trackin the selection state, but aren't sure we want to blow the
+   * annomatrix cache.
+   */
+  dropDimension(field, query) {
+    const { annoMatrix } = this;
+    let { obsCrossfilter } = this;
+    const keys = annoMatrix
+      .getCacheKeys(field, query)
+      .filter((k) => k !== undefined);
+    const dimName = _dimensionName(field, keys);
+    if (obsCrossfilter.hasDimension(dimName)) {
+      obsCrossfilter = obsCrossfilter.delDimension(dimName);
+    }
+    return new AnnoMatrixObsCrossfilter(annoMatrix, obsCrossfilter);
+  }
+
   /**
   Selection state - API is identical to ImmutableTypedCrossfilter, as these
   are just wrappers to lazy create indices.
@@ -164,9 +182,7 @@ export default class AnnoMatrixObsCrossfilter {
 		*/
     const { annoMatrix } = this;
     const currentDims = this.obsCrossfilter.dimensionNames();
-    const obsCrossfilter = currentDims.reduce((xfltr, dim) => {
-      return xfltr.select(dim, { mode: "all" });
-    }, this.obsCrossfilter);
+    const obsCrossfilter = currentDims.reduce((xfltr, dim) => xfltr.select(dim, { mode: "all" }), this.obsCrossfilter);
     return new AnnoMatrixObsCrossfilter(annoMatrix, obsCrossfilter);
   }
 

client/src/annoMatrix/fetchHelpers.js

@@ -1,4 +1,4 @@
-export { doBinaryRequest } from "../util/actionHelpers";
+export { doBinaryRequest, doFetch } from "../util/actionHelpers";
 
 /* double URI encode - needed for query-param filters */
 export function _dubEncURIComp(s) {
@@ -19,9 +19,7 @@ export function _fetchResult(promise) {
     }
   );
 
-  res.status = () => {
-    return _status;
-  };
+  res.status = () => _status;
 
   return res;
 }

client/src/annoMatrix/loader.js

@@ -1,6 +1,6 @@
-import { doBinaryRequest, _dubEncURIComp } from "./fetchHelpers";
+import { doBinaryRequest, doFetch } from "./fetchHelpers";
 import { matrixFBSToDataframe } from "../util/stateManager/matrix";
-import { _getColumnSchema, _normalizeCategoricalSchema } from "./schema";
+import { _getColumnSchema } from "./schema";
 import {
   addObsAnnoColumn,
   removeObsAnnoColumn,
@@ -12,6 +12,17 @@ import { isArrayOrTypedArray } from "../util/typeHelpers";
 import { _whereCacheCreate } from "./whereCache";
 import AnnoMatrix from "./annoMatrix";
 import PromiseLimit from "../util/promiseLimit";
+import {
+  _expectSimpleQuery,
+  _expectComplexQuery,
+  _urlEncodeLabelQuery,
+  _urlEncodeComplexQuery,
+  _hashStringValues,
+} from "./query";
+import {
+  normalizeResponse,
+  normalizeWritableCategoricalSchema,
+} from "./normalize";
 
 const promiseThrottle = new PromiseLimit(5);
 
@@ -115,7 +126,7 @@ export default class AnnoMatrixLoader extends AnnoMatrix {
       data = new Ctor(this.nObs).fill(value);
     }
     newAnnoMatrix._cache.obs = this._cache.obs.withCol(colName, data);
-    _normalizeCategoricalSchema(
+    normalizeWritableCategoricalSchema(
       colSchema,
       newAnnoMatrix._cache.obs.col(colName)
     );
@@ -223,27 +234,23 @@ export default class AnnoMatrixLoader extends AnnoMatrix {
     /*
     _doLoad - evaluates the query against the field. Returns:
       * whereCache update: column query map mapping the query to the column labels
-      * Dataframe containing the new colums (one per dimension)
+      * Dataframe containing the new columns (one per dimension)
     */
-    let urlQuery;
-    let urlBase;
+    let doRequest;
     let priority = 10; // default fetch priority
 
     switch (field) {
       case "obs":
       case "var": {
-        urlBase = `${this.baseURL}annotations/${field}`;
-        urlQuery = _encodeQuery("annotation-name", query);
+        doRequest = _obsOrVarLoader(this.baseURL, field, query);
         break;
       }
       case "X": {
-        urlBase = `${this.baseURL}data/var`;
-        urlQuery = _encodeQuery(undefined, query);
+        doRequest = _XLoader(this.baseURL, field, query);
         break;
       }
       case "emb": {
-        urlBase = `${this.baseURL}layout/obs`;
-        urlQuery = _encodeQuery("layout-name", query);
+        doRequest = _embLoader(this.baseURL, field, query);
         priority = 0; // high prio load for embeddings
         break;
       }
@@ -251,13 +258,8 @@ export default class AnnoMatrixLoader extends AnnoMatrix {
         throw new Error("Unknown field name");
     }
 
-    const url = `${urlBase}?${urlQuery}`;
-    const buffer = await promiseThrottle.priorityAdd(
-      priority,
-      doBinaryRequest,
-      url
-    );
-    const result = matrixFBSToDataframe(buffer);
+    const buffer = await promiseThrottle.priorityAdd(priority, doRequest);
+    let result = matrixFBSToDataframe(buffer);
     if (!result || result.isEmpty()) throw Error("Unknown field/col");
 
     const whereCacheUpdate = _whereCacheCreate(
@@ -266,13 +268,7 @@ export default class AnnoMatrixLoader extends AnnoMatrix {
       result.colIndex.labels()
     );
 
-    if (field === "obs") {
-      /* cough, cough - see comment on method */
-      _normalizeCategoricalSchema(
-        this.schema.annotations.obsByName[query],
-        result.col(query)
-      );
-    }
+    result = normalizeResponse(field, query, this.schema, result);
 
     return [whereCacheUpdate, result];
   }
@@ -282,17 +278,6 @@ export default class AnnoMatrixLoader extends AnnoMatrix {
 Utility functions below
 */
 
-function _encodeQuery(colKey, q) {
-  if (typeof q === "object") {
-    const { field: queryField, column: queryColumn, value: queryValue } = q;
-    return `${_dubEncURIComp(queryField)}:${_dubEncURIComp(
-      queryColumn
-    )}=${_dubEncURIComp(queryValue)}`;
-  }
-  if (!colKey) throw new Error("Unsupported query by name");
-  return `${colKey}=${encodeURIComponent(q)}`;
-}
-
 function _writableCheck(colSchema) {
   if (!colSchema?.writable) {
     throw new Error("Unknown or readonly obs column");
@@ -305,3 +290,57 @@ function _writableCategoryTypeCheck(colSchema) {
     throw new Error("column must be categorical");
   }
 }
+
+function _embLoader(baseURL, _field, query) {
+  _expectSimpleQuery(query);
+
+  const urlBase = `${baseURL}layout/obs`;
+  const urlQuery = _urlEncodeLabelQuery("layout-name", query);
+  const url = `${urlBase}?${urlQuery}`;
+  return () => doBinaryRequest(url);
+}
+
+function _obsOrVarLoader(baseURL, field, query) {
+  _expectSimpleQuery(query);
+
+  const urlBase = `${baseURL}annotations/${field}`;
+  const urlQuery = _urlEncodeLabelQuery("annotation-name", query);
+  const url = `${urlBase}?${urlQuery}`;
+  return () => doBinaryRequest(url);
+}
+
+function _XLoader(baseURL, field, query) {
+  _expectComplexQuery(query);
+
+  if (query.where) {
+    const urlBase = `${baseURL}data/var`;
+    const urlQuery = _urlEncodeComplexQuery(query);
+    const url = `${urlBase}?${urlQuery}`;
+    return () => doBinaryRequest(url);
+  }
+
+  if (query.summarize) {
+    const urlBase = `${baseURL}summarize/var`;
+    const urlQuery = _urlEncodeComplexQuery(query);
+
+    if (urlBase.length + urlQuery.length < 2000) {
+      const url = `${urlBase}?${urlQuery}`;
+      return () => doBinaryRequest(url);
+    }
+
+    const url = `${urlBase}?key=${_hashStringValues([urlQuery])}`;
+    return async () => {
+      const res = await doFetch(url, {
+        method: "POST",
+        body: urlQuery,
+        headers: new Headers({
+          Accept: "application/octet-stream",
+          "Content-Type": "application/x-www-form-urlencoded",
+        }),
+      });
+      return res.arrayBuffer();
+    };
+  }
+
+  throw new Error("Unknown query structure");
+}

client/src/annoMatrix/normalize.js

@@ -0,0 +1,150 @@
+import { _getColumnSchema, _isIndex } from "./schema";
+import catLabelSort from "../util/catLabelSort";
+import {
+  unassignedCategoryLabel,
+  overflowCategoryLabel,
+  globalConfig,
+} from "../globals";
+
+export function normalizeResponse(field, query, schema, response) {
+  /**
+   * There are a number of assumptions in the front-end about data typing and data
+   * characteristics. This routine will normalize a server response dataframe
+   * to match front-end expectations and UI conventions. This includes cast/transform
+   * of the data and schema updates.
+   *
+   * This consolidates all assumptions into one location, for ease of update.
+   *
+   * Currently, this includes normalization for obs/var columns only:
+   *
+   * - Dataframe columns in var/obs that are declared type: boolean may be sent by
+   *   the server in a variety of formats (eg uint8, etc).  Cast to JS Array[boolean]
+   *
+   * - "Categorical" columns may not have all categories represented in the server-provided
+   *   schema (for valid reasons, eg, floating point rounding differences).  For all
+   *   types we treat as categorical in the UI (string, boolean, categorical), update
+   *   the schema to contain all categories as a convenience.
+   *
+   * - "Categorical" columns (ie, string, boolean, categorical) may contain an excess
+   *   of category values (aka labels).  Consolidate any excess into an "all other"
+   *   category.
+   */
+
+  // currently no data or schema normalization necessary for X or emb
+  if (field !== "obs" && field !== "var") return response;
+
+  const colLabels = response.colIndex.labels();
+  for (const colLabel of colLabels) {
+    const colSchema = _getColumnSchema(schema, field, colLabel);
+    const isIndex = _isIndex(schema, field, colLabel);
+    const { type, writable } = colSchema;
+
+    // Boolean data -- cast entire array to Array[bool]
+    if (type === "boolean") {
+      response = castColumnToBoolean(response, colLabel);
+    }
+
+    // Types that are categorical in UI (string, boolean, categorical) OR are writable
+    // are introspected to ensure the schema `categories` field and data values match,
+    // and that we do not have an excess of category values (for non-writable columns)
+    const isEnumType =
+      type === "boolean" ||
+      type === "string" ||
+      type === "categorical" ||
+      writable;
+    if (!isIndex && isEnumType) {
+      response = normalizeCategorical(response, colLabel, colSchema);
+    }
+  }
+  return response;
+}
+
+function castColumnToBoolean(df, label) {
+  const colData = df.col(label).asArray();
+  const newColData = new Array(colData.length);
+  for (let i = 0; i < colData.length; i += 1) newColData[i] = !!colData[i];
+  df = df.replaceColData(label, newColData);
+  return df;
+}
+
+export function normalizeWritableCategoricalSchema(colSchema, col) {
+  /*
+  Ensure all enum writable / categorical schema have a categories array, that
+  the categories array contains all unique values in the data array, AND that
+  the array is UI sorted.
+  */
+  const categorySet = new Set(
+    col.summarizeCategorical().categories.concat(colSchema.categories ?? [])
+  );
+  if (!categorySet.has(unassignedCategoryLabel)) {
+    categorySet.add(unassignedCategoryLabel);
+  }
+  colSchema.categories = catLabelSort(true, Array.from(categorySet));
+  return colSchema;
+}
+
+export function normalizeCategorical(df, colLabel, colSchema) {
+  /*
+  If writable, ensure schema matches data and we have an unassigned label
+  If not writable, ensure schema matches data and that we consolidate labels in excess
+  of "top N" into an overflow labels.
+  */
+  const { writable } = colSchema;
+  const col = df.col(colLabel);
+
+  if (writable) {
+    // writable (aka user) annotations
+    normalizeWritableCategoricalSchema(colSchema, col);
+    return df;
+  }
+
+  // else read-only, categorical columns
+  const TopN = globalConfig.maxCategoricalOptionsToDisplay;
+
+  // consolidate all categories from data and schema into a single list
+  const colDataSummary = col.summarizeCategorical();
+  const allCategories = new Set(
+    colDataSummary.categories.concat(colSchema.categories ?? [])
+  );
+
+  // if no overflow, just UI sort schema categories and return
+  if (allCategories.size <= TopN) {
+    colSchema.categories = catLabelSort(writable, [...allCategories.keys()]);
+    return df;
+  }
+
+  // Otherwise, pick top N categories by count and rewrite data
+
+  // choose unique overflow category label
+  let overflowCatName = `${colLabel}${overflowCategoryLabel}`;
+  while (allCategories.has(overflowCatName)) {
+    overflowCatName += "_";
+  }
+
+  // pick top N category labels and add overflow label
+  const topNCategories = new Set(
+    [...colDataSummary.categoryCounts.keys()].slice(0, TopN)
+  );
+  topNCategories.add(overflowCatName);
+
+  // rewrite data - consolidate all excess labels into overflow label
+  const newColData = Array.from(col.asArray());
+  for (let i = 0; i < newColData.length; i += 1) {
+    if (!topNCategories.has(newColData[i])) {
+      newColData[i] = overflowCatName;
+    }
+  }
+
+  // replace data in dataframe
+  df = df.replaceColData(colLabel, newColData);
+
+  // Update schema with categories, in UI sort order. Ensure overflow label is at end
+  // of list for display purposes.
+  const revisedCategories = df.col(colLabel).summarizeCategorical().categories;
+  revisedCategories.push(
+    revisedCategories.splice(revisedCategories.indexOf(overflowCatName), 1)[0]
+  );
+  colSchema.categories = catLabelSort(writable, revisedCategories);
+
+  return df;
+}

client/src/annoMatrix/query.js

@@ -0,0 +1,126 @@
+import sha1 from "sha1";
+import { _dubEncURIComp } from "./fetchHelpers";
+
+/**
+ * Query utilities, mostly for debugging support and validation.
+ */
+
+/**
+ * Normalize & error check the query.
+ * @param {object | string} query - the query
+ * @returns {object | string} - the normalized query
+ */
+export function _queryValidate(query) {
+  if (typeof query !== "object") return query;
+
+  if (query.where && query.summarize)
+    throw new Error("query may not specify both where and summarize");
+  if (query.where) {
+    const {
+      field: queryField,
+      column: queryColumn,
+      value: queryValue,
+    } = query.where;
+    if (!queryField || !queryColumn || !queryValue)
+      throw new Error("Incomplete where query");
+    return query;
+  }
+  if (query.summarize) {
+    const {
+      field: queryField,
+      column: queryColumn,
+      values: queryValues,
+    } = query.summarize;
+    if (!queryField || !queryColumn || !queryValues)
+      throw new Error("Incomplete where query");
+    if (!Array.isArray(queryValues))
+      throw new Error("Summarize query values must be an array");
+    return query;
+  }
+  throw new Error("query must specify one of where or summarize");
+}
+
+export function _expectSimpleQuery(query) {
+  if (typeof query === "object") throw new Error("expected simple query");
+}
+
+export function _expectComplexQuery(query) {
+  if (typeof query !== "object") throw new Error("expected complex query");
+}
+
+/**
+ * Generate a unique key which can be used to reference this query.
+ *
+ * @param {string} field
+ * @param {string|object} query
+ * @returns the key
+ */
+export function _queryCacheKey(field, query) {
+  if (typeof query === "object") {
+    // complex query
+    if (query.where) {
+      const {
+        field: queryField,
+        column: queryColumn,
+        value: queryValue,
+      } = query.where;
+      return `${field}/${queryField}/${queryColumn}/${queryValue}`;
+    }
+    if (query.summarize) {
+      const {
+        method,
+        field: queryField,
+        column: queryColumn,
+        values: queryValues,
+      } = query.summarize;
+      return `${field}/${method}/${queryField}/${queryColumn}/${queryValues.join(
+        ","
+      )}`;
+    }
+    throw new Error("Unrecognized complex query type");
+  }
+
+  // simple query
+  return `${field}/${query}`;
+}
+
+function _urlEncodeWhereQuery(q) {
+  const { field: queryField, column: queryColumn, value: queryValue } = q;
+  return `${_dubEncURIComp(queryField)}:${_dubEncURIComp(
+    queryColumn
+  )}=${_dubEncURIComp(queryValue)}`;
+}
+
+function _urlEncodeSummarizeQuery(q) {
+  const { method, field, column, values } = q;
+  const filter = values
+    .map((value) => _urlEncodeWhereQuery({ field, column, value }))
+    .join("&");
+  return `method=${method}&${filter}`;
+}
+
+export function _urlEncodeComplexQuery(q) {
+  if (typeof q === "object") {
+    if (q.where) {
+      return _urlEncodeWhereQuery(q.where);
+    }
+    if (q.summarize) {
+      return _urlEncodeSummarizeQuery(q.summarize);
+    }
+  }
+  throw new Error("Unrecognized complex query type");
+}
+
+export function _urlEncodeLabelQuery(colKey, q) {
+  if (!colKey) throw new Error("Unsupported query by name");
+  if (typeof q !== "string") throw new Error("Query must be a simple label.");
+  return `${colKey}=${encodeURIComponent(q)}`;
+}
+
+/**
+ * Generate the column key the server will send us for this query.
+ */
+export function _hashStringValues(arrayOfString) {
+  const hash = sha1(arrayOfString.join(""));
+  return hash;
+}

client/src/annoMatrix/schema.js

@@ -1,9 +1,6 @@
 /*
 Private helper functions related to schema
 */
-import catLabelSort from "../util/catLabelSort";
-import { unassignedCategoryLabel } from "../globals";
-
 export function _getColumnSchema(schema, field, col) {
   /* look up the column definition */
   switch (field) {
@@ -26,10 +23,15 @@ export function _getColumnSchema(schema, field, col) {
   }
 }
 
+export function _isIndex(schema, field, col) {
+  const index = schema.annotations?.[field].index;
+  return index && index === col;
+}
+
 export function _getColumnDimensionNames(schema, field, col) {
   /*
 		field/col may be an alias for multiple columns. Currently used to map ND 
-		values to 1D dataframe columns for embeddings/layout. Signfied by the presence
+		values to 1D dataframe columns for embeddings/layout. Signified by the presence
 		of the "dims" value in the schema.
 		*/
   const colSchema = _getColumnSchema(schema, field, col);
@@ -63,26 +65,3 @@ export function _isContinuousType(schema) {
   const { type } = schema;
   return !(type === "string" || type === "boolean" || type === "categorical");
 }
-
-export function _normalizeCategoricalSchema(colSchema, col) {
-  const { type, writable } = colSchema;
-  if (
-    type === "string" ||
-    type === "boolean" ||
-    type === "categorical" ||
-    writable
-  ) {
-    const categorySet = new Set(
-      col.summarizeCategorical().categories.concat(colSchema.categories ?? [])
-    );
-    if (writable && !categorySet.has(unassignedCategoryLabel)) {
-      categorySet.add(unassignedCategoryLabel);
-    }
-    colSchema.categories = Array.from(categorySet);
-  }
-
-  if (colSchema.categories) {
-    colSchema.categories = catLabelSort(writable, colSchema.categories);
-  }
-  return colSchema;
-}

client/src/annoMatrix/viewCreators.js

@@ -58,10 +58,11 @@ function _maskToList(mask) {
   if (!mask) {
     return null;
   }
-  const list = new Int32Array(mask.length);
+  const [...m] = mask;
+  const list = new Int32Array(m.length);
   let elems = 0;
-  for (let i = 0, l = mask.length; i < l; i += 1) {
-    if (mask[i]) {
+  for (let i = 0, l = m.length; i < l; i += 1) {
+    if (m[i]) {
       list[elems] = i;
       elems += 1;
     }

client/src/annoMatrix/whereCache.js

@@ -1,50 +1,88 @@
 /*
 Private support functions.
 
-Support for a "where" query, eg,
+This implements a query resolver cache, mapping a query onto the column labels
+resolved by that query. These labels are then used to manage the acutal data cache,
+which stores data by the resolved label.
 
-	{ where: { field: "var", column: "gene", value: "FOXP2" }}
+There are three query forms:
+  * primitive (string, number) - which is just reference the column label of same value
+  * where query (object) - eg, { where: { field: "var", column: "gene", value: "FOXP2" }}
+  * summary query (object) - eg, { summarize: { method: "mean", field: "var", column: "gene", values: ["FOXP2", "GNE", "F5"]}}
 
-These evaluate to a given column label.
+These queries all resolve to one or more column labels on a field.  This
+cache maintains a record of this, allowing direct access to the data caches
+without a server round-trip.
 
-The "where cache" is a map that saves evaluated queries and points 
-to the column label they resolve to.
+The data structure for where queries, the following query against X as an example:
+  { where: { field: "var", column: "column_label_in_var", value: "value_in_var_column" } }
+results in the following cached entry:
+{
+  where: {
+    X: {
+      var: Map(
+        column_label_in_var => Map(
+          value_in_var_column => [column_label_in_X, ...]
+        )
+      )
+    }
+  },
+  summarize: {},
+}
 
-Data structure, using X as the example field being queried, and var as
-the index.
-
-{ 
-	X: {
-		var: Map(
-			column_label_in_var => Map(value_in_var_column => [column_label_in_X, ...])
-		)
-	}
+And for summarize queries, for the following summary on X:
+  { summarize: { method: "mean", field: "var", column: "gene", values: ["G1", "G2"]}}
+creates a cache entry of:
+{
+  where: {},
+  summarize: {
+    X: {
+      mean: {
+        var: Map(
+          "gene" => Map(
+            "G1,G2" => [summary_column_label, ...]
+          )
+        )
+      }
+    },
+  },
 }
 */
 import { _getColumnDimensionNames } from "./schema";
+import { _hashStringValues } from "./query";
 
 export function _whereCacheGet(whereCache, schema, field, query) {
-  /* 
+  /*
 	query will either be an where query (object) or a column name (string).
 
 	Return array of column labels or undefined.
 	*/
 
   if (typeof query === "object") {
-    const { field: queryField, column: queryColumn, value: queryValue } = query;
-
-    const columnMap = whereCache?.[field]?.[queryField];
-    if (columnMap === undefined) return [undefined];
-
-    const valueMap = columnMap.get(queryColumn);
-    if (valueMap === undefined) return [undefined];
-
-    const columnLabels = valueMap.get(queryValue);
-    return columnLabels === undefined ? [undefined] : columnLabels;
+    if (query.where) {
+      const {
+        field: queryField,
+        column: queryColumn,
+        value: queryValue,
+      } = query.where;
+      const columnMap = whereCache?.where?.[field]?.[queryField];
+      return columnMap?.get(queryColumn)?.get(queryValue) ?? [undefined];
+    }
+    if (query.summarize) {
+      const {
+        method,
+        field: queryField,
+        column: queryColumn,
+        values: queryValues,
+      } = query.summarize;
+      const columnMap = whereCache?.summarize?.[field]?.[method]?.[queryField];
+      const queryValueHash = _hashStringValues(queryValues);
+      return columnMap?.get(queryColumn)?.get(queryValueHash) ?? [undefined];
+    }
+    return [undefined];
   }
 
-  const colDims = _getColumnDimensionNames(schema, field, query);
-  return colDims === undefined ? [undefined] : colDims;
+  return _getColumnDimensionNames(schema, field, query) ?? [undefined];
 }
 
 export function _whereCacheCreate(field, query, columnLabels) {
@@ -53,40 +91,84 @@ export function _whereCacheCreate(field, query, columnLabels) {
 	*/
   if (typeof query !== "object") return null;
 
-  const { field: queryField, column: queryColumn, value: queryValue } = query;
-  const whereCache = {
-    [field]: {
-      [queryField]: new Map([
-        [queryColumn, new Map([[queryValue, columnLabels]])],
-      ]),
-    },
-  };
-  return whereCache;
+  const { where, summarize } = query;
+  if (where) {
+    const { field: queryField, column: queryColumn, value: queryValue } = where;
+    return {
+      where: {
+        [field]: {
+          [queryField]: new Map([
+            [queryColumn, new Map([[queryValue, columnLabels]])],
+          ]),
+        },
+      },
+    };
+  }
+  if (summarize) {
+    const {
+      method,
+      field: queryField,
+      column: queryColumn,
+      values: queryValues,
+    } = summarize;
+    const queryValueHash = _hashStringValues(queryValues);
+    return {
+      summarize: {
+        [field]: {
+          [method]: {
+            [queryField]: new Map([
+              [queryColumn, new Map([[queryValueHash, columnLabels]])],
+            ]),
+          },
+        },
+      },
+    };
+  }
+  // oops, not sure what that query is!
+  return {};
+}
+
+function __mergeQueries(dst, src) {
+  for (const [queryField, columnMap] of Object.entries(src)) {
+    dst[queryField] = dst[queryField] || new Map();
+    for (const [queryColumn, valueMap] of columnMap) {
+      if (!dst[queryField].has(queryColumn))
+        dst[queryField].set(queryColumn, new Map());
+      for (const [queryValue, columnLabels] of valueMap) {
+        dst[queryField].get(queryColumn).set(queryValue, columnLabels);
+      }
+    }
+  }
 }
 
 function __whereCacheMerge(dst, src) {
   /*
 	merge src into dst (modifies dst)
 	*/
-  if (!dst) dst = {};
   if (!src || typeof src !== "object") return dst;
-  Object.entries(src).forEach(([field, query]) => {
-    if (!Object.prototype.hasOwnProperty.call(dst, field)) dst[field] = {};
-    Object.entries(query).forEach(([queryField, columnMap]) => {
-      if (!Object.prototype.hasOwnProperty.call(dst[field], queryField))
-        dst[field][queryField] = new Map();
-      columnMap.forEach((valueMap, queryColumn) => {
-        if (!dst[field][queryField].has(queryColumn))
-          dst[field][queryField].set(queryColumn, new Map());
-        valueMap.forEach((columnLabels, queryValue) => {
-          dst[field][queryField].get(queryColumn).set(queryValue, columnLabels);
-        });
-      });
-    });
-  });
+
+  if (src.where) {
+    dst.where = dst.where || {};
+    for (const [field, query] of Object.entries(src.where)) {
+      dst.where[field] = dst.where[field] || {};
+      __mergeQueries(dst.where[field], query);
+    }
+  }
+  if (src.summarize) {
+    dst.summarize = dst.summarize || {};
+    for (const [field, method] of Object.entries(src.summarize)) {
+      dst.summarize[field] = dst.summarize[field] || {};
+      for (const [methodName, query] of Object.entries(method)) {
+        dst.summarize[field][methodName] =
+          dst.summarize[field][methodName] || {};
+        __mergeQueries(dst.summarize[field][methodName], query);
+      }
+    }
+  }
   return dst;
 }
 
 export function _whereCacheMerge(...caches) {
-  return caches.reduce((dst, src) => __whereCacheMerge(dst, src), {});
+  // eslint-disable-next-line compat/compat -- not using web APIs
+  return caches.reduce(__whereCacheMerge, {});
 }

client/src/components/annoDialog.js

@@ -18,6 +18,8 @@ class AnnoDialog extends React.PureComponent {
       validationError,
       annoSelect,
       annoInput,
+      secondaryInstructions,
+      secondaryInput,
       handleCancel,
       handleSubmit,
       primaryButtonText,
@@ -40,12 +42,19 @@ class AnnoDialog extends React.PureComponent {
               <p
                 style={{
                   marginTop: 7,
-                  visibility: validationError ? "visible" : "hidden",
+                  visibility: errorMessage !== "" ? "visible" : "hidden",
                   color: Colors.ORANGE3,
                 }}
               >
                 {errorMessage}
               </p>
+              {/* we might rename, secondary button and secondary input are not related */}
+              {secondaryInstructions && (
+                <p style={{ marginTop: secondaryInstructions ? 20 : 0 }}>
+                  {secondaryInstructions}
+                </p>
+              )}
+              {secondaryInput || null}
             </div>
             {annoSelect || null}
           </div>
@@ -54,6 +63,7 @@ class AnnoDialog extends React.PureComponent {
               <Tooltip content={cancelTooltipContent}>
                 <Button onClick={handleCancel}>Cancel</Button>
               </Tooltip>
+              {/* we might rename, secondary button and secondary input are not related */}
               {handleSecondaryButtonSubmit && secondaryButtonText ? (
                 <Button
                   onClick={handleSecondaryButtonSubmit}

client/src/components/app.js

@@ -1,4 +1,3 @@
-// jshint esversion: 6
 import React from "react";
 import Helmet from "react-helmet";
 import { connect } from "react-redux";
@@ -12,7 +11,6 @@ import Graph from "./graph/graph";
 import MenuBar from "./menubar";
 import Autosave from "./autosave";
 import Embedding from "./embedding";
-import TermsOfServicePrompt from "./termsPrompt";
 
 import actions from "../actions";
 
@@ -43,7 +41,7 @@ class App extends React.Component {
     const { loading, error, graphRenderCounter } = this.props;
     return (
       <Container>
-        <Helmet title="cellxgene" />
+        <Helmet title="CELL&times;GENE | Annotate" />
         {loading ? (
           <div
             style={{
@@ -76,7 +74,6 @@ class App extends React.Component {
                 <MenuBar />
                 <Embedding />
                 <Autosave />
-                <TermsOfServicePrompt />
                 <Legend viewportRef={viewportRef} />
                 <Graph key={graphRenderCounter} viewportRef={viewportRef} />
               </>

client/src/components/autosave/filenameDialog.js

@@ -3,18 +3,21 @@ import { connect } from "react-redux";
 
 import {
   Button,
-  Tooltip,
-  InputGroup,
-  Dialog,
   Classes,
+  Code,
   Colors,
+  Dialog,
+  InputGroup,
+  Tooltip,
 } from "@blueprintjs/core";
 
 @connect((state) => ({
   idhash: state.config?.parameters?.["annotations-user-data-idhash"] ?? null,
   annotations: state.annotations,
-  auth: state.config?.authentication,
   writableCategoriesEnabled: state.config?.parameters?.annotations ?? false,
+  writableGenesetsEnabled: !(
+    state.config?.parameters?.annotations_genesets_readonly ?? true
+  ),
 }))
 class FilenameDialog extends React.Component {
   constructor(props) {
@@ -91,16 +94,21 @@ class FilenameDialog extends React.Component {
   };
 
   render() {
-    const { writableCategoriesEnabled, annotations, idhash, auth } = this.props;
+    const {
+      writableCategoriesEnabled,
+      writableGenesetsEnabled,
+      annotations,
+      idhash,
+    } = this.props;
     const { filenameText } = this.state;
 
-    return writableCategoriesEnabled &&
+    return (writableCategoriesEnabled || writableGenesetsEnabled) &&
+      annotations.promptForFilename &&
       !annotations.dataCollectionNameIsReadOnly &&
-      !annotations.dataCollectionName &&
-      auth.is_authenticated ? (
+      !annotations.dataCollectionName ? (
       <Dialog
         icon="tag"
-        title="Annotations Collection"
+        title="User Generated Data Directory"
         isOpen={!annotations.dataCollectionName}
         onClose={this.dismissFilenameDialog}
       >
@@ -112,7 +120,7 @@ class FilenameDialog extends React.Component {
         >
           <div className={Classes.DIALOG_BODY} data-testid="annotation-dialog">
             <div style={{ marginBottom: 20 }}>
-              <p>Name your annotations collection:</p>
+              <p>Name your user generated data directory:</p>
               <InputGroup
                 autoFocus
                 value={filenameText}
@@ -137,10 +145,16 @@ class FilenameDialog extends React.Component {
             </div>
             <div>
               <p>
-                Your annotations are stored in this file:
-                <code className="bp3-code">
-                  {filenameText}-{idhash}.csv
-                </code>
+                {"Your annotations are stored in this file: "}
+                <Code>
+                  {filenameText}-cell-labels-{idhash}.csv
+                </Code>
+              </p>
+              <p>
+                {"Your gene sets are stored in this file: "}
+                <Code>
+                  {filenameText}-gene-sets-{idhash}.csv
+                </Code>
               </p>
               <p style={{ fontStyle: "italic" }}>
                 (We added a unique ID to your filename)
@@ -159,7 +173,7 @@ class FilenameDialog extends React.Component {
                 type="submit"
                 data-testid="submit-annotation"
               >
-                Create annotations collection
+                Create user generated data directory
               </Button>
             </div>
           </div>

client/src/components/autosave/index.js

@@ -5,11 +5,18 @@ import FilenameDialog from "./filenameDialog";
 
 @connect((state) => ({
   annotations: state.annotations,
-  saveInProgress: state.autosave?.saveInProgress ?? false,
+  obsAnnotationSaveInProgress:
+    state.autosave?.obsAnnotationSaveInProgress ?? false,
+  genesetSaveInProgress: state.autosave?.genesetSaveInProgress ?? false,
   error: state.autosave?.error,
   writableCategoriesEnabled: state.config?.parameters?.annotations ?? false,
+  writableGenesetsEnabled: !(
+    state.config?.parameters?.annotations_genesets_readonly ?? true
+  ),
   annoMatrix: state.annoMatrix,
+  genesets: state.genesets,
   lastSavedAnnoMatrix: state.autosave?.lastSavedAnnoMatrix,
+  lastSavedGenesets: state.autosave?.lastSavedGenesets,
 }))
 class Autosave extends React.Component {
   constructor(props) {
@@ -20,11 +27,11 @@ class Autosave extends React.Component {
   }
 
   componentDidMount() {
-    const { writableCategoriesEnabled } = this.props;
+    const { writableCategoriesEnabled, writableGenesetsEnabled } = this.props;
 
     let { timer } = this.state;
     if (timer) clearInterval(timer);
-    if (writableCategoriesEnabled) {
+    if (writableCategoriesEnabled || writableGenesetsEnabled) {
       timer = setInterval(this.tick, 2500);
     } else {
       timer = null;
@@ -38,18 +45,37 @@ class Autosave extends React.Component {
   }
 
   tick = () => {
-    const { dispatch, saveInProgress } = this.props;
-    if (this.needToSave() && !saveInProgress) {
+    const { dispatch, obsAnnotationSaveInProgress, genesetSaveInProgress } =
+      this.props;
+    if (!obsAnnotationSaveInProgress && this.needToSaveObsAnnotations()) {
       dispatch(actions.saveObsAnnotationsAction());
     }
+    if (!genesetSaveInProgress && this.needToSaveGenesets()) {
+      dispatch(actions.saveGenesetsAction());
+    }
   };
 
-  needToSave = () => {
-    /* return true if we need to save, false if we don't */
+  needToSaveObsAnnotations = () => {
+    /* return true if we need to save obs cell labels, false if we don't */
     const { annoMatrix, lastSavedAnnoMatrix } = this.props;
     return actions.needToSaveObsAnnotations(annoMatrix, lastSavedAnnoMatrix);
   };
 
+  needToSaveGenesets = () => {
+    /* return true if we need to save gene ses, false if we do not */
+    const { genesets, lastSavedGenesets } = this.props;
+    return genesets.initialized && genesets.genesets !== lastSavedGenesets;
+  };
+
+  needToSave() {
+    return this.needToSaveGenesets() || this.needToSaveObsAnnotations();
+  }
+
+  saveInProgress() {
+    const { obsAnnotationSaveInProgress, genesetSaveInProgress } = this.props;
+    return obsAnnotationSaveInProgress || genesetSaveInProgress;
+  }
+
   statusMessage() {
     const { error } = this.props;
     if (error) {
@@ -61,12 +87,12 @@ class Autosave extends React.Component {
   render() {
     const {
       writableCategoriesEnabled,
-      saveInProgress,
+      writableGenesetsEnabled,
       lastSavedAnnoMatrix,
     } = this.props;
     const initialDataLoadComplete = lastSavedAnnoMatrix;
 
-    if (!writableCategoriesEnabled) return null;
+    if (!writableCategoriesEnabled && !writableGenesetsEnabled) return null;
 
     return (
       <div
@@ -74,7 +100,7 @@ class Autosave extends React.Component {
         data-testclass={
           !initialDataLoadComplete
             ? "autosave-init"
-            : this.needToSave() || saveInProgress
+            : this.saveInProgress() || this.needToSave()
             ? "autosave-incomplete"
             : "autosave-complete"
         }

client/src/components/brushableHistogram/error.js

@@ -0,0 +1,15 @@
+import React from "react";
+import * as globals from "../../globals";
+
+const ErrorLoading = ({ displayName, zebra }) => (
+    <div
+      style={{
+        backgroundColor: zebra ? globals.lightestGrey : "white",
+        fontStyle: "italic",
+      }}
+    >
+      <span>{`Failure loading ${displayName}`}</span>
+    </div>
+  );
+
+export default ErrorLoading;

client/src/components/brushableHistogram/footer.js

@@ -0,0 +1,63 @@
+import React from "react";
+
+const HistogramFooter = React.memo(
+  ({
+    displayName,
+    hideRanges,
+    rangeMin,
+    rangeMax,
+    rangeColorMin,
+    rangeColorMax,
+    isObs,
+    isGeneSetSummary,
+  }) => 
+    /*
+    Footer of each histogram.  Will render range and title.
+
+    Required props:
+      * displayName - the displayName, aka "n_genes", "FOXP2", etc.
+      * hideRanges - true/false, enables/disable rendering of ranges
+      * range - length two array, [min, max], containing the range values to display
+      * rangeColor - length two array, [mincolor, maxcolor], each a CSS color
+    */
+     (
+      <div>
+        <div
+          style={{
+            display: "flex",
+            justifyContent: hideRanges ? "center" : "space-between",
+          }}
+        >
+          <span
+            style={{
+              color: rangeColorMin,
+              display: hideRanges ? "none" : "block",
+            }}
+          >
+            min {rangeMin.toPrecision(4)}
+          </span>
+          <span
+            data-testclass="brushable-histogram-field-name"
+            style={{ fontStyle: "italic" }}
+          >
+            {isObs && displayName}
+            {isGeneSetSummary && "gene set mean expression"}
+          </span>
+          <div style={{ display: hideRanges ? "block" : "none" }}>
+            : {rangeMin}
+          </div>
+          <span
+            style={{
+              color: rangeColorMax,
+              display: hideRanges ? "none" : "block",
+            }}
+          >
+            max {rangeMax.toPrecision(4)}
+          </span>
+        </div>
+      </div>
+    )
+  
+);
+
+export default HistogramFooter;

client/src/components/brushableHistogram/header.js

@@ -0,0 +1,100 @@
+import React, { useCallback } from "react";
+import { Button, ButtonGroup, Tooltip, Icon } from "@blueprintjs/core";
+import { IconNames } from "@blueprintjs/icons";
+import * as globals from "../../globals";
+
+const HistogramHeader = React.memo(
+  ({
+    fieldId,
+    isColorBy,
+    onColorByClick,
+    onRemoveClick,
+    isScatterPlotX,
+    isScatterPlotY,
+    onScatterPlotXClick,
+    onScatterPlotYClick,
+    isObs,
+  }) => {
+    /*
+        Render the toolbar for the histogram.  Props:
+          * fieldId - field identifier, used for various IDs
+          * isColorBy - true/false, is this the current color-by
+          * onColorByClick - color-by click handler
+          * onRemoveClick - optional handler for remove.  Button will not render if not defined.
+          * isScatterPlotX - optional, true/false if currently the X scatterplot field
+          * isScatterPlotY - optional, true/false if currently the Y scatterplot field
+          * onScatterPlotXClick - optional, handler for scatterPlot X button.
+          * onScatterPlotYClick - optional, handler for scatterPlot X button.
+  
+        Scatterplot controls will not render if either handler unspecified.
+      */
+
+    const memoizedColorByCallback = useCallback(
+      () => onColorByClick(fieldId, isObs),
+      [fieldId, isObs]
+    );
+
+    return (
+      <div
+        style={{
+          display: "flex",
+          justifyContent: "flex-end",
+          paddingBottom: "8px",
+        }}
+      >
+        {onScatterPlotXClick && onScatterPlotYClick ? (
+          <span>
+            <Icon icon={IconNames.SCATTER_PLOT} style={{ marginRight: 7 }} />
+            <ButtonGroup style={{ marginRight: 7 }}>
+              <Button
+                data-testid={`plot-x-${fieldId}`}
+                onClick={onScatterPlotXClick}
+                active={isScatterPlotX}
+                intent={isScatterPlotX ? "primary" : "none"}
+              >
+                plot x
+              </Button>
+              <Button
+                data-testid={`plot-y-${fieldId}`}
+                onClick={onScatterPlotYClick}
+                active={isScatterPlotY}
+                intent={isScatterPlotY ? "primary" : "none"}
+              >
+                plot y
+              </Button>
+            </ButtonGroup>
+          </span>
+        ) : null}
+        {onRemoveClick ? (
+          <Button
+            minimal
+            onClick={onRemoveClick}
+            style={{
+              color: globals.blue,
+              cursor: "pointer",
+              marginLeft: 7,
+            }}
+          >
+            remove
+          </Button>
+        ) : null}
+        <Tooltip
+          content="Use as color scale"
+          position="bottom"
+          hoverOpenDelay={globals.tooltipHoverOpenDelay}
+        >
+          <Button
+            onClick={memoizedColorByCallback}
+            active={isColorBy}
+            intent={isColorBy ? "primary" : "none"}
+            data-testclass="colorby"
+            data-testid={`colorby-${fieldId}`}
+            icon="tint"
+          />
+        </Tooltip>
+      </div>
+    );
+  }
+);
+
+export default HistogramHeader;

client/src/components/brushableHistogram/histogram.js

@@ -0,0 +1,191 @@
+import React, { useEffect, useRef, useState } from "react";
+import { interpolateCool } from "d3-scale-chromatic";
+import * as d3 from "d3";
+
+import maybeScientific from "../../util/maybeScientific";
+import clamp from "../../util/clamp";
+
+const Histogram = ({
+  field,
+  fieldForId,
+  display,
+  histogram,
+  width,
+  height,
+  onBrush,
+  onBrushEnd,
+  margin,
+  isColorBy,
+  selectionRange,
+  mini,
+}) => {
+  const svgRef = useRef(null);
+  const [brush, setBrush] = useState(null);
+
+  useEffect(() => {
+    /*
+      Create the d3 histogram
+      */
+    //  This is just a constant that's flipped by parent's `mini` boolean
+    const {
+      LEFT: marginLeft,
+      RIGHT: marginRight,
+      BOTTOM: marginBottom,
+      TOP: marginTop,
+    } = margin;
+    const { x, y, bins, binStart, binEnd, binWidth } = histogram;
+    const svg = d3.select(svgRef.current);
+    const binPadding = mini ? 0 : -1;
+    const defaultBarColor = mini ? "black" : "#bbb";
+
+    /* Remove everything */
+    svg.selectAll("*").remove();
+
+    /* Set margins within the SVG */
+    const container = svg
+      .attr("width", width + marginLeft + marginRight)
+      .attr("height", height + marginTop + marginBottom)
+      .append("g")
+      .attr("class", "histogram-container")
+      .attr("transform", `translate(${marginLeft},${marginTop})`);
+
+    const colorScale = d3
+      .scaleSequential(interpolateCool)
+      .domain([0, bins.length]);
+
+    const histogramScale = d3
+      .scaleLinear()
+      .domain(x.domain())
+      .range([
+        colorScale.domain()[1],
+        colorScale.domain()[0],
+      ]); /* we flip this to make colors dark if high in the color scale */
+
+    if (binWidth > 0) {
+      /* BINS */
+      container
+        .insert("g", "*")
+        .selectAll("rect")
+        .data(bins)
+        .enter()
+        .append("rect")
+        .attr("x", (d, i) => x(binStart(i)) + 1)
+        .attr("y", (d) => y(d))
+        .attr("width", (d, i) => x(binEnd(i)) - x(binStart(i)) - binPadding)
+        .attr("height", (d) => y(0) - y(d))
+        .style(
+          "fill",
+          isColorBy
+            ? (d, i) => colorScale(histogramScale(binStart(i)))
+            : defaultBarColor
+        );
+    }
+
+    if (!mini) {
+      // BRUSH
+      // Note the brushable area is bounded by the data on three sides, but goes down to cover the x-axis
+      const brushX = d3
+        .brushX()
+        .extent([
+          [x.range()[0], y.range()[1]],
+          [x.range()[1], marginTop + height + marginBottom],
+        ])
+        /*
+        emit start so that the Undoable history can save an undo point
+        upon drag start, and ignore the subsequent intermediate drag events.
+        */
+        .on("start", onBrush(field, x.invert, "start"))
+        .on("brush", onBrush(field, x.invert, "brush"))
+        .on("end", onBrushEnd(field, x.invert));
+
+      const brushXselection = container
+        .insert("g")
+        .attr("class", "brush")
+        .attr("data-testid", `${svgRef.current.dataset.testid}-brushable-area`)
+        .call(brushX);
+
+      /* X AXIS */
+      container
+        .insert("g")
+        .attr("class", "axis axis--x")
+        .attr("transform", `translate(0,${marginTop + height})`)
+        .call(
+          d3
+            .axisBottom(x)
+            .ticks(4)
+            .tickFormat(d3.format(maybeScientific(x)))
+        );
+
+      /* Y AXIS */
+      container
+        .insert("g")
+        .attr("class", "axis axis--y")
+        .attr("transform", `translate(${marginLeft + width},0)`)
+        .call(
+          d3
+            .axisRight(y)
+            .ticks(3)
+            .tickFormat(
+              d3.format(
+                y.domain().some((n) => Math.abs(n) >= 10000) ? ".0e" : ","
+              )
+            )
+        );
+
+      /* axis style */
+      svg.selectAll(".axis text").style("fill", "rgb(80,80,80)");
+      svg.selectAll(".axis path").style("stroke", "rgb(230,230,230)");
+      svg.selectAll(".axis line").style("stroke", "rgb(230,230,230)");
+
+      setBrush({ brushX, brushXselection });
+    }
+  }, [histogram, isColorBy]);
+
+  useEffect(() => {
+    /*
+      paint/update selection brush
+      */
+    if (!brush) return;
+    const { brushX, brushXselection } = brush;
+    const selection = d3.brushSelection(brushXselection.node());
+    if (!selectionRange && selection) {
+      /* no active selection - clear brush */
+      brushXselection.call(brushX.move, null);
+    } else if (selectionRange) {
+      const { x, domain } = histogram;
+      const [min, max] = domain;
+      const x0 = x(clamp(selectionRange[0], [min, max]));
+      const x1 = x(clamp(selectionRange[1], [min, max]));
+      if (!selection) {
+        /* there is an active selection, but no brush - set the brush */
+        brushXselection.call(brushX.move, [x0, x1]);
+      } else {
+        /* there is an active selection and a brush - make sure they match */
+        const moveDeltaThreshold = 1;
+        const dX0 = Math.abs(x0 - selection[0]);
+        const dX1 = Math.abs(x1 - selection[1]);
+        /*
+          only update the brush if it is grossly incorrect,
+          as defined by the moveDeltaThreshold
+          */
+        if (dX0 > moveDeltaThreshold || dX1 > moveDeltaThreshold) {
+          brushXselection.call(brushX.move, [x0, x1]);
+        }
+      }
+    }
+  }, [brush, selectionRange]);
+
+  return (
+    <svg
+      style={{ display }}
+      width={width}
+      height={height}
+      id={`histogram_${fieldForId}_svg`}
+      data-testclass="histogram-plot"
+      data-testid={`histogram-${field}-plot`}
+      ref={svgRef}
+    />
+  );
+};
+
+export default Histogram;

client/src/components/brushableHistogram/index.js

@@ -1,449 +1,38 @@
-/*
-https://bl.ocks.org/mbostock/4341954
-https://bl.ocks.org/mbostock/34f08d5e11952a80609169b7917d4172
-https://bl.ocks.org/SpaceActuary/2f004899ea1b2bd78d6f1dbb2febf771
-https://bl.ocks.org/mbostock/3019563
-*/
-import React, { useEffect, useRef, useState, useCallback } from "react";
-import { Button, ButtonGroup, Tooltip } from "@blueprintjs/core";
-import { connect } from "react-redux";
+import React from "react";
+import { connect, shallowEqual } from "react-redux";
 import * as d3 from "d3";
-import { interpolateCool } from "d3-scale-chromatic";
 import Async from "react-async";
 import memoize from "memoize-one";
 import * as globals from "../../globals";
 import actions from "../../actions";
-import { histogramContinuous } from "../../util/dataframe/histogram";
 import { makeContinuousDimensionName } from "../../util/nameCreators";
-import significantDigits from "../../util/significantDigits";
+import HistogramHeader from "./header";
+import Histogram from "./histogram";
+import HistogramFooter from "./footer";
+import StillLoading from "./loading";
+import ErrorLoading from "./error";
 
-function clamp(val, rng) {
-  return Math.max(Math.min(val, rng[1]), rng[0]);
-}
-
-function maybeScientific(x) {
-  let format = ",";
-  const _ticks = x.ticks(4);
-
-  if (x.domain().some((n) => Math.abs(n) >= 10000)) {
-    /* 
-      heuristic: if the last tick d3 wants to render has one significant
-      digit ie., 2000, render 2e+3, but if it's anything else ie., 42000000 render
-      4.20e+n
-    */
-    format = significantDigits(_ticks[_ticks.length - 1]) === 1 ? ".0e" : ".2e";
-  }
-
-  return format;
-}
-
-const StillLoading = ({ zebra, displayName }) => {
-  /*
-  Render a loading indicator for the field.
-  */
-  return (
-    <div
-      style={{
-        padding: globals.leftSidebarSectionPadding,
-        backgroundColor: zebra ? globals.lightestGrey : "white",
-      }}
-    >
-      <div
-        style={{
-          display: "flex",
-          justifyContent: "space-between",
-          justifyItems: "center",
-          alignItems: "center",
-        }}
-      >
-        <div style={{ minWidth: 30 }} />
-        <div style={{ display: "flex", alignSelf: "center" }}>
-          <span style={{ fontStyle: "italic" }}>{displayName}</span>
-        </div>
-        <div
-          style={{
-            display: "flex",
-            justifyContent: "flex-end",
-          }}
-        >
-          <Button minimal loading intent="primary" />
-        </div>
-      </div>
-    </div>
-  );
+const MARGIN = {
+  LEFT: 10, // Space for 0 tick label on X axis
+  RIGHT: 54, // space for Y axis & labels
+  BOTTOM: 25, // space for X axis & labels
+  TOP: 3,
 };
-
-const ErrorLoading = ({ displayName, error, zebra }) => {
-  console.log(error); // log to console as this is unexpected
-  return (
-    <div
-      style={{
-        padding: globals.leftSidebarSectionPadding,
-        backgroundColor: zebra ? globals.lightestGrey : "white",
-      }}
-    >
-      <span>{`Failure loading ${displayName}`}</span>
-    </div>
-  );
-};
-
-const HistogramFooter = React.memo(
-  ({
-    displayName,
-    hideRanges,
-    rangeMin,
-    rangeMax,
-    rangeColorMin,
-    rangeColorMax,
-    logFoldChange,
-    pvalAdj,
-  }) => {
-    /*
-  Footer of each histogram.  Will render range, title, and optionally 
-  differential expression info.
-
-  Required props:
-    * displayName - the displayName, aka "n_genes", "FOXP2", etc.
-    * hideRanges - true/false, enables/disable rendering of ranges
-    * range - length two array, [min, max], containing the range values to display
-    * rangeColor - length two array, [mincolor, maxcolor], each a CSS color
-    * logFoldChange - lfc to display, optional.
-    * pValue - pValue to display, optional.
-  */
-    return (
-      <div>
-        <div
-          style={{
-            display: "flex",
-            justifyContent: hideRanges ? "center" : "space-between",
-          }}
-        >
-          <span
-            style={{
-              color: rangeColorMin,
-              display: hideRanges ? "none" : "block",
-            }}
-          >
-            min {rangeMin.toPrecision(4)}
-          </span>
-          <span
-            data-testclass="brushable-histogram-field-name"
-            style={{ fontStyle: "italic" }}
-          >
-            {displayName}
-          </span>
-          <div style={{ display: hideRanges ? "block" : "none" }}>
-            : {rangeMin}
-          </div>
-          <span
-            style={{
-              color: rangeColorMax,
-              display: hideRanges ? "none" : "block",
-            }}
-          >
-            max {rangeMax.toPrecision(4)}
-          </span>
-        </div>
-
-        {logFoldChange && pvalAdj ? (
-          <div
-            style={{
-              display: "flex",
-              justifyContent: "center",
-              alignItems: "baseline",
-            }}
-          >
-            <span>
-              <strong>log fold change:</strong>
-              {` ${logFoldChange.toPrecision(4)}`}
-            </span>
-            <span
-              style={{
-                marginLeft: 7,
-                padding: 2,
-              }}
-            >
-              <strong>p-value (adj):</strong>
-              {pvalAdj < 0.0001 ? " < 0.0001" : ` ${pvalAdj.toFixed(4)}`}
-            </span>
-          </div>
-        ) : null}
-      </div>
-    );
-  }
-);
-
-const HistogramHeader = React.memo(
-  ({
-    fieldId,
-    isColorBy,
-    onColorByClick,
-    onRemoveClick,
-    isScatterPlotX,
-    isScatterPlotY,
-    onScatterPlotXClick,
-    onScatterPlotYClick,
-    isObs,
-  }) => {
-    /*
-      Render the toolbar for the histogram.  Props:
-        * fieldId - field identifier, used for various IDs
-        * isColorBy - true/false, is this the current color-by
-        * onColorByClick - color-by click handler
-        * onRemoveClick - optional handler for remove.  Button will not render if not defined.
-        * isScatterPlotX - optional, true/false if currently the X scatterplot field
-        * isScatterPlotY - optional, true/false if currently the Y scatterplot field
-        * onScatterPlotXClick - optional, handler for scatterPlot X button.
-        * onScatterPlotYClick - optional, handler for scatterPlot X button.
-
-      Scatterplot controls will not render if either handler unspecified.
-    */
-
-    const memoizedColorByCallback = useCallback(
-      () => onColorByClick(fieldId, isObs),
-      [fieldId, isObs]
-    );
-
-    return (
-      <div
-        style={{
-          display: "flex",
-          justifyContent: "flex-end",
-          paddingBottom: "8px",
-        }}
-      >
-        {onScatterPlotXClick && onScatterPlotYClick ? (
-          <span>
-            <span
-              style={{ marginRight: 7 }}
-              className="bp3-icon-standard bp3-icon-scatter-plot"
-            />
-            <ButtonGroup style={{ marginRight: 7 }}>
-              <Button
-                data-testid={`plot-x-${fieldId}`}
-                onClick={onScatterPlotXClick}
-                active={isScatterPlotX}
-                intent={isScatterPlotX ? "primary" : "none"}
-              >
-                plot x
-              </Button>
-              <Button
-                data-testid={`plot-y-${fieldId}`}
-                onClick={onScatterPlotYClick}
-                active={isScatterPlotY}
-                intent={isScatterPlotY ? "primary" : "none"}
-              >
-                plot y
-              </Button>
-            </ButtonGroup>
-          </span>
-        ) : null}
-        {onRemoveClick ? (
-          <Button
-            minimal
-            onClick={onRemoveClick}
-            style={{
-              color: globals.blue,
-              cursor: "pointer",
-              marginLeft: 7,
-            }}
-          >
-            remove
-          </Button>
-        ) : null}
-        <Tooltip
-          content="Use as color scale"
-          position="bottom"
-          hoverOpenDelay={globals.tooltipHoverOpenDelay}
-        >
-          <Button
-            onClick={memoizedColorByCallback}
-            active={isColorBy}
-            intent={isColorBy ? "primary" : "none"}
-            data-testclass="colorby"
-            data-testid={`colorby-${fieldId}`}
-            icon="tint"
-          />
-        </Tooltip>
-      </div>
-    );
-  }
-);
-
-const Histogram = ({
-  field,
-  fieldForId,
-  display,
-  histogram,
-  width,
-  height,
-  onBrush,
-  onBrushEnd,
-  margin,
-  isColorBy,
-  selectionRange,
-}) => {
-  const svgRef = useRef(null);
-  const [brush, setBrush] = useState(null);
-
-  useEffect(() => {
-    /*
-    Create the d3 histogram
-    */
-    const { marginLeft, marginRight, marginBottom, marginTop } = margin;
-    const { x, y, bins, binStart, binEnd, binWidth } = histogram;
-    const svg = d3.select(svgRef.current);
-
-    /* Remove everything */
-    svg.selectAll("*").remove();
-
-    /* Set margins within the SVG */
-    const container = svg
-      .attr("width", width + marginLeft + marginRight)
-      .attr("height", height + marginTop + marginBottom)
-      .append("g")
-      .attr("class", "histogram-container")
-      .attr("transform", `translate(${marginLeft},${marginTop})`);
-
-    const colorScale = d3
-      .scaleSequential(interpolateCool)
-      .domain([0, bins.length]);
-
-    const histogramScale = d3
-      .scaleLinear()
-      .domain(x.domain())
-      .range([
-        colorScale.domain()[1],
-        colorScale.domain()[0],
-      ]); /* we flip this to make colors dark if high in the color scale */
-
-    if (binWidth > 0) {
-      /* BINS */
-      container
-        .insert("g", "*")
-        .selectAll("rect")
-        .data(bins)
-        .enter()
-        .append("rect")
-        .attr("x", (d, i) => x(binStart(i)) + 1)
-        .attr("y", (d) => y(d))
-        .attr("width", (d, i) => x(binEnd(i)) - x(binStart(i)) - 1)
-        .attr("height", (d) => y(0) - y(d))
-        .style(
-          "fill",
-          isColorBy ? (d, i) => colorScale(histogramScale(binStart(i))) : "#bbb"
-        );
-    }
-
-    // BRUSH
-    // Note the brushable area is bounded by the data on three sides, but goes down to cover the x-axis
-    const brushX = d3
-      .brushX()
-      .extent([
-        [x.range()[0], y.range()[1]],
-        [x.range()[1], marginTop + height + marginBottom],
-      ])
-      /*
-      emit start so that the Undoable history can save an undo point
-      upon drag start, and ignore the subsequent intermediate drag events.
-      */
-      .on("start", onBrush(field, x.invert, "start"))
-      .on("brush", onBrush(field, x.invert, "brush"))
-      .on("end", onBrushEnd(field, x.invert));
-
-    const brushXselection = container
-      .insert("g")
-      .attr("class", "brush")
-      .attr("data-testid", `${svgRef.current.dataset.testid}-brushable-area`)
-      .call(brushX);
-
-    /* X AXIS */
-    container
-      .insert("g")
-      .attr("class", "axis axis--x")
-      .attr("transform", `translate(0,${marginTop + height})`)
-      .call(
-        d3
-          .axisBottom(x)
-          .ticks(4)
-          .tickFormat(d3.format(maybeScientific(x)))
-      );
-
-    /* Y AXIS */
-    container
-      .insert("g")
-      .attr("class", "axis axis--y")
-      .attr("transform", `translate(${marginLeft + width},0)`)
-      .call(
-        d3
-          .axisRight(y)
-          .ticks(3)
-          .tickFormat(
-            d3.format(
-              y.domain().some((n) => Math.abs(n) >= 10000) ? ".0e" : ","
-            )
-          )
-      );
-
-    /* axis style */
-    svg.selectAll(".axis text").style("fill", "rgb(80,80,80)");
-    svg.selectAll(".axis path").style("stroke", "rgb(230,230,230)");
-    svg.selectAll(".axis line").style("stroke", "rgb(230,230,230)");
-
-    setBrush({ brushX, brushXselection });
-  }, [histogram, isColorBy]);
-
-  useEffect(() => {
-    /*
-    paint/update selection brush
-    */
-    if (!brush) return;
-    const { brushX, brushXselection } = brush;
-    const selection = d3.brushSelection(brushXselection.node());
-    if (!selectionRange && selection) {
-      /* no active selection - clear brush */
-      brushXselection.call(brushX.move, null);
-    } else if (selectionRange) {
-      const { x, domain } = histogram;
-      const [min, max] = domain;
-      const x0 = x(clamp(selectionRange[0], [min, max]));
-      const x1 = x(clamp(selectionRange[1], [min, max]));
-      if (!selection) {
-        /* there is an active selection, but no brush - set the brush */
-        brushXselection.call(brushX.move, [x0, x1]);
-      } else {
-        /* there is an active selection and a brush - make sure they match */
-        const moveDeltaThreshold = 1;
-        const dX0 = Math.abs(x0 - selection[0]);
-        const dX1 = Math.abs(x1 - selection[1]);
-        /*
-        only update the brush if it is grossly incorrect,
-        as defined by the moveDeltaThreshold
-        */
-        if (dX0 > moveDeltaThreshold || dX1 > moveDeltaThreshold) {
-          brushXselection.call(brushX.move, [x0, x1]);
-        }
-      }
-    }
-  }, [brush, selectionRange]);
-
-  return (
-    <svg
-      style={{ display }}
-      width={width}
-      height={height}
-      id={`histogram_${fieldForId}_svg`}
-      data-testclass="histogram-plot"
-      data-testid={`histogram-${field}-plot`}
-      ref={svgRef}
-    />
-  );
+const WIDTH = 340 - MARGIN.LEFT - MARGIN.RIGHT;
+const HEIGHT = 135 - MARGIN.TOP - MARGIN.BOTTOM;
+const MARGIN_MINI = {
+  LEFT: 0, // Space for 0 tick label on X axis
+  RIGHT: 0, // space for Y axis & labels
+  BOTTOM: 0, // space for X axis & labels
+  TOP: 0,
 };
+const WIDTH_MINI = 120 - MARGIN_MINI.LEFT - MARGIN_MINI.RIGHT;
+const HEIGHT_MINI = 15 - MARGIN_MINI.TOP - MARGIN_MINI.BOTTOM;
 
 @connect((state, ownProps) => {
-  const { isObs, isUserDefined, isDiffExp, field } = ownProps;
+  const { isObs, isUserDefined, isGeneSetSummary, field } = ownProps;
   const myName = makeContinuousDimensionName(
-    { isObs, isUserDefined, isDiffExp },
+    { isObs, isUserDefined, isGeneSetSummary },
     field
   );
   return {
@@ -455,6 +44,10 @@ const Histogram = ({
   };
 })
 class HistogramBrush extends React.PureComponent {
+  static watchAsync(props, prevProps) {
+    return !shallowEqual(props.watchProps, prevProps.watchProps);
+  }
+
   /* memoized closure to prevent HistogramHeader unecessary repaint */
   handleColorAction = memoize((dispatch) => (field, isObs) => {
     if (isObs) {
@@ -467,27 +60,11 @@ class HistogramBrush extends React.PureComponent {
     }
   });
 
-  constructor(props) {
-    super(props);
-
-    const marginLeft = 10; // Space for 0 tick label on X axis
-    const marginRight = 54; // space for Y axis & labels
-    const marginBottom = 25; // space for X axis & labels
-    const marginTop = 3;
-    this.margin = {
-      marginLeft,
-      marginRight,
-      marginBottom,
-      marginTop,
-    };
-    this.width = 340 - marginLeft - marginRight;
-    this.height = 135 - marginTop - marginBottom;
-  }
-
   onBrush = (selection, x, eventType) => {
     const type = `continuous metadata histogram ${eventType}`;
     return () => {
-      const { dispatch, field, isObs, isUserDefined, isDiffExp } = this.props;
+      const { dispatch, field, isObs, isUserDefined, isGeneSetSummary } =
+        this.props;
 
       // ignore programmatically generated events
       if (!d3.event.sourceEvent) return;
@@ -503,7 +80,7 @@ class HistogramBrush extends React.PureComponent {
         continuousNamespace: {
           isObs,
           isUserDefined,
-          isDiffExp,
+          isGeneSetSummary,
         },
       };
       dispatch(
@@ -512,9 +89,9 @@ class HistogramBrush extends React.PureComponent {
     };
   };
 
-  onBrushEnd = (selection, x) => {
-    return () => {
-      const { dispatch, field, isObs, isUserDefined, isDiffExp } = this.props;
+  onBrushEnd = (selection, x) => () => {
+      const { dispatch, field, isObs, isUserDefined, isGeneSetSummary } =
+        this.props;
       const minAllowedBrushSize = 10;
       const smallAmountToAvoidInfiniteLoop = 0.1;
 
@@ -553,14 +130,13 @@ class HistogramBrush extends React.PureComponent {
         continuousNamespace: {
           isObs,
           isUserDefined,
-          isDiffExp,
+          isGeneSetSummary,
         },
       };
       dispatch(
         actions.selectContinuousMetadataAction(type, query, range, otherProps)
       );
     };
-  };
 
   handleSetGeneAsScatterplotX = () => {
     const { dispatch, field } = this.props;
@@ -610,7 +186,8 @@ class HistogramBrush extends React.PureComponent {
   };
 
   fetchAsyncProps = async () => {
-    const { annoMatrix } = this.props;
+    const { annoMatrix, width } = this.props;
+
     const { isClipped } = annoMatrix;
 
     const query = this.createQuery();
@@ -640,9 +217,15 @@ class HistogramBrush extends React.PureComponent {
 
     const histogram = this.calcHistogramCache(
       column,
-      this.margin,
-      this.width,
-      this.height
+      MARGIN,
+      width || WIDTH,
+      HEIGHT
+    );
+    const miniHistogram = this.calcHistogramCache(
+      column,
+      MARGIN_MINI,
+      width || WIDTH_MINI,
+      HEIGHT_MINI
     );
 
     const isSingleValue = summary.min === summary.max;
@@ -656,6 +239,7 @@ class HistogramBrush extends React.PureComponent {
 
     return {
       histogram,
+      miniHistogram,
       range,
       unclippedRange,
       unclippedRangeColor,
@@ -665,27 +249,33 @@ class HistogramBrush extends React.PureComponent {
   };
 
   // eslint-disable-next-line class-methods-use-this -- instance method allows for memoization per annotation
-  calcHistogramCache(col, margin, width, height) {
+  calcHistogramCache(col, newMargin, newWidth, newHeight) {
     /*
      recalculate expensive stuff, notably bins, summaries, etc.
     */
-    const histogramCache = {};
-    const summary = col.summarize();
+    const histogramCache = {}; /* maybe change this so that it computes ... */
+    const summary =
+      col.summarize(); /* this is memoized, so it's free the second time you call it */
     const { min: domainMin, max: domainMax } = summary;
     const numBins = 40;
-    const { marginTop, marginLeft } = margin;
+    const { TOP: topMargin, LEFT: leftMargin } =
+      newMargin; /* changes with mini */
 
-    histogramCache.domain = [domainMin, domainMax];
+    histogramCache.domain = [
+      domainMin,
+      domainMax,
+    ]; /* doesn't change with mini */
 
     histogramCache.x = d3
       .scaleLinear()
       .domain([domainMin, domainMax])
-      .range([marginLeft, marginLeft + width]);
+      .range([leftMargin, leftMargin + newWidth]);
 
-    histogramCache.bins = histogramContinuous(col, numBins, [
+    histogramCache.bins = col.histogram(numBins, [
       domainMin,
       domainMax,
-    ]);
+    ]); /* memoized */
+
     histogramCache.binWidth = (domainMax - domainMin) / numBins;
 
     histogramCache.binStart = (i) => domainMin + i * histogramCache.binWidth;
@@ -697,25 +287,43 @@ class HistogramBrush extends React.PureComponent {
     histogramCache.y = d3
       .scaleLinear()
       .domain([0, yMax])
-      .range([marginTop + height, marginTop]);
+      .range([topMargin + newHeight, topMargin]);
 
     return histogramCache;
   }
 
   createQuery() {
-    const { isObs, field, annoMatrix } = this.props;
+    const { isObs, isGeneSetSummary, field, setGenes, annoMatrix } = this.props;
     const { schema } = annoMatrix;
     if (isObs) {
       return ["obs", field];
     }
     const varIndex = schema?.annotations?.var?.index;
     if (!varIndex) return null;
+
+    if (isGeneSetSummary) {
+      return [
+        "X",
+        {
+          summarize: {
+            method: "mean",
+            field: "var",
+            column: varIndex,
+            values: [...setGenes.keys()],
+          },
+        },
+      ];
+    }
+
+    // else, we assume it is a gene expression
     return [
       "X",
       {
-        field: "var",
-        column: varIndex,
-        value: field,
+        where: {
+          field: "var",
+          column: varIndex,
+          value: field,
+        },
       },
     ];
   }
@@ -727,20 +335,34 @@ class HistogramBrush extends React.PureComponent {
       field,
       isColorAccessor,
       isUserDefined,
-      isDiffExp,
-      logFoldChange,
-      pvalAdj,
+      isGeneSetSummary,
       isScatterplotXXaccessor,
       isScatterplotYYaccessor,
       zebra,
       continuousSelectionRange,
       isObs,
+      mini,
+      setGenes,
     } = this.props;
+
+    let { width } = this.props;
+    if (!width) {
+      width = mini ? WIDTH_MINI : WIDTH;
+    }
+
     const fieldForId = field.replace(/\s/g, "_");
-    const showScatterPlot = isDiffExp || isUserDefined;
+    const showScatterPlot = isUserDefined;
+
+    let testClass = "histogram-continuous-metadata";
+    if (isUserDefined) testClass = "histogram-user-gene";
+    else if (isGeneSetSummary) testClass = "histogram-gene-set-summary";
 
     return (
-      <Async watch={annoMatrix} promiseFn={this.fetchAsyncProps}>
+      <Async
+        watchFn={HistogramBrush.watchAsync}
+        promiseFn={this.fetchAsyncProps}
+        watchProps={{ annoMatrix, setGenes }}
+      >
         <Async.Pending initial>
           <StillLoading displayName={field} zebra={zebra} />
         </Async.Pending>
@@ -755,56 +377,57 @@ class HistogramBrush extends React.PureComponent {
               <div
                 id={`histogram_${fieldForId}`}
                 data-testid={`histogram-${field}`}
-                data-testclass={
-                  isDiffExp
-                    ? "histogram-diffexp"
-                    : isUserDefined
-                    ? "histogram-user-gene"
-                    : "histogram-continuous-metadata"
-                }
+                data-testclass={testClass}
                 style={{
-                  padding: globals.leftSidebarSectionPadding,
+                  padding: mini ? 0 : globals.leftSidebarSectionPadding,
                   backgroundColor: zebra ? globals.lightestGrey : "white",
                 }}
               >
-                <HistogramHeader
-                  fieldId={field}
-                  isColorBy={isColorAccessor}
-                  isObs={isObs}
-                  onColorByClick={this.handleColorAction(dispatch)}
-                  onRemoveClick={isUserDefined ? this.removeHistogram : null}
-                  isScatterPlotX={isScatterplotXXaccessor}
-                  isScatterPlotY={isScatterplotYYaccessor}
-                  onScatterPlotXClick={
-                    showScatterPlot ? this.handleSetGeneAsScatterplotX : null
-                  }
-                  onScatterPlotYClick={
-                    showScatterPlot ? this.handleSetGeneAsScatterplotY : null
-                  }
-                />
+                {!mini && isObs ? (
+                  <HistogramHeader
+                    fieldId={field}
+                    isColorBy={isColorAccessor}
+                    isObs={isObs}
+                    onColorByClick={this.handleColorAction(dispatch)}
+                    onRemoveClick={isUserDefined ? this.removeHistogram : null}
+                    isScatterPlotX={isScatterplotXXaccessor}
+                    isScatterPlotY={isScatterplotYYaccessor}
+                    onScatterPlotXClick={
+                      showScatterPlot ? this.handleSetGeneAsScatterplotX : null
+                    }
+                    onScatterPlotYClick={
+                      showScatterPlot ? this.handleSetGeneAsScatterplotY : null
+                    }
+                  />
+                ) : null}
                 <Histogram
                   field={field}
                   fieldForId={fieldForId}
                   display={asyncProps.isSingleValue ? "none" : "block"}
-                  histogram={asyncProps.histogram}
-                  width={this.width}
-                  height={this.height}
+                  histogram={
+                    mini ? asyncProps.miniHistogram : asyncProps.histogram
+                  }
+                  width={width}
+                  height={mini ? HEIGHT_MINI : HEIGHT}
                   onBrush={this.onBrush}
                   onBrushEnd={this.onBrushEnd}
-                  margin={this.margin}
+                  margin={mini ? MARGIN_MINI : MARGIN}
                   isColorBy={isColorAccessor}
                   selectionRange={continuousSelectionRange}
+                  mini={mini}
                 />
-                <HistogramFooter
-                  displayName={field}
-                  hideRanges={asyncProps.isSingleValue}
-                  rangeMin={asyncProps.unclippedRange[0]}
-                  rangeMax={asyncProps.unclippedRange[1]}
-                  rangeColorMin={asyncProps.unclippedRangeColor[0]}
-                  rangeColorMax={asyncProps.unclippedRangeColor[1]}
-                  logFoldChange={logFoldChange}
-                  pvalAdj={pvalAdj}
-                />
+                {!mini && (
+                  <HistogramFooter
+                    isGeneSetSummary={isGeneSetSummary}
+                    isObs={isObs}
+                    displayName={field}
+                    hideRanges={asyncProps.isSingleValue}
+                    rangeMin={asyncProps.unclippedRange[0]}
+                    rangeMax={asyncProps.unclippedRange[1]}
+                    rangeColorMin={asyncProps.unclippedRangeColor[0]}
+                    rangeColorMax={asyncProps.unclippedRangeColor[1]}
+                  />
+                )}
               </div>
             ) : null
           }

client/src/components/brushableHistogram/loading.js

@@ -0,0 +1,43 @@
+import React from "react";
+import { Button } from "@blueprintjs/core";
+
+import * as globals from "../../globals";
+
+const StillLoading = ({ zebra, displayName }) => 
+  /*
+    Render a loading indicator for the field.
+    */
+   (
+    <div
+      data-testclass="gene-loading-spinner"
+      style={{
+        padding: globals.leftSidebarSectionPadding,
+        backgroundColor: zebra ? globals.lightestGrey : "white",
+      }}
+    >
+      <div
+        style={{
+          display: "flex",
+          justifyContent: "space-between",
+          justifyItems: "center",
+          alignItems: "center",
+        }}
+      >
+        <div style={{ minWidth: 30 }} />
+        <div style={{ display: "flex", alignSelf: "center" }}>
+          <span style={{ fontStyle: "italic" }}>{displayName}</span>
+        </div>
+        <div
+          style={{
+            display: "flex",
+            justifyContent: "flex-end",
+          }}
+        >
+          <Button minimal loading intent="primary" />
+        </div>
+      </div>
+    </div>
+  )
+;
+
+export default StillLoading;

client/src/components/categorical/annoSelect.js

@@ -26,16 +26,14 @@ class DuplicateCategorySelect extends React.PureComponent {
             [] /* this is a placeholder, could be  a subcomponent to avoid this */
           }
           filterable={false}
-          itemRenderer={(d, { handleClick }) => {
-            return (
+          itemRenderer={(d, { handleClick }) => (
               <MenuItem
                 data-testclass="duplicate-category-dropdown-option"
                 onClick={handleClick}
                 key={d}
                 text={d}
               />
-            );
-          }}
+            )}
           noResults={<MenuItem disabled text="No results." />}
           onItemSelect={(d) => {
             handleModalDuplicateCategorySelection(d);

client/src/components/categorical/category/annoDialogAddLabel.js

@@ -1,14 +1,13 @@
 import React from "react";
 import { connect } from "react-redux";
-import AnnoDialog from "../annoDialog";
-import LabelInput from "../labelInput";
+import AnnoDialog from "../../annoDialog";
+import LabelInput from "../../labelInput";
 import { labelPrompt, isLabelErroneous } from "../labelUtil";
 import actions from "../../../actions";
 
 @connect((state) => ({
   annotations: state.annotations,
   schema: state.annoMatrix?.schema,
-  ontology: state.ontology,
   obsCrossfilter: state.obsCrossfilter,
 }))
 class Category extends React.PureComponent {
@@ -57,13 +56,11 @@ class Category extends React.PureComponent {
   };
 
   labelNameError = (name) => {
-    const { metadataField, ontology, schema } = this.props;
-    return isLabelErroneous(name, metadataField, ontology, schema);
+    const { metadataField, schema } = this.props;
+    return isLabelErroneous(name, metadataField, schema);
   };
 
-  instruction = (label) => {
-    return labelPrompt(this.labelNameError(label), "New, unique label", ":");
-  };
+  instruction = (label) => labelPrompt(this.labelNameError(label), "New, unique label", ":");
 
   handleChangeOrSelect = (label) => {
     this.setState({ newLabelText: label });
@@ -71,8 +68,7 @@ class Category extends React.PureComponent {
 
   render() {
     const { newLabelText } = this.state;
-    const { metadataField, annotations, ontology, obsCrossfilter } = this.props;
-    const ontologyEnabled = ontology?.enabled ?? false;
+    const { metadataField, annotations, obsCrossfilter } = this.props;
 
     return (
       <>
@@ -97,7 +93,7 @@ class Category extends React.PureComponent {
           handleCancel={this.disableAddNewLabelMode}
           annoInput={
             <LabelInput
-              labelSuggestions={ontologyEnabled ? ontology.terms : null}
+              labelSuggestions={null}
               onChange={this.handleChangeOrSelect}
               onSelect={this.handleChangeOrSelect}
               inputProps={{

client/src/components/categorical/category/annoDialogEditCategoryName.js

@@ -1,7 +1,7 @@
 import React from "react";
 import { connect } from "react-redux";
-import AnnoDialog from "../annoDialog";
-import LabelInput from "../labelInput";
+import AnnoDialog from "../../annoDialog";
+import LabelInput from "../../labelInput";
 import { labelPrompt } from "../labelUtil";
 
 import { AnnotationsHelpers } from "../../../util/stateManager";
@@ -10,7 +10,6 @@ import actions from "../../../actions";
 @connect((state) => ({
   annotations: state.annotations,
   schema: state.annoMatrix?.schema,
-  ontology: state.ontology,
 }))
 class AnnoDialogEditCategoryName extends React.PureComponent {
   constructor(props) {
@@ -90,13 +89,11 @@ class AnnoDialogEditCategoryName extends React.PureComponent {
     return false;
   };
 
-  instruction = (name) => {
-    return labelPrompt(
+  instruction = (name) => labelPrompt(
       this.editedCategoryNameError(name),
       "New, unique category name",
       ":"
     );
-  };
 
   allCategoryNames() {
     const { schema } = this.props;
@@ -105,8 +102,7 @@ class AnnoDialogEditCategoryName extends React.PureComponent {
 
   render() {
     const { newCategoryText } = this.state;
-    const { metadataField, annotations, ontology } = this.props;
-    const ontologyEnabled = ontology?.enabled ?? false;
+    const { metadataField, annotations } = this.props;
 
     return (
       <>
@@ -132,7 +128,7 @@ class AnnoDialogEditCategoryName extends React.PureComponent {
           annoInput={
             <LabelInput
               label={newCategoryText}
-              labelSuggestions={ontologyEnabled ? ontology.terms : null}
+              labelSuggestions={null}
               onChange={this.handleChangeOrSelect}
               onSelect={this.handleChangeOrSelect}
               inputProps={{

client/src/components/categorical/category/annoMenuCategory.js

@@ -9,7 +9,9 @@ import {
   Tooltip,
   Icon,
   PopoverInteractionKind,
+  Intent,
 } from "@blueprintjs/core";
+import { IconNames } from "@blueprintjs/icons";
 
 import * as globals from "../../../globals";
 import actions from "../../../actions";
@@ -89,8 +91,8 @@ class AnnoMenuCategory extends React.PureComponent {
                     text={editText}
                   />
                   <MenuItem
-                    icon="delete"
-                    intent="danger"
+                    icon={IconNames.TRASH}
+                    intent={Intent.DANGER}
                     data-testclass="handleDeleteCategory"
                     data-testid={`${metadataField}:delete-category`}
                     onClick={this.handleDeleteCategory}

client/src/components/categorical/category/index.js

@@ -1,7 +1,13 @@
 import React, { useRef, useEffect } from "react";
 import { connect, shallowEqual } from "react-redux";
 import { FaChevronRight, FaChevronDown } from "react-icons/fa";
-import { AnchorButton, Button, Tooltip, Position } from "@blueprintjs/core";
+import {
+  AnchorButton,
+  Button,
+  Classes,
+  Position,
+  Tooltip,
+} from "@blueprintjs/core";
 import { Flipper, Flipped } from "react-flip-toolkit";
 import Async from "react-async";
 import memoize from "memoize-one";
@@ -38,6 +44,7 @@ const LABEL_WIDTH_ANNO = LABEL_WIDTH - ANNO_BUTTON_WIDTH;
     schema,
     crossfilter: state.obsCrossfilter,
     isUserAnno,
+    genesets: state.genesets.genesets,
   };
 })
 class Category extends React.PureComponent {
@@ -136,9 +143,15 @@ class Category extends React.PureComponent {
     */
     const { schema } = annoMatrix;
     const { colorAccessor, colorMode } = colors;
+    const { genesets } = this.props;
     let colorDataPromise = Promise.resolve(null);
     if (colorAccessor) {
-      const query = createColorQuery(colorMode, colorAccessor, schema);
+      const query = createColorQuery(
+        colorMode,
+        colorAccessor,
+        schema,
+        genesets
+      );
       if (query) colorDataPromise = annoMatrix.fetch(...query);
     }
     const [categoryData, colorData] = await Promise.all([
@@ -245,14 +258,12 @@ class Category extends React.PureComponent {
                 isColorAccessor,
                 handleCategoryToggleAllClick,
               } = asyncProps;
-              const isTruncated = !!categorySummary?.isTruncated;
               const selectionState = this.getSelectionState(categorySummary);
               return (
                 <CategoryRender
                   metadataField={metadataField}
                   checkboxID={checkboxID}
                   isUserAnno={isUserAnno}
-                  isTruncated={isTruncated}
                   isExpanded={isExpanded}
                   isColorAccessor={isColorAccessor}
                   selectionState={selectionState}
@@ -277,54 +288,54 @@ class Category extends React.PureComponent {
 
 export default Category;
 
-const StillLoading = ({ metadataField, checkboxID }) => {
+const StillLoading = ({ metadataField, checkboxID }) => (
   /*
   We are still loading this category, so render a "busy" signal.
   */
-  return (
+  <div
+    style={{
+      maxWidth: globals.maxControlsWidth,
+    }}
+  >
     <div
       style={{
-        maxWidth: globals.maxControlsWidth,
+        display: "flex",
+        justifyContent: "space-between",
+        alignItems: "baseline",
       }}
     >
       <div
         style={{
           display: "flex",
-          justifyContent: "space-between",
-          alignItems: "baseline",
+          justifyContent: "flex-start",
+          alignItems: "flex-start",
         }}
       >
-        <div
-          style={{
-            display: "flex",
-            justifyContent: "flex-start",
-            alignItems: "flex-start",
-          }}
+        <label
+          htmlFor={checkboxID}
+          className={`${Classes.CONTROL} ${Classes.CHECKBOX}`}
         >
-          <label htmlFor={checkboxID} className="bp3-control bp3-checkbox">
-            <input disabled id={checkboxID} checked type="checkbox" />
-            <span className="bp3-control-indicator" />
-          </label>
-          <Truncate>
-            <span
-              style={{
-                cursor: "pointer",
-                display: "inline-block",
-                width: LABEL_WIDTH,
-              }}
-            >
-              {metadataField}
-            </span>
-          </Truncate>
-        </div>
-        <div>
-          <Button minimal loading intent="primary" />
-        </div>
+          <input disabled id={checkboxID} checked type="checkbox" />
+          <span className={Classes.CONTROL_INDICATOR} />
+        </label>
+        <Truncate>
+          <span
+            style={{
+              cursor: "pointer",
+              display: "inline-block",
+              width: LABEL_WIDTH,
+            }}
+          >
+            {metadataField}
+          </span>
+        </Truncate>
+      </div>
+      <div>
+        <Button minimal loading intent="primary" />
       </div>
     </div>
-  );
-};
-
+  </div>
+);
 const ErrorLoading = ({ metadataField, error }) => {
   console.error(error); // log error to console as it is unexpected.
   return (
@@ -348,7 +359,6 @@ const CategoryHeader = React.memo(
     metadataField,
     checkboxID,
     isUserAnno,
-    isTruncated,
     isColorAccessor,
     isExpanded,
     selectionState,
@@ -375,7 +385,10 @@ const CategoryHeader = React.memo(
             alignItems: "flex-start",
           }}
         >
-          <label className="bp3-control bp3-checkbox" htmlFor={checkboxID}>
+          <label
+            className={`${Classes.CONTROL} ${Classes.CHECKBOX}`}
+            htmlFor={checkboxID}
+          >
             <input
               id={checkboxID}
               data-testclass="category-select"
@@ -385,7 +398,7 @@ const CategoryHeader = React.memo(
               checked={selectionState === "all"}
               type="checkbox"
             />
-            <span className="bp3-control-indicator" />
+            <span className={Classes.CONTROL_INDICATOR} />
           </label>
           <span
             role="menuitem"
@@ -404,6 +417,7 @@ const CategoryHeader = React.memo(
                   maxWidth: isUserAnno ? LABEL_WIDTH_ANNO : LABEL_WIDTH,
                 }}
                 data-testid={`${metadataField}:category-label`}
+                tabIndex="-1"
               >
                 {metadataField}
               </span>
@@ -421,8 +435,8 @@ const CategoryHeader = React.memo(
             )}
           </span>
         </div>
-        {<AnnoDialogEditCategoryName metadataField={metadataField} />}
-        {<AnnoDialogAddLabel metadataField={metadataField} />}
+        <AnnoDialogEditCategoryName metadataField={metadataField} />
+        <AnnoDialogAddLabel metadataField={metadataField} />
         <div>
           <AnnoMenu
             metadataField={metadataField}
@@ -433,11 +447,7 @@ const CategoryHeader = React.memo(
           />
 
           <Tooltip
-            content={
-              isTruncated
-                ? `Coloring by ${metadataField} is disabled, as it exceeds the limit of ${globals.maxCategoricalOptionsToDisplay} labels`
-                : "Use as color scale"
-            }
+            content="Use as color scale"
             position={Position.LEFT}
             usePortal
             hoverOpenDelay={globals.tooltipHoverOpenDelay}
@@ -452,7 +462,6 @@ const CategoryHeader = React.memo(
               onClick={onColorChangeClick}
               active={isColorAccessor}
               intent={isColorAccessor ? "primary" : "none"}
-              disabled={isTruncated}
               icon="tint"
             />
           </Tooltip>
@@ -467,7 +476,6 @@ const CategoryRender = React.memo(
     metadataField,
     checkboxID,
     isUserAnno,
-    isTruncated,
     isColorAccessor,
     isExpanded,
     selectionState,
@@ -486,7 +494,6 @@ const CategoryRender = React.memo(
     */
     const { numCategoryValues } = categorySummary;
     const isSingularValue = !isUserAnno && numCategoryValues === 1;
-
     if (isSingularValue) {
       /*
       Entire category has a single value, special case.
@@ -528,7 +535,6 @@ const CategoryRender = React.memo(
             metadataField={metadataField}
             checkboxID={checkboxID}
             isUserAnno={isUserAnno}
-            isTruncated={isTruncated}
             isExpanded={isExpanded}
             isColorAccessor={isColorAccessor}
             selectionState={selectionState}
@@ -554,11 +560,6 @@ const CategoryRender = React.memo(
             ) : null
           }
         </div>
-        <div>
-          {isExpanded && isTruncated ? (
-            <p style={{ paddingLeft: 15 }}>... truncated list ...</p>
-          ) : null}
-        </div>
       </div>
     );
   }

client/src/components/categorical/index.js

@@ -1,20 +1,18 @@
-// jshint esversion: 6
 import React from "react";
-import { Button } from "@blueprintjs/core";
+import { AnchorButton, Tooltip, Position } from "@blueprintjs/core";
 import { connect } from "react-redux";
 import * as globals from "../../globals";
 import Category from "./category";
 import { AnnotationsHelpers, ControlsHelpers } from "../../util/stateManager";
-import AnnoDialog from "./annoDialog";
+import AnnoDialog from "../annoDialog";
 import AnnoSelect from "./annoSelect";
-import LabelInput from "./labelInput";
+import LabelInput from "../labelInput";
 import { labelPrompt } from "./labelUtil";
 import actions from "../../actions";
 
 @connect((state) => ({
   writableCategoriesEnabled: state.config?.parameters?.annotations ?? false,
   schema: state.annoMatrix?.schema,
-  ontology: state.ontology,
 }))
 class Categories extends React.Component {
   constructor(props) {
@@ -99,13 +97,8 @@ class Categories extends React.Component {
     this.setState({ newCategoryText: name });
   };
 
-  instruction = (name) => {
-    return labelPrompt(
-      this.categoryNameError(name),
-      "New, unique category name",
-      ":"
-    );
-  };
+  instruction = (name) =>
+    labelPrompt(this.categoryNameError(name), "New, unique category name", ":");
 
   onExpansionChange = (catName) => {
     const { expandedCats } = this.state;
@@ -127,12 +120,10 @@ class Categories extends React.Component {
       newCategoryText,
       expandedCats,
     } = this.state;
-    const { writableCategoriesEnabled, schema, ontology } = this.props;
-    const ontologyEnabled = ontology?.enabled ?? false;
+    const { writableCategoriesEnabled, schema } = this.props;
     /* all names, sorted in display order.  Will be rendered in this order */
-    const allCategoryNames = ControlsHelpers.selectableCategoryNames(
-      schema
-    ).sort();
+    const allCategoryNames =
+      ControlsHelpers.selectableCategoryNames(schema).sort();
 
     return (
       <div
@@ -153,7 +144,7 @@ class Categories extends React.Component {
           handleCancel={this.handleDisableAnnoMode}
           annoInput={
             <LabelInput
-              labelSuggestions={ontologyEnabled ? ontology.terms : null}
+              labelSuggestions={null}
               onChange={this.handleChange}
               onSelect={this.handleSelect}
               inputProps={{
@@ -176,18 +167,43 @@ class Categories extends React.Component {
           }
         />
 
+        {writableCategoriesEnabled ? (
+          <div style={{ marginBottom: 10 }}>
+            <Tooltip
+              content="Create a new category"
+              position={Position.RIGHT}
+              boundary="viewport"
+              hoverOpenDelay={globals.tooltipHoverOpenDelay}
+              modifiers={{
+                preventOverflow: { enabled: false },
+                hide: { enabled: false },
+              }}
+            >
+              <AnchorButton
+                type="button"
+                data-testid="open-annotation-dialog"
+                onClick={this.handleEnableAnnoMode}
+                intent="primary"
+              >
+                Create new <strong>category</strong>
+              </AnchorButton>
+            </Tooltip>
+          </div>
+        ) : null}
+
         {/* READ ONLY CATEGORICAL FIELDS */}
         {/* this is duplicative but flat, could be abstracted */}
-        {allCategoryNames.map((catName) =>
-          !schema.annotations.obsByName[catName].writable ? (
-            <Category
-              key={catName}
-              metadataField={catName}
-              onExpansionChange={this.onExpansionChange}
-              isExpanded={expandedCats.has(catName)}
-              createAnnoModeActive={createAnnoModeActive}
-            />
-          ) : null
+        {allCategoryNames.map(
+          (catName) =>
+            !schema.annotations.obsByName[catName].writable && (
+              <Category
+                key={catName}
+                metadataField={catName}
+                onExpansionChange={this.onExpansionChange}
+                isExpanded={expandedCats.has(catName)}
+                createAnnoModeActive={createAnnoModeActive}
+              />
+            )
         )}
         {/* WRITEABLE FIELDS */}
         {allCategoryNames.map((catName) =>
@@ -201,18 +217,6 @@ class Categories extends React.Component {
             />
           ) : null
         )}
-
-        {writableCategoriesEnabled ? (
-          <div>
-            <Button
-              data-testid="open-annotation-dialog"
-              onClick={this.handleEnableAnnoMode}
-              intent="primary"
-            >
-              Create new category
-            </Button>
-          </div>
-        ) : null}
       </div>
     );
   }