chore: CCIE-4984 conform to open sourcing guidelines

Trademark

.bumpversion.cfg

@@ -1,17 +1,14 @@
 [bumpversion]
-current_version = 0.16.0
+current_version = 1.3.0
 commit = True
-# The below regex details an acceptable version number by naming the groups (major, minor, patch, prerel, and
-# prerelversion) and also specifying the valid values for each group (integers, `\d+`,  for major, minor, patch, and
-# prerelversion and only `rc` as the acceptable value for prerel).
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(?:-(?P<prerel>rc)\.(?P<prerelversion>\d+))?
-serialize =
+serialize = 
 	{major}.{minor}.{patch}-{prerel}.{prerelversion}
 	{major}.{minor}.{patch}
 
 [bumpversion:part:prerel]
 optional_value = release
-values =
+values = 
 	rc
 	release
 

.github/ISSUE_TEMPLATE/tech-issue.md

@@ -0,0 +1,23 @@
+---
+name: Tech Issue
+about: Engineering-specific technical work that is not product-specific. Engineering team "owns" these issues.
+title: ""
+labels: tech
+assignees: ""
+---
+
+## Motivation
+
+Why is this work important to engineers?
+
+## Definition of Done
+
+What should the end result look like? What will have been changed?
+
+## Tasks
+
+Detail the specific tasks that can be used to accomplish the desired changes.
+If detailed steps cannot be provided at this time, please file a [Tech Proposal](https://docs.google.com/document/d/1o2vuvl-kXwRJN1nBoPzJS_MAQgDGYnjmPZWa4qRDi-I/edit#heading=h.7dvzhm7gqc3v) instead.
+
+- [ ]
+- [ ]

.github/workflows/close-stale-prs.yml

@@ -0,0 +1,23 @@
+name: Close inactive pull requests
+on:
+  schedule:
+    - cron: "30 1 * * *"
+
+jobs:
+  close-issues:
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write
+      pull-requests: write
+    steps:
+      - uses: actions/stale@v5
+        with:
+          days-before-issue-stale: -1 # Do not mark any issues as stale
+          days-before-pr-stale: 14
+          days-before-pr-close: 3
+          stale-pr-message: "This PR has not seen any activity in the past 2 weeks; if no one comments or reviews it in the next 3 days, this PR will be closed."
+          close-pr-message: "This PR was closed because it has been inactive for 17 days, 3 days since being marked as stale. Please re-open if you still need this to be addressed."
+          stale-pr-label: "stale"
+          close-pr-label: "autoclosed"
+          exempt-draft-pr: true
+          repo-token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/compatibility_tests.yml

@@ -2,7 +2,7 @@ name: Compatibility Tests
 
 on:
   schedule:
-    - cron: '0 8 7 * 2'
+    - cron: "0 8 7 * 2"
   push:
     branches:
       - main
@@ -14,95 +14,99 @@ jobs:
   docker-build:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v1
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
       - name: Build docker image
         run: docker build .
 
-  cellxgene-main-with-python-and-anndata-versions:
-    name: python versions x anndata versions
-    runs-on: ubuntu-latest
-    continue-on-error: true
+  matrix-compatibility-test:
+    name: cxg:${{ matrix.cellxgene_build }} os:${{ matrix.os }} py:${{ matrix.python-version }} anndata:${{ matrix.anndata_version || 'latest' }}
+    runs-on: ${{ matrix.os }}
     strategy:
+      fail-fast: false
       matrix:
-        python-version: [3.6, 3.7] # As of Oct 2020 Anndata is not compatible with 3.8
-        anndata-version: [0.7.0, 0.7.1, 0.7.2, 0.7.3, 0.7.4, 0.7.5]
-        test-suite: [smoke-test, smoke-test-annotations]
+        os: [ubuntu-latest, macos-latest, macos-13]
+        python-version: ["3.10", "3.11", "3.12"]
+        cellxgene_build: [main, latest]
+        # add anndata pinned version test for subset of matrix configurations,
+        # in order to reduce matrix cross-product explosion
+        include:
+          - python-version: 3.12
+            cellxgene_build: latest
+            # TODO: dynamically use the literal version in requirements.txt,
+            # to avoid having to update this in manually in the future
+            # TODO: Do not bother running this if anndata latest version
+            # matches this pinned version, to avoid a redundant test
+            anndata_version: "==0.10.9"
     steps:
-    - uses: actions/checkout@v2
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v1
-      with:
-        python-version: ${{ matrix.python-version }}
-    - name: Install dependencies
-      run: |
-        # 1. only install the dev requirements on top of what is in the cellxgene pip package
-        sudo apt-get update && sudo apt-get install -y libhdf5-serial-dev
-        sed -i 's/-r requirements.txt//' server/requirements-dev.txt
-        pip install -r server/requirements-dev.txt
-        # 2. install cellxgene
-        make pydist install-dist
-        # 3. install anndata
-        pip install anndata==${{ matrix.anndata-version }}
-        # workaround for anndata 0.6.22.post1 bug
-        [[ "0.6.22.post1" = "${{ matrix.anndata-version }}" ]] && pip install h5py==2.9.0 || true
-    - name: Tests
-      run: make unit-test ${{ matrix.test-suite }}
-
-  cellxgene-release-with-anndata-master:
-    name: cellxgene release with anndata master
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        test-suite: [smoke-test, smoke-test-annotations]
-    steps:
-      - uses: actions/checkout@v2
-      - name: Set up Python 3.7
-        uses: actions/setup-python@v1
+      - uses: actions/checkout@v4
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
         with:
-          python-version: 3.7
-      - name: Checkout
-        uses: actions/checkout@v2
+          python-version: ${{ matrix.python-version }}
+      - name: Cache env vars
+        run: echo "PIP_CACHE=`python -m pip cache dir`" >> $GITHUB_ENV
+      - name: Cache env vars (MacOS)
+        if: startsWith(matrix.os, 'macos')
+        run: echo "BREW_CACHE=`brew --cache`" >> $GITHUB_ENV
+      # FIXME: Only working for Linux
+      - name: Python cache
+        uses: actions/cache@v4
         with:
-          path: cellxgene
-      - name: Install dependencies
+          path: ${{ env.PIP_CACHE }}
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}
+          restore-keys: |
+            ${{ runner.os }}-pip-
+      - name: Node cache
+        uses: actions/cache@v4
+        with:
+          path: ~/.npm
+          key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
+          restore-keys: |
+            ${{ runner.os }}-node-
+      - name: Brew cache (MacOS)
+        if: startsWith(matrix.os, 'macos')
+        uses: actions/cache@v4
+        with:
+          path: ${{ env.BREW_CACHE }}
+          key: ${{ runner.os }}-brew-
+      - name: Install dependencies (Ubuntu Linux)
+        if: startsWith(matrix.os, 'ubuntu')
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y libhdf5-serial-dev
+      - name: Install dependencies (MacOS)
+        if: startsWith(matrix.os, 'macos')
+        run: brew install hdf5
+      - name: Install cellxgene from `main` branch
+        if: matrix.cellxgene_build == 'main'
         run: |
-          cd cellxgene
-          # 1. only install the dev requirements on top of what is in the cellxgene pip package
-          make dev-env-client
-          sed -i 's/-r requirements.txt//' server/requirements-dev.txt
           pip install -r server/requirements-dev.txt
-          # 2. install cellxgene
-          pip install --upgrade cellxgene
-          # 3. install anndata
-          pip install git+https://github.com/theislab/anndata
-      - name: Tests
-        run: cd cellxgene && make unit-test ${{ matrix.test-suite }}
-
-  cellxgene-main-with-anndata-master:
-    name: cellxgene main with anndata master
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        test-suite: [smoke-test, smoke-test-annotations]
-    steps:
-      - uses: actions/checkout@v2
-      - name: Set up Python 3.7
-        uses: actions/setup-python@v1
-        with:
-          python-version: 3.7
-      - name: Checkout
-        uses: actions/checkout@v2
-        with:
-          path: cellxgene
-      - name: Install dependencies
+          make pydist install-dist
+      - name: Install cellxgene from latest release (pypi.org)
+        if: matrix.cellxgene_build == 'latest'
         run: |
-          cd cellxgene
-          sed -i -E 's/^anndata[>=]=[0-9]+.[0-9]+.[0-9]+$/anndata/g' server/requirements.txt
-          make pydist install-dist dev-env
-          pip install git+https://github.com/theislab/anndata
-      - name: Tests
-        run: cd cellxgene && make unit-test ${{ matrix.test-suite }}
+          pip install --upgrade cellxgene
+          # install the additional dev requirements on top of what is in the
+          # cellxgene pip package, which are needed for testing, but otherwise
+          # keep same pip pkg versions as in the cxg release
+          sed -i'' -e 's/-r requirements.txt//' server/requirements-dev.txt
+          pip install -r server/requirements-dev.txt
+          pip install --force-reinstall numpy==2.0.1 numba>=0.60.0 pandas flatbuffers==2.0.7
+      - name: Install anndata version per matrix variable
+        run: pip install anndata${{ matrix.anndata_version }}
+      - name: Install node
+        run: make dev-env-client
+      # Run different types of test separately, to facilitate troubleshooting
+      - name: Unit Tests - client
+        run: make unit-test-client
+      - name: Unit Tests - server
+        run: make unit-test-server
+      - name: Smoke Tests
+        run: make smoke-test
+    # FIXME: Fails intermittently. See https://app.zenhub.com/workspaces/single-cell-5e2a191dad828d52cc78b028/issues/chanzuckerberg/cellxgene/2415
+    # - name: Smoke Tests with Annotations
+    #   run: make smoke-test-annotations

.github/workflows/deploy.yml

@@ -1,13 +0,0 @@
-name: Deploy via single cell infra repo
-
-on:
-  push:
-    branches: main
-
-jobs:
-  deploy:
-    runs-on: ubuntu-latest
-    steps:
-      - name: repository dispatch
-        run: |
-          curl -XPOST -u czi-sci-single-cell-eng:${{secrets.SCI_GITHUB_TOKEN}} -H "Accept: application/vnd.github.everest-preview+json"  -H "Content-Type: application/json" https://api.github.com/repos/chanzuckerberg/single-cell-infra/dispatches --data '{"event_type": "cellxgene-hook"}'

.github/workflows/lint-pr-commit-message.yml

@@ -0,0 +1,19 @@
+
+name: "Lint PR commit message"
+
+on:
+  pull_request_target:
+    types:
+      - opened
+      - edited
+      - synchronize
+
+jobs:
+  main:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: amannn/action-semantic-pull-request@v3.4.1
+        with:
+          validateSingleCommit: true
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/push_tests.yml

@@ -14,15 +14,15 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - run: |
           git fetch --depth=1 origin +${{github.base_ref}}
-      - name: Set up Python 3.7
-        uses: actions/setup-python@v1
+      - name: Set up Python 3.12
+        uses: actions/setup-python@v5
         with:
-          python-version: 3.7
+          python-version: 3.12
       - name: Node cache
-        uses: actions/cache@v1
+        uses: actions/cache@v4
         with:
           path: ~/.npm
           key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
@@ -40,25 +40,27 @@ jobs:
       - name: Lint src with eslint
         working-directory: ./client
         run: |
-          make lint
+          npx eslint src __tests__
 
   unit-test:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
-      - name: Set up Python 3.7
-        uses: actions/setup-python@v1
+      - uses: actions/checkout@v4
+      - name: Set up Python 3.12 (pyenv) # pyenv needed for mlflow in cli annotate tests
+        uses: gabrielfalcao/pyenv-action@v9
         with:
-          python-version: 3.7
+          default: 3.12
+          command: pip install -U pip # upgrade pip after installing python
+      - run: pip install virtualenv # virtualenv needed for mlflow in cli annotate tests
       - name: Python cache
-        uses: actions/cache@v1
+        uses: actions/cache@v4
         with:
           path: ~/.cache/pip
           key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}
           restore-keys: |
             ${{ runner.os }}-pip-
       - name: Node cache
-        uses: actions/cache@v1
+        uses: actions/cache@v4
         with:
           path: ~/.npm
           key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
@@ -68,66 +70,69 @@ jobs:
         run: make pydist install-dist dev-env-server
       - name: Unit tests
         run: |
-          make unit-test
-          bash <(curl -s https://codecov.io/bash) -y .codecov.yml -k server -cF backend,python,unitTest
+          make unit-test-server unit-test-client
+          bash <(curl -s https://codecov.io/bash) -y .codecov.yml -k server -cF server,python,unitTest
           cd client && ./node_modules/codecov/bin/codecov --yml=../.codecov.yml --root=../ --gcov-root=../ -C -F frontend,javascript,unitTest
 
   smoke-tests:
     runs-on: macos-latest
     timeout-minutes: 20
     steps:
-      - uses: actions/checkout@v2
-      - name: Set up Python 3.7
-        uses: actions/setup-python@v1
+      - uses: actions/checkout@v4
+      - name: Set up Python 3.12
+        uses: actions/setup-python@v5
         with:
-          python-version: 3.7
+          python-version: 3.12
       - name: Python cache
-        uses: actions/cache@v1
+        uses: actions/cache@v4
         with:
           path: ~/.cache/pip
           key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}
           restore-keys: |
             ${{ runner.os }}-pip-
       - name: Node cache
-        uses: actions/cache@v1
+        uses: actions/cache@v4
         with:
           path: ~/.npm
           key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
           restore-keys: |
             ${{ runner.os }}-node-
       - name: Install dependencies
-        run: make pydist install-dist
+        run: |
+          pip install setuptools
+          make pydist install-dist
       - name: Smoke tests (without annotations feature)
         run: |
           cd client && make smoke-test
           ./node_modules/codecov/bin/codecov --yml=../.codecov.yml --root=../ --gcov-root=../ -C -F frontend,javascript,smokeTest
 
-  smoke-tests-annotations:
-    runs-on: ubuntu-latest
-    timeout-minutes: 20
-    steps:
-      - uses: actions/checkout@v2
-      - name: Set up Python 3.7
-        uses: actions/setup-python@v1
-        with:
-          python-version: 3.7
-      - name: Python cache
-        uses: actions/cache@v1
-        with:
-          path: ~/.cache/pip
-          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}
-          restore-keys: |
-            ${{ runner.os }}-pip-
-      - name: Node cache
-        uses: actions/cache@v1
-        with:
-          path: ~/.npm
-          key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
-          restore-keys: |
-            ${{ runner.os }}-node-
-      - name: Install dependencies
-        run: make pydist install-dist
-      - name: Smoke tests (with annotations feature)
-        run: |
-          cd client && make smoke-test-annotations
-          ./node_modules/codecov/bin/codecov --yml=../.codecov.yml --root=../ --gcov-root=../ -C -F frontend,javascript,smokeTestAnnotations
+  # TODO: reinstate: https://github.com/chanzuckerberg/cellxgene/issues/2544
+  # smoke-tests-annotations:
+  #   runs-on: ubuntu-latest
+  #   timeout-minutes: 20
+  #   steps:
+  #     - uses: actions/checkout@v2
+  #     - name: Set up Python 3.9
+  #       uses: actions/setup-python@v4
+  #       with:
+  #         python-version: 3.9
+  #     - name: Python cache
+  #       uses: actions/cache@v1
+  #       with:
+  #         path: ~/.cache/pip
+  #         key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}
+  #         restore-keys: |
+  #           ${{ runner.os }}-pip-
+  #     - name: Node cache
+  #       uses: actions/cache@v1
+  #       with:
+  #         path: ~/.npm
+  #         key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
+  #         restore-keys: |
+  #           ${{ runner.os }}-node-
+  #     - name: Install dependencies
+  #       run: make pydist install-dist
+  #     - name: Smoke tests (with annotations feature)
+  #       run: |
+  #         cd client && make smoke-test-annotations
+  #         ./node_modules/codecov/bin/codecov --yml=../.codecov.yml --root=../ --gcov-root=../ -C -F frontend,javascript,smokeTestAnnotations

.github/workflows/scale-test.yml

@@ -1,30 +0,0 @@
-name: "Scale test cellxgene APIs for initial loading"
-
-on:
-  schedule:
-    - cron: "0 0 * * Sun"
-
-jobs:
-  locust-build:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - name: Set up Python 3.7
-        uses: actions/setup-python@v1
-        with:
-          python-version: 3.7
-      - name: Install dependencies
-        run: |
-          pip install -r server/test/locust/requirements-locust.txt
-      - name: Dev Scale Test
-        run: |
-          locust -f server/test/locust/locustfile.py --headless -u 30 -r 10 --host https://api.cellxgene.dev.single-cell.czi.technology/cellxgene/e/ --run-time 5m 2>&1 | tee locust_dev_stats.txt
-      - name: Slack success webhook
-        env:
-          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
-        run: |
-          DEV_STATS=$(tail -n 61 locust_dev_stats.txt)
-          DEV_MSG="\`\`\`CELLXGENE EXPLORER DEV SCALE TEST RESULTS: ${DEV_STATS}\`\`\`"
-          curl -X POST -H 'Content-type: application/json' --data "{'text':'${DEV_MSG}'}" $SLACK_WEBHOOK
-
-

.gitignore

@@ -15,7 +15,7 @@ dist/
 *.egg-info
 
 # Environments
-venv/
+venv*/
 cellxgene/
 
 # client build
@@ -54,3 +54,6 @@ client/.eslintcache
 
 # E2E Testing
 ignoreE2E*
+
+# annotate subcmd
+.models_cache

CONTRIBUTING.md

@@ -1,3 +1,3 @@
 We warmly welcome contributions from the community!
 
-Whether you want to contribute ideas, requests, documentation, or code, you can get started by visiting our [contribution guide](https://chanzuckerberg.github.io/cellxgene/posts/contribute).
+Whether you want to contribute ideas, requests, documentation, or code, you can get started by visiting our [contribution guide](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/contribute.md).

LICENSE

@@ -0,0 +1,20 @@
+The MIT License (MIT)
+
+Copyright (c) 2017-2023 Chan Zuckerberg Initiative
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

LICENSE.txt

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2017-2020 Chan Zuckerberg Initiative
+Copyright (c) 2017-2023 Chan Zuckerberg Initiative
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of
 this software and associated documentation files (the "Software"), to deal in

MANIFEST.in

@@ -3,5 +3,6 @@ recursive-include server/common/web/static *
 
 include server/requirements.txt
 include server/requirements-prepare.txt
+include server/requirements-annotate.txt
 include server/converters/schema/hgnc_complete_set.txt.gz
 include server/converters/schema/schema_definitions/*

Makefile

@@ -16,8 +16,13 @@ clean: clean-lite clean-server clean-client
 clean-lite:
 	rm -rf $(CLEANFILES)
 
-clean-%:
-	cd $(*) && $(MAKE) clean
+.PHONY: clean-client
+clean-client:
+	cd client && $(MAKE) clean
+
+.PHONY: clean-server
+clean-server:
+	cd server && $(MAKE) clean
 
 
 # BUILDING PACKAGE
@@ -28,20 +33,20 @@ build-client:
 
 .PHONY: build
 build: clean build-client
-	git ls-files server/ | grep -v 'server/test/' | cpio -pdm $(BUILDDIR)
+	git ls-files server/ | cpio -pdm $(BUILDDIR)
 	cp -r client/build/  $(CLIENTBUILD)
 	$(call copy_client_assets,$(CLIENTBUILD),$(SERVERBUILD))
 	cp MANIFEST.in README.md setup.cfg setup.py $(BUILDDIR)
 
 # If you are actively developing in the server folder use this, dirties the source tree
 .PHONY: build-for-server-dev
-build-for-server-dev: clean-server build-client
-	$(call copy_client_assets,client/build,server)
+build-for-server-dev: clean-server build-client copy-client-assets
 
 .PHONY: copy-client-assets
 copy-client-assets:
 	$(call copy_client_assets,client/build,server)
 
+
 # TESTING
 .PHONY: test
 test: unit-test smoke-test
@@ -49,8 +54,22 @@ test: unit-test smoke-test
 .PHONY: unit-test
 unit-test: unit-test-server unit-test-client
 
-unit-test-%:
-	cd $(*) && $(MAKE) unit-test
+.PHONY: test-server
+test-server: unit-test-server smoke-test
+
+.PHONY: unit-test-client
+unit-test-client:
+	cd client && $(MAKE) unit-test
+
+.PHONY: unit-test-server
+unit-test-server:
+	PYTHONWARNINGS=ignore:ResourceWarning coverage run \
+		--source=server \
+		--omit=.coverage,venv \
+		-m unittest discover \
+		--start-directory test/unit \
+		--verbose; test_result=$$?; \
+	exit $$test_result \
 
 .PHONY: smoke-test
 smoke-test:
@@ -60,14 +79,9 @@ smoke-test:
 smoke-test-annotations:
 	cd client && $(MAKE) smoke-test-annotations
 
-.PHONY: test-db
-test-db:
-	cd server && $(MAKE) test-db
-
-
 # FORMATTING CODE
 
-.PHOHY: fmt
+.PHONY: fmt
 fmt: fmt-client fmt-py
 
 .PHONY: fmt-client
@@ -81,10 +95,10 @@ fmt-py:
 .PHONY: lint
 lint: lint-server lint-client
 
+
 .PHONY: lint-server
 lint-server: fmt-py
-	flake8 server --per-file-ignores='server/test/fixtures/dataset_config_outline.py:F821 server/test/fixtures/server_config_outline.py:F821 server/test/performance/scale_test_annotations.py:E501'
-
+	flake8 server --per-file-ignores='test/fixtures/dataset_config_outline.py:F821 test/fixtures/server_config_outline.py:F821 test/performance/scale_test_annotations.py:E501'
 
 .PHONY: lint-client
 lint-client:
@@ -97,29 +111,34 @@ pydist: build
 	cd $(BUILDDIR); python setup.py sdist -d ../dist
 	@echo "done"
 
-
 # RELEASE HELPERS
 
+# Set PART=[major, minor, patch] as param to make bump.
+# This will create a release candidate. (i.e. 0.16.1 -> 0.16.2-rc.0 for a patch bump)
+.PHONY: bump-version
+bump-version:
+	bumpversion --config-file .bumpversion.cfg $(PART)
+
 # Create new version to commit to main
 .PHONY: create-release-candidate
-create-release-candidate: dev-env bump-version clean-lite gen-package-lock
+create-release-candidate: bump-version clean-lite gen-package-lock
 	@echo "Version bumped part:$(PART) and client built. Ready to commit and push"
 
 # Bump the release candidate version if needed (i.e. the previous release candidate had errors).
 .PHONY: recreate-release-candidate
-recreate-release-candidate: dev-env bump-release-candidate clean-lite gen-package-lock
+recreate-release-candidate: bump-release-candidate clean-lite gen-package-lock
 	@echo "Version bumped part:$(PART) and client built. Ready to commit and push"
 
 # Build dist and release to Test PyPI
 .PHONY: release-candidate-to-test-pypi
-release-candidate-to-test-pypi: dev-env pydist twine
+release-candidate-to-test-pypi: pydist twine
 	@echo "Dist built and uploaded to test.pypi.org"
 	@echo "Test the install:"
 	@echo "    make install-release-test"
 
 # Build final dist (gets rid of the rc tag) and release final candidate to TestPyPI
 .PHONY: release-final-to-test-pypi
-release-final-to-test-pypi: dev-env bump-release clean-lite gen-package-lock pydist twine
+release-final-to-test-pypi: bump-release clean-lite gen-package-lock pydist twine
 	@echo "Final release dist built and uploaded to test.pypi.org"
 	@echo "Test the install:"
 	@echo "    make install-release-test"
@@ -129,9 +148,9 @@ release-final: twine-prod
 	@echo "Release uploaded to pypi.org"
 
 # DANGER: releases directly to prod
-# use this if you accidently burned a test release version number,
+# use this if you accidentally burned a test release version number,
 .PHONY: release-directly-to-prod
-release-directly-to-prod: dev-env pydist twine-prod
+release-directly-to-prod: pydist twine-prod
 	@echo "Dist built and uploaded to pypi.org"
 	@echo "Test the install:"
 	@echo "    make install-release"
@@ -147,12 +166,6 @@ dev-env-client:
 dev-env-server:
 	pip install -r server/requirements-dev.txt
 
-# Set PART=[major, minor, patch] as param to make bump.
-# This will create a release candidate. (i.e. 0.16.1 -> 0.16.2-rc.0 for a patch bump)
-.PHONY: bump-version
-bump-version:
-	bumpversion --config-file .bumpversion.cfg $(PART)
-
 # Increments the release candidate version (i.e. 0.16.2-rc.1 -> 0.16.2-rc.2)
 .PHONY: bump-release-candidate
 bump-release-candidate:
@@ -187,7 +200,7 @@ install-dev: uninstall
 # install from test.pypi to test your release
 .PHONY: install-release-test
 install-release-test: uninstall
-	pip install --no-cache-dir --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple cellxgene
+	pip install --no-cache-dir --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple cellxgene==$(VERSION)
 	@echo "Installed cellxgene from test.pypi.org, now run and smoke test"
 
 # install from pypi to test your release

README.md

@@ -7,27 +7,27 @@ _an interactive explorer for single-cell transcriptomics data_
 [![Compatibility Tests](https://github.com/chanzuckerberg/cellxgene/workflows/Compatibility%20Tests/badge.svg)](https://github.com/chanzuckerberg/cellxgene/actions?query=workflow%3A%22Compatibility+Tests%22)
 ![Code Coverage](https://codecov.io/gh/chanzuckerberg/cellxgene/branch/main/graph/badge.svg)
 
-cellxgene (pronounced "cell-by-gene") is an interactive data explorer for single-cell transcriptomics datasets, such as those coming from the [Human Cell Atlas](https://humancellatlas.org). Leveraging modern web development techniques to enable fast visualizations of at least 1 million cells, we hope to enable biologists and computational researchers to explore their data.
+CZ CELLxGENE Annotate (pronounced "cell-by-gene") is an interactive data explorer for single-cell datasets, such as those coming from the [Human Cell Atlas](https://humancellatlas.org). Leveraging modern web development techniques to enable fast visualizations of at least 1 million cells, we hope to enable biologists and computational researchers to explore their data.
 
-Whether you need to visualize one thousand cells or one million, cellxgene helps you gain insight into your single-cell data.
+Whether you need to visualize one thousand cells or one million, CELLxGENE Annotate helps you gain insight into your single-cell data.
 
 <img src="https://github.com/chanzuckerberg/cellxgene/raw/main/docs/images/crossfilter.gif" width="350" height="200" hspace="30"><img src="https://github.com/chanzuckerberg/cellxgene/raw/main/docs/images/category-breakdown.gif" width="350" height="200" hspace="30">
 
 # Getting started
 
-### The comprehensive guide to cellxgene
+### The comprehensive guide to CZ CELLxGENE Annotate
 
-[The cellxgene documentation is your one-stop-shop for information about cellxgene](https://chanzuckerberg.github.io/cellxgene/)! You may be particularly interested in:
+[The CZ CELLxGENE Annotate documentation is your one-stop-shop for information about CELLxGENE Annotate](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/README.md)! You may be particularly interested in:
 
-- Seeing [what cellxgene can do](https://chanzuckerberg.github.io/cellxgene/posts/gallery)
-- Learning more about cellxgene [installation](https://chanzuckerberg.github.io/cellxgene/posts/install) and [usage](https://chanzuckerberg.github.io/cellxgene/posts/launch)
-- [Preparing your own data](https://chanzuckerberg.github.io/cellxgene/posts/prepare) for use in cellxgene
-- Checking out [our roadmap](https://chanzuckerberg.github.io/cellxgene/posts/roadmap) for future development
-- [Contributing](https://chanzuckerberg.github.io/cellxgene/posts/contribute) to cellxgene
+- Seeing [what Annotate can do](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/explore-data/explorer-tutorials.md)
+- Learning more about Annotate [installation](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/desktop/install.md) and [usage](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/desktop/quick-start.md#quick-start-1)
+- [Preparing your own data](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/desktop/data-reqs.md) for use in Annotate
+- Checking out [our roadmap](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/roadmap.md) for future development
+- [Contributing](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/contribute.md) to Annotate
 
 ### Quick start
 
-To install cellxgene you need Python 3.6+. We recommend [installing cellxgene into a conda or virtual environment.](https://chanzuckerberg.github.io/cellxgene/posts/install)
+To install CELLxGENE Annotate you need Python 3.10+. We recommend [installing Annotate into a conda or virtual environment.](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/desktop/install.md)
 
 Install the package.
 
@@ -35,54 +35,59 @@ Install the package.
 pip install cellxgene
 ```
 
-Launch cellxgene with an example [anndata](https://anndata.readthedocs.io/en/latest/) file
+Launch Annotate with an example [anndata](https://anndata.readthedocs.io/en/latest/) file
 
 ```bash
 cellxgene launch https://cellxgene-example-data.czi.technology/pbmc3k.h5ad
 ```
 
-To explore more datasets already formatted for cellxgene, check out the [Demo data](https://chanzuckerberg.github.io/cellxgene/posts/demo-data) or
-see [Preparing your data](https://chanzuckerberg.github.io/cellxgene/posts/prepare) to learn more about formatting your own
-data for cellxgene.
+To explore more datasets already formatted for Annotate, check out the [Demo data](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/desktop/quick-start.md#example-datasets) or
+see [Preparing your data](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/desktop/data-reqs.md) to learn more about formatting your own
+data for CELLxGENE Annotate.
 
 ### Supported browsers
 
-cellxgene currently supports the following browsers:
+CELLxGENE Annotate currently supports the following browsers:
 
 - Google Chrome 61+
 - Edge 15+
 - Firefox 60+
-- Safari 10.1+
 
 Please [file an issue](https://github.com/chanzuckerberg/cellxgene/issues/new/choose) if you would like us to add support for an unsupported browser.
 
 ### Finding help
 
 We'd love to hear from you!
-For questions, suggestions, or accolades, [join the `#cellxgene-users` channel on the CZI Science Slack](https://join-cellxgene-users.herokuapp.com/) and say "hi!".
+For questions, suggestions, or accolades, join the `#cellxgene-users` channel on the [CZI Science Community Slack](https://czi.co/science-slack) and say "hi!".
 
 For any errors, [report bugs on Github](https://github.com/chanzuckerberg/cellxgene/issues).
 
-# Developing with cellxgene
+# Developing with CZ CELLxGENE Annotate
 
 ### Contributing
 
-We warmly welcome contributions from the community! Please see our [contributing guide](https://chanzuckerberg.github.io/cellxgene/posts/contribute) and don't hesitate to open an issue or send a pull request to improve cellxgene. Please see the [dev_docs](https://github.com/chanzuckerberg/cellxgene/tree/main/dev_docs) for pull request suggestions, unit test details, local documentation preview, and other development specifics. 
+We warmly welcome contributions from the community! Please see our [contributing guide](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/contribute.md) and don't hesitate to open an issue or send a pull request to improve CELLxGENE Annotate. Please see the [dev_docs](https://github.com/chanzuckerberg/cellxgene/tree/main/dev_docs) for pull request suggestions, unit test details, local documentation preview, and other development specifics.
 
 This project adheres to the Contributor Covenant [code of conduct](https://github.com/chanzuckerberg/.github/blob/master/CODE_OF_CONDUCT.md). By participating, you are expected to uphold this code. Please report unacceptable behavior to opensource@chanzuckerberg.com.
 
 ### Reuse
 
-This project was started with the sole goal of empowering the scientific community to explore and understand their data. 
-As such, we encourage other scientific tool builders in academia or industry to adopt the patterns, tools, and code from 
+This project was started with the sole goal of empowering the scientific community to explore and understand their data.
+As such, we encourage other scientific tool builders in academia or industry to adopt the patterns, tools, and code from
 this project. All code is freely available for reuse under the [MIT license](https://opensource.org/licenses/MIT).
 
+Before extending CELLxGENE Annotate, we encourage you to reach out to us with ideas or questions. It might be possible that an
+extension could be directly contributed, which would make it available for a wider audience, or that it's on our
+[roadmap](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/roadmap.md) and under active development.
 
-Before extending cellxgene, we encourage you to reach out to us with ideas or questions. It might be possible that an 
-extension could be directly contributed, which would make it available for a wider audience, or that it's on our 
-[roadmap](./docs/posts/roadmap.md) and under active development. 
+See the [CELLxGENE extensions](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/community-extensions.md) section of our documentation for examples of community use and CELLxGENE extensions.
+
+### Trademarks
+
+CZ CELLXGENE, CZ CELLXGENE DISCOVER, and CZ CELLXGENE ANNOTATE are trademarks of the Chan Zuckerberg Initiative. All rights reserved.
+
+Use, reuse, modification, and re-distribution of the source code in this repository is subject to the terms of the applicable open source [license](LICENSE.txt). However, that license does not grant permission to use the trademarks without separate, express permission from the Chan Zuckerberg Initiative.
 
-See the [cellxgene extensions](./docs/posts/extensions.md) section of our documentation for examples of community use and cellxgene extensions. 
 
 ### Security
 

SECURITY.md

@@ -0,0 +1,2 @@
+## Reporting Security Issues
+If you believe you have found a security issue, please responsibly disclose by contacting us at [security@chanzuckerberg.com](mailto:security@chanzuckerberg.com).

client/.husky/pre-commit

@@ -0,0 +1,5 @@
+#!/bin/sh
+. "$(dirname "$0")/_/husky.sh"
+
+cd client
+npx --no-install lint-staged --config "./configuration/lint-staged/lint-staged.config.js"

client/.nvmrc

@@ -0,0 +1 @@
+18.17.0

client/Makefile

@@ -1,9 +1,13 @@
 include ../common.mk
 
-ANNOTATIONS := $(if $(ANNOTATIONS),$(ANNOTATIONS),../server/test/fixtures/pbmc3k-annotations.csv)
+ANNOTATIONS := $(if $(ANNOTATIONS),$(ANNOTATIONS),../test/fixtures/pbmc3k-annotations.csv)
+GENE_SETS := $(if $(GENE_SETS),$(GENE_SETS),../test/fixtures/pbmc3k-genesets.csv)
 ANNOTATIONS_FILENAME := $(shell basename $(ANNOTATIONS))
+GENE_SETS_FILENAME := $(shell basename $(GENE_SETS))
+
+CXG_CONFIG := $(if $(CXG_CONFIG),$(CXG_CONFIG),./__tests__/e2e/test_config.yaml)
+
 
-CXG_CONFIG := $(if $(CXG_CONFIG), $(CXG_CONFIG), ./__tests__/e2e/test_config.yaml)
 
 # Packaging
 .PHONY: clean
@@ -35,15 +39,16 @@ smoke-test:
 	start_server_and_test \
 		'CXG_OPTIONS="--config-file $(CXG_CONFIG)" $(MAKE) start-server' \
 		$(CXG_SERVER_PORT) \
-		'CXG_URL_BASE="http://localhost:$(CXG_SERVER_PORT)" CXG_AUTH_TYPE="test" npm run e2e -- --verbose false'
+		'CXG_URL_BASE="http://localhost:$(CXG_SERVER_PORT)" npm run e2e -- --verbose false'
 
 # start an instance of cellxgene and run the end-to-end annotations tests
 .PHONY: smoke-test-annotations
 smoke-test-annotations:
 	$(eval TMP_DIR := $(shell  mktemp -d /tmp/cellxgene_XXXXXX))
 	cp $(ANNOTATIONS) $(TMP_DIR)/ && \
+	cp $(GENE_SETS) $(TMP_DIR)/ && \
 	start_server_and_test \
-		'CXG_OPTIONS="--annotations-file $(TMP_DIR)/$(ANNOTATIONS_FILENAME)" $(MAKE) start-server' \
+		'CXG_OPTIONS="--annotations-file $(TMP_DIR)/$(ANNOTATIONS_FILENAME) --gene-sets-file $(TMP_DIR)/$(GENE_SETS_FILENAME)" $(MAKE) start-server' \
 		$(CXG_SERVER_PORT) \
 		'CXG_URL_BASE="http://localhost:$(CXG_SERVER_PORT)" npm run e2e-annotations -- --verbose false'
 	rm -rf $(TMP_DIR)

client/__tests__/e2e/__snapshots__/e2e.test.js.snap

@@ -2,4 +2,4 @@
 
 exports[`did launch page launched 1`] = `"<span style=\\"max-width: 155px; display: flex; overflow: hidden; justify-content: flex-start; width: 100%; padding: 0px;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">pbm</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">c3k</span><span style=\\"position: absolute; right: 0px; color: inherit;\\">c3k</span></span></span>"`;
 
-exports[`metadata loads categories and values from dataset appear 1`] = `"<div style=\\"display: flex; justify-content: space-between; align-items: baseline;\\"><div style=\\"display: flex; justify-content: flex-start; align-items: flex-start;\\"><label class=\\"bp3-control bp3-checkbox\\" for=\\"category-select-louvain\\"><input id=\\"category-select-louvain\\" data-testclass=\\"category-select\\" data-testid=\\"louvain:category-select\\" type=\\"checkbox\\" checked=\\"\\"><span class=\\"bp3-control-indicator\\"></span></label><span role=\\"menuitem\\" tabindex=\\"0\\" data-testclass=\\"category-expand\\" data-testid=\\"louvain:category-expand\\" style=\\"cursor: pointer;\\"><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\" style=\\"max-width: 265px;\\"><span data-testid=\\"louvain:category-label\\" aria-label=\\"louvain\\" class=\\"\\" tabindex=\\"0\\" style=\\"max-width: 265px;\\"><span style=\\"max-width: 265px; display: flex; overflow: hidden; justify-content: flex-start; width: 100%; padding: 0px;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">lou</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">vain</span><span style=\\"position: absolute; right: 0px; color: inherit;\\">vain</span></span></span></span></span></span><svg stroke=\\"currentColor\\" fill=\\"currentColor\\" stroke-width=\\"0\\" viewBox=\\"0 0 320 512\\" data-testclass=\\"category-expand-is-not-expanded\\" height=\\"1em\\" width=\\"1em\\" xmlns=\\"http://www.w3.org/2000/svg\\" style=\\"font-size: 10px; margin-left: 5px;\\"><path d=\\"M285.476 272.971L91.132 467.314c-9.373 9.373-24.569 9.373-33.941 0l-22.667-22.667c-9.357-9.357-9.375-24.522-.04-33.901L188.505 256 34.484 101.255c-9.335-9.379-9.317-24.544.04-33.901l22.667-22.667c9.373-9.373 24.569-9.373 33.941 0L285.475 239.03c9.373 9.372 9.373 24.568.001 33.941z\\"></path></svg></span></div><div><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\"><a role=\\"button\\" data-testclass=\\"colorby\\" data-testid=\\"colorby-louvain\\" class=\\"bp3-button\\" tabindex=\\"0\\"><span icon=\\"tint\\" class=\\"bp3-icon bp3-icon-tint\\"><svg data-icon=\\"tint\\" width=\\"16\\" height=\\"16\\" viewBox=\\"0 0 16 16\\"><desc>tint</desc><path d=\\"M7.88 1s-4.9 6.28-4.9 8.9c.01 2.82 2.34 5.1 4.99 5.1 2.65-.01 5.03-2.3 5.03-5.13C12.99 7.17 7.88 1 7.88 1z\\" fill-rule=\\"evenodd\\"></path></svg></span></a></span></span></div></div><div style=\\"margin-left: 26px;\\"></div><div></div>"`;
+exports[`metadata loads categories and values from dataset appear 1`] = `"<div style=\\"display: flex; justify-content: space-between; align-items: baseline;\\"><div style=\\"display: flex; justify-content: flex-start; align-items: flex-start;\\"><label class=\\"bp3-control bp3-checkbox\\" for=\\"category-select-louvain\\"><input id=\\"category-select-louvain\\" data-testclass=\\"category-select\\" data-testid=\\"louvain:category-select\\" type=\\"checkbox\\" checked=\\"\\"><span class=\\"bp3-control-indicator\\"></span></label><span role=\\"menuitem\\" tabindex=\\"0\\" data-testclass=\\"category-expand\\" data-testid=\\"louvain:category-expand\\" style=\\"cursor: pointer;\\"><span aria-haspopup=\\"true\\" class=\\"bp3-popover2-target\\"><span data-testid=\\"louvain:category-label\\" tabindex=\\"-1\\" aria-label=\\"louvain\\" class=\\"\\" style=\\"max-width: 265px;\\"><span style=\\"max-width: 265px; display: flex; overflow: hidden; justify-content: flex-start; width: 100%; padding: 0px;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">lou</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">vain</span><span style=\\"position: absolute; right: 0px; color: inherit;\\">vain</span></span></span></span></span><svg stroke=\\"currentColor\\" fill=\\"currentColor\\" stroke-width=\\"0\\" viewBox=\\"0 0 320 512\\" data-testclass=\\"category-expand-is-not-expanded\\" height=\\"1em\\" width=\\"1em\\" xmlns=\\"http://www.w3.org/2000/svg\\" style=\\"font-size: 10px; margin-left: 5px;\\"><path d=\\"M285.476 272.971L91.132 467.314c-9.373 9.373-24.569 9.373-33.941 0l-22.667-22.667c-9.357-9.357-9.375-24.522-.04-33.901L188.505 256 34.484 101.255c-9.335-9.379-9.317-24.544.04-33.901l22.667-22.667c9.373-9.373 24.569-9.373 33.941 0L285.475 239.03c9.373 9.372 9.373 24.568.001 33.941z\\"></path></svg></span></div><div><span class=\\"bp3-popover-wrapper\\"><span aria-haspopup=\\"true\\" class=\\"bp3-popover-target\\"><a role=\\"button\\" data-testclass=\\"colorby\\" data-testid=\\"colorby-louvain\\" class=\\"bp3-button\\" tabindex=\\"0\\"><span icon=\\"tint\\" aria-hidden=\\"true\\" tabindex=\\"0\\" class=\\"bp3-icon bp3-icon-tint\\"><svg data-icon=\\"tint\\" width=\\"16\\" height=\\"16\\" viewBox=\\"0 0 16 16\\"><path d=\\"M7.88 1s-4.9 6.28-4.9 8.9c.01 2.82 2.34 5.1 4.99 5.1 2.65-.01 5.03-2.3 5.03-5.13C12.99 7.17 7.88 1 7.88 1z\\" fill-rule=\\"evenodd\\"></path></svg></span></a></span></span></div></div><div style=\\"margin-left: 26px;\\"></div>"`;

client/__tests__/e2e/cellxgeneActions.js

@@ -13,11 +13,8 @@ import {
   getTestClass,
   getTestId,
   isElementPresent,
-  goToPage,
 } from "./puppeteerUtils";
 
-import { appUrlBase } from "./config";
-
 export async function drag(testId, start, end, lasso = false) {
   const layout = await waitByID(testId);
   const elBox = await layout.boxModel();
@@ -61,11 +58,9 @@ export async function getAllHistograms(testclass, testIds) {
   const allHistograms = await getAllByClass(testclass);
 
   const testIDs = await Promise.all(
-    allHistograms.map((hist) => {
-      return page.evaluate((elem) => {
-        return elem.dataset.testid;
-      }, hist);
-    })
+    allHistograms.map((hist) =>
+      page.evaluate((elem) => elem.dataset.testid, hist)
+    )
   );
 
   return testIDs.map((id) => id.replace(/^histogram-/, ""));
@@ -176,6 +171,137 @@ export async function createCategory(categoryName) {
   await clickOn("submit-category");
 }
 
+/*
+
+  GENESET
+
+*/
+
+export async function colorByGeneset(genesetName) {
+  await clickOn(`${genesetName}:colorby-entire-geneset`);
+}
+
+export async function colorByGene(gene) {
+  await clickOn(`colorby-${gene}`);
+}
+
+export async function assertColorLegendLabel(label) {
+  const handle = await waitByID("continuous_legend_color_by_label");
+
+  const result = await handle.evaluate((node) =>
+    node.getAttribute("aria-label")
+  );
+
+  return expect(result).toBe(label);
+}
+
+export async function expandGeneset(genesetName) {
+  const expand = await waitByID(`${genesetName}:geneset-expand`);
+  const notExpanded = await expand.$(
+    "[data-testclass='geneset-expand-is-not-expanded']"
+  );
+  if (notExpanded) await clickOn(`${genesetName}:geneset-expand`);
+}
+
+export async function createGeneset(genesetName) {
+  await clickOnUntil("open-create-geneset-dialog", async () => {
+    await expect(page).toMatchElement(getTestId("create-geneset-input"));
+  });
+
+  await typeInto("create-geneset-input", genesetName);
+  await clickOn("submit-geneset");
+  await waitByClass("autosave-complete");
+}
+
+export async function editGenesetName(genesetName, editText) {
+  const editButton = `${genesetName}:edit-genesetName-mode`;
+  const submitButton = `${genesetName}:submit-geneset`;
+  await clickOnUntil(`${genesetName}:see-actions`, async () => {
+    await expect(page).toMatchElement(getTestId(editButton));
+  });
+  await clickOn(editButton);
+  await typeInto("rename-geneset-modal", editText);
+  await clickOn(submitButton);
+}
+
+export async function deleteGeneset(genesetName) {
+  const targetId = `${genesetName}:delete-geneset`;
+
+  await clickOnUntil(`${genesetName}:see-actions`, async () => {
+    await expect(page).toMatchElement(getTestId(targetId));
+  });
+
+  await clickOn(targetId);
+
+  await assertGenesetDoesNotExist(genesetName);
+  await waitByClass("autosave-complete");
+}
+
+export async function assertGenesetDoesNotExist(genesetName) {
+  const result = await isElementPresent(
+    getTestId(`${genesetName}:geneset-name`)
+  );
+  await expect(result).toBe(false);
+}
+
+export async function assertGenesetExists(genesetName) {
+  const handle = await waitByID(`${genesetName}:geneset-name`);
+
+  const result = await handle.evaluate((node) =>
+    node.getAttribute("aria-label")
+  );
+
+  return expect(result).toBe(genesetName);
+}
+
+/*
+
+  GENE
+
+*/
+
+export async function addGeneToSet(genesetName, geneToAddToSet) {
+  const submitButton = `${genesetName}:submit-gene`;
+
+  await clickOn(`${genesetName}:add-new-gene-to-geneset`);
+  await typeInto("add-genes", geneToAddToSet);
+  await clickOn(submitButton);
+}
+
+export async function removeGene(geneSymbol) {
+  const targetId = `delete-from-geneset:${geneSymbol}`;
+
+  await clickOn(targetId);
+
+  await waitByClass("autosave-complete");
+}
+
+export async function assertGeneExistsInGeneset(geneSymbol) {
+  const handle = await waitByID(`${geneSymbol}:gene-label`);
+
+  const result = await handle.evaluate((node) =>
+    node.getAttribute("aria-label")
+  );
+
+  return expect(result).toBe(geneSymbol);
+}
+
+export async function assertGeneDoesNotExist(geneSymbol) {
+  const result = await isElementPresent(getTestId(`${geneSymbol}:gene-label`));
+
+  await expect(result).toBe(false);
+}
+
+export async function expandGene(geneSymbol) {
+  await clickOn(`maximize-${geneSymbol}`);
+}
+
+/*
+
+  CATEGORY
+
+*/
+
 export async function duplicateCategory(categoryName) {
   await clickOn("open-annotation-dialog");
 
@@ -238,7 +364,7 @@ export async function createLabel(categoryName, labelName) {
    * 4. You will see `123` is persisted in the input box
    * 5. Expected behavior is to get an empty input box
    */
-  await page.waitFor(500);
+  await page.waitForTimeout(500);
 
   await clickOn(`${categoryName}:see-actions`);
 
@@ -316,73 +442,4 @@ export async function assertCategoryDoesNotExist(categoryName) {
   await expect(result).toBe(false);
 }
 
-export async function login() {
-  const email = `cellxgene-smoke-test+${process.env.DEPLOYMENT_STAGE}@chanzuckerberg.com`;
-  const password = "Test1111";
-
-  await goToPage(appUrlBase);
-
-  await clickOn("log-in");
-
-  // (thuang): Auth0 form is unstable and unsafe for input until verified
-  await waitUntilFormFieldStable('[name="email"]');
-
-  await expect(page).toFillForm("form", {
-    email,
-    password,
-  });
-
-  await Promise.all([
-    page.waitForNavigation({ waitUntil: "networkidle0" }),
-    expect(page).toClick('[name="submit"]'),
-  ]);
-
-  expect(page.url()).toContain(appUrlBase);
-}
-
-export async function logout() {
-  await clickOnUntil("user-info", async () => {
-    await waitByID("log-out");
-    await Promise.all([
-      page.waitForNavigation({ waitUntil: "networkidle0" }),
-      clickOn("log-out"),
-    ]);
-  });
-
-  await waitByID("log-in");
-}
-
-async function waitUntilFormFieldStable(selector) {
-  const MAX_RETRY = 10;
-  const WAIT_FOR_MS = 200;
-
-  const EXPECTED_VALUE = "aaa";
-
-  let retry = 0;
-
-  while (retry < MAX_RETRY) {
-    try {
-      await expect(page).toFill(selector, EXPECTED_VALUE);
-
-      const fieldHandle = await expect(page).toMatchElement(selector);
-
-      const fieldValue = await page.evaluate(
-        (input) => input.value,
-        fieldHandle
-      );
-
-      expect(fieldValue).toBe(EXPECTED_VALUE);
-
-      break;
-    } catch (error) {
-      retry += 1;
-
-      await page.waitFor(WAIT_FOR_MS);
-    }
-  }
-
-  if (retry === MAX_RETRY) {
-    throw Error("clickOnUntil() assertion failed!");
-  }
-}
 /* eslint-enable no-await-in-loop -- await in loop is needed to emulate sequential user actions */

client/__tests__/e2e/config.js

@@ -6,3 +6,5 @@ export const appUrlBase =
 export const DATASET = "pbmc3k";
 export const isDev = jestEnv === ENV_DEFAULT.DEV;
 export const isDebug = jestEnv === ENV_DEFAULT.DEBUG;
+export const TEST_EMAIL = "user@example.com";
+export const TEST_PASSWORD = process.env.TEST_ACCOUNT_PASS ?? "";

client/__tests__/e2e/diffexpGeneSets.js

@@ -0,0 +1,105 @@
+export const diffexpPop1Genes = [
+  "CD79A",
+  "HLA-DRB1",
+  "HLA-DQA1",
+  "HLA-DPB1",
+  "HLA-DQB1",
+  "HLA-DPA1",
+  "MS4A1",
+  "LTB",
+  "CD79B",
+  "CD37",
+  "HLA-DMA",
+  "TCL1A",
+  "LINC00926",
+  "HLA-DMB",
+  "HVCN1",
+  "EAF2",
+  "FCRLA",
+  "IRF8",
+  "PKIG",
+  "P2RX5",
+  "HLA-DOB",
+  "SPIB",
+  "BLNK",
+  "SWAP70",
+  "PNOC",
+  "CD19",
+  "SMIM14",
+  "CD72",
+  "KIAA0125",
+  "IGLL5",
+  "ARHGAP24",
+  "COTL1",
+  "C16orf74",
+  "BTK",
+  "SNX29P2",
+  "ADAM28",
+  "FCGR2B",
+  "PLD4",
+  "PPP1R14A",
+  "MZB1",
+  "KIAA0040",
+  "PHACTR1",
+  "FCRL2",
+  "RIC3",
+  "P2RY10",
+  "SCPEP1",
+  "DRAM2",
+  "RP5-887A10.1",
+  "CD82",
+  "GPX1",
+];
+
+export const diffexpPop2Genes = [
+  "NKG7",
+  "GZMB",
+  "CTSW",
+  "PRF1",
+  "GNLY",
+  "GZMA",
+  "CST7",
+  "FGFBP2",
+  "SRGN",
+  "CD247",
+  "FCGR3A",
+  "TYROBP",
+  "FCER1G",
+  "ID2",
+  "SPON2",
+  "CCL4",
+  "CCL5",
+  "GZMH",
+  "GIMAP7",
+  "CLIC3",
+  "HOPX",
+  "XCL2",
+  "LGALS1",
+  "IGFBP7",
+  "AKR1C3",
+  "IL32",
+  "EFHD2",
+  "PRSS23",
+  "TTC38",
+  "ZAP70",
+  "S1PR5",
+  "SAMD3",
+  "GIMAP4",
+  "CCL3",
+  "ABI3",
+  "XCL1",
+  "S100A6",
+  "UBB",
+  "GPR56",
+  "PDIA3",
+  "S100A11",
+  "APOBEC3G",
+  "HAVCR2",
+  "PLEKHF1",
+  "LITAF",
+  "ARPC5L",
+  "PTGDR",
+  "PRMT2",
+  "GSTP1",
+  "FCRL6",
+];

client/__tests__/e2e/e2e.test.js

@@ -15,25 +15,16 @@ import {
   getOneElementInnerHTML,
   getTestId,
   goToPage,
-  typeInto,
   waitByID,
-  clickOnUntil,
 } from "./puppeteerUtils";
 
 import {
-  addGeneToSearch,
-  bulkAddGenes,
   calcDragCoordinates,
   clip,
   drag,
   getAllCategoriesAndCounts,
-  getAllHistograms,
   getCellSetCount,
-  runDiffExp,
   selectCategory,
-  subset,
-  login,
-  logout,
 } from "./cellxgeneActions";
 
 const data = datasets[DATASET];
@@ -149,47 +140,6 @@ describe("cell selection", () => {
   });
 });
 
-describe("gene entry", () => {
-  test("search for single gene", async () => {
-    await goToPage(appUrlBase);
-    await addGeneToSearch(data.genes.search);
-  });
-
-  test("bulk add genes", async () => {
-    await goToPage(appUrlBase);
-
-    const testGenes = data.genes.bulkadd;
-
-    await bulkAddGenes(testGenes);
-    const allHistograms = await getAllHistograms(
-      "histogram-user-gene",
-      testGenes
-    );
-
-    expect(allHistograms).toEqual(expect.arrayContaining(testGenes));
-    expect(allHistograms).toHaveLength(testGenes.length);
-  });
-});
-
-describe("differential expression", () => {
-  test("selects cells, saves them and performs diffexp", async () => {
-    await goToPage(appUrlBase);
-
-    await runDiffExp(data.diffexp.cellset1, data.diffexp.cellset2);
-
-    const allHistograms = await getAllHistograms(
-      "histogram-diffexp",
-      data.diffexp["gene-results"]
-    );
-
-    expect(allHistograms).toEqual(
-      expect.arrayContaining(data.diffexp["gene-results"])
-    );
-
-    expect(allHistograms).toHaveLength(data.diffexp["gene-results"].length);
-  });
-});
-
 describe("subset", () => {
   test("subset - cell count matches", async () => {
     await goToPage(appUrlBase);
@@ -236,94 +186,6 @@ describe("subset", () => {
     const cellCount = await getCellSetCount(1);
     expect(cellCount).toBe(data.subset.lasso.count);
   });
-
-  test("undo selection appends the top diff exp genes to user defined genes", async () => {
-    await goToPage(appUrlBase);
-
-    const userDefinedGenes = data.genes.bulkadd;
-    const diffExpGenes = data.diffexp["gene-results"];
-
-    await bulkAddGenes(userDefinedGenes);
-    const userDefinedHistograms = await getAllHistograms(
-      "histogram-user-gene",
-      userDefinedGenes
-    );
-
-    expect(userDefinedHistograms).toEqual(
-      expect.arrayContaining(userDefinedGenes)
-    );
-
-    await subset({ x1: 0.15, y1: 0.1, x2: 0.98, y2: 0.98 });
-    await runDiffExp(data.diffexp.cellset1, data.diffexp.cellset2);
-
-    const diffExpHistograms = await getAllHistograms(
-      "histogram-diffexp",
-      diffExpGenes
-    );
-
-    expect(diffExpHistograms).toEqual(expect.arrayContaining(diffExpGenes));
-
-    await clickOn("reset-subset-button");
-    const expected = [].concat(userDefinedGenes, diffExpGenes);
-    const userDefinedHistogramsAfterSubset = await getAllHistograms(
-      "histogram-user-gene",
-      expected
-    );
-
-    expect(userDefinedHistogramsAfterSubset).toEqual(
-      expect.arrayContaining(expected)
-    );
-  });
-
-  test("subset selection appends the top diff exp genes to user defined genes", async () => {
-    await goToPage(appUrlBase);
-
-    const userDefinedGenes = data.genes.bulkadd;
-    const diffExpGenes = data.diffexp["gene-results"];
-
-    await bulkAddGenes(userDefinedGenes);
-    const userDefinedHistograms = await getAllHistograms(
-      "histogram-user-gene",
-      userDefinedGenes
-    );
-
-    expect(userDefinedHistograms).toEqual(
-      expect.arrayContaining(userDefinedGenes)
-    );
-
-    await subset({ x1: 0.15, y1: 0.1, x2: 0.98, y2: 0.98 });
-    await runDiffExp(data.diffexp.cellset1, data.diffexp.cellset2);
-
-    const diffExpHistograms = await getAllHistograms(
-      "histogram-diffexp",
-      diffExpGenes
-    );
-
-    expect(diffExpHistograms).toEqual(expect.arrayContaining(diffExpGenes));
-
-    await subset({ x1: 0.16, y1: 0.11, x2: 0.97, y2: 0.97 });
-
-    const expected = [].concat(userDefinedGenes, diffExpGenes);
-    const userDefinedHistogramsAfterSubset = await getAllHistograms(
-      "histogram-user-gene",
-      expected
-    );
-
-    expect(userDefinedHistogramsAfterSubset).toEqual(
-      expect.arrayContaining(expected)
-    );
-  });
-});
-
-describe("scatter plot", () => {
-  test("scatter plot appears", async () => {
-    await goToPage(appUrlBase);
-
-    await bulkAddGenes(Object.values(data.scatter.genes));
-    await clickOn(`plot-x-${data.scatter.genes.x}`);
-    await clickOn(`plot-y-${data.scatter.genes.y}`);
-    await waitByID("scatterplot");
-  });
 });
 
 describe("clipping", () => {
@@ -340,30 +202,6 @@ describe("clipping", () => {
     const cellCount = await getCellSetCount(1);
     expect(cellCount).toBe(data.clip.count);
   });
-
-  test("clip gene", async () => {
-    await goToPage(appUrlBase);
-
-    await typeInto("gene-search", data.clip.gene);
-    await page.keyboard.press("Enter");
-
-    await page.waitForSelector(`[data-testid='histogram-${data.clip.gene}']`);
-
-    await clip(data.clip.min, data.clip.max);
-
-    const histBrushableAreaId = `histogram-${data.clip.gene}-plot-brushable-area`;
-
-    const coords = await calcDragCoordinates(
-      histBrushableAreaId,
-      data.clip["coordinates-as-percent"]
-    );
-
-    await drag(histBrushableAreaId, coords.start, coords.end);
-
-    const cellCount = await getCellSetCount(1);
-
-    expect(cellCount).toBe(data.clip["gene-cell-count"]);
-  });
 });
 
 // interact with UI elements just that they do not break
@@ -381,17 +219,6 @@ describe("ui elements don't error", () => {
     }
   });
 
-  test("color by for gene", async () => {
-    await goToPage(appUrlBase);
-
-    await typeInto("gene-search", data.genes.search);
-    await page.keyboard.press("Enter");
-    await page.waitForSelector(
-      `[data-testid='histogram-${data.genes.search}']`
-    );
-    await clickOn(`colorby-${data.genes.search}`);
-  });
-
   test("pan and zoom", async () => {
     await goToPage(appUrlBase);
 
@@ -521,34 +348,4 @@ test("lasso moves after pan", async () => {
 
   expect(panCount).toBe(initialCount);
 });
-
-const describeIfCalledByMakeFileTarget =
-  process.env.CXG_AUTH_TYPE?.toLowerCase() === "test"
-    ? describe
-    : describe.skip;
-
-describeIfCalledByMakeFileTarget("auth buttons", () => {
-  test("login then logout", async () => {
-    await goToPage(appUrlBase);
-    await clickOnUntil("log-in", async () => {
-      await page.waitForNavigation({ waitUntil: "networkidle0" });
-      await waitByID("user-info");
-    });
-    await logout();
-  });
-});
-
-const conditionalDescribe =
-  process.env.TEST_AUTH_INTEGRATION === "true" ? describe : describe.skip;
-
-conditionalDescribe("AuthN Integration", () => {
-  it("logs in", async () => {
-    await login();
-  });
-
-  it("logs out", async () => {
-    await login();
-    await logout();
-  });
-});
 /* eslint-enable no-await-in-loop -- await in loop is needed to emulate sequential user actions */

client/__tests__/e2e/e2eAnnotations.test.js

@@ -12,6 +12,8 @@ import {
   getTestId,
   getTestClass,
   getAllByClass,
+  clickOnUntil,
+  getOneElementInnerHTML,
 } from "./puppeteerUtils";
 
 import {
@@ -27,6 +29,21 @@ import {
   renameLabel,
   subset,
   duplicateCategory,
+  createGeneset,
+  deleteGeneset,
+  assertGenesetExists,
+  assertGenesetDoesNotExist,
+  getCellSetCount,
+  expandGeneset,
+  editGenesetName,
+  addGeneToSet,
+  assertGeneExistsInGeneset,
+  removeGene,
+  assertGeneDoesNotExist,
+  expandGene,
+  colorByGeneset,
+  assertColorLegendLabel,
+  colorByGene,
 } from "./cellxgeneActions";
 
 const data = datasets[DATASET];
@@ -34,12 +51,45 @@ const data = datasets[DATASET];
 const perTestCategoryName = "TEST-CATEGORY";
 const perTestLabelName = "TEST-LABEL";
 
+// geneset CRUD
+const genesetToDeleteName = "geneset_to_delete";
+const preExistingGenesetName = "fifth_dataset";
+const meanExpressionBrushGenesetName = "second_gene_set";
+const meanExpressionBrushCellsSelected = "557";
+const subsetMeanExpressionBrushCellsSelected = "452";
+
+// initial text, the text we type in, the result
+const editableGenesetName = "geneset_to_edit";
+const editText = "_111";
+const newGenesetName = "geneset_to_edit_111";
+
+// add gene to set
+const geneToAddToSet = "RER1";
+const setToAddGeneTo = "fill_this_geneset";
+
+// remove gene from set
+const geneToRemove = "SIK1";
+const setToRemoveFrom = "empty_this_geneset";
+
+// brush a gene
+const geneToBrushAndColorBy = "SIK1";
+const brushThisGeneGeneset = "brush_this_gene";
+const geneBrushedCellCount = "109";
+const subsetGeneBrushedCellCount = "96";
+
+const genesetDescriptionID =
+  "geneset-description-tooltip-fourth_gene_set: fourth description";
+const genesetDescriptionString = "fourth_gene_set: fourth description";
+const genesetToCheckForDescription = "fourth_gene_set";
+
 async function setup(config) {
   await goToPage(appUrlBase);
 
-  // setup the test fixtures
-  await createCategory(perTestCategoryName);
-  await createLabel(perTestCategoryName, perTestLabelName);
+  if (config.categoricalAnno) {
+    // setup the test fixtures
+    await createCategory(perTestCategoryName);
+    await createLabel(perTestCategoryName, perTestLabelName);
+  }
 
   if (config.withSubset) {
     await subset({ x1: 0.1, y1: 0.1, x2: 0.8, y2: 0.8 });
@@ -51,6 +101,200 @@ async function setup(config) {
 describe.each([
   { withSubset: true, tag: "subset" },
   { withSubset: false, tag: "whole" },
+])("geneSET crud operations and interactions", (config) => {
+  test("genesets load from csv", async () => {
+    await setup(config);
+
+    await assertGenesetExists(preExistingGenesetName);
+  });
+  test("brush on geneset mean", async () => {
+    await setup(config);
+
+    await expandGeneset(meanExpressionBrushGenesetName);
+
+    const histBrushableAreaId = `histogram-${meanExpressionBrushGenesetName}-plot-brushable-area`;
+
+    const coords = await calcDragCoordinates(histBrushableAreaId, {
+      x1: 0.25,
+      y1: 0.5,
+      x2: 0.55,
+      y2: 0.5,
+    });
+
+    await drag(histBrushableAreaId, coords.start, coords.end);
+
+    const cellCount = await getCellSetCount(1);
+    if (config.withSubset) {
+      expect(cellCount).toBe(subsetMeanExpressionBrushCellsSelected);
+    } else {
+      expect(cellCount).toBe(meanExpressionBrushCellsSelected);
+    }
+  });
+  test("color by mean expression", async () => {
+    await setup(config);
+
+    await colorByGeneset(meanExpressionBrushGenesetName);
+    await assertColorLegendLabel(meanExpressionBrushGenesetName);
+  });
+  test("diffexp", async () => {
+    if (config.withSubset) return;
+
+    await setup(config);
+
+    // set the two cell sets to b cells vs nk cells
+    await expandCategory(`louvain`);
+    await clickOn(`louvain:category-select`);
+    await clickOn(`categorical-value-select-louvain-B cells`);
+    await clickOn(`cellset-button-1`);
+    await clickOn(`categorical-value-select-louvain-B cells`);
+    await clickOn(`categorical-value-select-louvain-NK cells`);
+    await clickOn(`cellset-button-2`);
+
+    // run diffexp
+    await clickOn(`diffexp-button`);
+    await waitByClass("pop-1-geneset-expand");
+    await expect(page).toClick(getTestClass("pop-1-geneset-expand"));
+
+    await page.waitForFunction(
+      (selector) => !document.querySelector(selector),
+      {},
+      getTestClass("gene-loading-spinner")
+    );
+
+    let genesHTML = await getOneElementInnerHTML(
+      getTestClass("gene-set-genes")
+    );
+
+    expect(genesHTML).toMatchSnapshot();
+
+    await expect(page).toClick(getTestClass("pop-1-geneset-expand"));
+    await expect(page).toClick(getTestClass("pop-2-geneset-expand"));
+
+    await page.waitForFunction(
+      (selector) => !document.querySelector(selector),
+      {},
+      getTestClass("gene-loading-spinner")
+    );
+
+    genesHTML = await getOneElementInnerHTML(getTestClass("gene-set-genes"));
+
+    expect(genesHTML).toMatchSnapshot();
+  });
+  test("create a new geneset and undo/redo", async () => {
+    if (config.withSubset) return;
+
+    await setup(config);
+
+    const genesetName = `test-geneset-foo-123`;
+    await assertGenesetDoesNotExist(genesetName);
+    await createGeneset(genesetName);
+    /* note: as of June 2021, the aria label is in the truncate component which clones the element */
+    await assertGenesetExists(genesetName);
+    await clickOn("undo");
+    await assertGenesetDoesNotExist(genesetName);
+    await clickOn("redo");
+    await assertGenesetExists(genesetName);
+  });
+  test("edit geneset name and undo/redo", async () => {
+    await setup(config);
+
+    await editGenesetName(editableGenesetName, editText);
+    await assertGenesetExists(newGenesetName);
+    await clickOn("undo");
+    await assertGenesetExists(editableGenesetName);
+    await clickOn("redo");
+    await assertGenesetExists(newGenesetName);
+  });
+  test("delete a geneset and undo/redo", async () => {
+    if (config.withSubset) return;
+
+    await setup(config);
+
+    await deleteGeneset(genesetToDeleteName);
+    await clickOn("undo");
+    await assertGenesetExists(genesetToDeleteName);
+    await clickOn("redo");
+    await assertGenesetDoesNotExist(genesetToDeleteName);
+  });
+  test("geneset description", async () => {
+    if (config.withSubset) return;
+
+    await setup(config);
+
+    await clickOnUntil(
+      `${genesetToCheckForDescription}:geneset-expand`,
+      async () => {
+        expect(page).toMatchElement(getTestId(genesetDescriptionID), {
+          text: genesetDescriptionString,
+        });
+      }
+    );
+  });
+});
+
+describe.each([
+  { withSubset: true, tag: "subset" },
+  { withSubset: false, tag: "whole" },
+])("GENE crud operations and interactions", (config) => {
+  test("add a gene to geneset and undo/redo", async () => {
+    await setup(config);
+
+    await addGeneToSet(setToAddGeneTo, geneToAddToSet);
+    await expandGeneset(setToAddGeneTo);
+    await assertGeneExistsInGeneset(geneToAddToSet);
+    await clickOn("undo");
+    await assertGeneDoesNotExist(geneToAddToSet);
+    await clickOn("redo");
+    await assertGeneExistsInGeneset(geneToAddToSet);
+  });
+  test("expand gene and brush", async () => {
+    await setup(config);
+
+    await expandGeneset(brushThisGeneGeneset);
+    await expandGene(geneToBrushAndColorBy);
+    const histBrushableAreaId = `histogram-${geneToBrushAndColorBy}-plot-brushable-area`;
+
+    const coords = await calcDragCoordinates(histBrushableAreaId, {
+      x1: 0.25,
+      y1: 0.5,
+      x2: 0.55,
+      y2: 0.5,
+    });
+    await drag(histBrushableAreaId, coords.start, coords.end);
+    const cellCount = await getCellSetCount(1);
+    if (config.withSubset) {
+      expect(cellCount).toBe(subsetGeneBrushedCellCount);
+    } else {
+      expect(cellCount).toBe(geneBrushedCellCount);
+    }
+  });
+  test("color by gene in geneset", async () => {
+    await setup(config);
+
+    await expandGeneset(meanExpressionBrushGenesetName);
+
+    await colorByGene(geneToBrushAndColorBy);
+    await assertColorLegendLabel(geneToBrushAndColorBy);
+  });
+  test("delete gene from geneset and undo/redo", async () => {
+    // We've already deleted the gene
+    if (config.withSubset) return;
+
+    await setup(config);
+
+    await expandGeneset(setToRemoveFrom);
+    await removeGene(geneToRemove);
+    await assertGeneDoesNotExist(geneToRemove);
+    await clickOn("undo");
+    await assertGeneExistsInGeneset(geneToRemove);
+    await clickOn("redo");
+    await assertGeneDoesNotExist(geneToRemove);
+  });
+});
+
+describe.each([
+  { withSubset: true, tag: "subset", categoricalAnno: true },
+  { withSubset: false, tag: "whole", categoricalAnno: true },
 ])("annotations", (config) => {
   test("create a category", async () => {
     await setup(config);
@@ -287,11 +531,7 @@ describe.each([
     const labels = await getAllByClass("categorical-row");
 
     const result = await Promise.all(
-      labels.map((label) => {
-        return page.evaluate((element) => {
-          return element.outerHTML;
-        }, label);
-      })
+      labels.map((label) => page.evaluate((element) => element.outerHTML, label))
     );
 
     expect(result).toMatchSnapshot();

client/__tests__/e2e/e2eJestConfig.json

@@ -4,7 +4,7 @@
   "testMatch": ["**/__tests__/**/?(*.)(spec|test).js?(x)"],
   "setupFiles": ["../setupMissingGlobals.js"],
   "setupFilesAfterEnv": ["expect-puppeteer", "./puppeteer.setup.js"],
-  "globalSetup": "jest-environment-puppeteer/setup",
+  "globalSetup": "../globalSetup.js",
   "globalTeardown": "jest-environment-puppeteer/teardown",
   "testEnvironment": "./screenshot_env.js"
 }

client/__tests__/e2e/puppeteer.setup.js

@@ -13,7 +13,7 @@ import * as ENV_DEFAULT from "../../../environment.default.json";
 // a test can take more time to finish, so we don't want
 // jest to shut off the test too soon
 jest.setTimeout(2 * 60 * 1000);
-setDefaultOptions({ timeout: 20 * 1000 });
+setDefaultOptions({ timeout: 60 * 1000 });
 
 jest.retryTimes(ENV_DEFAULT.RETRY_ATTEMPTS);
 

client/__tests__/e2e/puppeteerUtils.js

@@ -12,7 +12,7 @@ export async function waitByID(testId, props = {}) {
 }
 
 export async function waitByClass(testClass, props = {}) {
-  await page.waitForSelector(`[data-testclass='${testClass}']`, props);
+  return page.waitForSelector(`[data-testclass='${testClass}']`, props);
 }
 
 export async function waitForAllByIds(testIds) {
@@ -32,7 +32,7 @@ export async function typeInto(testId, text) {
   const selector = getTestId(testId);
   // type ahead can be annoying if you don't pause before you type
   await page.click(selector);
-  await page.waitFor(200);
+  await page.waitForTimeout(200);
   await page.type(selector, text);
 }
 
@@ -42,7 +42,7 @@ export async function clearInputAndTypeInto(testId, text) {
   // only works for text without special characters
   // type ahead can be annoying if you don't pause before you type
   await page.click(selector);
-  await page.waitFor(200);
+  await page.waitForTimeout(200);
   // select all
   await page.click(selector, { clickCount: 3 });
   await page.keyboard.press("Backspace");
@@ -72,7 +72,7 @@ export async function clickOnUntil(testId, assert) {
     } catch (error) {
       retry += 1;
 
-      await page.waitFor(WAIT_FOR_MS);
+      await page.waitForTimeout(WAIT_FOR_MS);
     }
   }
 
@@ -100,12 +100,6 @@ export async function getElementCoordinates(testId) {
   });
 }
 
-async function clickTermsOfService() {
-  if (!(await isElementPresent(getTestId("tos-cookies-accept")))) return;
-
-  await clickOn("tos-cookies-accept");
-}
-
 async function nameNewAnnotation() {
   if (await isElementPresent(getTestId("annotation-dialog"))) {
     await typeInto("new-annotation-name", "ignoreE2E");
@@ -122,7 +116,6 @@ export async function goToPage(url) {
   });
 
   await nameNewAnnotation();
-  await clickTermsOfService();
 }
 
 export async function isElementPresent(selector, options) {

client/__tests__/e2e/test_config.yaml

@@ -1,33 +1,9 @@
 server:
   app:
     force_https: true
-
-    # By default, cellxgene will serve api requests from the same base url as the webpage.
-    # In general api_base_url and web_base_url will not need to be set.
-    # There are two reasons to set these parameters:
-    #  1. Oauth authentication is used; the oauth server will redirect back to the api_base_url after login,
-    #     which then redirects back to the web_base_url.  If the web_base_url is not set, it will default to
-    #     the api_base_url.  If oauth authentication is used, the api_base_url must be set.
-    #     For a local test (where the server runs on "http://localhost:<port>"), then the api_base_url may be
-    #     set to the string "local".
-    #  2. The cellxgene deploymnent is in an environment where the webpage and api have
-    #     different base urls.  In this case both api_base_url and web_base_url must be set.
-    #     It is up to the server admin to ensure that the networking is setup correctly for this environment.
-    api_base_url: http://localhost:5005
-    web_base_url: http://localhost:3000
-
-  authentication:
-    # The authentication types may be "none", "session", "oauth"
-    # none:  No authentication support, features like user_annotations must not be enabled.
-    # session:  A session based userid is automatically generated. (no params needed)
-    # oauth: oauth2 is used for authentication;  parameters are defined in params_oauth.
-    type: test
+    port: 5005
 
 dataset:
-  app:
-    about_legal_tos: null
-    about_legal_privacy: null
-
   presentation:
     max_categories: 1000
     custom_colors: true
@@ -38,10 +14,6 @@ dataset:
     local_file_csv:
       directory: null
       file: null
-    ontology:
-      enable: false
-      obo_location: null
 
   embeddings:
     names: []
-    enable_reembedding: false

client/__tests__/globalSetup.js

@@ -0,0 +1,5 @@
+const { setup } = require("jest-environment-puppeteer");
+
+module.exports = async () => {
+  await setup();
+};

client/__tests__/reducers/genesets.test.js

@@ -0,0 +1,524 @@
+import genesetsReducer from "../../src/reducers/genesets";
+
+describe("initial reducer state", () => {
+  test("some other action", () => {
+    expect(genesetsReducer(undefined, { type: "foo" })).toMatchObject({
+      initialized: false,
+      lastTid: undefined,
+      genesets: new Map(),
+    });
+  });
+});
+
+describe("geneset: initial load", () => {
+  test("missing JSON response", () => {
+    expect(() =>
+      genesetsReducer(undefined, {
+        type: "geneset: initial load",
+      })
+    ).toThrow("missing or malformed JSON response");
+  });
+
+  test("empty geneset", () => {
+    expect(
+      genesetsReducer(undefined, {
+        type: "geneset: initial load",
+        data: {
+          tid: 0,
+          genesets: [],
+        },
+      })
+    ).toMatchObject({
+      initialized: true,
+      lastTid: 0,
+      genesets: new Map(),
+    });
+  });
+
+  test("non-empty geneset", () => {
+    expect(
+      genesetsReducer(undefined, {
+        type: "geneset: initial load",
+        data: {
+          tid: 99,
+          genesets: [
+            {
+              geneset_name: "G1",
+              genes: [{ gene_symbol: "F5" }],
+            },
+            {
+              geneset_name: "G2",
+              geneset_description: "G2 desc",
+              genes: [{ gene_symbol: "F6" }],
+            },
+            {
+              geneset_name: "G3",
+              geneset_description: "G3 desc",
+              genes: [{ gene_symbol: "F7", gene_description: "gene desc" }],
+            },
+          ],
+        },
+      })
+    ).toMatchObject({
+      initialized: true,
+      lastTid: 99,
+      genesets: new Map([
+        [
+          "G1",
+          {
+            genesetName: "G1",
+            genesetDescription: "",
+            genes: new Map([["F5", { geneSymbol: "F5", geneDescription: "" }]]),
+          },
+        ],
+        [
+          "G2",
+          {
+            genesetName: "G2",
+            genesetDescription: "G2 desc",
+            genes: new Map([["F6", { geneSymbol: "F6", geneDescription: "" }]]),
+          },
+        ],
+        [
+          "G3",
+          {
+            genesetName: "G3",
+            genesetDescription: "G3 desc",
+            genes: new Map([
+              ["F7", { geneSymbol: "F7", geneDescription: "gene desc" }],
+            ]),
+          },
+        ],
+      ]),
+    });
+  });
+});
+
+describe("geneset: create", () => {
+  const initialState = genesetsReducer(undefined, {
+    type: "geneset: initial load",
+    data: {
+      tid: 0,
+      genesets: [],
+    },
+  });
+
+  test("simple create", () => {
+    expect(
+      genesetsReducer(initialState, {
+        type: "geneset: create",
+        genesetName: "a geneset",
+        genesetDescription: "",
+      })
+    ).toMatchObject({
+      ...initialState,
+      genesets: new Map([
+        [
+          "a geneset",
+          {
+            genesetName: "a geneset",
+            genesetDescription: "",
+            genes: new Map(),
+          },
+        ],
+      ]),
+    });
+  });
+
+  test("error - duplicate name", () => {
+    expect(() => {
+      genesetsReducer(
+        genesetsReducer(initialState, {
+          type: "geneset: create",
+          genesetName: "foo",
+          genesetDescription: "foo",
+        }),
+        {
+          type: "geneset: create",
+          genesetName: "foo",
+          genesetDescription: "bar",
+        }
+      );
+    }).toThrow("name already defined");
+  });
+
+  test("error - missing required action values", () => {
+    expect(() => {
+      genesetsReducer(initialState, {
+        type: "geneset: create",
+        genesetDescription: "foo",
+      });
+    }).toThrow();
+    expect(() => {
+      genesetsReducer(initialState, {
+        type: "geneset: create",
+        genesetName: "foo",
+      });
+    }).toThrow("name or description unspecified");
+  });
+});
+
+describe("geneset: delete", () => {
+  const initialState = genesetsReducer(undefined, {
+    type: "geneset: initial load",
+    data: {
+      tid: 0,
+      genesets: [],
+    },
+  });
+
+  test("simple delete", () => {
+    expect(
+      genesetsReducer(
+        genesetsReducer(initialState, {
+          type: "geneset: create",
+          genesetName: "foo",
+          genesetDescription: "foo",
+        }),
+        {
+          type: "geneset: delete",
+          genesetName: "foo",
+        }
+      )
+    ).toMatchObject({
+      initialized: true,
+      lastTid: 0,
+      genesets: new Map(),
+    });
+  });
+
+  test("error - missing name", () => {
+    expect(() => {
+      genesetsReducer(initialState, {
+        type: "geneset: delete",
+        genesetName: "foo",
+      });
+    }).toThrow("name does not exist");
+  });
+});
+
+describe("geneset: update", () => {
+  const initialState = genesetsReducer(undefined, {
+    type: "geneset: initial load",
+    data: {
+      tid: 0,
+      genesets: [],
+    },
+  });
+
+  test("simple update", () => {
+    expect(
+      genesetsReducer(
+        genesetsReducer(
+          genesetsReducer(initialState, {
+            type: "geneset: create",
+            genesetName: "foo1",
+            genesetDescription: "foo1",
+          }),
+          {
+            type: "geneset: create",
+            genesetName: "foo2",
+            genesetDescription: "foo2",
+          }
+        ),
+        {
+          type: "geneset: update",
+          genesetName: "foo1",
+          update: {
+            genesetName: "bar",
+            genesetDescription: "bar",
+          },
+        }
+      )
+    ).toMatchObject({
+      initialized: true,
+      lastTid: 0,
+      genesets: new Map([
+        [
+          "bar",
+          { genesetName: "bar", genesetDescription: "bar", genes: new Map() },
+        ],
+        [
+          "foo2",
+          { genesetName: "foo2", genesetDescription: "foo2", genes: new Map() },
+        ],
+      ]),
+    });
+  });
+
+  test("error - unknown name", () => {
+    expect(() => {
+      genesetsReducer(initialState, {
+        type: "geneset: update",
+        genesetName: "foo",
+        update: {
+          genesetName: "foo",
+          genesetDescription: "bar",
+        },
+      });
+    }).toThrow("name unspecified or does not exist");
+  });
+
+  test("error - duplicate name", () => {
+    expect(() => {
+      genesetsReducer(
+        genesetsReducer(initialState, {
+          type: "geneset: create",
+          genesetName: "foo",
+          genesetDescription: "foo",
+        }),
+        {
+          type: "geneset: update",
+          genesetName: "foo",
+          update: {
+            genesetName: "foo",
+            genesetDescription: "foo",
+          },
+        }
+      );
+    }).toThrow("update specified existing name and description");
+  });
+});
+
+describe("geneset: add genes", () => {
+  const initialState = genesetsReducer(
+    genesetsReducer(undefined, {
+      type: "geneset: initial load",
+      data: {
+        tid: 0,
+        genesets: [],
+      },
+    }),
+    {
+      type: "geneset: create",
+      genesetName: "test",
+      genesetDescription: "",
+    }
+  );
+
+  test("add a gene", () => {
+    expect(
+      genesetsReducer(initialState, {
+        type: "geneset: add genes",
+        genesetName: "test",
+        genes: [{ geneSymbol: "F5" }],
+      })
+    ).toMatchObject({
+      ...initialState,
+      genesets: new Map([
+        [
+          "test",
+          {
+            genesetName: "test",
+            genesetDescription: "",
+            genes: new Map([["F5", { geneSymbol: "F5", geneDescription: "" }]]),
+          },
+        ],
+      ]),
+    });
+
+    expect(
+      genesetsReducer(initialState, {
+        type: "geneset: add genes",
+        genesetName: "test",
+        genes: [
+          { geneSymbol: "F5", geneDescription: "desc" },
+          { geneSymbol: "SET1", geneDescription: "" },
+        ],
+      })
+    ).toMatchObject({
+      ...initialState,
+      genesets: new Map([
+        [
+          "test",
+          {
+            genesetName: "test",
+            genesetDescription: "",
+            genes: new Map([
+              ["F5", { geneSymbol: "F5", geneDescription: "desc" }],
+              ["SET1", { geneSymbol: "SET1", geneDescription: "" }],
+            ]),
+          },
+        ],
+      ]),
+    });
+  });
+
+  test("no such geneset error", () => {
+    expect(() => {
+      genesetsReducer(initialState, {
+        type: "geneset: add genes",
+        genesetName: "mumble",
+        genes: [],
+      });
+    }).toThrow("geneset name does not exist");
+  });
+});
+
+describe("geneset: delete genes", () => {
+  const initialState = genesetsReducer(
+    genesetsReducer(
+      genesetsReducer(undefined, {
+        type: "geneset: initial load",
+        data: {
+          tid: 0,
+          genesets: [],
+        },
+      }),
+      {
+        type: "geneset: create",
+        genesetName: "test",
+        genesetDescription: "",
+      }
+    ),
+    {
+      type: "geneset: add genes",
+      genesetName: "test",
+      genes: [{ geneSymbol: "F5" }],
+    }
+  );
+
+  test("simple", () => {
+    expect(
+      genesetsReducer(initialState, {
+        type: "geneset: delete genes",
+        genesetName: "test",
+        geneSymbols: ["F5"],
+      })
+    ).toMatchObject({
+      ...initialState,
+      genesets: new Map([
+        [
+          "test",
+          {
+            genesetName: "test",
+            genesetDescription: "",
+            genes: new Map(),
+          },
+        ],
+      ]),
+    });
+  });
+
+  test("no such geneset error", () => {
+    expect(() => {
+      genesetsReducer(initialState, {
+        type: "geneset: delete genes",
+        genesetName: "mumble",
+        geneSymbols: [],
+      });
+    }).toThrow("name does not exist");
+  });
+});
+
+describe("geneset: set gene description", () => {
+  const initialState = genesetsReducer(
+    genesetsReducer(
+      genesetsReducer(undefined, {
+        type: "geneset: initial load",
+        data: {
+          tid: 0,
+          genesets: [],
+        },
+      }),
+      {
+        type: "geneset: create",
+        genesetName: "test",
+        genesetDescription: "",
+      }
+    ),
+    {
+      type: "geneset: add genes",
+      genesetName: "test",
+      genes: [{ geneSymbol: "F5" }],
+    }
+  );
+
+  test("simple set", () => {
+    expect(
+      genesetsReducer(initialState, {
+        type: "geneset: set gene description",
+        genesetName: "test",
+        update: {
+          geneSymbol: "F5",
+          geneDescription: "mumble",
+        },
+      })
+    ).toMatchObject({
+      ...initialState,
+      genesets: new Map([
+        [
+          "test",
+          {
+            genesetName: "test",
+            genesetDescription: "",
+            genes: new Map([
+              ["F5", { geneSymbol: "F5", geneDescription: "mumble" }],
+            ]),
+          },
+        ],
+      ]),
+    });
+  });
+
+  test("no such geneset error", () => {
+    expect(() => {
+      genesetsReducer(initialState, {
+        type: "geneset: set gene description",
+        genesetName: "does not exist",
+        update: {
+          geneSymbol: "F5",
+          geneDescription: "mumble",
+        },
+      });
+    }).toThrow("geneset name does not exist");
+  });
+
+  test("no such gene error", () => {
+    expect(() => {
+      genesetsReducer(initialState, {
+        type: "geneset: set gene description",
+        genesetName: "test",
+        update: {
+          geneSymbol: "NO SUCH GENE",
+          geneDescription: "mumble",
+        },
+      });
+    }).toThrow("no such gene");
+  });
+});
+
+describe("geneset: set tid", () => {
+  test("simple set", () => {
+    expect(
+      genesetsReducer(undefined, {
+        type: "geneset: set tid",
+        tid: 1,
+      })
+    ).toMatchObject({ lastTid: 1 });
+  });
+
+  test("not a number error", () => {
+    expect(() => {
+      genesetsReducer(
+        { lastTid: 1 },
+        {
+          type: "geneset: set tid",
+          tid: "0",
+        }
+      );
+    }).toThrow("must be a positive integer");
+  });
+
+  test("decrement error", () => {
+    expect(() => {
+      genesetsReducer(
+        { lastTid: 1 },
+        {
+          type: "geneset: set tid",
+          tid: 0,
+        }
+      );
+    }).toThrow("may not be decremented");
+  });
+});

client/__tests__/reducers/genesetsUI.test.js

@@ -0,0 +1,75 @@
+import genesetsUIReducer from "../../src/reducers/genesetsUI";
+
+// Format: GeneSetsUI(state,action)
+
+const initialState = {
+  createGenesetModeActive: false,
+  isEditingGenesetName: false,
+  isAddingGenesToGeneset: false,
+};
+
+/* initial */
+describe("geneset UI states", () => {
+  test("initial state, some other action", () => {
+    expect(
+      genesetsUIReducer(undefined, {
+        type: "foo",
+      })
+    ).toMatchObject(initialState);
+  });
+  test("geneset: activate add new geneset mode", () => {
+    expect(
+      genesetsUIReducer(initialState, {
+        type: "geneset: activate add new geneset mode",
+      })
+    ).toMatchObject({
+      createGenesetModeActive: true,
+      isEditingGenesetName: false,
+      isAddingGenesToGeneset: false,
+    });
+  });
+  test("geneset: disable create geneset mode", () => {
+    expect(
+      genesetsUIReducer(undefined, { isEditingGenesetName: false })
+    ).toMatchObject(initialState);
+  });
+
+  test("activate add new genes mode", () => {
+    expect(
+      genesetsUIReducer(undefined, {
+        type: "geneset: activate add new genes mode",
+        geneset: "a geneset name",
+      })
+    ).toMatchObject({
+      createGenesetModeActive: false,
+      isEditingGenesetName: false,
+      isAddingGenesToGeneset: "a geneset name",
+    });
+  });
+  test("disable create geneset mode", () => {
+    expect(
+      genesetsUIReducer(undefined, {
+        type: "geneset: disable create geneset mode",
+      })
+    ).toMatchObject(initialState);
+  });
+  test("activate rename geneset mode", () => {
+    expect(
+      genesetsUIReducer(undefined, {
+        type: "geneset: activate rename geneset mode",
+        data: "a geneset name",
+      })
+    ).toMatchObject({
+      createGenesetModeActive: false,
+      isEditingGenesetName: "a geneset name",
+      isAddingGenesToGeneset: false,
+    });
+  });
+  test("disable rename geneset mode", () => {
+    expect(
+      genesetsUIReducer(undefined, {
+        type: "geneset: disable rename geneset mode",
+      })
+    ).toMatchObject(initialState);
+  });
+});

client/__tests__/reducers/undoable.test.js

@@ -23,9 +23,7 @@ describe("create", () => {
 describe("undo", () => {
   test("expected state modifications", () => {
     const initialState = { a: 0, b: 1000 };
-    const reducer = (state) => {
-      return { a: state.a + 1, b: state.b + 1 };
-    };
+    const reducer = (state) => ({ a: state.a + 1, b: state.b + 1 });
     const undoableReducer = undoable(reducer, ["a"]);
 
     const s1 = undoableReducer(initialState, { type: "test" });
@@ -43,9 +41,7 @@ describe("undo", () => {
 
 describe("redo", () => {
   const initialState = { a: 0, b: 1000 };
-  const reducer = (state) => {
-    return { a: state.a + 1, b: state.b + 1 };
-  };
+  const reducer = (state) => ({ a: state.a + 1, b: state.b + 1 });
   let UR;
 
   beforeEach(() => {

client/__tests__/util/actionHelpers.test.js

@@ -11,18 +11,14 @@ describe("rangeEncodeIndices", () => {
 
   test("sorted flag", () => {
     expect(rangeEncodeIndices([1, 9, 432], 10, true)).toMatchObject([
-      1,
-      9,
-      432,
+      1, 9, 432,
     ]);
     expect(rangeEncodeIndices([1, 9, 432], 10, false)).toMatchObject([
-      1,
-      9,
-      432,
+      1, 9, 432,
     ]);
-    expect(
-      rangeEncodeIndices([0, 1, 2, 3, 9, 10, 432], 2, true)
-    ).toMatchObject([[0, 3], [9, 10], 432]);
+    expect(rangeEncodeIndices([0, 1, 2, 3, 9, 10, 432], 2, true)).toMatchObject(
+      [[0, 3], [9, 10], 432]
+    );
     expect(
       rangeEncodeIndices([0, 1, 2, 3, 9, 10, 432], 2, false)
     ).toMatchObject([[0, 3], [9, 10], 432]);

client/__tests__/util/annoMatrix/annoMatrix.test.js

@@ -85,9 +85,11 @@ describe("AnnoMatrix", () => {
       fetch.once(serverMocks.responder);
       await expect(
         annoMatrix.fetch("X", {
-          field: "var",
-          column: annoMatrix.schema.annotations.var.index,
-          value: "TYMP",
+          where: {
+            field: "var",
+            column: annoMatrix.schema.annotations.var.index,
+            value: "TYMP",
+          },
         })
       ).resolves.toBeInstanceOf(Dataframe);
 
@@ -103,14 +105,18 @@ describe("AnnoMatrix", () => {
       await expect(
         annoMatrix.fetch("X", [
           {
-            field: "var",
-            column: varIndex,
-            value: "SUMO3",
+            where: {
+              field: "var",
+              column: varIndex,
+              value: "SUMO3",
+            },
           },
           {
-            field: "var",
-            column: varIndex,
-            value: "TYMP",
+            where: {
+              field: "var",
+              column: varIndex,
+              value: "TYMP",
+            },
           },
         ])
       ).resolves.toBeInstanceOf(Dataframe);

client/__tests__/util/annoMatrix/crossfilter.test.js

@@ -170,9 +170,11 @@ describe("AnnoMatrixCrossfilter", () => {
       const xfltr = await crossfilter.select(
         "X",
         {
-          field: "var",
-          column: varIndex,
-          value: "TYMP",
+          where: {
+            field: "var",
+            column: varIndex,
+            value: "TYMP",
+          },
         },
         {
           mode: "range",
@@ -186,9 +188,11 @@ describe("AnnoMatrixCrossfilter", () => {
       expect(xfltr.countSelected()).toEqual(501);
 
       const df = await annoMatrix.fetch("X", {
-        field: "var",
-        column: varIndex,
-        value: "TYMP",
+        where: {
+          field: "var",
+          column: varIndex,
+          value: "TYMP",
+        },
       });
       const values = df.icol(0).asArray();
       const selected = xfltr.allSelectedMask();

client/__tests__/util/annoMatrix/serverMocks/index.js

@@ -1,11 +1,4 @@
 export const baseDataURL = "https://a.fake.url/api/v0.2";
 
-window.CELLXGENE = {
-  API: {
-    prefix: baseDataURL,
-    version: "v0.2/",
-  },
-};
-
 export { schema } from "./schema";
 export * from "./routes";

client/__tests__/util/annoMatrix/whereCache.test.js

@@ -1,3 +1,4 @@
+import sha1 from "sha1";
 import {
   _whereCacheGet,
   _whereCacheCreate,
@@ -7,37 +8,82 @@ import {
 const schema = {};
 
 describe("whereCache", () => {
-  test("whereCacheGet - missing cache values", () => {
+  test("whereCacheGet - where query, missing cache values", () => {
     expect(
       _whereCacheGet({}, schema, "X", {
-        field: "var",
-        column: "foo",
-        value: "bar",
+        where: {
+          field: "var",
+          column: "foo",
+          value: "bar",
+        },
       })
     ).toEqual([undefined]);
     expect(
-      _whereCacheGet({ X: {} }, schema, "X", {
-        field: "var",
-        column: "foo",
-        value: "bar",
+      _whereCacheGet({}, schema, "X", {
+        summarize: {
+          field: "var",
+          column: "foo",
+          values: ["bar"],
+        },
       })
     ).toEqual([undefined]);
     expect(
-      _whereCacheGet({ X: { var: new Map() } }, schema, "X", {
-        field: "var",
-        column: "foo",
-        value: "bar",
+      _whereCacheGet({ where: { X: {} } }, schema, "X", {
+        where: {
+          field: "var",
+          column: "foo",
+          value: "bar",
+        },
+      })
+    ).toEqual([undefined]);
+    expect(
+      _whereCacheGet({ where: { X: { var: new Map() } } }, schema, "X", {
+        where: {
+          field: "var",
+          column: "foo",
+          value: "bar",
+        },
       })
     ).toEqual([undefined]);
     expect(
       _whereCacheGet(
-        { X: { var: new Map([["foo", new Map()]]) } },
+        { where: { X: { var: new Map([["foo", new Map()]]) } } },
         schema,
         "X",
         {
+          where: {
+            field: "var",
+            column: "foo",
+            value: "bar",
+          },
+        }
+      )
+    ).toEqual([undefined]);
+  });
+
+  test("whereCacheGet - summarize query, missing cache values", () => {
+    expect(
+      _whereCacheGet({}, schema, "X", {
+        summarize: {
+          method: "mean",
           field: "var",
           column: "foo",
-          value: "bar",
+          values: ["bar"],
+        },
+      })
+    ).toEqual([undefined]);
+    expect(
+      _whereCacheGet(
+        { summarize: { X: { mean: { var: new Map() } } } },
+        schema,
+        "X",
+        {
+          summarize: {
+            method: "mean",
+            field: "var",
+            column: "foo",
+            values: ["bar"],
+          },
         }
       )
     ).toEqual([undefined]);
@@ -45,140 +91,292 @@ describe("whereCache", () => {
 
   test("whereCacheGet - varied lookups", () => {
     const whereCache = {
-      X: {
-        var: new Map([
-          [
-            "foo",
-            new Map([
-              ["bar", [0]],
-              ["baz", [1, 2]],
+      where: {
+        X: {
+          var: new Map([
+            [
+              "foo",
+              new Map([
+                ["bar", [0]],
+                ["baz", [1, 2]],
+              ]),
+            ],
+          ]),
+        },
+      },
+      summarize: {
+        X: {
+          mean: {
+            var: new Map([
+              [
+                "foo",
+                new Map([
+                  [sha1("bar"), [0]],
+                  [sha1("baz"), [1, 2]],
+                ]),
+              ],
             ]),
-          ],
-        ]),
+          },
+        },
       },
     };
 
     expect(
       _whereCacheGet(whereCache, schema, "X", {
-        field: "var",
-        column: "foo",
-        value: "bar",
+        where: {
+          field: "var",
+          column: "foo",
+          value: "bar",
+        },
       })
     ).toEqual([0]);
     expect(
       _whereCacheGet(whereCache, schema, "X", {
-        field: "var",
-        column: "foo",
-        value: "baz",
+        summarize: {
+          method: "mean",
+          field: "var",
+          column: "foo",
+          values: ["bar"],
+        },
+      })
+    ).toEqual([0]);
+    expect(
+      _whereCacheGet(whereCache, schema, "X", {
+        where: {
+          field: "var",
+          column: "foo",
+          value: "baz",
+        },
+      })
+    ).toEqual([1, 2]);
+    expect(
+      _whereCacheGet(whereCache, schema, "X", {
+        summarize: {
+          method: "mean",
+          field: "var",
+          column: "foo",
+          values: ["baz"],
+        },
       })
     ).toEqual([1, 2]);
     expect(_whereCacheGet(whereCache, schema, "Y", {})).toEqual([undefined]);
     expect(
       _whereCacheGet(whereCache, schema, "X", {
-        field: "whoknows",
-        column: "whatever",
-        value: "snork",
+        where: {
+          field: "whoknows",
+          column: "whatever",
+          value: "snork",
+        },
       })
     ).toEqual([undefined]);
     expect(
       _whereCacheGet(whereCache, schema, "X", {
-        field: "var",
-        column: "whatever",
-        value: "snork",
+        where: {
+          field: "var",
+          column: "whatever",
+          value: "snork",
+        },
       })
     ).toEqual([undefined]);
     expect(
       _whereCacheGet(whereCache, schema, "X", {
-        field: "var",
-        column: "foo",
-        value: "snork",
+        where: {
+          field: "var",
+          column: "foo",
+          value: "snork",
+        },
       })
     ).toEqual([undefined]);
   });
 
-  test("whereCacheCreate", () => {
+  test("whereCacheCreate, where query", () => {
     const query = {
-      field: "queryField",
-      column: "queryColumn",
-      value: "queryValue",
+      where: {
+        field: "queryField",
+        column: "queryColumn",
+        value: "queryValue",
+      },
     };
     const wc = _whereCacheCreate(
       "field",
-      { field: "queryField", column: "queryColumn", value: "queryValue" },
+      {
+        where: {
+          field: "queryField",
+          column: "queryColumn",
+          value: "queryValue",
+        },
+      },
       [0, 1, 2]
     );
     expect(wc).toBeDefined();
     expect(wc).toEqual(
       expect.objectContaining({
-        field: {
-          queryField: expect.any(Map),
+        where: {
+          field: {
+            queryField: expect.any(Map),
+          },
         },
       })
     );
-    expect(wc.field.queryField.has("queryColumn")).toEqual(true);
-    expect(wc.field.queryField.get("queryColumn")).toBeInstanceOf(Map);
-    expect(wc.field.queryField.get("queryColumn").has("queryValue")).toEqual(
-      true
-    );
+    expect(wc.where.field.queryField.has("queryColumn")).toEqual(true);
+    expect(wc.where.field.queryField.get("queryColumn")).toBeInstanceOf(Map);
+    expect(
+      wc.where.field.queryField.get("queryColumn").has("queryValue")
+    ).toEqual(true);
     expect(_whereCacheGet(wc, schema, "field", query)).toEqual([0, 1, 2]);
   });
 
-  test("whereCacheMerge", () => {
+  test("whereCacheCreate, summarize query", () => {
+    const query = {
+      summarize: {
+        method: "method",
+        field: "queryField",
+        column: "queryColumn",
+        values: ["queryValue"],
+      },
+    };
+    const wc = _whereCacheCreate("field", query, [0, 1, 2]);
+    expect(_whereCacheGet(wc, schema, "field", query)).toEqual([0, 1, 2]);
+  });
+
+  test("whereCacheCreate, unknown query type", () => {
+    expect(_whereCacheCreate("field", { foobar: true }, [1])).toEqual({});
+  });
+
+  test("whereCacheMerge, where queries", () => {
     let wc;
 
     // remember, will mutate dst
     const src = _whereCacheCreate(
       "field",
-      { field: "queryField", column: "queryColumn", value: "foo" },
+      { where: { field: "queryField", column: "queryColumn", value: "foo" } },
       ["foo"]
     );
 
     const dst1 = _whereCacheCreate(
       "field",
-      { field: "queryField", column: "queryColumn", value: "bar" },
+      { where: { field: "queryField", column: "queryColumn", value: "bar" } },
       ["dst1"]
     );
     wc = _whereCacheMerge(dst1, src);
     expect(
       _whereCacheGet(wc, schema, "field", {
-        field: "queryField",
-        column: "queryColumn",
-        value: "foo",
+        where: {
+          field: "queryField",
+          column: "queryColumn",
+          value: "foo",
+        },
       })
     ).toEqual(["foo"]);
     expect(
       _whereCacheGet(wc, schema, "field", {
-        field: "queryField",
-        column: "queryColumn",
-        value: "bar",
+        where: {
+          field: "queryField",
+          column: "queryColumn",
+          value: "bar",
+        },
       })
     ).toEqual(["dst1"]);
 
     const dst2 = _whereCacheCreate(
       "field",
-      { field: "queryField", column: "queryColumn", value: "bar" },
+      { where: { field: "queryField", column: "queryColumn", value: "bar" } },
       ["dst2"]
     );
     wc = _whereCacheMerge(dst2, dst1, src);
     expect(
       _whereCacheGet(wc, schema, "field", {
-        field: "queryField",
-        column: "queryColumn",
-        value: "foo",
+        where: {
+          field: "queryField",
+          column: "queryColumn",
+          value: "foo",
+        },
       })
     ).toEqual(["foo"]);
     expect(
       _whereCacheGet(wc, schema, "field", {
-        field: "queryField",
-        column: "queryColumn",
-        value: "bar",
+        where: {
+          field: "queryField",
+          column: "queryColumn",
+          value: "bar",
+        },
       })
     ).toEqual(["dst1"]);
 
     wc = _whereCacheMerge({}, src);
     expect(wc).toEqual(src);
 
-    wc = _whereCacheMerge({ field: { queryField: new Map() } }, src);
+    wc = _whereCacheMerge({ where: { field: { queryField: new Map() } } }, src);
     expect(wc).toEqual(src);
   });
+
+  test("whereCacheMerge, mixed queries", () => {
+    const wc = _whereCacheMerge(
+      _whereCacheCreate(
+        "field",
+        {
+          where: {
+            field: "queryField",
+            column: "queryColumn",
+            value: "foo",
+          },
+        },
+        ["a"]
+      ),
+      _whereCacheCreate(
+        "field",
+        {
+          summarize: {
+            method: "mean",
+            field: "queryField",
+            column: "queryColumn",
+            values: ["foo", "bar", "baz"],
+          },
+        },
+        ["b"]
+      )
+    );
+
+    expect(
+      _whereCacheGet(wc, schema, "field", {
+        where: {
+          field: "queryField",
+          column: "queryColumn",
+          value: "foo",
+        },
+      })
+    ).toEqual(["a"]);
+
+    expect(
+      _whereCacheGet(wc, schema, "field", {
+        summarize: {
+          method: "mean",
+          field: "queryField",
+          column: "queryColumn",
+          values: ["foo", "bar", "baz"],
+        },
+      })
+    ).toEqual(["b"]);
+
+    expect(
+      _whereCacheGet(wc, schema, "field", {
+        where: {
+          field: "queryField",
+          column: "queryColumn",
+          value: "does-not-exist",
+        },
+      })
+    ).toEqual([undefined]);
+
+    expect(
+      _whereCacheGet(wc, schema, "field", {
+        summarize: {
+          method: "no-such-method",
+          field: "queryField",
+          column: "queryColumn",
+          values: ["does-not-exist"],
+        },
+      })
+    ).toEqual([undefined]);
+  });
 });

client/__tests__/util/centroid.test.js

@@ -1,11 +1,11 @@
-import _ from "lodash";
+import cloneDeep from "lodash.clonedeep";
 
 import calcCentroid from "../../src/util/centroid";
 import quantile from "../../src/util/quantile";
 import { matrixFBSToDataframe } from "../../src/util/stateManager/matrix";
 import * as REST from "./stateManager/sampleResponses";
 import { indexEntireSchema } from "../../src/util/stateManager/schemaHelpers";
-import { _normalizeCategoricalSchema } from "../../src/annoMatrix/schema";
+import { normalizeWritableCategoricalSchema } from "../../src/annoMatrix/normalize";
 
 describe("centroid", () => {
   let schema;
@@ -13,11 +13,11 @@ describe("centroid", () => {
   let obsLayout;
 
   beforeAll(() => {
-    schema = indexEntireSchema(_.cloneDeep(REST.schema.schema));
+    schema = indexEntireSchema(cloneDeep(REST.schema.schema));
     obsAnnotations = matrixFBSToDataframe(REST.annotationsObs);
     obsLayout = matrixFBSToDataframe(REST.layoutObs);
 
-    _normalizeCategoricalSchema(
+    normalizeWritableCategoricalSchema(
       schema.annotations.obsByName.field3,
       obsAnnotations.col("field3")
     );

client/__tests__/util/dataframe/dataframe.test.js

@@ -793,9 +793,7 @@ describe("dataframe factories", () => {
         [3, 3],
         [new Array(3).fill(0), new Array(3).fill(0), new Array(3).fill(0)]
       );
-      const dfB = dfA.mapColumns(() => {
-        return new Array(3).fill(1);
-      });
+      const dfB = dfA.mapColumns(() => new Array(3).fill(1));
       expect(dfA).not.toBe(dfB);
       expect(dfB.iat(0, 0)).toEqual(1);
       expect(dfB.iat(0, 1)).toEqual(1);

client/__tests__/util/dataframe/histogram.test.js

@@ -88,16 +88,7 @@ describe("Dataframe column histogram", () => {
     expect(df.col(1).histogram(5, [0, 100])).toEqual([5, 1, 0, 0, 2]);
     expect(df.col(0).histogram(2, [0, 10])).toEqual([2, 2]);
     expect(df.col(0).histogram(10, [0, 100])).toEqual([
-      3,
-      2,
-      1,
-      0,
-      0,
-      0,
-      0,
-      0,
-      0,
-      2,
+      3, 2, 1, 0, 0, 0, 0, 0, 0, 2,
     ]);
   });
 });

client/__tests__/util/quantile.test.js

@@ -19,11 +19,7 @@ describe("quantile", () => {
   test("multi q", () => {
     const arr = new Float32Array([9, 3, 5, 6, 0]);
     expect(quantile([0, 0.25, 0.5, 0.75, 1.0], arr)).toMatchObject([
-      0,
-      3,
-      5,
-      6,
-      9,
+      0, 3, 5, 6, 9,
     ]);
   });
 });

client/__tests__/util/stateManager/controlsHelpers.test.js

@@ -1,38 +1,9 @@
 /*
 test controls helpers
 */
-import { subsetAndResetGeneLists } from "../../../src/util/stateManager/controlsHelpers";
-import * as globals from "../../../src/globals";
+// TODO #2227 test: improve test coverage on control helper functions
+// (`isSelectableCategoryName()`, `selectableCategoryNames()`, `createCategorySummaryFromDfCol()`, `createCategoricalSelection()`, )
 
 describe("controls helpers", () => {
-  test("subsetAndResetGeneLists", () => {
-    const geneList = [];
-    const genRandGene = () => Math.random().toString(36).substring(2, 6);
-
-    // build a unique set of genes
-    for (let i = 0; i < 150; i += 1) {
-      let randGene = genRandGene();
-      while (geneList.includes(randGene)) randGene = genRandGene();
-      geneList.push(randGene);
-    }
-    // insert duplicates
-    geneList[0] = "dupl";
-    geneList[20] = "dupl";
-    const state = {
-      userDefinedGenes: geneList.slice(0, 20),
-      diffexpGenes: geneList.slice(20),
-    };
-    const [newUserDefinedGenes, newDiffExpGenes] = subsetAndResetGeneLists(
-      state
-    );
-    const expectedNewUserDefinedGenes = [
-      ...geneList.slice(0, 20),
-      ...geneList.slice(21),
-    ].slice(0, globals.maxGenes);
-    expect(globals.maxUserDefinedGenes).toBeLessThan(globals.maxGenes);
-    expect(geneList.length).toBeGreaterThan(globals.maxGenes);
-    expect(newUserDefinedGenes).toHaveLength(globals.maxGenes);
-    expect(newUserDefinedGenes).toStrictEqual(expectedNewUserDefinedGenes);
-    expect(newDiffExpGenes).toStrictEqual([]);
-  });
+  test("placeholder", () => {});
 });

client/__tests__/util/stateManager/sampleResponses.js

@@ -1,3 +1,7 @@
+import every from "lodash.every";
+import map from "lodash.map";
+import isNumber from "lodash.isnumber";
+import zip from "lodash.zip";
 import _ from "lodash";
 import { flatbuffers } from "flatbuffers";
 import { NetEncoding } from "../../../src/util/stateManager/matrix_generated";
@@ -121,10 +125,10 @@ function encodeMatrix(columns, colIndex = undefined) {
   */
   const utf8Encoder = new TextEncoder("utf-8");
   const builder = new flatbuffers.Builder(1024);
-  const cols = _.map(columns, (carr) => {
+  const cols = map(columns, (carr) => {
     let uType;
     let tarr;
-    if (_.every(carr, _.isNumber)) {
+    if (every(carr, isNumber)) {
       uType = NetEncoding.TypedArray.Float32Array;
       tarr = encodeTypedArray(builder, uType, new Float32Array(carr));
     } else {
@@ -167,12 +171,12 @@ function encodeMatrix(columns, colIndex = undefined) {
 }
 
 const anAnnotationsObsFBSResponse = (() => {
-  const columns = _.zip(...anAnnotationsObsJSONResponse.data).slice(1);
+  const columns = zip(...anAnnotationsObsJSONResponse.data).slice(1);
   return encodeMatrix(columns, anAnnotationsObsJSONResponse.names);
 })();
 
 const anAnnotationsVarFBSResponse = (() => {
-  const columns = _.zip(...anAnnotationsVarJSONResponse.data).slice(1);
+  const columns = zip(...anAnnotationsVarJSONResponse.data).slice(1);
   return encodeMatrix(columns, anAnnotationsVarJSONResponse.names);
 })();
 

client/__tests__/util/typedCrossfilter/bitArray.test.js

@@ -1,5 +1,3 @@
-// jshint esversion: 6
-
 import BitArray from "../../../src/util/typedCrossfilter/bitArray";
 
 const defaultTestLength = 8;

client/__tests__/util/typedCrossfilter/crossfilter.test.js

@@ -1,4 +1,5 @@
-import _ from "lodash";
+import filter from "lodash.filter";
+import zip from "lodash.zip";
 
 import Crossfilter from "../../../src/util/typedCrossfilter";
 
@@ -251,12 +252,12 @@ describe("ImmutableTypedCrossfilter", () => {
     test.each([[[]], [[2]], [[2, 1]], [[9, 82]], [[0, 1]]])("exact: %p", (v) =>
       expect(
         p.select("quantity", { mode: "exact", values: v }).countSelected()
-      ).toEqual(_.filter(someData, (d) => v.includes(d.quantity)).length)
+      ).toEqual(filter(someData, (d) => v.includes(d.quantity)).length)
     );
     test("single value exact", () => {
       expect(
         p.select("quantity", { mode: "exact", values: 2 }).countSelected()
-      ).toEqual(_.filter(someData, (d) => d.quantity === 2).length);
+      ).toEqual(filter(someData, (d) => d.quantity === 2).length);
     });
     test.each([
       [0, 1],
@@ -267,7 +268,7 @@ describe("ImmutableTypedCrossfilter", () => {
       expect(
         p.select("quantity", { mode: "range", lo, hi }).countSelected()
       ).toEqual(
-        _.filter(someData, (d) => d.quantity >= lo && d.quantity < hi).length
+        filter(someData, (d) => d.quantity >= lo && d.quantity < hi).length
       )
     );
     test("bad mode", () => {
@@ -298,12 +299,12 @@ describe("ImmutableTypedCrossfilter", () => {
     ])("exact: %p", (v) =>
       expect(
         p.select("type", { mode: "exact", values: v }).countSelected()
-      ).toEqual(_.filter(someData, (d) => v.includes(d.type)).length)
+      ).toEqual(filter(someData, (d) => v.includes(d.type)).length)
     );
     test("single value exact", () => {
       expect(
         p.select("type", { mode: "exact", values: "tab" }).countSelected()
-      ).toEqual(_.filter(someData, (d) => d.type === "tab").length);
+      ).toEqual(filter(someData, (d) => d.type === "tab").length);
     });
     test("range", () => {
       expect(() => p.select("type", { mode: "range", lo: 0, hi: 9 })).toThrow(
@@ -341,7 +342,7 @@ describe("ImmutableTypedCrossfilter", () => {
           .select("coords", { mode: "within-rect", minX, minY, maxX, maxY })
           .allSelected()
       ).toEqual(
-        _.filter(someData, (d) => {
+        filter(someData, (d) => {
           const [x, y] = d.coords;
           return minX <= x && x < maxX && minY <= y && y < maxY;
         })
@@ -397,7 +398,7 @@ describe("ImmutableTypedCrossfilter", () => {
       expect(
         p.select("coords", { mode: "within-polygon", polygon }).allSelected()
       ).toEqual(
-        _.zip(someData, expected)
+        zip(someData, expected)
           .filter((x) => x[1])
           .map((x) => x[0])
       );

client/__tests__/util/typedCrossfilter/positiveInterval.test.js

@@ -1,5 +1,3 @@
-// jshint esversion: 6
-
 // const PositiveIntervals = require("../../src/util/typedCrossfilter/positiveIntervals");
 import PositiveIntervals from "../../../src/util/typedCrossfilter/positiveIntervals";
 
@@ -153,9 +151,9 @@ describe("intersection", () => {
       [1, 2],
       [6, 9],
     ]);
-    expect(
-      PositiveIntervals.intersection([[0, 2638]], [[1363, 2638]])
-    ).toEqual([[1363, 2638]]);
+    expect(PositiveIntervals.intersection([[0, 2638]], [[1363, 2638]])).toEqual(
+      [[1363, 2638]]
+    );
     expect(PositiveIntervals.intersection([[1, 2]], [[1, 2]])).toEqual([
       [1, 2],
     ]);

client/configuration/babel/babel.dev.js

@@ -16,6 +16,8 @@ module.exports = {
     "@babel/plugin-proposal-function-bind",
     ["@babel/plugin-proposal-decorators", { legacy: true }],
     ["@babel/plugin-proposal-class-properties", { loose: true }],
+    ["@babel/plugin-transform-private-methods", { loose: true }],
+    ["@babel/plugin-transform-private-property-in-object", { loose: true }],
     "@babel/plugin-proposal-export-namespace-from",
     "@babel/plugin-proposal-optional-chaining",
     "@babel/plugin-proposal-nullish-coalescing-operator",

client/configuration/babel/babel.prod.js

@@ -15,6 +15,8 @@ module.exports = {
     "@babel/plugin-proposal-function-bind",
     ["@babel/plugin-proposal-decorators", { legacy: true }],
     ["@babel/plugin-proposal-class-properties", { loose: true }],
+    ["@babel/plugin-transform-private-methods", { loose: true }],
+    ["@babel/plugin-transform-private-property-in-object", { loose: true }],
     "@babel/plugin-proposal-export-namespace-from",
     "@babel/plugin-transform-react-constant-elements",
     "@babel/plugin-transform-runtime",

client/configuration/eslint/eslint.js

@@ -1,24 +1,21 @@
 module.exports = {
   root: true,
-  parser: "babel-eslint",
+
   extends: [
     "airbnb",
     "plugin:eslint-comments/recommended",
     "plugin:@blueprintjs/recommended",
     "plugin:compat/recommended",
-    "plugin:prettier/recommended",
-    "prettier/react",
+    "plugin:jsx-a11y/recommended",
+    // (thuang) disable eslint formatting rules, so prettier can do its job
+    // Do not use `plugin:prettier/recommended` per doc below:
+    // https://prettier.io/docs/en/integrating-with-linters.html
+    "prettier",
   ],
   settings: {
-    polyfills: [
-      "TextDecoder",
-      "TextEncoder",
-      "fetch",
-      "Request",
-      "Response",
-      "Headers",
-      "AbortController",
-    ],
+    // AbortController is not supported in iOS Safari 10.3, Chrome 61
+    // Headers is not supported in iOS Safari 10.3
+    polyfills: ["Headers", "AbortController"],
   },
   env: { browser: true, commonjs: true, es6: true },
   globals: {
@@ -31,6 +28,7 @@ module.exports = {
     context: true,
     beforeEach: true,
   },
+  parser: "@babel/eslint-parser",
   parserOptions: {
     ecmaVersion: 2017,
     sourceType: "module",
@@ -38,6 +36,9 @@ module.exports = {
       jsx: true,
       generators: true,
     },
+    babelOptions: {
+      configFile: "./configuration/babel/babel.prod.js",
+    },
   },
   rules: {
     "react/jsx-no-target-blank": "off",

client/configuration/lint-staged/lint-staged.config.js

@@ -1,3 +1,4 @@
 module.exports = {
   "*.js": "eslint --fix",
+  "**/*": "prettier --write --ignore-unknown",
 };

client/configuration/webpack/obsoleteHTMLTemplate.html

@@ -56,13 +56,6 @@
         />
         <div>Chrome &gt; 60</div>
       </a>
-      <a href="https://www.apple.com/safari/" aria-label="Download Safari">
-        <img
-          src="https://cellxgene.cziscience.com/s3/cellxgene/static/images/safari.png"
-          style="width: 80px; height: 80px;"
-        />
-        <div>Safari ≥ 10.1</div>
-      </a>
       <a href="https://www.mozilla.com/firefox/" aria-label="Download Firefox">
         <img
           src="https://cellxgene.cziscience.com/s3/cellxgene/static/images/firefox.png"

client/configuration/webpack/webpack.config.dev.js

@@ -1,8 +1,6 @@
 const path = require("path");
 const webpack = require("webpack");
 const HtmlWebpackPlugin = require("html-webpack-plugin");
-const FaviconsWebpackPlugin = require("favicons-webpack-plugin");
-const ScriptExtHtmlWebpackPlugin = require("script-ext-html-webpack-plugin");
 const MiniCssExtractPlugin = require("mini-css-extract-plugin");
 
 const { merge } = require("webpack-merge");
@@ -11,6 +9,7 @@ const sharedConfig = require("./webpack.config.shared");
 const babelOptions = require("../babel/babel.dev");
 
 const fonts = path.resolve("src/fonts");
+const images = path.resolve("src/images");
 const nodeModules = path.resolve("node_modules");
 
 const devConfig = {
@@ -30,11 +29,11 @@ const devConfig = {
       {
         test: /\.(jpg|png|gif|eot|svg|ttf|woff|woff2|otf)$/i,
         loader: "file-loader",
-        include: [nodeModules, fonts],
-        query: {
+        include: [nodeModules, fonts, images],
+        options: {
           name: "static/assets/[name].[ext]",
           // (thuang): This is needed to make sure @font url path is '/static/assets/'
-          publicPath: "/",
+          publicPath: "..",
         },
       },
     ],
@@ -44,21 +43,6 @@ const devConfig = {
       inject: true,
       template: path.resolve("index.html"),
     }),
-    new FaviconsWebpackPlugin({
-      logo: "./favicon.png",
-      prefix: "static/img/",
-      favicons: {
-        icons: {
-          android: false,
-          appleIcon: false,
-          appleStartup: false,
-          coast: false,
-          firefox: false,
-          windows: false,
-          yandex: false,
-        },
-      },
-    }),
     new MiniCssExtractPlugin({
       filename: "static/[name].css",
     }),
@@ -67,14 +51,16 @@ const devConfig = {
       __REACT_DEVTOOLS_GLOBAL_HOOK__: "({ isDisabled: true })",
     }),
     new webpack.DefinePlugin({
-      "process.env.CXG_SERVER_PORT": JSON.stringify(
-        process.env.CXG_SERVER_PORT
-      ),
-    }),
-    new ScriptExtHtmlWebpackPlugin({
-      async: "obsolete",
+      // webpack 5 no longer polyfills NodeJS modules, so fake the one we need
+      "process.env": JSON.stringify({
+        NODE_ENV: process.env.NODE_ENV || "development",
+        CXG_SERVER_PORT: process.env.CXG_SERVER_PORT || "5005",
+      }),
     }),
   ],
+  infrastructureLogging: {
+    level: "warn",
+  },
 };
 
 module.exports = merge(sharedConfig, devConfig);

client/configuration/webpack/webpack.config.prod.js

@@ -1,10 +1,9 @@
 const path = require("path");
+const webpack = require("webpack");
 const HtmlWebpackPlugin = require("html-webpack-plugin");
 const { CleanWebpackPlugin } = require("clean-webpack-plugin");
 const TerserJSPlugin = require("terser-webpack-plugin");
-const CleanCss = require("clean-css");
-const OptimizeCSSAssetsPlugin = require("optimize-css-assets-webpack-plugin");
-const FaviconsWebpackPlugin = require("favicons-webpack-plugin");
+const CssMinimizerPlugin = require("css-minimizer-webpack-plugin");
 const MiniCssExtractPlugin = require("mini-css-extract-plugin");
 
 const { merge } = require("webpack-merge");
@@ -15,6 +14,7 @@ const CspHashPlugin = require("./cspHashPlugin");
 const sharedConfig = require("./webpack.config.shared");
 
 const fonts = path.resolve("src/fonts");
+const images = path.resolve("src/images");
 const nodeModules = path.resolve("node_modules");
 
 const prodConfig = {
@@ -28,8 +28,8 @@ const prodConfig = {
     minimize: true,
     minimizer: [
       new TerserJSPlugin({}),
-      new OptimizeCSSAssetsPlugin({
-        cssProcessor: CleanCss,
+      new CssMinimizerPlugin({
+        minify: CssMinimizerPlugin.cleanCssMinify,
       }),
     ],
   },
@@ -44,11 +44,11 @@ const prodConfig = {
       {
         test: /\.(jpg|png|gif|eot|svg|ttf|woff|woff2|otf)$/i,
         loader: "file-loader",
-        include: [nodeModules, fonts],
-        query: {
+        include: [nodeModules, fonts, images],
+        options: {
           name: "static/assets/[name]-[contenthash].[ext]",
           // (thuang): This is needed to make sure @font url path is '../static/assets/'
-          publicPath: "static/",
+          publicPath: "..",
         },
       },
     ],
@@ -65,27 +65,18 @@ const prodConfig = {
       protectWebpackAssets: false,
       cleanAfterEveryBuildPatterns: ["main.js", "main.css"],
     }),
-    new FaviconsWebpackPlugin({
-      logo: "./favicon.png",
-      prefix: "static/assets/",
-      favicons: {
-        icons: {
-          android: false,
-          appleIcon: false,
-          appleStartup: false,
-          coast: false,
-          firefox: false,
-          windows: false,
-          yandex: false,
-        },
-      },
-    }),
     new MiniCssExtractPlugin({
       filename: "static/[name]-[contenthash].css",
     }),
     new CspHashPlugin({
       filename: "csp-hashes.json",
     }),
+    new webpack.DefinePlugin({
+      // webpack 5 no longer polyfills NodeJS modules, so fake the one we need
+      "process.env": JSON.stringify({
+        NODE_ENV: "production",
+      }),
+    }),
   ],
   performance: {
     maxEntrypointSize: 2000000,

client/configuration/webpack/webpack.config.shared.js

@@ -1,8 +1,7 @@
 const path = require("path");
 const fs = require("fs");
 const MiniCssExtractPlugin = require("mini-css-extract-plugin");
-const ObsoleteWebpackPlugin = require("obsolete-webpack-plugin");
-const ScriptExtHtmlWebpackPlugin = require("script-ext-html-webpack-plugin");
+const ObsoleteWebpackPlugin = require("webpack-obsolete-plugin");
 
 const src = path.resolve("src");
 const nodeModules = path.resolve("node_modules");
@@ -20,7 +19,6 @@ module.exports = {
   entry: [
     "core-js",
     "regenerator-runtime/runtime",
-    "fastestsmallesttextencoderdecoder",
     "whatwg-fetch",
     "abort-controller/polyfill",
     "./src/index",
@@ -41,7 +39,7 @@ module.exports = {
             loader: "css-loader",
             options: {
               modules: {
-                localIdentName: "[name]__[local]___[hash:base64:5]",
+                localIdentName: "[name]__[local]___[contenthash:base64:5]",
               },
               importLoaders: 1,
             },
@@ -67,8 +65,5 @@ module.exports = {
       template: obsoleteHTMLTemplate,
       promptOnNonTargetBrowser: false,
     }),
-    new ScriptExtHtmlWebpackPlugin({
-      async: "obsolete",
-    }),
   ],
 };

client/index.html

@@ -3,7 +3,7 @@
   <head>
     <meta charset="utf-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1" />
-    <title>cell&times;gene</title>
+    <title>CELL&times;GENE | Annotate</title>
     <style>
       html,
       body,

client/index_template.html

@@ -3,7 +3,7 @@
   <head>
     <meta charset="utf-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1" />
-    <title>cell&times;gene</title>
+    <title>CELL&times;GENE | Annotate</title>
     <style>
       html,
       body,

client/jest-puppeteer.config.js

@@ -14,6 +14,7 @@ const DEFAULT_LAUNCH_CONFIG = {
   headless: !isHeadful,
   args: ["--ignore-certificate-errors", "--ignore-ssl-errors"],
   ignoreHTTPSErrors: true,
+  timeout: 90000,
   defaultViewport: {
     width: 1280,
     height: 960,

client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cellxgene",
-  "version": "0.16.0",
+  "version": "1.3.0",
   "license": "MIT",
   "description": "cellxgene is a web application for the interactive exploration of single cell sequence data.",
   "repository": "https://github.com/chanzuckerberg/cellxgene",
@@ -10,15 +10,16 @@
     "dev": "npm run build -- configuration/webpack/webpack.config.dev.js",
     "e2e": "jest --config __tests__/e2e/e2eJestConfig.json e2e/e2e.test.js",
     "e2e-annotations": "jest --config __tests__/e2e/e2eJestConfig.json e2e/e2eAnnotations.test.js",
-    "e2e-prod": "CXG_URL_BASE='https://cellxgene.cziscience.com/d/pbmc3k.cxg/' jest --config __tests__/e2e/e2eJestConfig.json e2e/e2e.test.js",
     "fmt": "eslint --fix src __tests__",
     "lint": "eslint --fix src __tests__",
     "prod": "npm run build -- configuration/webpack/webpack.config.prod.js",
-    "test": "jest --testPathIgnorePatterns e2e"
+    "test": "jest --testPathIgnorePatterns e2e",
+    "prepare": "cd .. && husky install client/.husky"
   },
   "engineStrict": true,
   "engines": {
-    "npm": ">=3.0.0"
+    "npm": ">=9.6.7",
+    "node": "^18.17.0"
   },
   "eslintConfig": {
     "extends": "./configuration/eslint/eslint.js"
@@ -31,113 +32,118 @@
   },
   "browserslist": [
     "Chrome > 60",
-    "Safari >= 10.1",
     "iOS >= 10.3",
     "Firefox >= 60",
     "Edge >= 79",
-    "not Explorer > 0"
+    "not Explorer > 0",
+    "not Safari > 0"
   ],
   "dependencies": {
-    "@blueprintjs/core": "^3.30.0",
+    "@babel/eslint-parser": "^7.15.0",
+    "@blueprintjs/core": "^3.44.0",
     "@blueprintjs/icons": "^3.19.0",
-    "@blueprintjs/select": "^3.13.5",
+    "@blueprintjs/popover2": "^0.11.2",
+    "@blueprintjs/select": "^3.16.0",
     "abort-controller": "^3.0.0",
-    "core-js": "^3.6.5",
+    "core-js": "^3.16.3",
     "d3": "^4.10.0",
     "d3-scale-chromatic": "^1.5.0",
-    "fastestsmallesttextencoderdecoder": "^1.0.22",
     "flatbuffers": "^1.11.0",
     "fuzzysort": "^1.1.4",
     "gl-mat4": "^1.2.0",
     "gl-matrix": "^3.3.0",
     "gl-vec3": "^1.1.3",
     "is-number": "^7.0.0",
-    "lodash": "^4.17.20",
+    "lodash.clonedeep": "^4.5.0",
+    "lodash.difference": "^4.5.0",
+    "lodash.foreach": "^4.5.0",
+    "lodash.pull": "^4.1.0",
+    "lodash.sortby": "^4.7.0",
+    "lodash.uniq": "^4.5.0",
     "memoize-one": "^5.1.1",
-    "pako": "^1.0.11",
-    "react": "^16.13.1",
+    "pako": "^2.0.3",
+    "react": "^17.0.2",
     "react-async": "^10.0.1",
-    "react-dom": "^16.13.1",
+    "react-dom": "^17.0.2",
     "react-flip-toolkit": "^7.0.12",
-    "react-helmet": "^5.2.1",
-    "react-icons": "^3.10.0",
+    "react-helmet": "^6.1.0",
+    "react-icons": "^4.2.0",
     "react-redux": "^7.2.0",
     "redux": "^4.0.5",
     "redux-thunk": "^2.3.0",
     "regenerator-runtime": "^0.13.7",
-    "regl": "^1.6.1",
-    "script-ext-html-webpack-plugin": "^2.1.4",
+    "regl": "^2.1.0",
+    "sha1": "^1.1.1",
     "tinyqueue": "^2.0.3",
-    "webpack-merge": "^5.0.9",
     "whatwg-fetch": "^3.2.0"
   },
   "devDependencies": {
-    "@babel/core": "^7.10.5",
+    "@babel/core": "^7.25.2",
     "@babel/plugin-proposal-class-properties": "^7.10.4",
-    "@babel/plugin-proposal-decorators": "^7.10.5",
+    "@babel/plugin-proposal-decorators": "^7.13.15",
     "@babel/plugin-proposal-export-namespace-from": "^7.10.4",
     "@babel/plugin-proposal-function-bind": "^7.10.5",
     "@babel/plugin-proposal-nullish-coalescing-operator": "^7.10.4",
     "@babel/plugin-proposal-optional-chaining": "^7.10.4",
-    "@babel/plugin-transform-react-constant-elements": "^7.10.4",
-    "@babel/plugin-transform-runtime": "^7.10.5",
-    "@babel/preset-env": "^7.10.4",
-    "@babel/preset-react": "^7.10.4",
-    "@babel/register": "^7.10.5",
-    "@babel/runtime": "^7.10.5",
+    "@babel/plugin-transform-private-property-in-object": "^7.22.11",
+    "@babel/plugin-transform-react-constant-elements": "^7.13.13",
+    "@babel/plugin-transform-runtime": "^7.13.15",
+    "@babel/preset-env": "^7.22.20",
+    "@babel/preset-react": "^7.13.13",
+    "@babel/register": "^7.13.16",
+    "@babel/runtime": "^7.13.16",
     "@blueprintjs/eslint-plugin": "^0.3.0",
-    "@sentry/webpack-plugin": "^1.12.0",
-    "babel-eslint": "^10.1.0",
+    "@sentry/webpack-plugin": "^1.15.0",
     "babel-jest": "^26.1.0",
     "babel-loader": "^8.1.0",
-    "babel-preset-modern-browsers": "^14.2.1",
+    "babel-preset-modern-browsers": "^15.0.2",
     "chalk": "^4.1.0",
-    "cheerio": "^1.0.0-rc.3",
-    "clean-css": "^4.2.3",
-    "clean-webpack-plugin": "^3.0.0",
+    "cheerio": "^1.0.0-rc.6",
+    "clean-css": "^5.1.2",
+    "clean-webpack-plugin": "^4.0.0-alpha.0",
     "codecov": "^3.7.1",
-    "connect-history-api-fallback": "^1.6.0",
-    "css-loader": "^3.6.0",
-    "eslint": "^7.4.0",
+    "css-loader": "^5.2.4",
+    "css-minimizer-webpack-plugin": "^4.0.0",
+    "eslint": "^7.24.0",
     "eslint-config-airbnb": "^18.2.0",
-    "eslint-config-prettier": "^6.11.0",
-    "eslint-loader": "^3.0.4",
-    "eslint-plugin-compat": "^3.8.0",
+    "eslint-config-prettier": "^8.2.0",
+    "eslint-plugin-compat": "^4.2.0",
     "eslint-plugin-eslint-comments": "^3.2.0",
     "eslint-plugin-filenames": "^1.3.2",
-    "eslint-plugin-import": "^2.22.0",
-    "eslint-plugin-jest": "^23.18.0",
+    "eslint-plugin-import": "^2.24.2",
+    "eslint-plugin-jest": "^24.3.5",
     "eslint-plugin-jsx-a11y": "^6.3.1",
-    "eslint-plugin-prettier": "^3.1.4",
-    "eslint-plugin-react": "^7.20.3",
+    "eslint-plugin-react": "^7.23.2",
     "eslint-plugin-react-hooks": "^4.0.8",
-    "expect-puppeteer": "^4.4.0",
+    "expect-puppeteer": "^5.0.0",
     "express": "^4.17.1",
-    "favicons-webpack-plugin": "^3.0.1",
     "file-loader": "^6.0.0",
-    "html-webpack-plugin": "^4.3.0",
-    "husky": "^4.2.5",
-    "jest": "^26.1.0",
-    "jest-circus": "^26.1.0",
-    "jest-environment-puppeteer": "^4.4.0",
+    "html-webpack-plugin": "^5.3.1",
+    "husky": "^7.0.2",
+    "jest": "^27.0.6",
+    "jest-circus": "^27.0.6",
+    "jest-environment-puppeteer": "^5.0.1",
     "jest-fetch-mock": "^3.0.3",
-    "jest-puppeteer": "^4.4.0",
+    "jest-puppeteer": "^6.2.0",
     "json-loader": "^0.5.7",
     "lint-staged": "^10.2.11",
-    "mini-css-extract-plugin": "^0.9.0",
-    "obsolete-webpack-plugin": "^0.5.6",
-    "optimize-css-assets-webpack-plugin": "^5.0.3",
+    "lodash": "^4.17.21",
+    "lodash.every": "^4.6.0",
+    "lodash.filter": "^4.6.0",
+    "lodash.isnumber": "^3.0.3",
+    "lodash.map": "^4.6.0",
+    "lodash.zip": "^4.2.0",
+    "mini-css-extract-plugin": "^1.5.0",
     "prettier": "^2.0.5",
-    "puppeteer": "^3.3.0",
+    "puppeteer": "^10.4.0",
     "rimraf": "^3.0.2",
     "serve-favicon": "^2.5.0",
-    "style-loader": "^1.2.1",
-    "sw-precache-webpack-plugin": "^1.0.0",
-    "terser-webpack-plugin": "^3.0.7",
-    "url-loader": "^4.1.0",
-    "webpack": "^4.43.0",
-    "webpack-cli": "^3.3.12",
-    "webpack-dev-middleware": "^3.7.2"
+    "terser-webpack-plugin": "^5.1.1",
+    "webpack": "^5.94.0",
+    "webpack-cli": "^4.6.0",
+    "webpack-dev-middleware": "^4.1.0",
+    "webpack-merge": "^5.0.9",
+    "webpack-obsolete-plugin": "^1.0.5"
   },
   "jest": {
     "testMatch": [
@@ -170,6 +176,18 @@
               "loose": true
             }
           ],
+          [
+            "@babel/plugin-transform-private-methods",
+            {
+              "loose": true
+            }
+          ],
+          [
+            "@babel/plugin-transform-private-property-in-object",
+            {
+              "loose": true
+            }
+          ],
           "@babel/plugin-proposal-export-namespace-from",
           "@babel/plugin-transform-react-constant-elements",
           "@babel/plugin-transform-runtime",
@@ -178,10 +196,5 @@
         ]
       }
     }
-  },
-  "husky": {
-    "hooks": {
-      "pre-commit": "lint-staged --config \"./configuration/lint-staged/lint-staged.config.js\""
-    }
   }
 }

client/server/development.js

@@ -32,7 +32,6 @@ const app = express();
 
 app.use(
   devMiddleware(compiler, {
-    logLevel: "warn",
     publicPath: config.output.publicPath,
     index: true,
   })
@@ -40,14 +39,6 @@ app.use(
 
 app.use(favicon("./favicon.png"));
 
-app.get("/login", async (req, res) => {
-  try {
-    res.redirect(`${API.prefix}login?dataset=http://localhost:${CLIENT_PORT}`);
-  } catch (err) {
-    console.error(err);
-  }
-});
-
 app.get("/logout", async (req, res) => {
   try {
     res.redirect(`${API.prefix}logout?dataset=http://localhost:${CLIENT_PORT}`);

client/server/utils.js

@@ -8,7 +8,9 @@ function isLikelyASyntaxError(message) {
   return message.indexOf(friendlySyntaxErrorLabel) !== -1;
 }
 
-function formatMessage(message) {
+function formatMessage(messageObject) {
+  let { message, details } = messageObject;
+  if (details) message = message + ": " + details;
   return message
     .replace("Module build failed: SyntaxError:", friendlySyntaxErrorLabel)
     .replace(

client/src/actions/annotation.js

@@ -1,288 +1,260 @@
 /*
 Action creators for user annotation
 */
-import _ from "lodash";
+import difference from "lodash.difference";
 import pako from "pako";
 import * as globals from "../globals";
 import { MatrixFBS, AnnotationsHelpers } from "../util/stateManager";
 
 const { isUserAnnotation } = AnnotationsHelpers;
 
-export const annotationCreateCategoryAction = (
-  newCategoryName,
-  categoryToDuplicate
-) => async (dispatch, getState) => {
-  /*
+export const annotationCreateCategoryAction =
+  (newCategoryName, categoryToDuplicate) => async (dispatch, getState) => {
+    /*
   Add a new user-created category to the obs annotations.
 
   Arguments:
     newCategoryName - string name for the category.
     categoryToDuplicate - obs category to use for initial values, or null.
   */
-  const {
-    annoMatrix: prevAnnoMatrix,
-    obsCrossfilter: prevObsCrossfilter,
-  } = getState();
-  if (!prevAnnoMatrix || !prevObsCrossfilter) return;
-  const { schema } = prevAnnoMatrix;
+    const { annoMatrix: prevAnnoMatrix, obsCrossfilter: prevObsCrossfilter } =
+      getState();
+    if (!prevAnnoMatrix || !prevObsCrossfilter) return;
+    const { schema } = prevAnnoMatrix;
 
-  /* name must be a string,  non-zero length */
-  if (typeof newCategoryName !== "string" || newCategoryName.length === 0)
-    throw new Error("user annotations require string name");
+    /* name must be a string,  non-zero length */
+    if (typeof newCategoryName !== "string" || newCategoryName.length === 0)
+      throw new Error("user annotations require string name");
 
-  /* ensure the name isn't already in use! */
-  if (schema.annotations.obsByName[newCategoryName])
-    throw new Error("name collision on annotation category create");
+    /* ensure the name isn't already in use! */
+    if (schema.annotations.obsByName[newCategoryName])
+      throw new Error("name collision on annotation category create");
 
-  let initialValue;
-  let newSchema;
-  let ctor;
-  if (categoryToDuplicate) {
-    /* if we are duplicating a category, retrieve it */
-    const catDupSchema = schema.annotations.obsByName[categoryToDuplicate];
-    const catDupType = catDupSchema?.type;
-    if (catDupType !== "string" && catDupType !== "categorical")
-      throw new Error("categoryToDuplicate does not exist or has invalid type");
+    let initialValue;
+    let newSchema;
+    let ctor;
+    if (categoryToDuplicate) {
+      /* if we are duplicating a category, retrieve it */
+      const catDupSchema = schema.annotations.obsByName[categoryToDuplicate];
+      const catDupType = catDupSchema?.type;
+      if (catDupType !== "string" && catDupType !== "categorical")
+        throw new Error(
+          "categoryToDuplicate does not exist or has invalid type"
+        );
 
-    const catToDupDf = await prevAnnoMatrix
-      .base()
-      .fetch("obs", categoryToDuplicate);
-    const col = catToDupDf.col(categoryToDuplicate);
-    initialValue = col.asArray();
-    const { categories } = col.summarizeCategorical();
-    // all user-created annotations must have the unassigned category
-    if (!categories.includes(globals.unassignedCategoryLabel)) {
-      categories.push(globals.unassignedCategoryLabel);
+      const catToDupDf = await prevAnnoMatrix
+        .base()
+        .fetch("obs", categoryToDuplicate);
+      const col = catToDupDf.col(categoryToDuplicate);
+      initialValue = col.asArray();
+      const { categories } = col.summarizeCategorical();
+      // all user-created annotations must have the unassigned category
+      if (!categories.includes(globals.unassignedCategoryLabel)) {
+        categories.push(globals.unassignedCategoryLabel);
+      }
+      ctor = initialValue.constructor;
+      newSchema = {
+        ...catDupSchema,
+        name: newCategoryName,
+        categories,
+        writable: true,
+      };
+    } else {
+      /* else assign to the standard default value */
+      initialValue = globals.unassignedCategoryLabel;
+      ctor = Array;
+      newSchema = {
+        name: newCategoryName,
+        type: "categorical",
+        categories: [globals.unassignedCategoryLabel],
+        writable: true,
+      };
     }
-    ctor = initialValue.constructor;
-    newSchema = {
-      ...catDupSchema,
-      name: newCategoryName,
-      categories,
-      writable: true,
-    };
-  } else {
-    /* else assign to the standard default value */
-    initialValue = globals.unassignedCategoryLabel;
-    ctor = Array;
-    newSchema = {
-      name: newCategoryName,
-      type: "categorical",
-      categories: [globals.unassignedCategoryLabel],
-      writable: true,
-    };
-  }
 
-  const obsCrossfilter = prevObsCrossfilter.addObsColumn(
-    newSchema,
-    ctor,
-    initialValue
-  );
+    const obsCrossfilter = prevObsCrossfilter.addObsColumn(
+      newSchema,
+      ctor,
+      initialValue
+    );
 
-  dispatch({
-    type: "annotation: create category",
-    data: newCategoryName,
-    categoryToDuplicate,
-    annoMatrix: obsCrossfilter.annoMatrix,
-    obsCrossfilter,
-  });
-};
+    dispatch({
+      type: "annotation: create category",
+      data: newCategoryName,
+      categoryToDuplicate,
+      annoMatrix: obsCrossfilter.annoMatrix,
+      obsCrossfilter,
+    });
+  };
 
-export const annotationRenameCategoryAction = (
-  oldCategoryName,
-  newCategoryName
-) => (dispatch, getState) => {
-  /*
+export const annotationRenameCategoryAction =
+  (oldCategoryName, newCategoryName) => (dispatch, getState) => {
+    /*
   Rename a user-created annotation category
   */
-  const {
-    annoMatrix: prevAnnoMatrix,
-    obsCrossfilter: prevObsCrossfilter,
-  } = getState();
-  if (!prevAnnoMatrix || !prevObsCrossfilter) return;
-  if (!isUserAnnotation(prevAnnoMatrix, oldCategoryName))
-    throw new Error("not a user annotation");
+    const { annoMatrix: prevAnnoMatrix, obsCrossfilter: prevObsCrossfilter } =
+      getState();
+    if (!prevAnnoMatrix || !prevObsCrossfilter) return;
+    if (!isUserAnnotation(prevAnnoMatrix, oldCategoryName))
+      throw new Error("not a user annotation");
 
-  /* name must be a string,  non-zero length */
-  if (typeof newCategoryName !== "string" || newCategoryName.length === 0)
-    throw new Error("user annotations require string name");
+    /* name must be a string,  non-zero length */
+    if (typeof newCategoryName !== "string" || newCategoryName.length === 0)
+      throw new Error("user annotations require string name");
 
-  if (oldCategoryName === newCategoryName) return;
+    if (oldCategoryName === newCategoryName) return;
 
-  const obsCrossfilter = prevObsCrossfilter.renameObsColumn(
-    oldCategoryName,
-    newCategoryName
-  );
+    const obsCrossfilter = prevObsCrossfilter.renameObsColumn(
+      oldCategoryName,
+      newCategoryName
+    );
 
-  dispatch({
-    type: "annotation: category edited",
-    annoMatrix: obsCrossfilter.annoMatrix,
-    obsCrossfilter,
-    metadataField: oldCategoryName,
-    newCategoryText: newCategoryName,
-    data: newCategoryName,
-  });
-};
+    dispatch({
+      type: "annotation: category edited",
+      annoMatrix: obsCrossfilter.annoMatrix,
+      obsCrossfilter,
+      metadataField: oldCategoryName,
+      newCategoryText: newCategoryName,
+      data: newCategoryName,
+    });
+  };
 
-export const annotationDeleteCategoryAction = (categoryName) => (
-  dispatch,
-  getState
-) => {
-  /*
+export const annotationDeleteCategoryAction =
+  (categoryName) => (dispatch, getState) => {
+    /*
   Delete a user-created category
   */
-  const {
-    annoMatrix: prevAnnoMatrix,
-    obsCrossfilter: prevObsCrossfilter,
-  } = getState();
-  if (!prevAnnoMatrix || !prevObsCrossfilter) return;
-  if (!isUserAnnotation(prevAnnoMatrix, categoryName))
-    throw new Error("not a user annotation");
+    const { annoMatrix: prevAnnoMatrix, obsCrossfilter: prevObsCrossfilter } =
+      getState();
+    if (!prevAnnoMatrix || !prevObsCrossfilter) return;
+    if (!isUserAnnotation(prevAnnoMatrix, categoryName))
+      throw new Error("not a user annotation");
 
-  const obsCrossfilter = prevObsCrossfilter.dropObsColumn(categoryName);
-  dispatch({
-    type: "annotation: delete category",
-    annoMatrix: obsCrossfilter.annoMatrix,
-    obsCrossfilter,
-    metadataField: categoryName,
-  });
-};
+    const obsCrossfilter = prevObsCrossfilter.dropObsColumn(categoryName);
+    dispatch({
+      type: "annotation: delete category",
+      annoMatrix: obsCrossfilter.annoMatrix,
+      obsCrossfilter,
+      metadataField: categoryName,
+    });
+  };
 
-export const annotationCreateLabelInCategory = (
-  categoryName,
-  labelName,
-  assignSelected
-) => async (dispatch, getState) => {
-  /*
+export const annotationCreateLabelInCategory =
+  (categoryName, labelName, assignSelected) => async (dispatch, getState) => {
+    /*
   Add a new label to a user-defined category.  If assignSelected is true, assign
   the label to all currently selected cells.
   */
-  const {
-    annoMatrix: prevAnnoMatrix,
-    obsCrossfilter: prevObsCrossfilter,
-  } = getState();
-  if (!prevAnnoMatrix || !prevObsCrossfilter) return;
-  if (!isUserAnnotation(prevAnnoMatrix, categoryName))
-    throw new Error("not a user annotation");
+    const { annoMatrix: prevAnnoMatrix, obsCrossfilter: prevObsCrossfilter } =
+      getState();
+    if (!prevAnnoMatrix || !prevObsCrossfilter) return;
+    if (!isUserAnnotation(prevAnnoMatrix, categoryName))
+      throw new Error("not a user annotation");
 
-  let obsCrossfilter = prevObsCrossfilter.addObsAnnoCategory(
-    categoryName,
-    labelName
-  );
-  if (assignSelected) {
-    obsCrossfilter = await obsCrossfilter.setObsColumnValues(
+    let obsCrossfilter = prevObsCrossfilter.addObsAnnoCategory(
+      categoryName,
+      labelName
+    );
+    if (assignSelected) {
+      obsCrossfilter = await obsCrossfilter.setObsColumnValues(
+        categoryName,
+        prevObsCrossfilter.allSelectedLabels(),
+        labelName
+      );
+    }
+
+    dispatch({
+      type: "annotation: add new label to category",
+      annoMatrix: obsCrossfilter.annoMatrix,
+      obsCrossfilter,
+      metadataField: categoryName,
+      newLabelText: labelName,
+      assignSelectedCells: assignSelected,
+    });
+  };
+
+export const annotationDeleteLabelFromCategory =
+  (categoryName, labelName) => async (dispatch, getState) => {
+    /*
+  delete a label from a user-defined category
+  */
+    const { annoMatrix: prevAnnoMatrix, obsCrossfilter: prevObsCrossfilter } =
+      getState();
+    if (!prevAnnoMatrix || !prevObsCrossfilter) return;
+    if (!isUserAnnotation(prevAnnoMatrix, categoryName))
+      throw new Error("not a user annotation");
+
+    const obsCrossfilter = await prevObsCrossfilter.removeObsAnnoCategory(
+      categoryName,
+      labelName,
+      globals.unassignedCategoryLabel
+    );
+
+    dispatch({
+      type: "annotation: delete label",
+      metadataField: categoryName,
+      label: labelName,
+      annoMatrix: obsCrossfilter.annoMatrix,
+      obsCrossfilter,
+    });
+  };
+
+export const annotationRenameLabelInCategory =
+  (categoryName, oldLabelName, newLabelName) => async (dispatch, getState) => {
+    /*
+  label name change
+  */
+    const { annoMatrix: prevAnnoMatrix, obsCrossfilter: prevObsCrossfilter } =
+      getState();
+    if (!prevAnnoMatrix || !prevObsCrossfilter) return;
+    if (!isUserAnnotation(prevAnnoMatrix, categoryName))
+      throw new Error("not a user annotation");
+
+    let obsCrossfilter = await prevObsCrossfilter.resetObsColumnValues(
+      categoryName,
+      oldLabelName,
+      newLabelName
+    );
+    obsCrossfilter = await obsCrossfilter.removeObsAnnoCategory(
+      categoryName,
+      oldLabelName,
+      globals.unassignedCategoryLabel
+    );
+
+    dispatch({
+      type: "annotation: label edited",
+      editedLabel: newLabelName,
+      metadataField: categoryName,
+      label: oldLabelName,
+      annoMatrix: obsCrossfilter.annoMatrix,
+      obsCrossfilter,
+    });
+  };
+
+export const annotationLabelCurrentSelection =
+  (categoryName, labelName) => async (dispatch, getState) => {
+    /*
+  set the label on all currently selected
+  */
+    const { annoMatrix: prevAnnoMatrix, obsCrossfilter: prevObsCrossfilter } =
+      getState();
+    if (!prevAnnoMatrix || !prevObsCrossfilter) return;
+    if (!isUserAnnotation(prevAnnoMatrix, categoryName))
+      throw new Error("not a user annotation");
+
+    const obsCrossfilter = await prevObsCrossfilter.setObsColumnValues(
       categoryName,
       prevObsCrossfilter.allSelectedLabels(),
       labelName
     );
-  }
 
-  dispatch({
-    type: "annotation: add new label to category",
-    annoMatrix: obsCrossfilter.annoMatrix,
-    obsCrossfilter,
-    metadataField: categoryName,
-    newLabelText: labelName,
-    assignSelectedCells: assignSelected,
-  });
-};
-
-export const annotationDeleteLabelFromCategory = (
-  categoryName,
-  labelName
-) => async (dispatch, getState) => {
-  /*
-  delete a label from a user-defined category
-  */
-  const {
-    annoMatrix: prevAnnoMatrix,
-    obsCrossfilter: prevObsCrossfilter,
-  } = getState();
-  if (!prevAnnoMatrix || !prevObsCrossfilter) return;
-  if (!isUserAnnotation(prevAnnoMatrix, categoryName))
-    throw new Error("not a user annotation");
-
-  const obsCrossfilter = await prevObsCrossfilter.removeObsAnnoCategory(
-    categoryName,
-    labelName,
-    globals.unassignedCategoryLabel
-  );
-
-  dispatch({
-    type: "annotation: delete label",
-    metadataField: categoryName,
-    label: labelName,
-    annoMatrix: obsCrossfilter.annoMatrix,
-    obsCrossfilter,
-  });
-};
-
-export const annotationRenameLabelInCategory = (
-  categoryName,
-  oldLabelName,
-  newLabelName
-) => async (dispatch, getState) => {
-  /*
-  label name change
-  */
-  const {
-    annoMatrix: prevAnnoMatrix,
-    obsCrossfilter: prevObsCrossfilter,
-  } = getState();
-  if (!prevAnnoMatrix || !prevObsCrossfilter) return;
-  if (!isUserAnnotation(prevAnnoMatrix, categoryName))
-    throw new Error("not a user annotation");
-
-  let obsCrossfilter = await prevObsCrossfilter.resetObsColumnValues(
-    categoryName,
-    oldLabelName,
-    newLabelName
-  );
-  obsCrossfilter = await obsCrossfilter.removeObsAnnoCategory(
-    categoryName,
-    oldLabelName,
-    globals.unassignedCategoryLabel
-  );
-
-  dispatch({
-    type: "annotation: label edited",
-    editedLabel: newLabelName,
-    metadataField: categoryName,
-    label: oldLabelName,
-    annoMatrix: obsCrossfilter.annoMatrix,
-    obsCrossfilter,
-  });
-};
-
-export const annotationLabelCurrentSelection = (
-  categoryName,
-  labelName
-) => async (dispatch, getState) => {
-  /*
-  set the label on all currently selected
-  */
-  const {
-    annoMatrix: prevAnnoMatrix,
-    obsCrossfilter: prevObsCrossfilter,
-  } = getState();
-  if (!prevAnnoMatrix || !prevObsCrossfilter) return;
-  if (!isUserAnnotation(prevAnnoMatrix, categoryName))
-    throw new Error("not a user annotation");
-
-  const obsCrossfilter = await prevObsCrossfilter.setObsColumnValues(
-    categoryName,
-    prevObsCrossfilter.allSelectedLabels(),
-    labelName
-  );
-
-  dispatch({
-    type: "annotation: label current cell selection",
-    metadataField: categoryName,
-    label: labelName,
-    obsCrossfilter,
-    annoMatrix: obsCrossfilter.annoMatrix,
-  });
-};
+    dispatch({
+      type: "annotation: label current cell selection",
+      metadataField: categoryName,
+      label: labelName,
+      obsCrossfilter,
+      annoMatrix: obsCrossfilter.annoMatrix,
+    });
+  };
 
 function writableAnnotations(annoMatrix) {
   return annoMatrix.schema.annotations.obs.columns
@@ -308,7 +280,7 @@ export const needToSaveObsAnnotations = (annoMatrix, lastSavedAnnoMatrix) => {
 
   // if the schema has changed, we need to save
   const currentWritable = writableAnnotations(annoMatrix);
-  if (_.difference(currentWritable, writableAnnotations(lastSavedAnnoMatrix))) {
+  if (difference(currentWritable, writableAnnotations(lastSavedAnnoMatrix))) {
     return true;
   }
 
@@ -387,3 +359,97 @@ export const saveObsAnnotationsAction = () => async (dispatch, getState) => {
     });
   }
 };
+
+export const saveGenesetsAction = () => async (dispatch, getState) => {
+  const state = getState();
+
+  // bail if gene sets not available, or in readonly mode.
+  const { config } = state;
+  const { lastTid, genesets } = state.genesets;
+
+  const genesetsAreAvailable =
+    config?.parameters?.annotations_genesets ?? false;
+  const genesetsReadonly =
+    config?.parameters?.annotations_genesets_readonly ?? true;
+  if (!genesetsAreAvailable || genesetsReadonly) {
+    // our non-save was completed!
+    return dispatch({
+      type: "autosave: genesets complete",
+      lastSavedGenesets: genesets,
+    });
+  }
+
+  dispatch({
+    type: "autosave: genesets started",
+  });
+
+  /* Create the JSON OTA data structure */
+  const tid = (lastTid ?? 0) + 1;
+  const genesetsOTA = [];
+  for (const [name, gs] of genesets) {
+    const genes = [];
+    for (const g of gs.genes.values()) {
+      genes.push({
+        gene_symbol: g.geneSymbol,
+        gene_description: g.geneDescription,
+      });
+    }
+    genesetsOTA.push({
+      geneset_name: name,
+      geneset_description: gs.genesetDescription,
+      genes,
+    });
+  }
+  const ota = {
+    tid,
+    genesets: genesetsOTA,
+  };
+
+  /* Save to server */
+  try {
+    const { dataCollectionNameIsReadOnly, dataCollectionName } =
+      state.annotations;
+    const queryString =
+      !dataCollectionNameIsReadOnly && !!dataCollectionName
+        ? `?annotation-collection-name=${encodeURIComponent(
+            dataCollectionName
+          )}`
+        : "";
+
+    const res = await fetch(
+      `${globals.API.prefix}${globals.API.version}genesets${queryString}`,
+      {
+        method: "PUT",
+        headers: new Headers({
+          Accept: "application/json",
+          "Content-Type": "application/json",
+        }),
+        body: JSON.stringify(ota),
+        credentials: "include",
+      }
+    );
+    if (!res.ok) {
+      return dispatch({
+        type: "autosave: genesets error",
+        message: `HTTP error ${res.status} - ${res.statusText}`,
+        res,
+      });
+    }
+    return Promise.all([
+      dispatch({
+        type: "autosave: genesets complete",
+        lastSavedGenesets: genesets,
+      }),
+      dispatch({
+        type: "geneset: set tid",
+        tid,
+      }),
+    ]);
+  } catch (error) {
+    return dispatch({
+      type: "autosave: genesets error",
+      message: error.toString(),
+      error,
+    });
+  }
+};

client/src/actions/embedding.js

@@ -11,7 +11,7 @@ export async function _switchEmbedding(
   newEmbeddingName
 ) {
   /*
-  DRY helper used by this and reembedding action creators
+  DRY helper used by embedding action creators
   */
   const base = prevAnnoMatrix.base();
   const embeddingDf = await base.fetch("emb", newEmbeddingName);
@@ -25,27 +25,23 @@ export async function _switchEmbedding(
   return [annoMatrix, obsCrossfilter];
 }
 
-export const layoutChoiceAction = (newLayoutChoice) => async (
-  dispatch,
-  getState
-) => {
-  /*
+export const layoutChoiceAction =
+  (newLayoutChoice) => async (dispatch, getState) => {
+    /*
   On layout choice, make sure we have selected all on the previous layout, AND the new
   layout.
   */
-  const {
-    annoMatrix: prevAnnoMatrix,
-    obsCrossfilter: prevCrossfilter,
-  } = getState();
-  const [annoMatrix, obsCrossfilter] = await _switchEmbedding(
-    prevAnnoMatrix,
-    prevCrossfilter,
-    newLayoutChoice
-  );
-  dispatch({
-    type: "set layout choice",
-    layoutChoice: newLayoutChoice,
-    obsCrossfilter,
-    annoMatrix,
-  });
-};
+    const { annoMatrix: prevAnnoMatrix, obsCrossfilter: prevCrossfilter } =
+      getState();
+    const [annoMatrix, obsCrossfilter] = await _switchEmbedding(
+      prevAnnoMatrix,
+      prevCrossfilter,
+      newLayoutChoice
+    );
+    dispatch({
+      type: "set layout choice",
+      layoutChoice: newLayoutChoice,
+      obsCrossfilter,
+      annoMatrix,
+    });
+  };

client/src/actions/geneset.js

@@ -0,0 +1,149 @@
+import { postUserErrorToast } from "../components/framework/toasters";
+/*
+Action creators for gene sets
+
+Primarily used to keep the crossfilter and underlying data in sync with the UI.
+
+The behavior manifest in these action creators:
+
+    Delete a gene set, will
+      * drop index & clear selection state on the gene set summary
+      * drop index & clear selection state of each gene in the geneset
+
+    Delete a gene from a gene set, will:
+      * drop index & clear selection state on the gene set summary
+      * drop index & clear selection state on the gene
+
+    Add a gene to a gene set, will:
+      * drop index & clear selection state on the gene set summary
+      * will NOT touch the selection state for the gene
+
+Note that crossfilter indices are lazy created, as needed.
+*/
+
+export const genesetDelete = (genesetName) => (dispatch, getState) => {
+  const state = getState();
+  const { genesets } = state;
+  const gs = genesets?.genesets?.get(genesetName) ?? {};
+  const geneSymbols = Array.from(gs.genes.keys());
+  const obsCrossfilter = dropGeneset(dispatch, state, genesetName, geneSymbols);
+  if (genesetName === state.colors.colorAccessor) {
+    dispatch({
+      type: "reset colorscale",
+    });
+  }
+  dispatch({
+    type: "geneset: delete",
+    genesetName,
+    obsCrossfilter,
+    annoMatrix: obsCrossfilter.annoMatrix,
+  });
+};
+
+export const genesetAddGenes =
+  (genesetName, genes) => async (dispatch, getState) => {
+    const state = getState();
+    const { obsCrossfilter: prevObsCrossfilter, annoMatrix } = state;
+    const { schema } = annoMatrix;
+    const varIndex = schema.annotations.var.index;
+    const df = await annoMatrix.fetch("var", varIndex);
+    const geneNames = df.col(varIndex).asArray();
+    genes = genes.reduce((acc, gene) => {
+      if (geneNames.indexOf(gene.geneSymbol) === -1) {
+        postUserErrorToast(
+          `${gene.geneSymbol} doesn't appear to be a valid gene name.`
+        );
+      } else acc.push(gene);
+      return acc;
+    }, []);
+
+    const obsCrossfilter = dropGenesetSummaryDimension(
+      prevObsCrossfilter,
+      state,
+      genesetName
+    );
+    dispatch({
+      type: "continuous metadata histogram cancel",
+      continuousNamespace: { isGeneSetSummary: true },
+      selection: genesetName,
+    });
+    return dispatch({
+      type: "geneset: add genes",
+      genesetName,
+      genes,
+      obsCrossfilter,
+      annoMatrix: obsCrossfilter.annoMatrix,
+    });
+  };
+
+export const genesetDeleteGenes =
+  (genesetName, geneSymbols) => (dispatch, getState) => {
+    const state = getState();
+    const obsCrossfilter = dropGeneset(
+      dispatch,
+      state,
+      genesetName,
+      geneSymbols
+    );
+    return dispatch({
+      type: "geneset: delete genes",
+      genesetName,
+      geneSymbols,
+      obsCrossfilter,
+      annoMatrix: obsCrossfilter.annoMatrix,
+    });
+  };
+
+/*
+Private
+*/
+
+function dropGenesetSummaryDimension(obsCrossfilter, state, genesetName) {
+  const { annoMatrix, genesets } = state;
+  const varIndex = annoMatrix.schema.annotations?.var?.index;
+  const gs = genesets?.genesets?.get(genesetName) ?? {};
+  const genes = Array.from(gs.genes.keys());
+  const query = {
+    summarize: {
+      method: "mean",
+      field: "var",
+      column: varIndex,
+      values: genes,
+    },
+  };
+  return obsCrossfilter.dropDimension("X", query);
+}
+
+function dropGeneDimension(obsCrossfilter, state, gene) {
+  const { annoMatrix } = state;
+  const varIndex = annoMatrix.schema.annotations?.var?.index;
+  const query = {
+    where: {
+      field: "var",
+      column: varIndex,
+      value: gene,
+    },
+  };
+  return obsCrossfilter.dropDimension("X", query);
+}
+
+function dropGeneset(dispatch, state, genesetName, geneSymbols) {
+  const { obsCrossfilter: prevObsCrossfilter } = state;
+  const obsCrossfilter = geneSymbols.reduce(
+    (crossfilter, gene) => dropGeneDimension(crossfilter, state, gene),
+    dropGenesetSummaryDimension(prevObsCrossfilter, state, genesetName)
+  );
+  dispatch({
+    type: "continuous metadata histogram cancel",
+    continuousNamespace: { isGeneSetSummary: true },
+    selection: genesetName,
+  });
+  geneSymbols.forEach((g) =>
+    dispatch({
+      type: "continuous metadata histogram cancel",
+      continuousNamespace: { isUserDefined: true },
+      selection: g,
+    })
+  );
+  return obsCrossfilter;
+}

client/src/actions/index.js

@@ -5,14 +5,22 @@ import {
   doJsonRequest,
   dispatchNetworkErrorMessageToUser,
 } from "../util/actionHelpers";
-import {
-  requestReembed /* , reembedResetWorldToUniverse -- disabled temporarily, TODO issue #1606 */,
-} from "./reembed";
 import { loadUserColorConfig } from "../util/stateManager/colorHelpers";
 import * as selnActions from "./selection";
 import * as annoActions from "./annotation";
 import * as viewActions from "./viewStack";
 import * as embActions from "./embedding";
+import * as genesetActions from "./geneset";
+
+function setGlobalConfig(config) {
+  /**
+   * Set any global run-time config not _exclusively_ managed by the config reducer.
+   * This should only set fields defined in globals.globalConfig.
+   */
+  globals.globalConfig.maxCategoricalOptionsToDisplay =
+    config?.parameters?.["max-category-items"] ??
+    globals.globalConfig.maxCategoricalOptionsToDisplay;
+}
 
 /*
 return promise fetching user-configured colors
@@ -33,6 +41,9 @@ async function schemaFetch() {
 async function configFetch(dispatch) {
   return fetchJson("config").then((response) => {
     const config = { ...globals.configDefaults, ...response.config };
+
+    setGlobalConfig(config);
+
     dispatch({
       type: "configuration load complete",
       config,
@@ -41,15 +52,25 @@ async function configFetch(dispatch) {
   });
 }
 
-async function userInfoFetch(dispatch) {
-  return fetchJson("userinfo").then((response) => {
-    const { userinfo: userInfo } = response || {};
-    dispatch({
-      type: "userInfo load complete",
-      userInfo,
+async function genesetsFetch(dispatch, config) {
+  /* request genesets ONLY if the backend supports the feature */
+  const defaultResponse = {
+    genesets: [],
+    tid: 0,
+  };
+  if (config?.parameters?.annotations_genesets ?? false) {
+    fetchJson("genesets").then((response) => {
+      dispatch({
+        type: "geneset: initial load",
+        data: response ?? defaultResponse,
+      });
     });
-    return userInfo;
-  });
+  } else {
+    dispatch({
+      type: "geneset: initial load",
+      data: defaultResponse,
+    });
+  }
 }
 
 function prefetchEmbeddings(annoMatrix) {
@@ -73,9 +94,10 @@ const doInitialDataLoad = () =>
         configFetch(dispatch),
         schemaFetch(dispatch),
         userColorsFetchAndLoad(dispatch),
-        userInfoFetch(dispatch),
       ]);
 
+      genesetsFetch(dispatch, config);
+
       const baseDataUrl = `${globals.API.prefix}${globals.API.version}`;
       const annoMatrix = new AnnoMatrixLoader(baseDataUrl, schema.schema);
       const obsCrossfilter = new AnnoMatrixObsCrossfilter(annoMatrix);
@@ -88,7 +110,7 @@ const doInitialDataLoad = () =>
       });
       dispatch({ type: "initial data load complete" });
 
-      const defaultEmbedding = config?.parameters?.["default_embedding"];
+      const defaultEmbedding = config?.parameters?.default_embedding;
       const layoutSchema = schema?.schema?.layout?.obs ?? [];
       if (
         defaultEmbedding &&
@@ -138,72 +160,74 @@ const dispatchDiffExpErrors = (dispatch, response) => {
   }
 };
 
-const requestDifferentialExpression = (set1, set2, num_genes = 10) => async (
-  dispatch,
-  getState
-) => {
-  dispatch({ type: "request differential expression started" });
-  try {
-    /*
+const requestDifferentialExpression =
+  (set1, set2, num_genes = 50) =>
+  async (dispatch, getState) => {
+    dispatch({ type: "request differential expression started" });
+    try {
+      /*
     Steps:
     1. get the most differentially expressed genes
     2. get expression data for each
     */
-    const { annoMatrix } = getState();
-    const varIndexName = annoMatrix.schema.annotations.var.index;
+      const { annoMatrix } = getState();
+      const varIndexName = annoMatrix.schema.annotations.var.index;
 
-    // Legal values are null, Array or TypedArray.  Null is initial state.
-    if (!set1) set1 = [];
-    if (!set2) set2 = [];
+      // Legal values are null, Array or TypedArray.  Null is initial state.
+      if (!set1) set1 = [];
+      if (!set2) set2 = [];
 
-    // These lines ensure that we convert any TypedArray to an Array.
-    // This is necessary because JSON.stringify() does some very strange
-    // things with TypedArrays (they are marshalled to JSON objects, rather
-    // than being marshalled as a JSON array).
-    set1 = Array.isArray(set1) ? set1 : Array.from(set1);
-    set2 = Array.isArray(set2) ? set2 : Array.from(set2);
+      // These lines ensure that we convert any TypedArray to an Array.
+      // This is necessary because JSON.stringify() does some very strange
+      // things with TypedArrays (they are marshalled to JSON objects, rather
+      // than being marshalled as a JSON array).
+      set1 = Array.isArray(set1) ? set1 : Array.from(set1);
+      set2 = Array.isArray(set2) ? set2 : Array.from(set2);
 
-    const res = await fetch(
-      `${globals.API.prefix}${globals.API.version}diffexp/obs`,
-      {
-        method: "POST",
-        headers: new Headers({
-          Accept: "application/json",
-          "Content-Type": "application/json",
-        }),
-        body: JSON.stringify({
-          mode: "topN",
-          count: num_genes,
-          set1: { filter: { obs: { index: set1 } } },
-          set2: { filter: { obs: { index: set2 } } },
-        }),
-        credentials: "include",
+      const res = await fetch(
+        `${globals.API.prefix}${globals.API.version}diffexp/obs`,
+        {
+          method: "POST",
+          headers: new Headers({
+            Accept: "application/json",
+            "Content-Type": "application/json",
+          }),
+          body: JSON.stringify({
+            mode: "topN",
+            count: num_genes,
+            set1: { filter: { obs: { index: set1 } } },
+            set2: { filter: { obs: { index: set2 } } },
+          }),
+          credentials: "include",
+        }
+      );
+
+      if (!res.ok || res.headers.get("Content-Type") !== "application/json") {
+        return dispatchDiffExpErrors(dispatch, res);
       }
-    );
 
-    if (!res.ok || res.headers.get("Content-Type") !== "application/json") {
-      return dispatchDiffExpErrors(dispatch, res);
+      const response = await res.json();
+      const varIndex = await annoMatrix.fetch("var", varIndexName);
+      const diffexpLists = { negative: [], positive: [] };
+      for (const polarity of Object.keys(diffexpLists)) {
+        diffexpLists[polarity] = response[polarity].map((v) => [
+          varIndex.at(v[0], varIndexName),
+          ...v.slice(1),
+        ]);
+      }
+
+      /* then send the success case action through */
+      return dispatch({
+        type: "request differential expression success",
+        data: diffexpLists,
+      });
+    } catch (error) {
+      return dispatch({
+        type: "request differential expression error",
+        error,
+      });
     }
-
-    const response = await res.json();
-    const varIndex = await annoMatrix.fetch("var", varIndexName);
-    const data = response.map((v) => [
-      varIndex.at(v[0], varIndexName),
-      ...v.slice(1),
-    ]);
-
-    /* then send the success case action through */
-    return dispatch({
-      type: "request differential expression success",
-      data,
-    });
-  } catch (error) {
-    return dispatch({
-      type: "request differential expression error",
-      error,
-    });
-  }
-};
+  };
 
 function fetchJson(pathAndQuery) {
   return doJsonRequest(
@@ -216,7 +240,6 @@ export default {
   requestDifferentialExpression,
   requestSingleGeneExpressionCountsForColoringPOST,
   requestUserDefinedGene,
-  requestReembed,
   selectContinuousMetadataAction: selnActions.selectContinuousMetadataAction,
   selectCategoricalMetadataAction: selnActions.selectCategoricalMetadataAction,
   selectCategoricalAllMetadataAction:
@@ -242,7 +265,11 @@ export default {
   annotationRenameLabelInCategory: annoActions.annotationRenameLabelInCategory,
   annotationLabelCurrentSelection: annoActions.annotationLabelCurrentSelection,
   saveObsAnnotationsAction: annoActions.saveObsAnnotationsAction,
+  saveGenesetsAction: annoActions.saveGenesetsAction,
   needToSaveObsAnnotations: annoActions.needToSaveObsAnnotations,
   layoutChoiceAction: embActions.layoutChoiceAction,
   setCellSetFromSelection: selnActions.setCellSetFromSelection,
+  genesetDelete: genesetActions.genesetDelete,
+  genesetAddGenes: genesetActions.genesetAddGenes,
+  genesetDeleteGenes: genesetActions.genesetDeleteGenes,
 };

client/src/actions/reembed.js

@@ -1,109 +0,0 @@
-import { API } from "../globals";
-import {
-  postNetworkErrorToast,
-  postAsyncSuccessToast,
-  postAsyncFailureToast,
-} from "../components/framework/toasters";
-import { _switchEmbedding } from "./embedding";
-
-function abortableFetch(request, opts, timeout = 0) {
-  const controller = new AbortController();
-  const { signal } = controller;
-
-  return {
-    abort: () => controller.abort(),
-    isAborted: () => signal.aborted,
-    ready: () => {
-      if (timeout) {
-        setTimeout(() => controller.abort(), timeout);
-      }
-      return fetch(request, { ...opts, signal });
-    },
-  };
-}
-
-async function doReembedFetch(dispatch, getState) {
-  const state = getState();
-  let cells = state.annoMatrix.rowIndex.labels();
-
-  // These lines ensure that we convert any TypedArray to an Array.
-  // This is necessary because JSON.stringify() does some very strange
-  // things with TypedArrays (they are marshalled to JSON objects, rather
-  // than being marshalled as a JSON array).
-  cells = Array.isArray(cells) ? cells : Array.from(cells);
-
-  const af = abortableFetch(
-    `${API.prefix}${API.version}layout/obs`,
-    {
-      method: "PUT",
-      headers: new Headers({
-        Accept: "application/octet-stream",
-        "Content-Type": "application/json",
-      }),
-      body: JSON.stringify({
-        method: "umap",
-        filter: { obs: { index: cells } },
-      }),
-      credentials: "include",
-    },
-    60000 // 1 minute timeout
-  );
-  dispatch({
-    type: "reembed: request start",
-    abortableFetch: af,
-  });
-  const res = await af.ready();
-
-  if (res.ok && res.headers.get("Content-Type").includes("application/json")) {
-    return res;
-  }
-
-  // else an error
-  let msg = `Unexpected HTTP response ${res.status}, ${res.statusText}`;
-  const body = await res.text();
-  if (body && body.length > 0) {
-    msg = `${msg} -- ${body}`;
-  }
-  throw new Error(msg);
-}
-
-/*
-functions below are dispatch-able
-*/
-export function requestReembed() {
-  return async (dispatch, getState) => {
-    try {
-      const res = await doReembedFetch(dispatch, getState);
-      const schema = await res.json();
-      dispatch({
-        type: "reembed: request completed",
-      });
-
-      const { annoMatrix: prevAnnoMatrix, obsCrossfilter: prevCrossfilter } = getState();
-      const base = prevAnnoMatrix.base().addEmbedding(schema);
-      const [annoMatrix, obsCrossfilter] = await _switchEmbedding(
-        base,
-        prevCrossfilter,
-        schema.name
-      );
-      dispatch({
-        type: "reembed: add reembedding",
-        schema,
-        annoMatrix,
-        obsCrossfilter,
-      });
-
-      postAsyncSuccessToast("Re-embedding has completed.");
-    } catch (error) {
-      dispatch({
-        type: "reembed: request aborted",
-      });
-      if (error.name === "AbortError") {
-        postAsyncFailureToast("Re-embedding calculation was aborted.");
-      } else {
-        postNetworkErrorToast(`Re-embedding: ${error.message}`);
-      }
-      console.log("Reembed exception:", error, error.name, error.message);
-    }
-  };
-}

client/src/actions/selection.js

@@ -1,104 +1,101 @@
 /*
 Action creators for selection 
 */
-export const selectContinuousMetadataAction = (
-  type,
-  query,
-  range,
-  oldProps = {}
-) => async (dispatch, getState) => {
-  const { obsCrossfilter: prevObsCrossfilter } = getState();
+export const selectContinuousMetadataAction =
+  (type, query, range, oldProps = {}) =>
+  async (dispatch, getState) => {
+    const { obsCrossfilter: prevObsCrossfilter } = getState();
 
-  const selection = range
-    ? {
-        mode: "range",
-        lo: range[0],
-        hi: range[1],
-        inclusive: true, // [lo, hi] incluisve selection
-      }
-    : { mode: "all" };
+    const selection = range
+      ? {
+          mode: "range",
+          lo: range[0],
+          hi: range[1],
+          inclusive: true, // [lo, hi] incluisve selection
+        }
+      : { mode: "all" };
 
-  const obsCrossfilter = await prevObsCrossfilter.select(...query, selection);
+    const obsCrossfilter = await prevObsCrossfilter.select(...query, selection);
 
-  dispatch({
-    type,
-    obsCrossfilter,
-    range,
-    ...oldProps,
-  });
-};
-
-export const selectCategoricalMetadataAction = (
-  type, // action type
-  metadataField, // annotation category name
-  labels,
-  label, // the label being selected/deselected
-  isSelected, // bool
-  oldProps = {}
-) => async (dispatch, getState) => {
-  const {
-    obsCrossfilter: prevObsCrossfilter,
-    categoricalSelection,
-  } = getState();
-
-  const labelSelectionState = new Map(categoricalSelection[metadataField]);
-  labels.forEach(
-    (l) => labelSelectionState.has(l) || labelSelectionState.set(l, true)
-  );
-  labelSelectionState.set(label, isSelected);
-
-  const values = Array.from(labelSelectionState.keys()).filter((k) =>
-    labelSelectionState.get(k)
-  );
-  const selection = {
-    mode: "exact",
-    values,
+    dispatch({
+      type,
+      obsCrossfilter,
+      range,
+      ...oldProps,
+    });
   };
-  const obsCrossfilter = await prevObsCrossfilter.select(
-    "obs",
-    metadataField,
-    selection
-  );
 
-  dispatch({
-    type,
-    obsCrossfilter,
-    metadataField,
-    labelSelectionState,
-    ...oldProps,
-  });
-};
+export const selectCategoricalMetadataAction =
+  (
+    type, // action type
+    metadataField, // annotation category name
+    labels,
+    label, // the label being selected/deselected
+    isSelected, // bool
+    oldProps = {}
+  ) =>
+  async (dispatch, getState) => {
+    const { obsCrossfilter: prevObsCrossfilter, categoricalSelection } =
+      getState();
 
-export const selectCategoricalAllMetadataAction = (
-  type, // action type
-  metadataField, // annotation category name
-  labels,
-  isSelected, // bool, select all or none
-  oldProps = {}
-) => async (dispatch, getState) => {
-  const {
-    obsCrossfilter: prevObsCrossfilter,
-    categoricalSelection,
-  } = getState();
+    const labelSelectionState = new Map(categoricalSelection[metadataField]);
+    labels.forEach(
+      (l) => labelSelectionState.has(l) || labelSelectionState.set(l, true)
+    );
+    labelSelectionState.set(label, isSelected);
 
-  const labelSelectionState = new Map(categoricalSelection[metadataField]);
-  labels.forEach((label) => labelSelectionState.set(label, isSelected));
+    const values = Array.from(labelSelectionState.keys()).filter((k) =>
+      labelSelectionState.get(k)
+    );
+    const selection = {
+      mode: "exact",
+      values,
+    };
+    const obsCrossfilter = await prevObsCrossfilter.select(
+      "obs",
+      metadataField,
+      selection
+    );
 
-  const selection = { mode: isSelected ? "all" : "none" };
-  const obsCrossfilter = await prevObsCrossfilter.select(
-    "obs",
-    metadataField,
-    selection
-  );
+    dispatch({
+      type,
+      obsCrossfilter,
+      metadataField,
+      labelSelectionState,
+      ...oldProps,
+    });
+  };
 
-  dispatch({
-    type,
-    obsCrossfilter,
-    metadataField,
-    labelSelectionState,
-    ...oldProps,
-  });
-};
+export const selectCategoricalAllMetadataAction =
+  (
+    type, // action type
+    metadataField, // annotation category name
+    labels,
+    isSelected, // bool, select all or none
+    oldProps = {}
+  ) =>
+  async (dispatch, getState) => {
+    const { obsCrossfilter: prevObsCrossfilter, categoricalSelection } =
+      getState();
+
+    const labelSelectionState = new Map(categoricalSelection[metadataField]);
+    labels.forEach((label) => labelSelectionState.set(label, isSelected));
+
+    const selection = { mode: isSelected ? "all" : "none" };
+    const obsCrossfilter = await prevObsCrossfilter.select(
+      "obs",
+      metadataField,
+      selection
+    );
+
+    dispatch({
+      type,
+      obsCrossfilter,
+      metadataField,
+      labelSelectionState,
+      ...oldProps,
+    });
+  };
 
 /**
  ** Graph selection-related actions
@@ -108,25 +105,23 @@ export const graphBrushStartAction = () =>
   /* no change to crossfilter until a change fires */
   ({ type: "graph brush start" });
 
-const _graphBrushWithinRectAction = (type, embName, brushCoords) => async (
-  dispatch,
-  getState
-) => {
-  const { obsCrossfilter: prevObsCrossfilter } = getState();
+const _graphBrushWithinRectAction =
+  (type, embName, brushCoords) => async (dispatch, getState) => {
+    const { obsCrossfilter: prevObsCrossfilter } = getState();
 
-  const selection = { mode: "within-rect", ...brushCoords };
-  const obsCrossfilter = await prevObsCrossfilter.select(
-    "emb",
-    embName,
-    selection
-  );
+    const selection = { mode: "within-rect", ...brushCoords };
+    const obsCrossfilter = await prevObsCrossfilter.select(
+      "emb",
+      embName,
+      selection
+    );
 
-  dispatch({
-    type,
-    obsCrossfilter,
-    brushCoords,
-  });
-};
+    dispatch({
+      type,
+      obsCrossfilter,
+      brushCoords,
+    });
+  };
 
 const _graphAllAction = (type, embName) => async (dispatch, getState) => {
   const { obsCrossfilter: prevObsCrossfilter } = getState();
@@ -162,28 +157,26 @@ export const graphLassoCancelAction = (embName) =>
 export const graphLassoDeselectAction = (embName) =>
   _graphAllAction("graph lasso cancel", embName);
 
-export const graphLassoEndAction = (embName, polygon) => async (
-  dispatch,
-  getState
-) => {
-  const { obsCrossfilter: prevObsCrossfilter } = getState();
+export const graphLassoEndAction =
+  (embName, polygon) => async (dispatch, getState) => {
+    const { obsCrossfilter: prevObsCrossfilter } = getState();
 
-  const selection = {
-    mode: "within-polygon",
-    polygon,
+    const selection = {
+      mode: "within-polygon",
+      polygon,
+    };
+    const obsCrossfilter = await prevObsCrossfilter.select(
+      "emb",
+      embName,
+      selection
+    );
+
+    dispatch({
+      type: "graph lasso end",
+      obsCrossfilter,
+      polygon,
+    });
   };
-  const obsCrossfilter = await prevObsCrossfilter.select(
-    "emb",
-    embName,
-    selection
-  );
-
-  dispatch({
-    type: "graph lasso end",
-    obsCrossfilter,
-    polygon,
-  });
-};
 
 /*
 Differential expression set selection

client/src/actions/viewStack.js

@@ -42,10 +42,8 @@ export const subsetAction = () => (dispatch, getState) => {
   By convention, a clip view is ALWAYS the top view, so if present, pop
   off and re-apply
   */
-  const {
-    annoMatrix: prevAnnoMatrix,
-    obsCrossfilter: prevObsCrossfilter,
-  } = getState();
+  const { annoMatrix: prevAnnoMatrix, obsCrossfilter: prevObsCrossfilter } =
+    getState();
   const annoMatrix = _userSubsetAnnoMatrix(
     prevAnnoMatrix,
     prevObsCrossfilter.allSelectedMask()

client/src/annoMatrix/annoMatrix.js

@@ -12,6 +12,7 @@ import {
 import { indexEntireSchema } from "../util/stateManager/schemaHelpers";
 import { _whereCacheGet, _whereCacheMerge } from "./whereCache";
 import _shallowClone from "./clone";
+import { _queryValidate, _queryCacheKey } from "./query";
 
 const _dataframeCache = dataframeMemo(128);
 
@@ -181,7 +182,7 @@ export default class AnnoMatrix {
 		Returns a Promise for the query result, which will resolve to a dataframe.
 
 		Field must be one of the matrix fields: 'obs', 'var', 'X', 'emb'.  Value
-		represents the underlying object upon which the query is occuring.
+		represents the underlying object upon which the query is occurring.
 
 		Query is one of:
 			* a string, representing a single column name from the field, eg,
@@ -197,12 +198,6 @@ export default class AnnoMatrix {
 		field/column, similar to a join.  Currently only supported on the var
 		dimension, allowing query of X columns by var value (eg, gene name)
 
-    The query filter	is a single value filter:
-			{ "field name": [
-				{name: "column name", values: [ list of values ]}
-			]}
-		One and only one value filter is allowed in a value query.
-
 		Examples:
 
     1. Fetch the "n_genes" column the "obs":
@@ -220,7 +215,9 @@ export default class AnnoMatrix {
        value "TYMP" in the var index.
 
 			fetch("X", { 
-				where: {field: "var", column: this.schema.annotations.var.index, value: "TYMP"}
+				where: {
+          field: "var", column: this.schema.annotations.var.index, value: "TYMP"
+        }
 			})
 
       In AnnData & Pandas DataFrame API, this is equivalent to:
@@ -410,6 +407,14 @@ export default class AnnoMatrix {
     _subclassResponsibility();
   }
 
+  getCacheKeys(field, query) {
+    /*
+    Return cache keys for columns associated with this query.  May return
+    [unknown] if no keys are known (ie, nothing is or was cached).
+    */
+    return _whereCacheGet(this._whereCache, this.schema, field, query);
+  }
+
   /**
    ** Private interfaces below.
    **/
@@ -427,6 +432,7 @@ export default class AnnoMatrix {
   async _fetch(field, q) {
     if (!AnnoMatrix.fields().includes(field)) return undefined;
     const queries = Array.isArray(q) ? q : [q];
+    queries.forEach(_queryValidate);
 
     /* find cached columns we need, and GC the rest */
     const cachedColumns = this._resolveCachedQueries(field, queries);
@@ -507,7 +513,7 @@ export default class AnnoMatrix {
     * obs, var and emb do not grow without bounds, and are needed constantly 
       for rendering.  
       a) There is no upside to GC'ing these in the base (loader)
-      b) The undo/redo cache can hold a large number in views, which is worht GC'ing
+      b) The undo/redo cache can hold a large number in views, which is worth GC'ing
     * X is often much larger than memory, and the UI allows add/del from
       this.  Most of the GC potential is here in both the base and views.
 
@@ -522,7 +528,7 @@ export default class AnnoMatrix {
   as much of the cache is pinned by that data structure.
   */
   _gcField(field, isHot, pinnedColumns) {
-    const maxColumns = isHot ? 256 : 10; // maybe to aggessive?
+    const maxColumns = isHot ? 256 : 10; // maybe to aggressive?
 
     const cache = this._cache[field];
     if (cache.colIndex.size() < maxColumns) return; // trivial rejection
@@ -590,23 +596,18 @@ export default class AnnoMatrix {
     called each time a query is performed, allowing the gc to update any bookkeeping
     information.  Currently, this is just a simple last-fetched timestamp, stored
     in a Map.
-
-    Map objects preserve order of insertion. This is leveraged as a cheap way to
-    do LRU, by removing and re-inserting keys.  IMPORTANT: the cleanup code assumes
-    the map insertion order is least-recently-used first.
     */
     const cols = dataframe.colIndex.labels();
     const { _gcInfo } = this;
     const now = Date.now();
     cols.forEach((c) => {
-      // gcInfo.delete(c);
       _gcInfo.set(_columnCacheKey(field, c), now);
     });
   }
 
   /**
-  Cloning sublcass protocol - we rely in cloning to preserve immutable
-  symantics while not causing races or other side effects in internal
+  Cloning subclass protocol - we rely in cloning to preserve immutable
+  semantics while not causing races or other side effects in internal
   cache management.
 
   Subclasses must override _cloneDeeper() if they have state which requires 
@@ -639,15 +640,6 @@ export default class AnnoMatrix {
 /*
 private utility functions below
 */
-
-function _queryCacheKey(field, query) {
-  if (typeof query === "object") {
-    const { field: queryField, column: queryColumn, value: queryValue } = query;
-    return `${field}/${queryField}/${queryColumn}/${queryValue}`;
-  }
-  return `${field}/${query}`;
-}
-
 function _columnCacheKey(field, column) {
   return `${field}/${column}`;
 }

client/src/annoMatrix/crossfilter.js

@@ -123,6 +123,24 @@ export default class AnnoMatrixObsCrossfilter {
     return new AnnoMatrixObsCrossfilter(annoMatrix, this.obsCrossfilter);
   }
 
+  /**
+   * Drop the crossfilter dimension. Do not change the annoMatrix. Useful when we
+   * want to stop trackin the selection state, but aren't sure we want to blow the
+   * annomatrix cache.
+   */
+  dropDimension(field, query) {
+    const { annoMatrix } = this;
+    let { obsCrossfilter } = this;
+    const keys = annoMatrix
+      .getCacheKeys(field, query)
+      .filter((k) => k !== undefined);
+    const dimName = _dimensionName(field, keys);
+    if (obsCrossfilter.hasDimension(dimName)) {
+      obsCrossfilter = obsCrossfilter.delDimension(dimName);
+    }
+    return new AnnoMatrixObsCrossfilter(annoMatrix, obsCrossfilter);
+  }
+
   /**
   Selection state - API is identical to ImmutableTypedCrossfilter, as these
   are just wrappers to lazy create indices.
@@ -164,9 +182,7 @@ export default class AnnoMatrixObsCrossfilter {
 		*/
     const { annoMatrix } = this;
     const currentDims = this.obsCrossfilter.dimensionNames();
-    const obsCrossfilter = currentDims.reduce((xfltr, dim) => {
-      return xfltr.select(dim, { mode: "all" });
-    }, this.obsCrossfilter);
+    const obsCrossfilter = currentDims.reduce((xfltr, dim) => xfltr.select(dim, { mode: "all" }), this.obsCrossfilter);
     return new AnnoMatrixObsCrossfilter(annoMatrix, obsCrossfilter);
   }
 

client/src/annoMatrix/fetchHelpers.js

@@ -1,4 +1,4 @@
-export { doBinaryRequest } from "../util/actionHelpers";
+export { doBinaryRequest, doFetch } from "../util/actionHelpers";
 
 /* double URI encode - needed for query-param filters */
 export function _dubEncURIComp(s) {
@@ -19,9 +19,7 @@ export function _fetchResult(promise) {
     }
   );
 
-  res.status = () => {
-    return _status;
-  };
+  res.status = () => _status;
 
   return res;
 }

client/src/annoMatrix/loader.js

@@ -1,6 +1,6 @@
-import { doBinaryRequest, _dubEncURIComp } from "./fetchHelpers";
+import { doBinaryRequest, doFetch } from "./fetchHelpers";
 import { matrixFBSToDataframe } from "../util/stateManager/matrix";
-import { _getColumnSchema, _normalizeCategoricalSchema } from "./schema";
+import { _getColumnSchema } from "./schema";
 import {
   addObsAnnoColumn,
   removeObsAnnoColumn,
@@ -12,6 +12,17 @@ import { isArrayOrTypedArray } from "../util/typeHelpers";
 import { _whereCacheCreate } from "./whereCache";
 import AnnoMatrix from "./annoMatrix";
 import PromiseLimit from "../util/promiseLimit";
+import {
+  _expectSimpleQuery,
+  _expectComplexQuery,
+  _urlEncodeLabelQuery,
+  _urlEncodeComplexQuery,
+  _hashStringValues,
+} from "./query";
+import {
+  normalizeResponse,
+  normalizeWritableCategoricalSchema,
+} from "./normalize";
 
 const promiseThrottle = new PromiseLimit(5);
 
@@ -115,7 +126,7 @@ export default class AnnoMatrixLoader extends AnnoMatrix {
       data = new Ctor(this.nObs).fill(value);
     }
     newAnnoMatrix._cache.obs = this._cache.obs.withCol(colName, data);
-    _normalizeCategoricalSchema(
+    normalizeWritableCategoricalSchema(
       colSchema,
       newAnnoMatrix._cache.obs.col(colName)
     );
@@ -223,27 +234,23 @@ export default class AnnoMatrixLoader extends AnnoMatrix {
     /*
     _doLoad - evaluates the query against the field. Returns:
       * whereCache update: column query map mapping the query to the column labels
-      * Dataframe containing the new colums (one per dimension)
+      * Dataframe containing the new columns (one per dimension)
     */
-    let urlQuery;
-    let urlBase;
+    let doRequest;
     let priority = 10; // default fetch priority
 
     switch (field) {
       case "obs":
       case "var": {
-        urlBase = `${this.baseURL}annotations/${field}`;
-        urlQuery = _encodeQuery("annotation-name", query);
+        doRequest = _obsOrVarLoader(this.baseURL, field, query);
         break;
       }
       case "X": {
-        urlBase = `${this.baseURL}data/var`;
-        urlQuery = _encodeQuery(undefined, query);
+        doRequest = _XLoader(this.baseURL, field, query);
         break;
       }
       case "emb": {
-        urlBase = `${this.baseURL}layout/obs`;
-        urlQuery = _encodeQuery("layout-name", query);
+        doRequest = _embLoader(this.baseURL, field, query);
         priority = 0; // high prio load for embeddings
         break;
       }
@@ -251,13 +258,8 @@ export default class AnnoMatrixLoader extends AnnoMatrix {
         throw new Error("Unknown field name");
     }
 
-    const url = `${urlBase}?${urlQuery}`;
-    const buffer = await promiseThrottle.priorityAdd(
-      priority,
-      doBinaryRequest,
-      url
-    );
-    const result = matrixFBSToDataframe(buffer);
+    const buffer = await promiseThrottle.priorityAdd(priority, doRequest);
+    let result = matrixFBSToDataframe(buffer);
     if (!result || result.isEmpty()) throw Error("Unknown field/col");
 
     const whereCacheUpdate = _whereCacheCreate(
@@ -266,13 +268,7 @@ export default class AnnoMatrixLoader extends AnnoMatrix {
       result.colIndex.labels()
     );
 
-    if (field === "obs") {
-      /* cough, cough - see comment on method */
-      _normalizeCategoricalSchema(
-        this.schema.annotations.obsByName[query],
-        result.col(query)
-      );
-    }
+    result = normalizeResponse(field, query, this.schema, result);
 
     return [whereCacheUpdate, result];
   }
@@ -282,17 +278,6 @@ export default class AnnoMatrixLoader extends AnnoMatrix {
 Utility functions below
 */
 
-function _encodeQuery(colKey, q) {
-  if (typeof q === "object") {
-    const { field: queryField, column: queryColumn, value: queryValue } = q;
-    return `${_dubEncURIComp(queryField)}:${_dubEncURIComp(
-      queryColumn
-    )}=${_dubEncURIComp(queryValue)}`;
-  }
-  if (!colKey) throw new Error("Unsupported query by name");
-  return `${colKey}=${encodeURIComponent(q)}`;
-}
-
 function _writableCheck(colSchema) {
   if (!colSchema?.writable) {
     throw new Error("Unknown or readonly obs column");
@@ -305,3 +290,57 @@ function _writableCategoryTypeCheck(colSchema) {
     throw new Error("column must be categorical");
   }
 }
+
+function _embLoader(baseURL, _field, query) {
+  _expectSimpleQuery(query);
+
+  const urlBase = `${baseURL}layout/obs`;
+  const urlQuery = _urlEncodeLabelQuery("layout-name", query);
+  const url = `${urlBase}?${urlQuery}`;
+  return () => doBinaryRequest(url);
+}
+
+function _obsOrVarLoader(baseURL, field, query) {
+  _expectSimpleQuery(query);
+
+  const urlBase = `${baseURL}annotations/${field}`;
+  const urlQuery = _urlEncodeLabelQuery("annotation-name", query);
+  const url = `${urlBase}?${urlQuery}`;
+  return () => doBinaryRequest(url);
+}
+
+function _XLoader(baseURL, field, query) {
+  _expectComplexQuery(query);
+
+  if (query.where) {
+    const urlBase = `${baseURL}data/var`;
+    const urlQuery = _urlEncodeComplexQuery(query);
+    const url = `${urlBase}?${urlQuery}`;
+    return () => doBinaryRequest(url);
+  }
+
+  if (query.summarize) {
+    const urlBase = `${baseURL}summarize/var`;
+    const urlQuery = _urlEncodeComplexQuery(query);
+
+    if (urlBase.length + urlQuery.length < 2000) {
+      const url = `${urlBase}?${urlQuery}`;
+      return () => doBinaryRequest(url);
+    }
+
+    const url = `${urlBase}?key=${_hashStringValues([urlQuery])}`;
+    return async () => {
+      const res = await doFetch(url, {
+        method: "POST",
+        body: urlQuery,
+        headers: new Headers({
+          Accept: "application/octet-stream",
+          "Content-Type": "application/x-www-form-urlencoded",
+        }),
+      });
+      return res.arrayBuffer();
+    };
+  }
+
+  throw new Error("Unknown query structure");
+}

client/src/annoMatrix/normalize.js

@@ -0,0 +1,150 @@
+import { _getColumnSchema, _isIndex } from "./schema";
+import catLabelSort from "../util/catLabelSort";
+import {
+  unassignedCategoryLabel,
+  overflowCategoryLabel,
+  globalConfig,
+} from "../globals";
+
+export function normalizeResponse(field, query, schema, response) {
+  /**
+   * There are a number of assumptions in the front-end about data typing and data
+   * characteristics. This routine will normalize a server response dataframe
+   * to match front-end expectations and UI conventions. This includes cast/transform
+   * of the data and schema updates.
+   *
+   * This consolidates all assumptions into one location, for ease of update.
+   *
+   * Currently, this includes normalization for obs/var columns only:
+   *
+   * - Dataframe columns in var/obs that are declared type: boolean may be sent by
+   *   the server in a variety of formats (eg uint8, etc).  Cast to JS Array[boolean]
+   *
+   * - "Categorical" columns may not have all categories represented in the server-provided
+   *   schema (for valid reasons, eg, floating point rounding differences).  For all
+   *   types we treat as categorical in the UI (string, boolean, categorical), update
+   *   the schema to contain all categories as a convenience.
+   *
+   * - "Categorical" columns (ie, string, boolean, categorical) may contain an excess
+   *   of category values (aka labels).  Consolidate any excess into an "all other"
+   *   category.
+   */
+
+  // currently no data or schema normalization necessary for X or emb
+  if (field !== "obs" && field !== "var") return response;
+
+  const colLabels = response.colIndex.labels();
+  for (const colLabel of colLabels) {
+    const colSchema = _getColumnSchema(schema, field, colLabel);
+    const isIndex = _isIndex(schema, field, colLabel);
+    const { type, writable } = colSchema;
+
+    // Boolean data -- cast entire array to Array[bool]
+    if (type === "boolean") {
+      response = castColumnToBoolean(response, colLabel);
+    }
+
+    // Types that are categorical in UI (string, boolean, categorical) OR are writable
+    // are introspected to ensure the schema `categories` field and data values match,
+    // and that we do not have an excess of category values (for non-writable columns)
+    const isEnumType =
+      type === "boolean" ||
+      type === "string" ||
+      type === "categorical" ||
+      writable;
+    if (!isIndex && isEnumType) {
+      response = normalizeCategorical(response, colLabel, colSchema);
+    }
+  }
+  return response;
+}
+
+function castColumnToBoolean(df, label) {
+  const colData = df.col(label).asArray();
+  const newColData = new Array(colData.length);
+  for (let i = 0; i < colData.length; i += 1) newColData[i] = !!colData[i];
+  df = df.replaceColData(label, newColData);
+  return df;
+}
+
+export function normalizeWritableCategoricalSchema(colSchema, col) {
+  /*
+  Ensure all enum writable / categorical schema have a categories array, that
+  the categories array contains all unique values in the data array, AND that
+  the array is UI sorted.
+  */
+  const categorySet = new Set(
+    col.summarizeCategorical().categories.concat(colSchema.categories ?? [])
+  );
+  if (!categorySet.has(unassignedCategoryLabel)) {
+    categorySet.add(unassignedCategoryLabel);
+  }
+  colSchema.categories = catLabelSort(true, Array.from(categorySet));
+  return colSchema;
+}
+
+export function normalizeCategorical(df, colLabel, colSchema) {
+  /*
+  If writable, ensure schema matches data and we have an unassigned label
+  If not writable, ensure schema matches data and that we consolidate labels in excess
+  of "top N" into an overflow labels.
+  */
+  const { writable } = colSchema;
+  const col = df.col(colLabel);
+
+  if (writable) {
+    // writable (aka user) annotations
+    normalizeWritableCategoricalSchema(colSchema, col);
+    return df;
+  }
+
+  // else read-only, categorical columns
+  const TopN = globalConfig.maxCategoricalOptionsToDisplay;
+
+  // consolidate all categories from data and schema into a single list
+  const colDataSummary = col.summarizeCategorical();
+  const allCategories = new Set(
+    colDataSummary.categories.concat(colSchema.categories ?? [])
+  );
+
+  // if no overflow, just UI sort schema categories and return
+  if (allCategories.size <= TopN) {
+    colSchema.categories = catLabelSort(writable, [...allCategories.keys()]);
+    return df;
+  }
+
+  // Otherwise, pick top N categories by count and rewrite data
+
+  // choose unique overflow category label
+  let overflowCatName = `${colLabel}${overflowCategoryLabel}`;
+  while (allCategories.has(overflowCatName)) {
+    overflowCatName += "_";
+  }
+
+  // pick top N category labels and add overflow label
+  const topNCategories = new Set(
+    [...colDataSummary.categoryCounts.keys()].slice(0, TopN)
+  );
+  topNCategories.add(overflowCatName);
+
+  // rewrite data - consolidate all excess labels into overflow label
+  const newColData = Array.from(col.asArray());
+  for (let i = 0; i < newColData.length; i += 1) {
+    if (!topNCategories.has(newColData[i])) {
+      newColData[i] = overflowCatName;
+    }
+  }
+
+  // replace data in dataframe
+  df = df.replaceColData(colLabel, newColData);
+
+  // Update schema with categories, in UI sort order. Ensure overflow label is at end
+  // of list for display purposes.
+  const revisedCategories = df.col(colLabel).summarizeCategorical().categories;
+  revisedCategories.push(
+    revisedCategories.splice(revisedCategories.indexOf(overflowCatName), 1)[0]
+  );
+  colSchema.categories = catLabelSort(writable, revisedCategories);
+
+  return df;
+}

client/src/annoMatrix/query.js

@@ -0,0 +1,126 @@
+import sha1 from "sha1";
+import { _dubEncURIComp } from "./fetchHelpers";
+
+/**
+ * Query utilities, mostly for debugging support and validation.
+ */
+
+/**
+ * Normalize & error check the query.
+ * @param {object | string} query - the query
+ * @returns {object | string} - the normalized query
+ */
+export function _queryValidate(query) {
+  if (typeof query !== "object") return query;
+
+  if (query.where && query.summarize)
+    throw new Error("query may not specify both where and summarize");
+  if (query.where) {
+    const {
+      field: queryField,
+      column: queryColumn,
+      value: queryValue,
+    } = query.where;
+    if (!queryField || !queryColumn || !queryValue)
+      throw new Error("Incomplete where query");
+    return query;
+  }
+  if (query.summarize) {
+    const {
+      field: queryField,
+      column: queryColumn,
+      values: queryValues,
+    } = query.summarize;
+    if (!queryField || !queryColumn || !queryValues)
+      throw new Error("Incomplete where query");
+    if (!Array.isArray(queryValues))
+      throw new Error("Summarize query values must be an array");
+    return query;
+  }
+  throw new Error("query must specify one of where or summarize");
+}
+
+export function _expectSimpleQuery(query) {
+  if (typeof query === "object") throw new Error("expected simple query");
+}
+
+export function _expectComplexQuery(query) {
+  if (typeof query !== "object") throw new Error("expected complex query");
+}
+
+/**
+ * Generate a unique key which can be used to reference this query.
+ *
+ * @param {string} field
+ * @param {string|object} query
+ * @returns the key
+ */
+export function _queryCacheKey(field, query) {
+  if (typeof query === "object") {
+    // complex query
+    if (query.where) {
+      const {
+        field: queryField,
+        column: queryColumn,
+        value: queryValue,
+      } = query.where;
+      return `${field}/${queryField}/${queryColumn}/${queryValue}`;
+    }
+    if (query.summarize) {
+      const {
+        method,
+        field: queryField,
+        column: queryColumn,
+        values: queryValues,
+      } = query.summarize;
+      return `${field}/${method}/${queryField}/${queryColumn}/${queryValues.join(
+        ","
+      )}`;
+    }
+    throw new Error("Unrecognized complex query type");
+  }
+
+  // simple query
+  return `${field}/${query}`;
+}
+
+function _urlEncodeWhereQuery(q) {
+  const { field: queryField, column: queryColumn, value: queryValue } = q;
+  return `${_dubEncURIComp(queryField)}:${_dubEncURIComp(
+    queryColumn
+  )}=${_dubEncURIComp(queryValue)}`;
+}
+
+function _urlEncodeSummarizeQuery(q) {
+  const { method, field, column, values } = q;
+  const filter = values
+    .map((value) => _urlEncodeWhereQuery({ field, column, value }))
+    .join("&");
+  return `method=${method}&${filter}`;
+}
+
+export function _urlEncodeComplexQuery(q) {
+  if (typeof q === "object") {
+    if (q.where) {
+      return _urlEncodeWhereQuery(q.where);
+    }
+    if (q.summarize) {
+      return _urlEncodeSummarizeQuery(q.summarize);
+    }
+  }
+  throw new Error("Unrecognized complex query type");
+}
+
+export function _urlEncodeLabelQuery(colKey, q) {
+  if (!colKey) throw new Error("Unsupported query by name");
+  if (typeof q !== "string") throw new Error("Query must be a simple label.");
+  return `${colKey}=${encodeURIComponent(q)}`;
+}
+
+/**
+ * Generate the column key the server will send us for this query.
+ */
+export function _hashStringValues(arrayOfString) {
+  const hash = sha1(arrayOfString.join(""));
+  return hash;
+}

client/src/annoMatrix/schema.js

@@ -1,9 +1,6 @@
 /*
 Private helper functions related to schema
 */
-import catLabelSort from "../util/catLabelSort";
-import { unassignedCategoryLabel } from "../globals";
-
 export function _getColumnSchema(schema, field, col) {
   /* look up the column definition */
   switch (field) {
@@ -26,10 +23,15 @@ export function _getColumnSchema(schema, field, col) {
   }
 }
 
+export function _isIndex(schema, field, col) {
+  const index = schema.annotations?.[field].index;
+  return index && index === col;
+}
+
 export function _getColumnDimensionNames(schema, field, col) {
   /*
 		field/col may be an alias for multiple columns. Currently used to map ND 
-		values to 1D dataframe columns for embeddings/layout. Signfied by the presence
+		values to 1D dataframe columns for embeddings/layout. Signified by the presence
 		of the "dims" value in the schema.
 		*/
   const colSchema = _getColumnSchema(schema, field, col);
@@ -63,26 +65,3 @@ export function _isContinuousType(schema) {
   const { type } = schema;
   return !(type === "string" || type === "boolean" || type === "categorical");
 }
-
-export function _normalizeCategoricalSchema(colSchema, col) {
-  const { type, writable } = colSchema;
-  if (
-    type === "string" ||
-    type === "boolean" ||
-    type === "categorical" ||
-    writable
-  ) {
-    const categorySet = new Set(
-      col.summarizeCategorical().categories.concat(colSchema.categories ?? [])
-    );
-    if (writable && !categorySet.has(unassignedCategoryLabel)) {
-      categorySet.add(unassignedCategoryLabel);
-    }
-    colSchema.categories = Array.from(categorySet);
-  }
-
-  if (colSchema.categories) {
-    colSchema.categories = catLabelSort(writable, colSchema.categories);
-  }
-  return colSchema;
-}

client/src/annoMatrix/viewCreators.js

@@ -58,10 +58,11 @@ function _maskToList(mask) {
   if (!mask) {
     return null;
   }
-  const list = new Int32Array(mask.length);
+  const [...m] = mask;
+  const list = new Int32Array(m.length);
   let elems = 0;
-  for (let i = 0, l = mask.length; i < l; i += 1) {
-    if (mask[i]) {
+  for (let i = 0, l = m.length; i < l; i += 1) {
+    if (m[i]) {
       list[elems] = i;
       elems += 1;
     }

client/src/annoMatrix/whereCache.js

@@ -1,50 +1,88 @@
 /*
 Private support functions.
 
-Support for a "where" query, eg,
+This implements a query resolver cache, mapping a query onto the column labels
+resolved by that query. These labels are then used to manage the acutal data cache,
+which stores data by the resolved label.
 
-	{ where: { field: "var", column: "gene", value: "FOXP2" }}
+There are three query forms:
+  * primitive (string, number) - which is just reference the column label of same value
+  * where query (object) - eg, { where: { field: "var", column: "gene", value: "FOXP2" }}
+  * summary query (object) - eg, { summarize: { method: "mean", field: "var", column: "gene", values: ["FOXP2", "GNE", "F5"]}}
 
-These evaluate to a given column label.
+These queries all resolve to one or more column labels on a field.  This
+cache maintains a record of this, allowing direct access to the data caches
+without a server round-trip.
 
-The "where cache" is a map that saves evaluated queries and points 
-to the column label they resolve to.
+The data structure for where queries, the following query against X as an example:
+  { where: { field: "var", column: "column_label_in_var", value: "value_in_var_column" } }
+results in the following cached entry:
+{
+  where: {
+    X: {
+      var: Map(
+        column_label_in_var => Map(
+          value_in_var_column => [column_label_in_X, ...]
+        )
+      )
+    }
+  },
+  summarize: {},
+}
 
-Data structure, using X as the example field being queried, and var as
-the index.
-
-{ 
-	X: {
-		var: Map(
-			column_label_in_var => Map(value_in_var_column => [column_label_in_X, ...])
-		)
-	}
+And for summarize queries, for the following summary on X:
+  { summarize: { method: "mean", field: "var", column: "gene", values: ["G1", "G2"]}}
+creates a cache entry of:
+{
+  where: {},
+  summarize: {
+    X: {
+      mean: {
+        var: Map(
+          "gene" => Map(
+            "G1,G2" => [summary_column_label, ...]
+          )
+        )
+      }
+    },
+  },
 }
 */
 import { _getColumnDimensionNames } from "./schema";
+import { _hashStringValues } from "./query";
 
 export function _whereCacheGet(whereCache, schema, field, query) {
-  /* 
+  /*
 	query will either be an where query (object) or a column name (string).
 
 	Return array of column labels or undefined.
 	*/
 
   if (typeof query === "object") {
-    const { field: queryField, column: queryColumn, value: queryValue } = query;
-
-    const columnMap = whereCache?.[field]?.[queryField];
-    if (columnMap === undefined) return [undefined];
-
-    const valueMap = columnMap.get(queryColumn);
-    if (valueMap === undefined) return [undefined];
-
-    const columnLabels = valueMap.get(queryValue);
-    return columnLabels === undefined ? [undefined] : columnLabels;
+    if (query.where) {
+      const {
+        field: queryField,
+        column: queryColumn,
+        value: queryValue,
+      } = query.where;
+      const columnMap = whereCache?.where?.[field]?.[queryField];
+      return columnMap?.get(queryColumn)?.get(queryValue) ?? [undefined];
+    }
+    if (query.summarize) {
+      const {
+        method,
+        field: queryField,
+        column: queryColumn,
+        values: queryValues,
+      } = query.summarize;
+      const columnMap = whereCache?.summarize?.[field]?.[method]?.[queryField];
+      const queryValueHash = _hashStringValues(queryValues);
+      return columnMap?.get(queryColumn)?.get(queryValueHash) ?? [undefined];
+    }
+    return [undefined];
   }
 
-  const colDims = _getColumnDimensionNames(schema, field, query);
-  return colDims === undefined ? [undefined] : colDims;
+  return _getColumnDimensionNames(schema, field, query) ?? [undefined];
 }
 
 export function _whereCacheCreate(field, query, columnLabels) {
@@ -53,40 +91,84 @@ export function _whereCacheCreate(field, query, columnLabels) {
 	*/
   if (typeof query !== "object") return null;
 
-  const { field: queryField, column: queryColumn, value: queryValue } = query;
-  const whereCache = {
-    [field]: {
-      [queryField]: new Map([
-        [queryColumn, new Map([[queryValue, columnLabels]])],
-      ]),
-    },
-  };
-  return whereCache;
+  const { where, summarize } = query;
+  if (where) {
+    const { field: queryField, column: queryColumn, value: queryValue } = where;
+    return {
+      where: {
+        [field]: {
+          [queryField]: new Map([
+            [queryColumn, new Map([[queryValue, columnLabels]])],
+          ]),
+        },
+      },
+    };
+  }
+  if (summarize) {
+    const {
+      method,
+      field: queryField,
+      column: queryColumn,
+      values: queryValues,
+    } = summarize;
+    const queryValueHash = _hashStringValues(queryValues);
+    return {
+      summarize: {
+        [field]: {
+          [method]: {
+            [queryField]: new Map([
+              [queryColumn, new Map([[queryValueHash, columnLabels]])],
+            ]),
+          },
+        },
+      },
+    };
+  }
+  // oops, not sure what that query is!
+  return {};
+}
+
+function __mergeQueries(dst, src) {
+  for (const [queryField, columnMap] of Object.entries(src)) {
+    dst[queryField] = dst[queryField] || new Map();
+    for (const [queryColumn, valueMap] of columnMap) {
+      if (!dst[queryField].has(queryColumn))
+        dst[queryField].set(queryColumn, new Map());
+      for (const [queryValue, columnLabels] of valueMap) {
+        dst[queryField].get(queryColumn).set(queryValue, columnLabels);
+      }
+    }
+  }
 }
 
 function __whereCacheMerge(dst, src) {
   /*
 	merge src into dst (modifies dst)
 	*/
-  if (!dst) dst = {};
   if (!src || typeof src !== "object") return dst;
-  Object.entries(src).forEach(([field, query]) => {
-    if (!Object.prototype.hasOwnProperty.call(dst, field)) dst[field] = {};
-    Object.entries(query).forEach(([queryField, columnMap]) => {
-      if (!Object.prototype.hasOwnProperty.call(dst[field], queryField))
-        dst[field][queryField] = new Map();
-      columnMap.forEach((valueMap, queryColumn) => {
-        if (!dst[field][queryField].has(queryColumn))
-          dst[field][queryField].set(queryColumn, new Map());
-        valueMap.forEach((columnLabels, queryValue) => {
-          dst[field][queryField].get(queryColumn).set(queryValue, columnLabels);
-        });
-      });
-    });
-  });
+
+  if (src.where) {
+    dst.where = dst.where || {};
+    for (const [field, query] of Object.entries(src.where)) {
+      dst.where[field] = dst.where[field] || {};
+      __mergeQueries(dst.where[field], query);
+    }
+  }
+  if (src.summarize) {
+    dst.summarize = dst.summarize || {};
+    for (const [field, method] of Object.entries(src.summarize)) {
+      dst.summarize[field] = dst.summarize[field] || {};
+      for (const [methodName, query] of Object.entries(method)) {
+        dst.summarize[field][methodName] =
+          dst.summarize[field][methodName] || {};
+        __mergeQueries(dst.summarize[field][methodName], query);
+      }
+    }
+  }
   return dst;
 }
 
 export function _whereCacheMerge(...caches) {
-  return caches.reduce((dst, src) => __whereCacheMerge(dst, src), {});
+  // eslint-disable-next-line compat/compat -- not using web APIs
+  return caches.reduce(__whereCacheMerge, {});
 }

client/src/components/annoDialog.js

@@ -18,6 +18,8 @@ class AnnoDialog extends React.PureComponent {
       validationError,
       annoSelect,
       annoInput,
+      secondaryInstructions,
+      secondaryInput,
       handleCancel,
       handleSubmit,
       primaryButtonText,
@@ -40,12 +42,19 @@ class AnnoDialog extends React.PureComponent {
               <p
                 style={{
                   marginTop: 7,
-                  visibility: validationError ? "visible" : "hidden",
+                  visibility: errorMessage !== "" ? "visible" : "hidden",
                   color: Colors.ORANGE3,
                 }}
               >
                 {errorMessage}
               </p>
+              {/* we might rename, secondary button and secondary input are not related */}
+              {secondaryInstructions && (
+                <p style={{ marginTop: secondaryInstructions ? 20 : 0 }}>
+                  {secondaryInstructions}
+                </p>
+              )}
+              {secondaryInput || null}
             </div>
             {annoSelect || null}
           </div>
@@ -54,6 +63,7 @@ class AnnoDialog extends React.PureComponent {
               <Tooltip content={cancelTooltipContent}>
                 <Button onClick={handleCancel}>Cancel</Button>
               </Tooltip>
+              {/* we might rename, secondary button and secondary input are not related */}
               {handleSecondaryButtonSubmit && secondaryButtonText ? (
                 <Button
                   onClick={handleSecondaryButtonSubmit}

client/src/components/app.js

@@ -1,4 +1,3 @@
-// jshint esversion: 6
 import React from "react";
 import Helmet from "react-helmet";
 import { connect } from "react-redux";
@@ -12,7 +11,6 @@ import Graph from "./graph/graph";
 import MenuBar from "./menubar";
 import Autosave from "./autosave";
 import Embedding from "./embedding";
-import TermsOfServicePrompt from "./termsPrompt";
 
 import actions from "../actions";
 
@@ -43,7 +41,7 @@ class App extends React.Component {
     const { loading, error, graphRenderCounter } = this.props;
     return (
       <Container>
-        <Helmet title="cellxgene" />
+        <Helmet title="CELL&times;GENE | Annotate" />
         {loading ? (
           <div
             style={{
@@ -76,7 +74,6 @@ class App extends React.Component {
                 <MenuBar />
                 <Embedding />
                 <Autosave />
-                <TermsOfServicePrompt />
                 <Legend viewportRef={viewportRef} />
                 <Graph key={graphRenderCounter} viewportRef={viewportRef} />
               </>

client/src/components/autosave/filenameDialog.js

@@ -14,9 +14,10 @@ import {
 @connect((state) => ({
   idhash: state.config?.parameters?.["annotations-user-data-idhash"] ?? null,
   annotations: state.annotations,
-  auth: state.config?.authentication,
-  userInfo: state.userInfo,
   writableCategoriesEnabled: state.config?.parameters?.annotations ?? false,
+  writableGenesetsEnabled: !(
+    state.config?.parameters?.annotations_genesets_readonly ?? true
+  ),
 }))
 class FilenameDialog extends React.Component {
   constructor(props) {
@@ -95,20 +96,19 @@ class FilenameDialog extends React.Component {
   render() {
     const {
       writableCategoriesEnabled,
+      writableGenesetsEnabled,
       annotations,
       idhash,
-      userInfo,
     } = this.props;
     const { filenameText } = this.state;
 
-    return writableCategoriesEnabled &&
+    return (writableCategoriesEnabled || writableGenesetsEnabled) &&
       annotations.promptForFilename &&
       !annotations.dataCollectionNameIsReadOnly &&
-      !annotations.dataCollectionName &&
-      userInfo.is_authenticated ? (
+      !annotations.dataCollectionName ? (
       <Dialog
         icon="tag"
-        title="Annotations Collection"
+        title="User Generated Data Directory"
         isOpen={!annotations.dataCollectionName}
         onClose={this.dismissFilenameDialog}
       >
@@ -120,7 +120,7 @@ class FilenameDialog extends React.Component {
         >
           <div className={Classes.DIALOG_BODY} data-testid="annotation-dialog">
             <div style={{ marginBottom: 20 }}>
-              <p>Name your annotations collection:</p>
+              <p>Name your user generated data directory:</p>
               <InputGroup
                 autoFocus
                 value={filenameText}
@@ -145,9 +145,15 @@ class FilenameDialog extends React.Component {
             </div>
             <div>
               <p>
-                Your annotations are stored in this file:
+                {"Your annotations are stored in this file: "}
                 <Code>
-                  {filenameText}-{idhash}.csv
+                  {filenameText}-cell-labels-{idhash}.csv
+                </Code>
+              </p>
+              <p>
+                {"Your gene sets are stored in this file: "}
+                <Code>
+                  {filenameText}-gene-sets-{idhash}.csv
                 </Code>
               </p>
               <p style={{ fontStyle: "italic" }}>
@@ -167,7 +173,7 @@ class FilenameDialog extends React.Component {
                 type="submit"
                 data-testid="submit-annotation"
               >
-                Create annotations collection
+                Create user generated data directory
               </Button>
             </div>
           </div>

client/src/components/autosave/index.js

@@ -5,11 +5,18 @@ import FilenameDialog from "./filenameDialog";
 
 @connect((state) => ({
   annotations: state.annotations,
-  saveInProgress: state.autosave?.saveInProgress ?? false,
+  obsAnnotationSaveInProgress:
+    state.autosave?.obsAnnotationSaveInProgress ?? false,
+  genesetSaveInProgress: state.autosave?.genesetSaveInProgress ?? false,
   error: state.autosave?.error,
   writableCategoriesEnabled: state.config?.parameters?.annotations ?? false,
+  writableGenesetsEnabled: !(
+    state.config?.parameters?.annotations_genesets_readonly ?? true
+  ),
   annoMatrix: state.annoMatrix,
+  genesets: state.genesets,
   lastSavedAnnoMatrix: state.autosave?.lastSavedAnnoMatrix,
+  lastSavedGenesets: state.autosave?.lastSavedGenesets,
 }))
 class Autosave extends React.Component {
   constructor(props) {
@@ -20,11 +27,11 @@ class Autosave extends React.Component {
   }
 
   componentDidMount() {
-    const { writableCategoriesEnabled } = this.props;
+    const { writableCategoriesEnabled, writableGenesetsEnabled } = this.props;
 
     let { timer } = this.state;
     if (timer) clearInterval(timer);
-    if (writableCategoriesEnabled) {
+    if (writableCategoriesEnabled || writableGenesetsEnabled) {
       timer = setInterval(this.tick, 2500);
     } else {
       timer = null;
@@ -38,18 +45,37 @@ class Autosave extends React.Component {
   }
 
   tick = () => {
-    const { dispatch, saveInProgress } = this.props;
-    if (this.needToSave() && !saveInProgress) {
+    const { dispatch, obsAnnotationSaveInProgress, genesetSaveInProgress } =
+      this.props;
+    if (!obsAnnotationSaveInProgress && this.needToSaveObsAnnotations()) {
       dispatch(actions.saveObsAnnotationsAction());
     }
+    if (!genesetSaveInProgress && this.needToSaveGenesets()) {
+      dispatch(actions.saveGenesetsAction());
+    }
   };
 
-  needToSave = () => {
-    /* return true if we need to save, false if we don't */
+  needToSaveObsAnnotations = () => {
+    /* return true if we need to save obs cell labels, false if we don't */
     const { annoMatrix, lastSavedAnnoMatrix } = this.props;
     return actions.needToSaveObsAnnotations(annoMatrix, lastSavedAnnoMatrix);
   };
 
+  needToSaveGenesets = () => {
+    /* return true if we need to save gene ses, false if we do not */
+    const { genesets, lastSavedGenesets } = this.props;
+    return genesets.initialized && genesets.genesets !== lastSavedGenesets;
+  };
+
+  needToSave() {
+    return this.needToSaveGenesets() || this.needToSaveObsAnnotations();
+  }
+
+  saveInProgress() {
+    const { obsAnnotationSaveInProgress, genesetSaveInProgress } = this.props;
+    return obsAnnotationSaveInProgress || genesetSaveInProgress;
+  }
+
   statusMessage() {
     const { error } = this.props;
     if (error) {
@@ -61,12 +87,12 @@ class Autosave extends React.Component {
   render() {
     const {
       writableCategoriesEnabled,
-      saveInProgress,
+      writableGenesetsEnabled,
       lastSavedAnnoMatrix,
     } = this.props;
     const initialDataLoadComplete = lastSavedAnnoMatrix;
 
-    if (!writableCategoriesEnabled) return null;
+    if (!writableCategoriesEnabled && !writableGenesetsEnabled) return null;
 
     return (
       <div
@@ -74,7 +100,7 @@ class Autosave extends React.Component {
         data-testclass={
           !initialDataLoadComplete
             ? "autosave-init"
-            : this.needToSave() || saveInProgress
+            : this.saveInProgress() || this.needToSave()
             ? "autosave-incomplete"
             : "autosave-complete"
         }

client/src/components/brushableHistogram/error.js

@@ -0,0 +1,15 @@
+import React from "react";
+import * as globals from "../../globals";
+
+const ErrorLoading = ({ displayName, zebra }) => (
+    <div
+      style={{
+        backgroundColor: zebra ? globals.lightestGrey : "white",
+        fontStyle: "italic",
+      }}
+    >
+      <span>{`Failure loading ${displayName}`}</span>
+    </div>
+  );
+
+export default ErrorLoading;

client/src/components/brushableHistogram/footer.js

@@ -0,0 +1,63 @@
+import React from "react";
+
+const HistogramFooter = React.memo(
+  ({
+    displayName,
+    hideRanges,
+    rangeMin,
+    rangeMax,
+    rangeColorMin,
+    rangeColorMax,
+    isObs,
+    isGeneSetSummary,
+  }) => 
+    /*
+    Footer of each histogram.  Will render range and title.
+
+    Required props:
+      * displayName - the displayName, aka "n_genes", "FOXP2", etc.
+      * hideRanges - true/false, enables/disable rendering of ranges
+      * range - length two array, [min, max], containing the range values to display
+      * rangeColor - length two array, [mincolor, maxcolor], each a CSS color
+    */
+     (
+      <div>
+        <div
+          style={{
+            display: "flex",
+            justifyContent: hideRanges ? "center" : "space-between",
+          }}
+        >
+          <span
+            style={{
+              color: rangeColorMin,
+              display: hideRanges ? "none" : "block",
+            }}
+          >
+            min {rangeMin.toPrecision(4)}
+          </span>
+          <span
+            data-testclass="brushable-histogram-field-name"
+            style={{ fontStyle: "italic" }}
+          >
+            {isObs && displayName}
+            {isGeneSetSummary && "gene set mean expression"}
+          </span>
+          <div style={{ display: hideRanges ? "block" : "none" }}>
+            : {rangeMin}
+          </div>
+          <span
+            style={{
+              color: rangeColorMax,
+              display: hideRanges ? "none" : "block",
+            }}
+          >
+            max {rangeMax.toPrecision(4)}
+          </span>
+        </div>
+      </div>
+    )
+  
+);
+
+export default HistogramFooter;

client/src/components/brushableHistogram/header.js

@@ -0,0 +1,100 @@
+import React, { useCallback } from "react";
+import { Button, ButtonGroup, Tooltip, Icon } from "@blueprintjs/core";
+import { IconNames } from "@blueprintjs/icons";
+import * as globals from "../../globals";
+
+const HistogramHeader = React.memo(
+  ({
+    fieldId,
+    isColorBy,
+    onColorByClick,
+    onRemoveClick,
+    isScatterPlotX,
+    isScatterPlotY,
+    onScatterPlotXClick,
+    onScatterPlotYClick,
+    isObs,
+  }) => {
+    /*
+        Render the toolbar for the histogram.  Props:
+          * fieldId - field identifier, used for various IDs
+          * isColorBy - true/false, is this the current color-by
+          * onColorByClick - color-by click handler
+          * onRemoveClick - optional handler for remove.  Button will not render if not defined.
+          * isScatterPlotX - optional, true/false if currently the X scatterplot field
+          * isScatterPlotY - optional, true/false if currently the Y scatterplot field
+          * onScatterPlotXClick - optional, handler for scatterPlot X button.
+          * onScatterPlotYClick - optional, handler for scatterPlot X button.
+  
+        Scatterplot controls will not render if either handler unspecified.
+      */
+
+    const memoizedColorByCallback = useCallback(
+      () => onColorByClick(fieldId, isObs),
+      [fieldId, isObs]
+    );
+
+    return (
+      <div
+        style={{
+          display: "flex",
+          justifyContent: "flex-end",
+          paddingBottom: "8px",
+        }}
+      >
+        {onScatterPlotXClick && onScatterPlotYClick ? (
+          <span>
+            <Icon icon={IconNames.SCATTER_PLOT} style={{ marginRight: 7 }} />
+            <ButtonGroup style={{ marginRight: 7 }}>
+              <Button
+                data-testid={`plot-x-${fieldId}`}
+                onClick={onScatterPlotXClick}
+                active={isScatterPlotX}
+                intent={isScatterPlotX ? "primary" : "none"}
+              >
+                plot x
+              </Button>
+              <Button
+                data-testid={`plot-y-${fieldId}`}
+                onClick={onScatterPlotYClick}
+                active={isScatterPlotY}
+                intent={isScatterPlotY ? "primary" : "none"}
+              >
+                plot y
+              </Button>
+            </ButtonGroup>
+          </span>
+        ) : null}
+        {onRemoveClick ? (
+          <Button
+            minimal
+            onClick={onRemoveClick}
+            style={{
+              color: globals.blue,
+              cursor: "pointer",
+              marginLeft: 7,
+            }}
+          >
+            remove
+          </Button>
+        ) : null}
+        <Tooltip
+          content="Use as color scale"
+          position="bottom"
+          hoverOpenDelay={globals.tooltipHoverOpenDelay}
+        >
+          <Button
+            onClick={memoizedColorByCallback}
+            active={isColorBy}
+            intent={isColorBy ? "primary" : "none"}
+            data-testclass="colorby"
+            data-testid={`colorby-${fieldId}`}
+            icon="tint"
+          />
+        </Tooltip>
+      </div>
+    );
+  }
+);
+
+export default HistogramHeader;

client/src/components/brushableHistogram/histogram.js

@@ -0,0 +1,191 @@
+import React, { useEffect, useRef, useState } from "react";
+import { interpolateCool } from "d3-scale-chromatic";
+import * as d3 from "d3";
+
+import maybeScientific from "../../util/maybeScientific";
+import clamp from "../../util/clamp";
+
+const Histogram = ({
+  field,
+  fieldForId,
+  display,
+  histogram,
+  width,
+  height,
+  onBrush,
+  onBrushEnd,
+  margin,
+  isColorBy,
+  selectionRange,
+  mini,
+}) => {
+  const svgRef = useRef(null);
+  const [brush, setBrush] = useState(null);
+
+  useEffect(() => {
+    /*
+      Create the d3 histogram
+      */
+    //  This is just a constant that's flipped by parent's `mini` boolean
+    const {
+      LEFT: marginLeft,
+      RIGHT: marginRight,
+      BOTTOM: marginBottom,
+      TOP: marginTop,
+    } = margin;
+    const { x, y, bins, binStart, binEnd, binWidth } = histogram;
+    const svg = d3.select(svgRef.current);
+    const binPadding = mini ? 0 : -1;
+    const defaultBarColor = mini ? "black" : "#bbb";
+
+    /* Remove everything */
+    svg.selectAll("*").remove();
+
+    /* Set margins within the SVG */
+    const container = svg
+      .attr("width", width + marginLeft + marginRight)
+      .attr("height", height + marginTop + marginBottom)
+      .append("g")
+      .attr("class", "histogram-container")
+      .attr("transform", `translate(${marginLeft},${marginTop})`);
+
+    const colorScale = d3
+      .scaleSequential(interpolateCool)
+      .domain([0, bins.length]);
+
+    const histogramScale = d3
+      .scaleLinear()
+      .domain(x.domain())
+      .range([
+        colorScale.domain()[1],
+        colorScale.domain()[0],
+      ]); /* we flip this to make colors dark if high in the color scale */
+
+    if (binWidth > 0) {
+      /* BINS */
+      container
+        .insert("g", "*")
+        .selectAll("rect")
+        .data(bins)
+        .enter()
+        .append("rect")
+        .attr("x", (d, i) => x(binStart(i)) + 1)
+        .attr("y", (d) => y(d))
+        .attr("width", (d, i) => x(binEnd(i)) - x(binStart(i)) - binPadding)
+        .attr("height", (d) => y(0) - y(d))
+        .style(
+          "fill",
+          isColorBy
+            ? (d, i) => colorScale(histogramScale(binStart(i)))
+            : defaultBarColor
+        );
+    }
+
+    if (!mini) {
+      // BRUSH
+      // Note the brushable area is bounded by the data on three sides, but goes down to cover the x-axis
+      const brushX = d3
+        .brushX()
+        .extent([
+          [x.range()[0], y.range()[1]],
+          [x.range()[1], marginTop + height + marginBottom],
+        ])
+        /*
+        emit start so that the Undoable history can save an undo point
+        upon drag start, and ignore the subsequent intermediate drag events.
+        */
+        .on("start", onBrush(field, x.invert, "start"))
+        .on("brush", onBrush(field, x.invert, "brush"))
+        .on("end", onBrushEnd(field, x.invert));
+
+      const brushXselection = container
+        .insert("g")
+        .attr("class", "brush")
+        .attr("data-testid", `${svgRef.current.dataset.testid}-brushable-area`)
+        .call(brushX);
+
+      /* X AXIS */
+      container
+        .insert("g")
+        .attr("class", "axis axis--x")
+        .attr("transform", `translate(0,${marginTop + height})`)
+        .call(
+          d3
+            .axisBottom(x)
+            .ticks(4)
+            .tickFormat(d3.format(maybeScientific(x)))
+        );
+
+      /* Y AXIS */
+      container
+        .insert("g")
+        .attr("class", "axis axis--y")
+        .attr("transform", `translate(${marginLeft + width},0)`)
+        .call(
+          d3
+            .axisRight(y)
+            .ticks(3)
+            .tickFormat(
+              d3.format(
+                y.domain().some((n) => Math.abs(n) >= 10000) ? ".0e" : ","
+              )
+            )
+        );
+
+      /* axis style */
+      svg.selectAll(".axis text").style("fill", "rgb(80,80,80)");
+      svg.selectAll(".axis path").style("stroke", "rgb(230,230,230)");
+      svg.selectAll(".axis line").style("stroke", "rgb(230,230,230)");
+
+      setBrush({ brushX, brushXselection });
+    }
+  }, [histogram, isColorBy]);
+
+  useEffect(() => {
+    /*
+      paint/update selection brush
+      */
+    if (!brush) return;
+    const { brushX, brushXselection } = brush;
+    const selection = d3.brushSelection(brushXselection.node());
+    if (!selectionRange && selection) {
+      /* no active selection - clear brush */
+      brushXselection.call(brushX.move, null);
+    } else if (selectionRange) {
+      const { x, domain } = histogram;
+      const [min, max] = domain;
+      const x0 = x(clamp(selectionRange[0], [min, max]));
+      const x1 = x(clamp(selectionRange[1], [min, max]));
+      if (!selection) {
+        /* there is an active selection, but no brush - set the brush */
+        brushXselection.call(brushX.move, [x0, x1]);
+      } else {
+        /* there is an active selection and a brush - make sure they match */
+        const moveDeltaThreshold = 1;
+        const dX0 = Math.abs(x0 - selection[0]);
+        const dX1 = Math.abs(x1 - selection[1]);
+        /*
+          only update the brush if it is grossly incorrect,
+          as defined by the moveDeltaThreshold
+          */
+        if (dX0 > moveDeltaThreshold || dX1 > moveDeltaThreshold) {
+          brushXselection.call(brushX.move, [x0, x1]);
+        }
+      }
+    }
+  }, [brush, selectionRange]);
+
+  return (
+    <svg
+      style={{ display }}
+      width={width}
+      height={height}
+      id={`histogram_${fieldForId}_svg`}
+      data-testclass="histogram-plot"
+      data-testid={`histogram-${field}-plot`}
+      ref={svgRef}
+    />
+  );
+};
+
+export default Histogram;

client/src/components/brushableHistogram/index.js

@@ -1,447 +1,38 @@
-/*
-https://bl.ocks.org/mbostock/4341954
-https://bl.ocks.org/mbostock/34f08d5e11952a80609169b7917d4172
-https://bl.ocks.org/SpaceActuary/2f004899ea1b2bd78d6f1dbb2febf771
-https://bl.ocks.org/mbostock/3019563
-*/
-import React, { useEffect, useRef, useState, useCallback } from "react";
-import { Button, ButtonGroup, Icon, Tooltip } from "@blueprintjs/core";
-import { connect } from "react-redux";
+import React from "react";
+import { connect, shallowEqual } from "react-redux";
 import * as d3 from "d3";
-import { interpolateCool } from "d3-scale-chromatic";
 import Async from "react-async";
 import memoize from "memoize-one";
-import { IconNames } from "@blueprintjs/icons";
 import * as globals from "../../globals";
 import actions from "../../actions";
-import { histogramContinuous } from "../../util/dataframe/histogram";
 import { makeContinuousDimensionName } from "../../util/nameCreators";
-import significantDigits from "../../util/significantDigits";
+import HistogramHeader from "./header";
+import Histogram from "./histogram";
+import HistogramFooter from "./footer";
+import StillLoading from "./loading";
+import ErrorLoading from "./error";
 
-function clamp(val, rng) {
-  return Math.max(Math.min(val, rng[1]), rng[0]);
-}
-
-function maybeScientific(x) {
-  let format = ",";
-  const _ticks = x.ticks(4);
-
-  if (x.domain().some((n) => Math.abs(n) >= 10000)) {
-    /*
-      heuristic: if the last tick d3 wants to render has one significant
-      digit ie., 2000, render 2e+3, but if it's anything else ie., 42000000 render
-      4.20e+n
-    */
-    format = significantDigits(_ticks[_ticks.length - 1]) === 1 ? ".0e" : ".2e";
-  }
-
-  return format;
-}
-
-const StillLoading = ({ zebra, displayName }) => {
-  /*
-  Render a loading indicator for the field.
-  */
-  return (
-    <div
-      style={{
-        padding: globals.leftSidebarSectionPadding,
-        backgroundColor: zebra ? globals.lightestGrey : "white",
-      }}
-    >
-      <div
-        style={{
-          display: "flex",
-          justifyContent: "space-between",
-          justifyItems: "center",
-          alignItems: "center",
-        }}
-      >
-        <div style={{ minWidth: 30 }} />
-        <div style={{ display: "flex", alignSelf: "center" }}>
-          <span style={{ fontStyle: "italic" }}>{displayName}</span>
-        </div>
-        <div
-          style={{
-            display: "flex",
-            justifyContent: "flex-end",
-          }}
-        >
-          <Button minimal loading intent="primary" />
-        </div>
-      </div>
-    </div>
-  );
+const MARGIN = {
+  LEFT: 10, // Space for 0 tick label on X axis
+  RIGHT: 54, // space for Y axis & labels
+  BOTTOM: 25, // space for X axis & labels
+  TOP: 3,
 };
-
-const ErrorLoading = ({ displayName, error, zebra }) => {
-  console.log(error); // log to console as this is unexpected
-  return (
-    <div
-      style={{
-        padding: globals.leftSidebarSectionPadding,
-        backgroundColor: zebra ? globals.lightestGrey : "white",
-      }}
-    >
-      <span>{`Failure loading ${displayName}`}</span>
-    </div>
-  );
-};
-
-const HistogramFooter = React.memo(
-  ({
-    displayName,
-    hideRanges,
-    rangeMin,
-    rangeMax,
-    rangeColorMin,
-    rangeColorMax,
-    logFoldChange,
-    pvalAdj,
-  }) => {
-    /*
-  Footer of each histogram.  Will render range, title, and optionally
-  differential expression info.
-
-  Required props:
-    * displayName - the displayName, aka "n_genes", "FOXP2", etc.
-    * hideRanges - true/false, enables/disable rendering of ranges
-    * range - length two array, [min, max], containing the range values to display
-    * rangeColor - length two array, [mincolor, maxcolor], each a CSS color
-    * logFoldChange - lfc to display, optional.
-    * pValue - pValue to display, optional.
-  */
-    return (
-      <div>
-        <div
-          style={{
-            display: "flex",
-            justifyContent: hideRanges ? "center" : "space-between",
-          }}
-        >
-          <span
-            style={{
-              color: rangeColorMin,
-              display: hideRanges ? "none" : "block",
-            }}
-          >
-            min {rangeMin.toPrecision(4)}
-          </span>
-          <span
-            data-testclass="brushable-histogram-field-name"
-            style={{ fontStyle: "italic" }}
-          >
-            {displayName}
-          </span>
-          <div style={{ display: hideRanges ? "block" : "none" }}>
-            : {rangeMin}
-          </div>
-          <span
-            style={{
-              color: rangeColorMax,
-              display: hideRanges ? "none" : "block",
-            }}
-          >
-            max {rangeMax.toPrecision(4)}
-          </span>
-        </div>
-
-        {logFoldChange !== undefined && pvalAdj !== undefined ? (
-          <div
-            style={{
-              display: "flex",
-              justifyContent: "center",
-              alignItems: "baseline",
-            }}
-          >
-            <span>
-              <strong>log fold change:</strong>
-              {` ${logFoldChange.toPrecision(4)}`}
-            </span>
-            <span
-              style={{
-                marginLeft: 7,
-                padding: 2,
-              }}
-            >
-              <strong>p-value (adj):</strong>
-              {pvalAdj < 0.0001 ? " < 0.0001" : ` ${pvalAdj.toFixed(4)}`}
-            </span>
-          </div>
-        ) : null}
-      </div>
-    );
-  }
-);
-
-const HistogramHeader = React.memo(
-  ({
-    fieldId,
-    isColorBy,
-    onColorByClick,
-    onRemoveClick,
-    isScatterPlotX,
-    isScatterPlotY,
-    onScatterPlotXClick,
-    onScatterPlotYClick,
-    isObs,
-  }) => {
-    /*
-      Render the toolbar for the histogram.  Props:
-        * fieldId - field identifier, used for various IDs
-        * isColorBy - true/false, is this the current color-by
-        * onColorByClick - color-by click handler
-        * onRemoveClick - optional handler for remove.  Button will not render if not defined.
-        * isScatterPlotX - optional, true/false if currently the X scatterplot field
-        * isScatterPlotY - optional, true/false if currently the Y scatterplot field
-        * onScatterPlotXClick - optional, handler for scatterPlot X button.
-        * onScatterPlotYClick - optional, handler for scatterPlot X button.
-
-      Scatterplot controls will not render if either handler unspecified.
-    */
-
-    const memoizedColorByCallback = useCallback(
-      () => onColorByClick(fieldId, isObs),
-      [fieldId, isObs]
-    );
-
-    return (
-      <div
-        style={{
-          display: "flex",
-          justifyContent: "flex-end",
-          paddingBottom: "8px",
-        }}
-      >
-        {onScatterPlotXClick && onScatterPlotYClick ? (
-          <span>
-            <Icon icon={IconNames.SCATTER_PLOT} style={{ marginRight: 7 }} />
-            <ButtonGroup style={{ marginRight: 7 }}>
-              <Button
-                data-testid={`plot-x-${fieldId}`}
-                onClick={onScatterPlotXClick}
-                active={isScatterPlotX}
-                intent={isScatterPlotX ? "primary" : "none"}
-              >
-                plot x
-              </Button>
-              <Button
-                data-testid={`plot-y-${fieldId}`}
-                onClick={onScatterPlotYClick}
-                active={isScatterPlotY}
-                intent={isScatterPlotY ? "primary" : "none"}
-              >
-                plot y
-              </Button>
-            </ButtonGroup>
-          </span>
-        ) : null}
-        {onRemoveClick ? (
-          <Button
-            minimal
-            onClick={onRemoveClick}
-            style={{
-              color: globals.blue,
-              cursor: "pointer",
-              marginLeft: 7,
-            }}
-          >
-            remove
-          </Button>
-        ) : null}
-        <Tooltip
-          content="Use as color scale"
-          position="bottom"
-          hoverOpenDelay={globals.tooltipHoverOpenDelay}
-        >
-          <Button
-            onClick={memoizedColorByCallback}
-            active={isColorBy}
-            intent={isColorBy ? "primary" : "none"}
-            data-testclass="colorby"
-            data-testid={`colorby-${fieldId}`}
-            icon="tint"
-          />
-        </Tooltip>
-      </div>
-    );
-  }
-);
-
-const Histogram = ({
-  field,
-  fieldForId,
-  display,
-  histogram,
-  width,
-  height,
-  onBrush,
-  onBrushEnd,
-  margin,
-  isColorBy,
-  selectionRange,
-}) => {
-  const svgRef = useRef(null);
-  const [brush, setBrush] = useState(null);
-
-  useEffect(() => {
-    /*
-    Create the d3 histogram
-    */
-    const { marginLeft, marginRight, marginBottom, marginTop } = margin;
-    const { x, y, bins, binStart, binEnd, binWidth } = histogram;
-    const svg = d3.select(svgRef.current);
-
-    /* Remove everything */
-    svg.selectAll("*").remove();
-
-    /* Set margins within the SVG */
-    const container = svg
-      .attr("width", width + marginLeft + marginRight)
-      .attr("height", height + marginTop + marginBottom)
-      .append("g")
-      .attr("class", "histogram-container")
-      .attr("transform", `translate(${marginLeft},${marginTop})`);
-
-    const colorScale = d3
-      .scaleSequential(interpolateCool)
-      .domain([0, bins.length]);
-
-    const histogramScale = d3
-      .scaleLinear()
-      .domain(x.domain())
-      .range([
-        colorScale.domain()[1],
-        colorScale.domain()[0],
-      ]); /* we flip this to make colors dark if high in the color scale */
-
-    if (binWidth > 0) {
-      /* BINS */
-      container
-        .insert("g", "*")
-        .selectAll("rect")
-        .data(bins)
-        .enter()
-        .append("rect")
-        .attr("x", (d, i) => x(binStart(i)) + 1)
-        .attr("y", (d) => y(d))
-        .attr("width", (d, i) => x(binEnd(i)) - x(binStart(i)) - 1)
-        .attr("height", (d) => y(0) - y(d))
-        .style(
-          "fill",
-          isColorBy ? (d, i) => colorScale(histogramScale(binStart(i))) : "#bbb"
-        );
-    }
-
-    // BRUSH
-    // Note the brushable area is bounded by the data on three sides, but goes down to cover the x-axis
-    const brushX = d3
-      .brushX()
-      .extent([
-        [x.range()[0], y.range()[1]],
-        [x.range()[1], marginTop + height + marginBottom],
-      ])
-      /*
-      emit start so that the Undoable history can save an undo point
-      upon drag start, and ignore the subsequent intermediate drag events.
-      */
-      .on("start", onBrush(field, x.invert, "start"))
-      .on("brush", onBrush(field, x.invert, "brush"))
-      .on("end", onBrushEnd(field, x.invert));
-
-    const brushXselection = container
-      .insert("g")
-      .attr("class", "brush")
-      .attr("data-testid", `${svgRef.current.dataset.testid}-brushable-area`)
-      .call(brushX);
-
-    /* X AXIS */
-    container
-      .insert("g")
-      .attr("class", "axis axis--x")
-      .attr("transform", `translate(0,${marginTop + height})`)
-      .call(
-        d3
-          .axisBottom(x)
-          .ticks(4)
-          .tickFormat(d3.format(maybeScientific(x)))
-      );
-
-    /* Y AXIS */
-    container
-      .insert("g")
-      .attr("class", "axis axis--y")
-      .attr("transform", `translate(${marginLeft + width},0)`)
-      .call(
-        d3
-          .axisRight(y)
-          .ticks(3)
-          .tickFormat(
-            d3.format(
-              y.domain().some((n) => Math.abs(n) >= 10000) ? ".0e" : ","
-            )
-          )
-      );
-
-    /* axis style */
-    svg.selectAll(".axis text").style("fill", "rgb(80,80,80)");
-    svg.selectAll(".axis path").style("stroke", "rgb(230,230,230)");
-    svg.selectAll(".axis line").style("stroke", "rgb(230,230,230)");
-
-    setBrush({ brushX, brushXselection });
-  }, [histogram, isColorBy]);
-
-  useEffect(() => {
-    /*
-    paint/update selection brush
-    */
-    if (!brush) return;
-    const { brushX, brushXselection } = brush;
-    const selection = d3.brushSelection(brushXselection.node());
-    if (!selectionRange && selection) {
-      /* no active selection - clear brush */
-      brushXselection.call(brushX.move, null);
-    } else if (selectionRange) {
-      const { x, domain } = histogram;
-      const [min, max] = domain;
-      const x0 = x(clamp(selectionRange[0], [min, max]));
-      const x1 = x(clamp(selectionRange[1], [min, max]));
-      if (!selection) {
-        /* there is an active selection, but no brush - set the brush */
-        brushXselection.call(brushX.move, [x0, x1]);
-      } else {
-        /* there is an active selection and a brush - make sure they match */
-        const moveDeltaThreshold = 1;
-        const dX0 = Math.abs(x0 - selection[0]);
-        const dX1 = Math.abs(x1 - selection[1]);
-        /*
-        only update the brush if it is grossly incorrect,
-        as defined by the moveDeltaThreshold
-        */
-        if (dX0 > moveDeltaThreshold || dX1 > moveDeltaThreshold) {
-          brushXselection.call(brushX.move, [x0, x1]);
-        }
-      }
-    }
-  }, [brush, selectionRange]);
-
-  return (
-    <svg
-      style={{ display }}
-      width={width}
-      height={height}
-      id={`histogram_${fieldForId}_svg`}
-      data-testclass="histogram-plot"
-      data-testid={`histogram-${field}-plot`}
-      ref={svgRef}
-    />
-  );
+const WIDTH = 340 - MARGIN.LEFT - MARGIN.RIGHT;
+const HEIGHT = 135 - MARGIN.TOP - MARGIN.BOTTOM;
+const MARGIN_MINI = {
+  LEFT: 0, // Space for 0 tick label on X axis
+  RIGHT: 0, // space for Y axis & labels
+  BOTTOM: 0, // space for X axis & labels
+  TOP: 0,
 };
+const WIDTH_MINI = 120 - MARGIN_MINI.LEFT - MARGIN_MINI.RIGHT;
+const HEIGHT_MINI = 15 - MARGIN_MINI.TOP - MARGIN_MINI.BOTTOM;
 
 @connect((state, ownProps) => {
-  const { isObs, isUserDefined, isDiffExp, field } = ownProps;
+  const { isObs, isUserDefined, isGeneSetSummary, field } = ownProps;
   const myName = makeContinuousDimensionName(
-    { isObs, isUserDefined, isDiffExp },
+    { isObs, isUserDefined, isGeneSetSummary },
     field
   );
   return {
@@ -453,6 +44,10 @@ const Histogram = ({
   };
 })
 class HistogramBrush extends React.PureComponent {
+  static watchAsync(props, prevProps) {
+    return !shallowEqual(props.watchProps, prevProps.watchProps);
+  }
+
   /* memoized closure to prevent HistogramHeader unecessary repaint */
   handleColorAction = memoize((dispatch) => (field, isObs) => {
     if (isObs) {
@@ -465,27 +60,11 @@ class HistogramBrush extends React.PureComponent {
     }
   });
 
-  constructor(props) {
-    super(props);
-
-    const marginLeft = 10; // Space for 0 tick label on X axis
-    const marginRight = 54; // space for Y axis & labels
-    const marginBottom = 25; // space for X axis & labels
-    const marginTop = 3;
-    this.margin = {
-      marginLeft,
-      marginRight,
-      marginBottom,
-      marginTop,
-    };
-    this.width = 340 - marginLeft - marginRight;
-    this.height = 135 - marginTop - marginBottom;
-  }
-
   onBrush = (selection, x, eventType) => {
     const type = `continuous metadata histogram ${eventType}`;
     return () => {
-      const { dispatch, field, isObs, isUserDefined, isDiffExp } = this.props;
+      const { dispatch, field, isObs, isUserDefined, isGeneSetSummary } =
+        this.props;
 
       // ignore programmatically generated events
       if (!d3.event.sourceEvent) return;
@@ -501,7 +80,7 @@ class HistogramBrush extends React.PureComponent {
         continuousNamespace: {
           isObs,
           isUserDefined,
-          isDiffExp,
+          isGeneSetSummary,
         },
       };
       dispatch(
@@ -510,9 +89,9 @@ class HistogramBrush extends React.PureComponent {
     };
   };
 
-  onBrushEnd = (selection, x) => {
-    return () => {
-      const { dispatch, field, isObs, isUserDefined, isDiffExp } = this.props;
+  onBrushEnd = (selection, x) => () => {
+      const { dispatch, field, isObs, isUserDefined, isGeneSetSummary } =
+        this.props;
       const minAllowedBrushSize = 10;
       const smallAmountToAvoidInfiniteLoop = 0.1;
 
@@ -551,14 +130,13 @@ class HistogramBrush extends React.PureComponent {
         continuousNamespace: {
           isObs,
           isUserDefined,
-          isDiffExp,
+          isGeneSetSummary,
         },
       };
       dispatch(
         actions.selectContinuousMetadataAction(type, query, range, otherProps)
       );
     };
-  };
 
   handleSetGeneAsScatterplotX = () => {
     const { dispatch, field } = this.props;
@@ -608,7 +186,8 @@ class HistogramBrush extends React.PureComponent {
   };
 
   fetchAsyncProps = async () => {
-    const { annoMatrix } = this.props;
+    const { annoMatrix, width } = this.props;
+
     const { isClipped } = annoMatrix;
 
     const query = this.createQuery();
@@ -638,9 +217,15 @@ class HistogramBrush extends React.PureComponent {
 
     const histogram = this.calcHistogramCache(
       column,
-      this.margin,
-      this.width,
-      this.height
+      MARGIN,
+      width || WIDTH,
+      HEIGHT
+    );
+    const miniHistogram = this.calcHistogramCache(
+      column,
+      MARGIN_MINI,
+      width || WIDTH_MINI,
+      HEIGHT_MINI
     );
 
     const isSingleValue = summary.min === summary.max;
@@ -654,6 +239,7 @@ class HistogramBrush extends React.PureComponent {
 
     return {
       histogram,
+      miniHistogram,
       range,
       unclippedRange,
       unclippedRangeColor,
@@ -663,27 +249,33 @@ class HistogramBrush extends React.PureComponent {
   };
 
   // eslint-disable-next-line class-methods-use-this -- instance method allows for memoization per annotation
-  calcHistogramCache(col, margin, width, height) {
+  calcHistogramCache(col, newMargin, newWidth, newHeight) {
     /*
      recalculate expensive stuff, notably bins, summaries, etc.
     */
-    const histogramCache = {};
-    const summary = col.summarize();
+    const histogramCache = {}; /* maybe change this so that it computes ... */
+    const summary =
+      col.summarize(); /* this is memoized, so it's free the second time you call it */
     const { min: domainMin, max: domainMax } = summary;
     const numBins = 40;
-    const { marginTop, marginLeft } = margin;
+    const { TOP: topMargin, LEFT: leftMargin } =
+      newMargin; /* changes with mini */
 
-    histogramCache.domain = [domainMin, domainMax];
+    histogramCache.domain = [
+      domainMin,
+      domainMax,
+    ]; /* doesn't change with mini */
 
     histogramCache.x = d3
       .scaleLinear()
       .domain([domainMin, domainMax])
-      .range([marginLeft, marginLeft + width]);
+      .range([leftMargin, leftMargin + newWidth]);
 
-    histogramCache.bins = histogramContinuous(col, numBins, [
+    histogramCache.bins = col.histogram(numBins, [
       domainMin,
       domainMax,
-    ]);
+    ]); /* memoized */
+
     histogramCache.binWidth = (domainMax - domainMin) / numBins;
 
     histogramCache.binStart = (i) => domainMin + i * histogramCache.binWidth;
@@ -695,25 +287,43 @@ class HistogramBrush extends React.PureComponent {
     histogramCache.y = d3
       .scaleLinear()
       .domain([0, yMax])
-      .range([marginTop + height, marginTop]);
+      .range([topMargin + newHeight, topMargin]);
 
     return histogramCache;
   }
 
   createQuery() {
-    const { isObs, field, annoMatrix } = this.props;
+    const { isObs, isGeneSetSummary, field, setGenes, annoMatrix } = this.props;
     const { schema } = annoMatrix;
     if (isObs) {
       return ["obs", field];
     }
     const varIndex = schema?.annotations?.var?.index;
     if (!varIndex) return null;
+
+    if (isGeneSetSummary) {
+      return [
+        "X",
+        {
+          summarize: {
+            method: "mean",
+            field: "var",
+            column: varIndex,
+            values: [...setGenes.keys()],
+          },
+        },
+      ];
+    }
+
+    // else, we assume it is a gene expression
     return [
       "X",
       {
-        field: "var",
-        column: varIndex,
-        value: field,
+        where: {
+          field: "var",
+          column: varIndex,
+          value: field,
+        },
       },
     ];
   }
@@ -725,20 +335,34 @@ class HistogramBrush extends React.PureComponent {
       field,
       isColorAccessor,
       isUserDefined,
-      isDiffExp,
-      logFoldChange,
-      pvalAdj,
+      isGeneSetSummary,
       isScatterplotXXaccessor,
       isScatterplotYYaccessor,
       zebra,
       continuousSelectionRange,
       isObs,
+      mini,
+      setGenes,
     } = this.props;
+
+    let { width } = this.props;
+    if (!width) {
+      width = mini ? WIDTH_MINI : WIDTH;
+    }
+
     const fieldForId = field.replace(/\s/g, "_");
-    const showScatterPlot = isDiffExp || isUserDefined;
+    const showScatterPlot = isUserDefined;
+
+    let testClass = "histogram-continuous-metadata";
+    if (isUserDefined) testClass = "histogram-user-gene";
+    else if (isGeneSetSummary) testClass = "histogram-gene-set-summary";
 
     return (
-      <Async watch={annoMatrix} promiseFn={this.fetchAsyncProps}>
+      <Async
+        watchFn={HistogramBrush.watchAsync}
+        promiseFn={this.fetchAsyncProps}
+        watchProps={{ annoMatrix, setGenes }}
+      >
         <Async.Pending initial>
           <StillLoading displayName={field} zebra={zebra} />
         </Async.Pending>
@@ -753,56 +377,57 @@ class HistogramBrush extends React.PureComponent {
               <div
                 id={`histogram_${fieldForId}`}
                 data-testid={`histogram-${field}`}
-                data-testclass={
-                  isDiffExp
-                    ? "histogram-diffexp"
-                    : isUserDefined
-                    ? "histogram-user-gene"
-                    : "histogram-continuous-metadata"
-                }
+                data-testclass={testClass}
                 style={{
-                  padding: globals.leftSidebarSectionPadding,
+                  padding: mini ? 0 : globals.leftSidebarSectionPadding,
                   backgroundColor: zebra ? globals.lightestGrey : "white",
                 }}
               >
-                <HistogramHeader
-                  fieldId={field}
-                  isColorBy={isColorAccessor}
-                  isObs={isObs}
-                  onColorByClick={this.handleColorAction(dispatch)}
-                  onRemoveClick={isUserDefined ? this.removeHistogram : null}
-                  isScatterPlotX={isScatterplotXXaccessor}
-                  isScatterPlotY={isScatterplotYYaccessor}
-                  onScatterPlotXClick={
-                    showScatterPlot ? this.handleSetGeneAsScatterplotX : null
-                  }
-                  onScatterPlotYClick={
-                    showScatterPlot ? this.handleSetGeneAsScatterplotY : null
-                  }
-                />
+                {!mini && isObs ? (
+                  <HistogramHeader
+                    fieldId={field}
+                    isColorBy={isColorAccessor}
+                    isObs={isObs}
+                    onColorByClick={this.handleColorAction(dispatch)}
+                    onRemoveClick={isUserDefined ? this.removeHistogram : null}
+                    isScatterPlotX={isScatterplotXXaccessor}
+                    isScatterPlotY={isScatterplotYYaccessor}
+                    onScatterPlotXClick={
+                      showScatterPlot ? this.handleSetGeneAsScatterplotX : null
+                    }
+                    onScatterPlotYClick={
+                      showScatterPlot ? this.handleSetGeneAsScatterplotY : null
+                    }
+                  />
+                ) : null}
                 <Histogram
                   field={field}
                   fieldForId={fieldForId}
                   display={asyncProps.isSingleValue ? "none" : "block"}
-                  histogram={asyncProps.histogram}
-                  width={this.width}
-                  height={this.height}
+                  histogram={
+                    mini ? asyncProps.miniHistogram : asyncProps.histogram
+                  }
+                  width={width}
+                  height={mini ? HEIGHT_MINI : HEIGHT}
                   onBrush={this.onBrush}
                   onBrushEnd={this.onBrushEnd}
-                  margin={this.margin}
+                  margin={mini ? MARGIN_MINI : MARGIN}
                   isColorBy={isColorAccessor}
                   selectionRange={continuousSelectionRange}
+                  mini={mini}
                 />
-                <HistogramFooter
-                  displayName={field}
-                  hideRanges={asyncProps.isSingleValue}
-                  rangeMin={asyncProps.unclippedRange[0]}
-                  rangeMax={asyncProps.unclippedRange[1]}
-                  rangeColorMin={asyncProps.unclippedRangeColor[0]}
-                  rangeColorMax={asyncProps.unclippedRangeColor[1]}
-                  logFoldChange={logFoldChange}
-                  pvalAdj={pvalAdj}
-                />
+                {!mini && (
+                  <HistogramFooter
+                    isGeneSetSummary={isGeneSetSummary}
+                    isObs={isObs}
+                    displayName={field}
+                    hideRanges={asyncProps.isSingleValue}
+                    rangeMin={asyncProps.unclippedRange[0]}
+                    rangeMax={asyncProps.unclippedRange[1]}
+                    rangeColorMin={asyncProps.unclippedRangeColor[0]}
+                    rangeColorMax={asyncProps.unclippedRangeColor[1]}
+                  />
+                )}
               </div>
             ) : null
           }

client/src/components/brushableHistogram/loading.js

@@ -0,0 +1,43 @@
+import React from "react";
+import { Button } from "@blueprintjs/core";
+
+import * as globals from "../../globals";
+
+const StillLoading = ({ zebra, displayName }) => 
+  /*
+    Render a loading indicator for the field.
+    */
+   (
+    <div
+      data-testclass="gene-loading-spinner"
+      style={{
+        padding: globals.leftSidebarSectionPadding,
+        backgroundColor: zebra ? globals.lightestGrey : "white",
+      }}
+    >
+      <div
+        style={{
+          display: "flex",
+          justifyContent: "space-between",
+          justifyItems: "center",
+          alignItems: "center",
+        }}
+      >
+        <div style={{ minWidth: 30 }} />
+        <div style={{ display: "flex", alignSelf: "center" }}>
+          <span style={{ fontStyle: "italic" }}>{displayName}</span>
+        </div>
+        <div
+          style={{
+            display: "flex",
+            justifyContent: "flex-end",
+          }}
+        >
+          <Button minimal loading intent="primary" />
+        </div>
+      </div>
+    </div>
+  )
+;
+
+export default StillLoading;

client/src/components/categorical/annoSelect.js

@@ -26,16 +26,14 @@ class DuplicateCategorySelect extends React.PureComponent {
             [] /* this is a placeholder, could be  a subcomponent to avoid this */
           }
           filterable={false}
-          itemRenderer={(d, { handleClick }) => {
-            return (
+          itemRenderer={(d, { handleClick }) => (
               <MenuItem
                 data-testclass="duplicate-category-dropdown-option"
                 onClick={handleClick}
                 key={d}
                 text={d}
               />
-            );
-          }}
+            )}
           noResults={<MenuItem disabled text="No results." />}
           onItemSelect={(d) => {
             handleModalDuplicateCategorySelection(d);

client/src/components/categorical/category/annoDialogAddLabel.js

@@ -1,14 +1,13 @@
 import React from "react";
 import { connect } from "react-redux";
-import AnnoDialog from "../annoDialog";
-import LabelInput from "../labelInput";
+import AnnoDialog from "../../annoDialog";
+import LabelInput from "../../labelInput";
 import { labelPrompt, isLabelErroneous } from "../labelUtil";
 import actions from "../../../actions";
 
 @connect((state) => ({
   annotations: state.annotations,
   schema: state.annoMatrix?.schema,
-  ontology: state.ontology,
   obsCrossfilter: state.obsCrossfilter,
 }))
 class Category extends React.PureComponent {
@@ -57,13 +56,11 @@ class Category extends React.PureComponent {
   };
 
   labelNameError = (name) => {
-    const { metadataField, ontology, schema } = this.props;
-    return isLabelErroneous(name, metadataField, ontology, schema);
+    const { metadataField, schema } = this.props;
+    return isLabelErroneous(name, metadataField, schema);
   };
 
-  instruction = (label) => {
-    return labelPrompt(this.labelNameError(label), "New, unique label", ":");
-  };
+  instruction = (label) => labelPrompt(this.labelNameError(label), "New, unique label", ":");
 
   handleChangeOrSelect = (label) => {
     this.setState({ newLabelText: label });
@@ -71,8 +68,7 @@ class Category extends React.PureComponent {
 
   render() {
     const { newLabelText } = this.state;
-    const { metadataField, annotations, ontology, obsCrossfilter } = this.props;
-    const ontologyEnabled = ontology?.enabled ?? false;
+    const { metadataField, annotations, obsCrossfilter } = this.props;
 
     return (
       <>
@@ -97,7 +93,7 @@ class Category extends React.PureComponent {
           handleCancel={this.disableAddNewLabelMode}
           annoInput={
             <LabelInput
-              labelSuggestions={ontologyEnabled ? ontology.terms : null}
+              labelSuggestions={null}
               onChange={this.handleChangeOrSelect}
               onSelect={this.handleChangeOrSelect}
               inputProps={{

client/src/components/categorical/category/annoDialogEditCategoryName.js

@@ -1,7 +1,7 @@
 import React from "react";
 import { connect } from "react-redux";
-import AnnoDialog from "../annoDialog";
-import LabelInput from "../labelInput";
+import AnnoDialog from "../../annoDialog";
+import LabelInput from "../../labelInput";
 import { labelPrompt } from "../labelUtil";
 
 import { AnnotationsHelpers } from "../../../util/stateManager";
@@ -10,7 +10,6 @@ import actions from "../../../actions";
 @connect((state) => ({
   annotations: state.annotations,
   schema: state.annoMatrix?.schema,
-  ontology: state.ontology,
 }))
 class AnnoDialogEditCategoryName extends React.PureComponent {
   constructor(props) {
@@ -90,13 +89,11 @@ class AnnoDialogEditCategoryName extends React.PureComponent {
     return false;
   };
 
-  instruction = (name) => {
-    return labelPrompt(
+  instruction = (name) => labelPrompt(
       this.editedCategoryNameError(name),
       "New, unique category name",
       ":"
     );
-  };
 
   allCategoryNames() {
     const { schema } = this.props;
@@ -105,8 +102,7 @@ class AnnoDialogEditCategoryName extends React.PureComponent {
 
   render() {
     const { newCategoryText } = this.state;
-    const { metadataField, annotations, ontology } = this.props;
-    const ontologyEnabled = ontology?.enabled ?? false;
+    const { metadataField, annotations } = this.props;
 
     return (
       <>
@@ -132,7 +128,7 @@ class AnnoDialogEditCategoryName extends React.PureComponent {
           annoInput={
             <LabelInput
               label={newCategoryText}
-              labelSuggestions={ontologyEnabled ? ontology.terms : null}
+              labelSuggestions={null}
               onChange={this.handleChangeOrSelect}
               onSelect={this.handleChangeOrSelect}
               inputProps={{

client/src/components/categorical/category/annoMenuCategory.js

@@ -9,7 +9,9 @@ import {
   Tooltip,
   Icon,
   PopoverInteractionKind,
+  Intent,
 } from "@blueprintjs/core";
+import { IconNames } from "@blueprintjs/icons";
 
 import * as globals from "../../../globals";
 import actions from "../../../actions";
@@ -89,8 +91,8 @@ class AnnoMenuCategory extends React.PureComponent {
                     text={editText}
                   />
                   <MenuItem
-                    icon="delete"
-                    intent="danger"
+                    icon={IconNames.TRASH}
+                    intent={Intent.DANGER}
                     data-testclass="handleDeleteCategory"
                     data-testid={`${metadataField}:delete-category`}
                     onClick={this.handleDeleteCategory}

client/src/components/categorical/category/index.js

@@ -44,6 +44,7 @@ const LABEL_WIDTH_ANNO = LABEL_WIDTH - ANNO_BUTTON_WIDTH;
     schema,
     crossfilter: state.obsCrossfilter,
     isUserAnno,
+    genesets: state.genesets.genesets,
   };
 })
 class Category extends React.PureComponent {
@@ -142,9 +143,15 @@ class Category extends React.PureComponent {
     */
     const { schema } = annoMatrix;
     const { colorAccessor, colorMode } = colors;
+    const { genesets } = this.props;
     let colorDataPromise = Promise.resolve(null);
     if (colorAccessor) {
-      const query = createColorQuery(colorMode, colorAccessor, schema);
+      const query = createColorQuery(
+        colorMode,
+        colorAccessor,
+        schema,
+        genesets
+      );
       if (query) colorDataPromise = annoMatrix.fetch(...query);
     }
     const [categoryData, colorData] = await Promise.all([
@@ -251,14 +258,12 @@ class Category extends React.PureComponent {
                 isColorAccessor,
                 handleCategoryToggleAllClick,
               } = asyncProps;
-              const isTruncated = !!categorySummary?.isTruncated;
               const selectionState = this.getSelectionState(categorySummary);
               return (
                 <CategoryRender
                   metadataField={metadataField}
                   checkboxID={checkboxID}
                   isUserAnno={isUserAnno}
-                  isTruncated={isTruncated}
                   isExpanded={isExpanded}
                   isColorAccessor={isColorAccessor}
                   selectionState={selectionState}
@@ -283,57 +288,54 @@ class Category extends React.PureComponent {
 
 export default Category;
 
-const StillLoading = ({ metadataField, checkboxID }) => {
+const StillLoading = ({ metadataField, checkboxID }) => (
   /*
   We are still loading this category, so render a "busy" signal.
   */
-  return (
+  <div
+    style={{
+      maxWidth: globals.maxControlsWidth,
+    }}
+  >
     <div
       style={{
-        maxWidth: globals.maxControlsWidth,
+        display: "flex",
+        justifyContent: "space-between",
+        alignItems: "baseline",
       }}
     >
       <div
         style={{
           display: "flex",
-          justifyContent: "space-between",
-          alignItems: "baseline",
+          justifyContent: "flex-start",
+          alignItems: "flex-start",
         }}
       >
-        <div
-          style={{
-            display: "flex",
-            justifyContent: "flex-start",
-            alignItems: "flex-start",
-          }}
+        <label
+          htmlFor={checkboxID}
+          className={`${Classes.CONTROL} ${Classes.CHECKBOX}`}
         >
-          <label
-            htmlFor={checkboxID}
-            className={`${Classes.CONTROL} ${Classes.CHECKBOX}`}
+          <input disabled id={checkboxID} checked type="checkbox" />
+          <span className={Classes.CONTROL_INDICATOR} />
+        </label>
+        <Truncate>
+          <span
+            style={{
+              cursor: "pointer",
+              display: "inline-block",
+              width: LABEL_WIDTH,
+            }}
           >
-            <input disabled id={checkboxID} checked type="checkbox" />
-            <span className={Classes.CONTROL_INDICATOR} />
-          </label>
-          <Truncate>
-            <span
-              style={{
-                cursor: "pointer",
-                display: "inline-block",
-                width: LABEL_WIDTH,
-              }}
-            >
-              {metadataField}
-            </span>
-          </Truncate>
-        </div>
-        <div>
-          <Button minimal loading intent="primary" />
-        </div>
+            {metadataField}
+          </span>
+        </Truncate>
+      </div>
+      <div>
+        <Button minimal loading intent="primary" />
       </div>
     </div>
-  );
-};
-
+  </div>
+);
 const ErrorLoading = ({ metadataField, error }) => {
   console.error(error); // log error to console as it is unexpected.
   return (
@@ -357,7 +359,6 @@ const CategoryHeader = React.memo(
     metadataField,
     checkboxID,
     isUserAnno,
-    isTruncated,
     isColorAccessor,
     isExpanded,
     selectionState,
@@ -416,6 +417,7 @@ const CategoryHeader = React.memo(
                   maxWidth: isUserAnno ? LABEL_WIDTH_ANNO : LABEL_WIDTH,
                 }}
                 data-testid={`${metadataField}:category-label`}
+                tabIndex="-1"
               >
                 {metadataField}
               </span>
@@ -433,8 +435,8 @@ const CategoryHeader = React.memo(
             )}
           </span>
         </div>
-        {<AnnoDialogEditCategoryName metadataField={metadataField} />}
-        {<AnnoDialogAddLabel metadataField={metadataField} />}
+        <AnnoDialogEditCategoryName metadataField={metadataField} />
+        <AnnoDialogAddLabel metadataField={metadataField} />
         <div>
           <AnnoMenu
             metadataField={metadataField}
@@ -445,11 +447,7 @@ const CategoryHeader = React.memo(
           />
 
           <Tooltip
-            content={
-              isTruncated
-                ? `Coloring by ${metadataField} is disabled, as it exceeds the limit of ${globals.maxCategoricalOptionsToDisplay} labels`
-                : "Use as color scale"
-            }
+            content="Use as color scale"
             position={Position.LEFT}
             usePortal
             hoverOpenDelay={globals.tooltipHoverOpenDelay}
@@ -464,7 +462,6 @@ const CategoryHeader = React.memo(
               onClick={onColorChangeClick}
               active={isColorAccessor}
               intent={isColorAccessor ? "primary" : "none"}
-              disabled={isTruncated}
               icon="tint"
             />
           </Tooltip>
@@ -479,7 +476,6 @@ const CategoryRender = React.memo(
     metadataField,
     checkboxID,
     isUserAnno,
-    isTruncated,
     isColorAccessor,
     isExpanded,
     selectionState,
@@ -498,12 +494,23 @@ const CategoryRender = React.memo(
     */
     const { numCategoryValues } = categorySummary;
     const isSingularValue = !isUserAnno && numCategoryValues === 1;
-
     if (isSingularValue) {
       /*
       Entire category has a single value, special case.
       */
-      return null;
+      const theOneValue = categorySummary.categoryValues[0];
+      return (
+        <div style={{ marginBottom: 10, marginTop: 4 }}>
+          <Truncate>
+            <span style={{ maxWidth: 150, fontWeight: 700 }}>
+              {metadataField}
+            </span>
+          </Truncate>
+          <Truncate>
+            <span style={{ maxWidth: 150 }}>{`: ${theOneValue}`}</span>
+          </Truncate>
+        </div>
+      );
     }
 
     /*
@@ -528,7 +535,6 @@ const CategoryRender = React.memo(
             metadataField={metadataField}
             checkboxID={checkboxID}
             isUserAnno={isUserAnno}
-            isTruncated={isTruncated}
             isExpanded={isExpanded}
             isColorAccessor={isColorAccessor}
             selectionState={selectionState}
@@ -554,11 +560,6 @@ const CategoryRender = React.memo(
             ) : null
           }
         </div>
-        <div>
-          {isExpanded && isTruncated ? (
-            <p style={{ paddingLeft: 15 }}>... truncated list ...</p>
-          ) : null}
-        </div>
       </div>
     );
   }

client/src/components/categorical/index.js

@@ -1,21 +1,18 @@
-// jshint esversion: 6
 import React from "react";
 import { AnchorButton, Tooltip, Position } from "@blueprintjs/core";
 import { connect } from "react-redux";
 import * as globals from "../../globals";
 import Category from "./category";
 import { AnnotationsHelpers, ControlsHelpers } from "../../util/stateManager";
-import AnnoDialog from "./annoDialog";
+import AnnoDialog from "../annoDialog";
 import AnnoSelect from "./annoSelect";
-import LabelInput from "./labelInput";
+import LabelInput from "../labelInput";
 import { labelPrompt } from "./labelUtil";
 import actions from "../../actions";
 
 @connect((state) => ({
   writableCategoriesEnabled: state.config?.parameters?.annotations ?? false,
   schema: state.annoMatrix?.schema,
-  ontology: state.ontology,
-  userInfo: state.userInfo,
 }))
 class Categories extends React.Component {
   constructor(props) {
@@ -100,13 +97,8 @@ class Categories extends React.Component {
     this.setState({ newCategoryText: name });
   };
 
-  instruction = (name) => {
-    return labelPrompt(
-      this.categoryNameError(name),
-      "New, unique category name",
-      ":"
-    );
-  };
+  instruction = (name) =>
+    labelPrompt(this.categoryNameError(name), "New, unique category name", ":");
 
   onExpansionChange = (catName) => {
     const { expandedCats } = this.state;
@@ -128,17 +120,10 @@ class Categories extends React.Component {
       newCategoryText,
       expandedCats,
     } = this.state;
-    const {
-      writableCategoriesEnabled,
-      schema,
-      ontology,
-      userInfo,
-    } = this.props;
-    const ontologyEnabled = ontology?.enabled ?? false;
+    const { writableCategoriesEnabled, schema } = this.props;
     /* all names, sorted in display order.  Will be rendered in this order */
-    const allCategoryNames = ControlsHelpers.selectableCategoryNames(
-      schema
-    ).sort();
+    const allCategoryNames =
+      ControlsHelpers.selectableCategoryNames(schema).sort();
 
     return (
       <div
@@ -159,7 +144,7 @@ class Categories extends React.Component {
           handleCancel={this.handleDisableAnnoMode}
           annoInput={
             <LabelInput
-              labelSuggestions={ontologyEnabled ? ontology.terms : null}
+              labelSuggestions={null}
               onChange={this.handleChange}
               onSelect={this.handleSelect}
               inputProps={{
@@ -182,20 +167,43 @@ class Categories extends React.Component {
           }
         />
 
+        {writableCategoriesEnabled ? (
+          <div style={{ marginBottom: 10 }}>
+            <Tooltip
+              content="Create a new category"
+              position={Position.RIGHT}
+              boundary="viewport"
+              hoverOpenDelay={globals.tooltipHoverOpenDelay}
+              modifiers={{
+                preventOverflow: { enabled: false },
+                hide: { enabled: false },
+              }}
+            >
+              <AnchorButton
+                type="button"
+                data-testid="open-annotation-dialog"
+                onClick={this.handleEnableAnnoMode}
+                intent="primary"
+              >
+                Create new <strong>category</strong>
+              </AnchorButton>
+            </Tooltip>
+          </div>
+        ) : null}
+
         {/* READ ONLY CATEGORICAL FIELDS */}
         {/* this is duplicative but flat, could be abstracted */}
-        {allCategoryNames.map((catName) =>
-          !schema.annotations.obsByName[catName].writable &&
-          (schema.annotations.obsByName[catName].categories?.length > 1 ||
-            !schema.annotations.obsByName[catName].categories) ? (
-            <Category
-              key={catName}
-              metadataField={catName}
-              onExpansionChange={this.onExpansionChange}
-              isExpanded={expandedCats.has(catName)}
-              createAnnoModeActive={createAnnoModeActive}
-            />
-          ) : null
+        {allCategoryNames.map(
+          (catName) =>
+            !schema.annotations.obsByName[catName].writable && (
+              <Category
+                key={catName}
+                metadataField={catName}
+                onExpansionChange={this.onExpansionChange}
+                isExpanded={expandedCats.has(catName)}
+                createAnnoModeActive={createAnnoModeActive}
+              />
+            )
         )}
         {/* WRITEABLE FIELDS */}
         {allCategoryNames.map((catName) =>
@@ -209,33 +217,6 @@ class Categories extends React.Component {
             />
           ) : null
         )}
-
-        {writableCategoriesEnabled ? (
-          <Tooltip
-            content={
-              userInfo.is_authenticated
-                ? "Create a new category"
-                : "You must be logged in to create new categorical fields"
-            }
-            position={Position.RIGHT}
-            boundary="viewport"
-            hoverOpenDelay={globals.tooltipHoverOpenDelay}
-            modifiers={{
-              preventOverflow: { enabled: false },
-              hide: { enabled: false },
-            }}
-          >
-            <AnchorButton
-              type="button"
-              data-testid="open-annotation-dialog"
-              onClick={this.handleEnableAnnoMode}
-              intent="primary"
-              disabled={!userInfo.is_authenticated}
-            >
-              Create new category
-            </AnchorButton>
-          </Tooltip>
-        ) : null}
       </div>
     );
   }

client/src/components/categorical/labelUtil.js

@@ -3,7 +3,7 @@ import { Colors } from "@blueprintjs/core";
 
 import { AnnotationsHelpers } from "../../util/stateManager";
 
-export function isLabelErroneous(label, metadataField, ontology, schema) {
+export function isLabelErroneous(label, metadataField, schema) {
   /*
     return false if this is a LEGAL/acceptable category name or NULL/empty string,
     or return an error type.
@@ -12,10 +12,9 @@ export function isLabelErroneous(label, metadataField, ontology, schema) {
   /* allow empty string */
   if (label === "") return false;
 
-  /* check for label syntax errors, but allow terms in ontology */
-  const termInOntology = ontology?.termSet.has(label) ?? false;
+  /* check for label syntax errors */
   const error = AnnotationsHelpers.annotationNameIsErroneous(label);
-  if (error && !termInOntology) return error;
+  if (error) return error;
 
   /* disallow duplicates */
   const { obsByName } = schema.annotations;

client/src/components/categorical/value/index.js

@@ -14,8 +14,8 @@ import {
 } from "@blueprintjs/core";
 import * as globals from "../../../globals";
 import styles from "../categorical.css";
-import AnnoDialog from "../annoDialog";
-import LabelInput from "../labelInput";
+import AnnoDialog from "../../annoDialog";
+import LabelInput from "../../labelInput";
 import Truncate from "../../util/truncate";
 
 import { AnnotationsHelpers } from "../../../util/stateManager";
@@ -25,8 +25,8 @@ import MiniHistogram from "../../miniHistogram";
 import MiniStackedBar from "../../miniStackedBar";
 import { CategoryCrossfilterContext } from "../categoryContext";
 
-const VALUE_HEIGHT = 11;
-const CHART_WIDTH = 100;
+const STACKED_BAR_HEIGHT = 11;
+const STACKED_BAR_WIDTH = 100;
 
 /* this is defined outside of the class so we can use it in connect() */
 function _currentLabelAsString(ownProps) {
@@ -50,7 +50,6 @@ function _currentLabelAsString(ownProps) {
   return {
     annotations: state.annotations,
     schema: state.annoMatrix?.schema,
-    ontology: state.ontology,
     isDilated,
     isSelected,
     label,
@@ -118,14 +117,12 @@ class CategoryValue extends React.Component {
   };
 
   labelNameError = (name) => {
-    const { metadataField, ontology, schema } = this.props;
+    const { metadataField, schema } = this.props;
     if (name === this.currentLabelAsString()) return false;
-    return isLabelErroneous(name, metadataField, ontology, schema);
+    return isLabelErroneous(name, metadataField, schema);
   };
 
-  instruction = (label) => {
-    return labelPrompt(this.labelNameError(label), "New, unique label", ":");
-  };
+  instruction = (label) => labelPrompt(this.labelNameError(label), "New, unique label", ":");
 
   activateEditLabelMode = () => {
     const { dispatch, metadataField, categoryIndex, label } = this.props;
@@ -151,12 +148,8 @@ class CategoryValue extends React.Component {
   };
 
   toggleOff = () => {
-    const {
-      dispatch,
-      metadataField,
-      categoryIndex,
-      categorySummary,
-    } = this.props;
+    const { dispatch, metadataField, categoryIndex, categorySummary } =
+      this.props;
     const label = categorySummary.categoryValues[categoryIndex];
     dispatch(
       actions.selectCategoricalMetadataAction(
@@ -221,12 +214,8 @@ class CategoryValue extends React.Component {
   };
 
   toggleOn = () => {
-    const {
-      dispatch,
-      metadataField,
-      categoryIndex,
-      categorySummary,
-    } = this.props;
+    const { dispatch, metadataField, categoryIndex, categorySummary } =
+      this.props;
     const label = categorySummary.categoryValues[categoryIndex];
     dispatch(
       actions.selectCategoricalMetadataAction(
@@ -410,7 +399,7 @@ class CategoryValue extends React.Component {
         label,
         colorTable,
         schema,
-        CHART_WIDTH
+        STACKED_BAR_WIDTH
       ) ?? {};
 
     if (!domainValues || !scale || !domain || !occupancy) {
@@ -426,8 +415,8 @@ class CategoryValue extends React.Component {
           domain,
           occupancy,
         }}
-        height={VALUE_HEIGHT}
-        width={CHART_WIDTH}
+        height={STACKED_BAR_HEIGHT}
+        width={STACKED_BAR_WIDTH}
       />
     );
   };
@@ -460,8 +449,8 @@ class CategoryValue extends React.Component {
         colorAccessor,
         colorData,
         label,
-        CHART_WIDTH,
-        VALUE_HEIGHT
+        STACKED_BAR_WIDTH,
+        STACKED_BAR_HEIGHT
       ) ?? {}; // if createHistogramBins returns empty object assign null to deconstructed
 
     if (!xScale || !yScale || !bins) return null;
@@ -476,8 +465,8 @@ class CategoryValue extends React.Component {
         }}
         obsOrVarContinuousFieldDisplayName={colorAccessor}
         domainLabel={label}
-        height={VALUE_HEIGHT}
-        width={CHART_WIDTH}
+        height={STACKED_BAR_HEIGHT}
+        width={STACKED_BAR_WIDTH}
       />
     );
   };
@@ -490,14 +479,12 @@ class CategoryValue extends React.Component {
       colorTable,
       isUserAnno,
       annotations,
-      ontology,
       isDilated,
       isSelected,
       categorySummary,
       label,
     } = this.props;
     const colorScale = colorTable?.scale;
-    const ontologyEnabled = ontology?.enabled ?? false;
 
     const { editedLabelText } = this.state;
 
@@ -534,7 +521,7 @@ class CategoryValue extends React.Component {
       colorAccessor && !isColorBy
         ? globals.leftSidebarWidth -
           otherElementsWidth -
-          CHART_WIDTH -
+          STACKED_BAR_WIDTH -
           CHART_MARGIN
         : globals.leftSidebarWidth - otherElementsWidth;
 
@@ -590,6 +577,7 @@ class CategoryValue extends React.Component {
               <span
                 data-testid={`categorical-value-${metadataField}-${displayString}`}
                 data-testclass="categorical-value"
+                tabIndex="-1"
                 style={{
                   width: labelWidth,
                   color:
@@ -633,7 +621,7 @@ class CategoryValue extends React.Component {
                   annoInput={
                     <LabelInput
                       label={editedLabelText}
-                      labelSuggestions={ontologyEnabled ? ontology.terms : null}
+                      labelSuggestions={null}
                       onChange={this.handleTextChange}
                       onSelect={this.handleTextChange}
                       inputProps={{
@@ -677,8 +665,8 @@ class CategoryValue extends React.Component {
               display={isColorBy && categoryValueIndices ? "auto" : "none"}
               style={{
                 marginLeft: 5,
-                width: VALUE_HEIGHT,
-                height: VALUE_HEIGHT,
+                width: 15,
+                height: 15,
                 backgroundColor:
                   isColorBy && categoryValueIndices
                     ? colorScale(categoryValueIndices.get(label))
@@ -739,7 +727,7 @@ class CategoryValue extends React.Component {
                       ) : null}
                       {displayString !== globals.unassignedCategoryLabel ? (
                         <MenuItem
-                          icon="delete"
+                          icon="trash"
                           intent="danger"
                           data-testclass="handleDeleteValue"
                           data-testid={`${metadataField}:${displayString}:delete-label`}

client/src/components/categorical/value/occupancy.js

@@ -1,4 +1,3 @@
-// jshint esversion: 6
 import React from "react";
 import { connect } from "react-redux";
 import * as d3 from "d3";
@@ -23,12 +22,8 @@ class Occupancy extends React.PureComponent {
       createHistogram fetches the continous data in relation to the cells releveant to the catagory value.
       It then seperates that data into 50 bins for drawing the mini-histogram
     */
-    const {
-      metadataField,
-      categoryData,
-      colorData,
-      categoryValue,
-    } = this.props;
+    const { metadataField, categoryData, colorData, categoryValue } =
+      this.props;
 
     if (!this.canvas) return;