server update

* Bump version: 0.19.0 → 1.0.0-rc.0

* Bump version: 1.0.0-rc.0 → 1.0.0

.bumpversion.cfg

@@ -1,17 +1,14 @@
 [bumpversion]
-current_version = 0.16.0
+current_version = 1.0.0
 commit = True
-# The below regex details an acceptable version number by naming the groups (major, minor, patch, prerel, and
-# prerelversion) and also specifying the valid values for each group (integers, `\d+`,  for major, minor, patch, and
-# prerelversion and only `rc` as the acceptable value for prerel).
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(?:-(?P<prerel>rc)\.(?P<prerelversion>\d+))?
-serialize =
+serialize = 
 	{major}.{minor}.{patch}-{prerel}.{prerelversion}
 	{major}.{minor}.{patch}
 
 [bumpversion:part:prerel]
 optional_value = release
-values =
+values = 
 	rc
 	release
 

.github/workflows/codeql-analysis.yml

@@ -0,0 +1,67 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL Scan"
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [ main ]
+  schedule:
+    - cron: '0 8 * * *'
+
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'javascript', 'python' ]
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ]
+        # Learn more:
+        # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v1
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+        # queries: ./path/to/local/query, your-org/your-repo/queries@main
+
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v1
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 https://git.io/JvXDl
+
+    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
+    #    and modify them (or add more) to build your code if your project
+    #    uses a compiled language
+
+    #- run: |
+    #   make bootstrap
+    #   make release
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v1

.github/workflows/compatibility_tests.yml

@@ -22,87 +22,102 @@ jobs:
       - name: Build docker image
         run: docker build .
 
-  cellxgene-main-with-python-and-anndata-versions:
-    name: python versions x anndata versions
-    runs-on: ubuntu-latest
-    continue-on-error: true
+  matrix-compatibility-test:
+    name: cxg:${{ matrix.cellxgene_build }} os:${{ matrix.os }} py:${{ matrix.python-version }} anndata:${{ matrix.anndata_version || 'latest' }}
+    runs-on: ${{ matrix.os }}
     strategy:
+      fail-fast: false
       matrix:
-        python-version: [3.6, 3.7, 3.8]
-        anndata-version: [0.6.22.post1, 0.7.1]
-        test-suite: [smoke-test, smoke-test-annotations]
+        # note: The `macos-latest` is latest Catalina version, and not Big Sur. So we explicitly ask for Big Sur (`macos-11`)
+        os: [ubuntu-latest, macos-latest, macos-11]
+        python-version: [3.6, 3.7, 3.8, 3.9]
+        cellxgene_build: [main, latest]
+        exclude:
+          # 3.6 no longer avail on Big Sur (`macos-11`)
+          - os: macos-11
+            python-version: 3.6
+          # no pypi build exists for macos+py3.9 and source install fails to
+          # install `tables` py pkg (a `scanpy` dependency), so we test py3.9
+          # only on ubuntu
+          - os: macos-11
+            python-version: 3.9
+          - os: macos-latest
+            python-version: 3.9
+        # add anndata pinned version test for subset of matrix configurations,
+        # in order to reduce matrix cross-product explosion
+        include:
+          - python-version: 3.8
+            cellxgene_build: latest
+            # TODO: dynamically use the literal version in requirements.txt,
+            # to avoid having to update this in manually in the future
+            # TODO: Do not bother running this if anndata latest version
+            # matches this pinned version, to avoid a redundant test
+            anndata_version: '==0.7.6'
     steps:
     - uses: actions/checkout@v2
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v1
       with:
         python-version: ${{ matrix.python-version }}
-    - name: Install dependencies
+    - name: Cache env vars
+      run: echo "PIP_CACHE=`python -m pip cache dir`" >> $GITHUB_ENV
+    - name: Cache env vars (MacOS)
+      if: startsWith(matrix.os, 'macos')
+      run: echo "BREW_CACHE=`brew --cache`" >> $GITHUB_ENV
+    # FIXME: Only working for Linux
+    - name: Python cache
+      uses: actions/cache@v1
+      with:
+        path: ${{ env.PIP_CACHE }}
+        key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}
+        restore-keys: |
+          ${{ runner.os }}-pip-
+    - name: Node cache
+      uses: actions/cache@v1
+      with:
+        path: ~/.npm
+        key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
+        restore-keys: |
+          ${{ runner.os }}-node-
+    - name: Brew cache (MacOS)
+      if: startsWith(matrix.os, 'macos')
+      uses: actions/cache@v1
+      with:
+        path: ${{ env.BREW_CACHE }}
+        key: ${{ runner.os }}-brew-
+    - name: Install dependencies (Ubuntu Linux)
+      if: startsWith(matrix.os, 'ubuntu')
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y libhdf5-serial-dev
+    - name: Install dependencies (MacOS)
+      if: startsWith(matrix.os, 'macos')
+      run: brew install hdf5
+    - name: Install cellxgene from `main` branch
+      if: matrix.cellxgene_build == 'main'
       run: |
-        # 1. only install the dev requirements on top of what is in the cellxgene pip package
-        sudo apt-get update && sudo apt-get install -y libhdf5-serial-dev
-        sed -i 's/-r requirements.txt//' server/requirements-dev.txt
         pip install -r server/requirements-dev.txt
-        # 2. install cellxgene
         make pydist install-dist
-        # 3. install anndata
-        pip install anndata==${{ matrix.anndata-version }}
-        # workaround for anndata 0.6.22.post1 bug
-        [[ "0.6.22.post1" = "${{ matrix.anndata-version }}" ]] && pip install h5py==2.9.0 || true
-    - name: Tests
-      run: make unit-test ${{ matrix.test-suite }}
-
-  cellxgene-release-with-anndata-master:
-    name: cellxgene release with anndata master
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        test-suite: [smoke-test, smoke-test-annotations]
-    steps:
-      - uses: actions/checkout@v2
-      - name: Set up Python 3.7
-        uses: actions/setup-python@v1
-        with:
-          python-version: 3.7
-      - name: Checkout
-        uses: actions/checkout@v2
-        with:
-          path: cellxgene
-      - name: Install dependencies
-        run: |
-          cd cellxgene
-          # 1. only install the dev requirements on top of what is in the cellxgene pip package
-          make dev-env-client
-          sed -i 's/-r requirements.txt//' server/requirements-dev.txt
-          pip install -r server/requirements-dev.txt
-          # 2. install cellxgene
-          pip install --upgrade cellxgene
-          # 3. install anndata
-          pip install git+https://github.com/theislab/anndata
-      - name: Tests
-        run: cd cellxgene && make unit-test ${{ matrix.test-suite }}
-
-  cellxgene-main-with-anndata-master:
-    name: cellxgene main with anndata master
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        test-suite: [smoke-test, smoke-test-annotations]
-    steps:
-      - uses: actions/checkout@v2
-      - name: Set up Python 3.7
-        uses: actions/setup-python@v1
-        with:
-          python-version: 3.7
-      - name: Checkout
-        uses: actions/checkout@v2
-        with:
-          path: cellxgene
-      - name: Install dependencies
-        run: |
-          cd cellxgene
-          sed -i -E 's/^anndata[>=]=[0-9]+.[0-9]+.[0-9]+$/anndata/g' server/requirements.txt
-          make pydist install-dist dev-env
-          pip install git+https://github.com/theislab/anndata
-      - name: Tests
-        run: cd cellxgene && make unit-test ${{ matrix.test-suite }}
+    - name: Install cellxgene from latest release (pypi.org)
+      if: matrix.cellxgene_build == 'latest'
+      run: |
+        pip install --upgrade cellxgene
+        # install the additional dev requirements on top of what is in the
+        # cellxgene pip package, which are needed for testing, but otherwise
+        # keep same pip pkg versions as in the cxg release
+        sed -i'' -e 's/-r requirements.txt//' server/requirements-dev.txt
+        pip install -r server/requirements-dev.txt
+    - name: Install anndata version per matrix variable
+      run: pip install anndata${{ matrix.anndata_version }}
+    - name: Install node
+      run: make dev-env-client
+    # Run different types of test separately, to facilitate troubleshooting
+    - name: Unit Tests - client
+      run: make unit-test-client
+    - name: Unit Tests - server
+      run: make unit-test-server
+    - name: Smoke Tests
+      run: make smoke-test
+    # FIXME: Fails intermittently. See https://app.zenhub.com/workspaces/single-cell-5e2a191dad828d52cc78b028/issues/chanzuckerberg/cellxgene/2415
+    # - name: Smoke Tests with Annotations
+    #   run: make smoke-test-annotations

.github/workflows/deploy.yml

@@ -1,13 +0,0 @@
-name: Deploy via single cell infra repo
-
-on:
-  push:
-    branches: main
-
-jobs:
-  deploy:
-    runs-on: ubuntu-latest
-    steps:
-      - name: repository dispatch
-        run: |
-          curl -XPOST -u czi-sci-single-cell-eng:${{secrets.SCI_GITHUB_TOKEN}} -H "Accept: application/vnd.github.everest-preview+json"  -H "Content-Type: application/json" https://api.github.com/repos/chanzuckerberg/single-cell-infra/dispatches --data '{"event_type": "cellxgene-hook"}'

.github/workflows/push_tests.yml

@@ -40,7 +40,7 @@ jobs:
       - name: Lint src with eslint
         working-directory: ./client
         run: |
-          make lint
+          npx eslint src __tests__
 
   unit-test:
     runs-on: ubuntu-latest
@@ -68,8 +68,8 @@ jobs:
         run: make pydist install-dist dev-env-server
       - name: Unit tests
         run: |
-          make unit-test
-          bash <(curl -s https://codecov.io/bash) -y .codecov.yml -k server -cF backend,python,unitTest
+          make unit-test-server unit-test-client
+          bash <(curl -s https://codecov.io/bash) -y .codecov.yml -k server -cF server,python,unitTest
           cd client && ./node_modules/codecov/bin/codecov --yml=../.codecov.yml --root=../ --gcov-root=../ -C -F frontend,javascript,unitTest
 
   smoke-tests:

.gitignore

@@ -15,7 +15,7 @@ dist/
 *.egg-info
 
 # Environments
-venv/
+venv*/
 cellxgene/
 
 # client build

CONTRIBUTING.md

@@ -1,3 +1,3 @@
 We warmly welcome contributions from the community!
 
-Whether you want to contribute ideas, requests, documentation, or code, you can get started by visiting our [contribution guide](https://chanzuckerberg.github.io/cellxgene/posts/contribute).
+Whether you want to contribute ideas, requests, documentation, or code, you can get started by visiting our [contribution guide](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/contribute.md).

LICENSE.txt

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2017-2020 Chan Zuckerberg Initiative
+Copyright (c) 2017-2021 Chan Zuckerberg Initiative
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of
 this software and associated documentation files (the "Software"), to deal in

MANIFEST.in

@@ -3,3 +3,5 @@ recursive-include server/common/web/static *
 
 include server/requirements.txt
 include server/requirements-prepare.txt
+include server/converters/schema/hgnc_complete_set.txt.gz
+include server/converters/schema/schema_definitions/*

Makefile

@@ -16,8 +16,13 @@ clean: clean-lite clean-server clean-client
 clean-lite:
 	rm -rf $(CLEANFILES)
 
-clean-%:
-	cd $(*) && $(MAKE) clean
+.PHONY: clean-client
+clean-client:
+	cd client && $(MAKE) clean
+
+.PHONY: clean-server
+clean-server:
+	cd server && $(MAKE) clean
 
 
 # BUILDING PACKAGE
@@ -28,20 +33,20 @@ build-client:
 
 .PHONY: build
 build: clean build-client
-	git ls-files server/ | grep -v 'server/test/' | cpio -pdm $(BUILDDIR)
+	git ls-files server/ | cpio -pdm $(BUILDDIR)
 	cp -r client/build/  $(CLIENTBUILD)
 	$(call copy_client_assets,$(CLIENTBUILD),$(SERVERBUILD))
 	cp MANIFEST.in README.md setup.cfg setup.py $(BUILDDIR)
 
 # If you are actively developing in the server folder use this, dirties the source tree
 .PHONY: build-for-server-dev
-build-for-server-dev: clean-server build-client
-	$(call copy_client_assets,client/build,server)
+build-for-server-dev: clean-server build-client copy-client-assets
 
 .PHONY: copy-client-assets
 copy-client-assets:
 	$(call copy_client_assets,client/build,server)
 
+
 # TESTING
 .PHONY: test
 test: unit-test smoke-test
@@ -49,8 +54,22 @@ test: unit-test smoke-test
 .PHONY: unit-test
 unit-test: unit-test-server unit-test-client
 
-unit-test-%:
-	cd $(*) && $(MAKE) unit-test
+.PHONY: test-server
+test-server: unit-test-server smoke-test
+
+.PHONY: unit-test-client
+unit-test-client:
+	cd client && $(MAKE) unit-test
+
+.PHONY: unit-test-server
+unit-test-server:
+	PYTHONWARNINGS=ignore:ResourceWarning coverage run \
+		--source=server \
+		--omit=.coverage,venv \
+		-m unittest discover \
+		--start-directory test/unit \
+		--verbose; test_result=$$?; \
+	exit $$test_result \
 
 .PHONY: smoke-test
 smoke-test:
@@ -60,14 +79,9 @@ smoke-test:
 smoke-test-annotations:
 	cd client && $(MAKE) smoke-test-annotations
 
-.PHONY: test-db
-test-db:
-	cd server && $(MAKE) test-db
-
-
 # FORMATTING CODE
 
-.PHOHY: fmt
+.PHONY: fmt
 fmt: fmt-client fmt-py
 
 .PHONY: fmt-client
@@ -81,10 +95,10 @@ fmt-py:
 .PHONY: lint
 lint: lint-server lint-client
 
+
 .PHONY: lint-server
 lint-server: fmt-py
-	flake8 server --per-file-ignores='server/test/fixtures/dataset_config_outline.py:F821 server/test/fixtures/server_config_outline.py:F821 server/test/performance/scale_test_annotations.py:E501'
-
+	flake8 server --per-file-ignores='test/fixtures/dataset_config_outline.py:F821 test/fixtures/server_config_outline.py:F821 test/performance/scale_test_annotations.py:E501'
 
 .PHONY: lint-client
 lint-client:
@@ -97,29 +111,34 @@ pydist: build
 	cd $(BUILDDIR); python setup.py sdist -d ../dist
 	@echo "done"
 
-
 # RELEASE HELPERS
 
+# Set PART=[major, minor, patch] as param to make bump.
+# This will create a release candidate. (i.e. 0.16.1 -> 0.16.2-rc.0 for a patch bump)
+.PHONY: bump-version
+bump-version:
+	bumpversion --config-file .bumpversion.cfg $(PART)
+
 # Create new version to commit to main
 .PHONY: create-release-candidate
-create-release-candidate: dev-env bump-version clean-lite gen-package-lock
+create-release-candidate: bump-version clean-lite gen-package-lock
 	@echo "Version bumped part:$(PART) and client built. Ready to commit and push"
 
 # Bump the release candidate version if needed (i.e. the previous release candidate had errors).
 .PHONY: recreate-release-candidate
-recreate-release-candidate: dev-env bump-release-candidate clean-lite gen-package-lock
+recreate-release-candidate: bump-release-candidate clean-lite gen-package-lock
 	@echo "Version bumped part:$(PART) and client built. Ready to commit and push"
 
 # Build dist and release to Test PyPI
 .PHONY: release-candidate-to-test-pypi
-release-candidate-to-test-pypi: dev-env pydist twine
+release-candidate-to-test-pypi: pydist twine
 	@echo "Dist built and uploaded to test.pypi.org"
 	@echo "Test the install:"
 	@echo "    make install-release-test"
 
 # Build final dist (gets rid of the rc tag) and release final candidate to TestPyPI
 .PHONY: release-final-to-test-pypi
-release-final-to-test-pypi: dev-env bump-release clean-lite gen-package-lock pydist twine
+release-final-to-test-pypi: bump-release clean-lite gen-package-lock pydist twine
 	@echo "Final release dist built and uploaded to test.pypi.org"
 	@echo "Test the install:"
 	@echo "    make install-release-test"
@@ -129,9 +148,9 @@ release-final: twine-prod
 	@echo "Release uploaded to pypi.org"
 
 # DANGER: releases directly to prod
-# use this if you accidently burned a test release version number,
+# use this if you accidentally burned a test release version number,
 .PHONY: release-directly-to-prod
-release-directly-to-prod: dev-env pydist twine-prod
+release-directly-to-prod: pydist twine-prod
 	@echo "Dist built and uploaded to pypi.org"
 	@echo "Test the install:"
 	@echo "    make install-release"
@@ -147,12 +166,6 @@ dev-env-client:
 dev-env-server:
 	pip install -r server/requirements-dev.txt
 
-# Set PART=[major, minor, patch] as param to make bump.
-# This will create a release candidate. (i.e. 0.16.1 -> 0.16.2-rc.0 for a patch bump)
-.PHONY: bump-version
-bump-version:
-	bumpversion --config-file .bumpversion.cfg $(PART)
-
 # Increments the release candidate version (i.e. 0.16.2-rc.1 -> 0.16.2-rc.2)
 .PHONY: bump-release-candidate
 bump-release-candidate:
@@ -187,7 +200,7 @@ install-dev: uninstall
 # install from test.pypi to test your release
 .PHONY: install-release-test
 install-release-test: uninstall
-	pip install --no-cache-dir --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple cellxgene
+	pip install --no-cache-dir --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple cellxgene==$(VERSION)
 	@echo "Installed cellxgene from test.pypi.org, now run and smoke test"
 
 # install from pypi to test your release

PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,11 @@
+#### Reviewers
+**Functional:** 
+
+**Readability:** 
+
+---
+
+## Changes
+- add
+- remove
+- modify

README-visium.md

@@ -0,0 +1,20 @@
+# Cellxgene Visium Beta
+
+## How it works
+1. Launch `cellxgene` as normal. 
+1. If the loaded dataset has spatial information available, the image data will be loaded on startup.
+1. On the toolbar, next to the Zoom icon, a `Toggle image` button will now appear. Click on it and the image will be added as an underlay.
+1. You can now use any `cellxgene` functionality and the image will still be present. If you pan and zoom, the image will also be panned and zoomed.
+1. If you want to hide the image, you can click on `Toggle image` again
+
+In order for the image to be displayed with the correct size and alignment, the H5AD needs to have a few requirements. See the following section to learn more.
+
+## h5ad requirements
+1. The spatial embedding layer should be contained in `obsm` and be named `X_spatial`. Other layers can exist, but only this one will have the spatial feature enabled.
+2. A `spatial` dict needs to be defined in the `uns` dictionary.
+3. Inside the `spatial` dict, an `images` dict must be defined. 
+4. The `images` dict must contain a `hires` key, which should reference an image encoded as an RGB matrix (i.e., a three-dimensional matrix of size `height x width x 3` where the final dimension has the RGB values for each pixel)
+5. The `images` dict must contain a `scalefactors` dict. This should in turn contain a `tissue_hires_scalef` key, which should reference a floating point number.
+
+Moreover, in order to have the image correctly aligned with the dots, the following must be true:
+1. `tissue_hires_scalef` should represent the ratio between the embedding layer `X_spatial` and the image matrix. In particular, if you multiply `X_spatial` by `tissue_hires_scalef`, you should obtain an array of points that ovelap the tissue image if you plot them in a plane.

README.md

@@ -7,27 +7,27 @@ _an interactive explorer for single-cell transcriptomics data_
 [![Compatibility Tests](https://github.com/chanzuckerberg/cellxgene/workflows/Compatibility%20Tests/badge.svg)](https://github.com/chanzuckerberg/cellxgene/actions?query=workflow%3A%22Compatibility+Tests%22)
 ![Code Coverage](https://codecov.io/gh/chanzuckerberg/cellxgene/branch/main/graph/badge.svg)
 
-cellxgene (pronounced "cell-by-gene") is an interactive data explorer for single-cell transcriptomics datasets, such as those coming from the [Human Cell Atlas](https://humancellatlas.org). Leveraging modern web development techniques to enable fast visualizations of at least 1 million cells, we hope to enable biologists and computational researchers to explore their data.
+cellxgene Desktop (pronounced "cell-by-gene") is an interactive data explorer for single-cell datasets, such as those coming from the [Human Cell Atlas](https://humancellatlas.org). Leveraging modern web development techniques to enable fast visualizations of at least 1 million cells, we hope to enable biologists and computational researchers to explore their data.
 
-Whether you need to visualize one thousand cells or one million, cellxgene helps you gain insight into your single-cell data.
+Whether you need to visualize one thousand cells or one million, cellxgene Desktop helps you gain insight into your single-cell data.
 
 <img src="https://github.com/chanzuckerberg/cellxgene/raw/main/docs/images/crossfilter.gif" width="350" height="200" hspace="30"><img src="https://github.com/chanzuckerberg/cellxgene/raw/main/docs/images/category-breakdown.gif" width="350" height="200" hspace="30">
 
 # Getting started
 
-### The comprehensive guide to cellxgene
+### The comprehensive guide to cellxgene Desktop
 
-[The cellxgene documentation is your one-stop-shop for information about cellxgene](https://chanzuckerberg.github.io/cellxgene/)! You may be particularly interested in:
+[The cellxgene documentation is your one-stop-shop for information about cellxgene Desktop](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/README.md)! You may be particularly interested in:
 
-- Seeing [what cellxgene can do](https://chanzuckerberg.github.io/cellxgene/posts/gallery)
-- Learning more about cellxgene [installation](https://chanzuckerberg.github.io/cellxgene/posts/install) and [usage](https://chanzuckerberg.github.io/cellxgene/posts/launch)
-- [Preparing your own data](https://chanzuckerberg.github.io/cellxgene/posts/prepare) for use in cellxgene
-- Checking out [our roadmap](https://chanzuckerberg.github.io/cellxgene/posts/roadmap) for future development
-- [Contributing](https://chanzuckerberg.github.io/cellxgene/posts/contribute) to cellxgene
+- Seeing [what cellxgene Desktop can do](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/explore-data/explorer-tutorials.md)
+- Learning more about cellxgene [installation](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/desktop/install.md) and [usage](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/desktop/quick-start.md#quick-start-1)
+- [Preparing your own data](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/desktop/data-reqs.md) for use in cellxgene Desktop
+- Checking out [our roadmap](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/roadmap.md) for future development
+- [Contributing](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/contribute.md) to cellxgene Desktop
 
 ### Quick start
 
-To install cellxgene you need Python 3.6+. We recommend [installing cellxgene into a conda or virtual environment.](https://chanzuckerberg.github.io/cellxgene/posts/install)
+To install cellxgene Desktop you need Python 3.6+. We recommend [installing cellxgene Desktop into a conda or virtual environment.](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/desktop/install.md)
 
 Install the package.
 
@@ -35,24 +35,23 @@ Install the package.
 pip install cellxgene
 ```
 
-Launch cellxgene with an example [anndata](https://anndata.readthedocs.io/en/latest/) file
+Launch cellxgene Desktop with an example [anndata](https://anndata.readthedocs.io/en/latest/) file
 
 ```bash
 cellxgene launch https://cellxgene-example-data.czi.technology/pbmc3k.h5ad
 ```
 
-To explore more datasets already formatted for cellxgene, check out the [Demo data](https://chanzuckerberg.github.io/cellxgene/posts/demo-data) or
-see [Preparing your data](https://chanzuckerberg.github.io/cellxgene/posts/prepare) to learn more about formatting your own
-data for cellxgene.
+To explore more datasets already formatted for cellxgene Desktop, check out the [Demo data](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/desktop/quick-start.md#example-datasets) or
+see [Preparing your data](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/desktop/data-reqs.md) to learn more about formatting your own
+data for cellxgene Desktop.
 
 ### Supported browsers
 
-cellxgene currently supports the following browsers:
+cellxgene Desktop currently supports the following browsers:
 
 - Google Chrome 61+
 - Edge 15+
 - Firefox 60+
-- Safari 10.1+
 
 Please [file an issue](https://github.com/chanzuckerberg/cellxgene/issues/new/choose) if you would like us to add support for an unsupported browser.
 
@@ -63,17 +62,26 @@ For questions, suggestions, or accolades, [join the `#cellxgene-users` channel o
 
 For any errors, [report bugs on Github](https://github.com/chanzuckerberg/cellxgene/issues).
 
-# Developing with cellxgene
+# Developing with cellxgene Desktop
 
 ### Contributing
 
-We warmly welcome contributions from the community! Please see our [contributing guide](https://chanzuckerberg.github.io/cellxgene/posts/contribute) and don't hesitate to open an issue or send a pull request to improve cellxgene.
+We warmly welcome contributions from the community! Please see our [contributing guide](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/contribute.md) and don't hesitate to open an issue or send a pull request to improve cellxgene Desktop. Please see the [dev_docs](https://github.com/chanzuckerberg/cellxgene/tree/main/dev_docs) for pull request suggestions, unit test details, local documentation preview, and other development specifics. 
 
 This project adheres to the Contributor Covenant [code of conduct](https://github.com/chanzuckerberg/.github/blob/master/CODE_OF_CONDUCT.md). By participating, you are expected to uphold this code. Please report unacceptable behavior to opensource@chanzuckerberg.com.
 
 ### Reuse
 
-This project was started with the sole goal of empowering the scientific community to explore and understand their data. As such, we encourage other scientific tool builders in academia or industry to adopt the patterns, tools, and code from this project, and reach out to us with ideas or questions. All code is freely available for reuse under the [MIT license](https://opensource.org/licenses/MIT).
+This project was started with the sole goal of empowering the scientific community to explore and understand their data. 
+As such, we encourage other scientific tool builders in academia or industry to adopt the patterns, tools, and code from 
+this project. All code is freely available for reuse under the [MIT license](https://opensource.org/licenses/MIT).
+
+
+Before extending cellxgene, we encourage you to reach out to us with ideas or questions. It might be possible that an 
+extension could be directly contributed, which would make it available for a wider audience, or that it's on our 
+[roadmap](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/roadmap.md) and under active development. 
+
+See the [cellxgene extensions](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/community-extensions.md) section of our documentation for examples of community use and cellxgene extensions. 
 
 ### Security
 

client/.husky/pre-commit

@@ -0,0 +1,5 @@
+#!/bin/sh
+. "$(dirname "$0")/_/husky.sh"
+
+cd client
+npx --no-install lint-staged --config "./configuration/lint-staged/lint-staged.config.js"

client/Makefile

@@ -1,9 +1,13 @@
 include ../common.mk
 
-ANNOTATIONS := $(if $(ANNOTATIONS),$(ANNOTATIONS),../server/test/fixtures/pbmc3k-annotations.csv)
+ANNOTATIONS := $(if $(ANNOTATIONS),$(ANNOTATIONS),../test/fixtures/pbmc3k-annotations.csv)
+GENE_SETS := $(if $(GENE_SETS),$(GENE_SETS),../test/fixtures/pbmc3k-genesets.csv)
 ANNOTATIONS_FILENAME := $(shell basename $(ANNOTATIONS))
+GENE_SETS_FILENAME := $(shell basename $(GENE_SETS))
+
+CXG_CONFIG := $(if $(CXG_CONFIG),$(CXG_CONFIG),./__tests__/e2e/test_config.yaml)
+
 
-CXG_CONFIG := $(if $(CXG_CONFIG), $(CXG_CONFIG), ./__tests__/e2e/test_config.yaml)
 
 # Packaging
 .PHONY: clean
@@ -35,15 +39,16 @@ smoke-test:
 	start_server_and_test \
 		'CXG_OPTIONS="--config-file $(CXG_CONFIG)" $(MAKE) start-server' \
 		$(CXG_SERVER_PORT) \
-		'CXG_URL_BASE="http://localhost:$(CXG_SERVER_PORT)" CXG_AUTH_TYPE="test" npm run e2e -- --verbose false'
+		'CXG_URL_BASE="http://localhost:$(CXG_SERVER_PORT)" npm run e2e -- --verbose false'
 
 # start an instance of cellxgene and run the end-to-end annotations tests
 .PHONY: smoke-test-annotations
 smoke-test-annotations:
 	$(eval TMP_DIR := $(shell  mktemp -d /tmp/cellxgene_XXXXXX))
 	cp $(ANNOTATIONS) $(TMP_DIR)/ && \
+	cp $(GENE_SETS) $(TMP_DIR)/ && \
 	start_server_and_test \
-		'CXG_OPTIONS="--annotations-file $(TMP_DIR)/$(ANNOTATIONS_FILENAME)" $(MAKE) start-server' \
+		'CXG_OPTIONS="--annotations-file $(TMP_DIR)/$(ANNOTATIONS_FILENAME) --gene-sets-file $(TMP_DIR)/$(GENE_SETS_FILENAME)" $(MAKE) start-server' \
 		$(CXG_SERVER_PORT) \
 		'CXG_URL_BASE="http://localhost:$(CXG_SERVER_PORT)" npm run e2e-annotations -- --verbose false'
 	rm -rf $(TMP_DIR)

client/__tests__/e2e/__snapshots__/e2e.test.js.snap

@@ -2,4 +2,4 @@
 
 exports[`did launch page launched 1`] = `"<span style=\\"max-width: 155px; display: flex; overflow: hidden; justify-content: flex-start; width: 100%; padding: 0px;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">pbm</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">c3k</span><span style=\\"position: absolute; right: 0px; color: inherit;\\">c3k</span></span></span>"`;
 
-exports[`metadata loads categories and values from dataset appear 1`] = `"<div style=\\"display: flex; justify-content: space-between; align-items: baseline;\\"><div style=\\"display: flex; justify-content: flex-start; align-items: flex-start;\\"><label class=\\"bp3-control bp3-checkbox\\" for=\\"category-select-louvain\\"><input id=\\"category-select-louvain\\" data-testclass=\\"category-select\\" data-testid=\\"louvain:category-select\\" type=\\"checkbox\\" checked=\\"\\"><span class=\\"bp3-control-indicator\\"></span></label><span role=\\"menuitem\\" tabindex=\\"0\\" data-testclass=\\"category-expand\\" data-testid=\\"louvain:category-expand\\" style=\\"cursor: pointer;\\"><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\" style=\\"max-width: 265px;\\"><span data-testid=\\"louvain:category-label\\" aria-label=\\"louvain\\" class=\\"\\" tabindex=\\"0\\" style=\\"max-width: 265px;\\"><span style=\\"max-width: 265px; display: flex; overflow: hidden; justify-content: flex-start; width: 100%; padding: 0px;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">lou</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">vain</span><span style=\\"position: absolute; right: 0px; color: inherit;\\">vain</span></span></span></span></span></span><svg stroke=\\"currentColor\\" fill=\\"currentColor\\" stroke-width=\\"0\\" viewBox=\\"0 0 320 512\\" data-testclass=\\"category-expand-is-not-expanded\\" height=\\"1em\\" width=\\"1em\\" xmlns=\\"http://www.w3.org/2000/svg\\" style=\\"font-size: 10px; margin-left: 5px;\\"><path d=\\"M285.476 272.971L91.132 467.314c-9.373 9.373-24.569 9.373-33.941 0l-22.667-22.667c-9.357-9.357-9.375-24.522-.04-33.901L188.505 256 34.484 101.255c-9.335-9.379-9.317-24.544.04-33.901l22.667-22.667c9.373-9.373 24.569-9.373 33.941 0L285.475 239.03c9.373 9.372 9.373 24.568.001 33.941z\\"></path></svg></span></div><div><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\"><a role=\\"button\\" data-testclass=\\"colorby\\" data-testid=\\"colorby-louvain\\" class=\\"bp3-button\\" tabindex=\\"0\\"><span icon=\\"tint\\" class=\\"bp3-icon bp3-icon-tint\\"><svg data-icon=\\"tint\\" width=\\"16\\" height=\\"16\\" viewBox=\\"0 0 16 16\\"><desc>tint</desc><path d=\\"M7.88 1s-4.9 6.28-4.9 8.9c.01 2.82 2.34 5.1 4.99 5.1 2.65-.01 5.03-2.3 5.03-5.13C12.99 7.17 7.88 1 7.88 1z\\" fill-rule=\\"evenodd\\"></path></svg></span></a></span></span></div></div><div style=\\"margin-left: 26px;\\"></div><div></div>"`;
+exports[`metadata loads categories and values from dataset appear 1`] = `"<div style=\\"display: flex; justify-content: space-between; align-items: baseline;\\"><div style=\\"display: flex; justify-content: flex-start; align-items: flex-start;\\"><label class=\\"bp3-control bp3-checkbox\\" for=\\"category-select-louvain\\"><input id=\\"category-select-louvain\\" data-testclass=\\"category-select\\" data-testid=\\"louvain:category-select\\" type=\\"checkbox\\" checked=\\"\\"><span class=\\"bp3-control-indicator\\"></span></label><span role=\\"menuitem\\" tabindex=\\"0\\" data-testclass=\\"category-expand\\" data-testid=\\"louvain:category-expand\\" style=\\"cursor: pointer;\\"><span aria-haspopup=\\"true\\" class=\\"bp3-popover2-target\\"><span data-testid=\\"louvain:category-label\\" tabindex=\\"-1\\" aria-label=\\"louvain\\" class=\\"\\" style=\\"max-width: 265px;\\"><span style=\\"max-width: 265px; display: flex; overflow: hidden; justify-content: flex-start; width: 100%; padding: 0px;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">lou</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">vain</span><span style=\\"position: absolute; right: 0px; color: inherit;\\">vain</span></span></span></span></span><svg stroke=\\"currentColor\\" fill=\\"currentColor\\" stroke-width=\\"0\\" viewBox=\\"0 0 320 512\\" data-testclass=\\"category-expand-is-not-expanded\\" height=\\"1em\\" width=\\"1em\\" xmlns=\\"http://www.w3.org/2000/svg\\" style=\\"font-size: 10px; margin-left: 5px;\\"><path d=\\"M285.476 272.971L91.132 467.314c-9.373 9.373-24.569 9.373-33.941 0l-22.667-22.667c-9.357-9.357-9.375-24.522-.04-33.901L188.505 256 34.484 101.255c-9.335-9.379-9.317-24.544.04-33.901l22.667-22.667c9.373-9.373 24.569-9.373 33.941 0L285.475 239.03c9.373 9.372 9.373 24.568.001 33.941z\\"></path></svg></span></div><div><span class=\\"bp3-popover-wrapper\\"><span aria-haspopup=\\"true\\" class=\\"bp3-popover-target\\"><a role=\\"button\\" data-testclass=\\"colorby\\" data-testid=\\"colorby-louvain\\" class=\\"bp3-button\\" tabindex=\\"0\\"><span icon=\\"tint\\" class=\\"bp3-icon bp3-icon-tint\\"><svg data-icon=\\"tint\\" width=\\"16\\" height=\\"16\\" viewBox=\\"0 0 16 16\\"><desc>tint</desc><path d=\\"M7.88 1s-4.9 6.28-4.9 8.9c.01 2.82 2.34 5.1 4.99 5.1 2.65-.01 5.03-2.3 5.03-5.13C12.99 7.17 7.88 1 7.88 1z\\" fill-rule=\\"evenodd\\"></path></svg></span></a></span></span></div></div><div style=\\"margin-left: 26px;\\"></div>"`;

client/__tests__/e2e/cellxgeneActions.js

@@ -13,11 +13,8 @@ import {
   getTestClass,
   getTestId,
   isElementPresent,
-  goToPage,
 } from "./puppeteerUtils";
 
-import { appUrlBase } from "./config";
-
 export async function drag(testId, start, end, lasso = false) {
   const layout = await waitByID(testId);
   const elBox = await layout.boxModel();
@@ -61,11 +58,9 @@ export async function getAllHistograms(testclass, testIds) {
   const allHistograms = await getAllByClass(testclass);
 
   const testIDs = await Promise.all(
-    allHistograms.map((hist) => {
-      return page.evaluate((elem) => {
-        return elem.dataset.testid;
-      }, hist);
-    })
+    allHistograms.map((hist) =>
+      page.evaluate((elem) => elem.dataset.testid, hist)
+    )
   );
 
   return testIDs.map((id) => id.replace(/^histogram-/, ""));
@@ -176,6 +171,137 @@ export async function createCategory(categoryName) {
   await clickOn("submit-category");
 }
 
+/*
+
+  GENESET
+
+*/
+
+export async function colorByGeneset(genesetName) {
+  await clickOn(`${genesetName}:colorby-entire-geneset`);
+}
+
+export async function colorByGene(gene) {
+  await clickOn(`colorby-${gene}`);
+}
+
+export async function assertColorLegendLabel(label) {
+  const handle = await waitByID("continuous_legend_color_by_label");
+
+  const result = await handle.evaluate((node) =>
+    node.getAttribute("aria-label")
+  );
+
+  return expect(result).toBe(label);
+}
+
+export async function expandGeneset(genesetName) {
+  const expand = await waitByID(`${genesetName}:geneset-expand`);
+  const notExpanded = await expand.$(
+    "[data-testclass='geneset-expand-is-not-expanded']"
+  );
+  if (notExpanded) await clickOn(`${genesetName}:geneset-expand`);
+}
+
+export async function createGeneset(genesetName) {
+  await clickOnUntil("open-create-geneset-dialog", async () => {
+    await expect(page).toMatchElement(getTestId("create-geneset-input"));
+  });
+
+  await typeInto("create-geneset-input", genesetName);
+  await clickOn("submit-geneset");
+  await waitByClass("autosave-complete");
+}
+
+export async function editGenesetName(genesetName, editText) {
+  const editButton = `${genesetName}:edit-genesetName-mode`;
+  const submitButton = `${genesetName}:submit-geneset`;
+  await clickOnUntil(`${genesetName}:see-actions`, async () => {
+    await expect(page).toMatchElement(getTestId(editButton));
+  });
+  await clickOn(editButton);
+  await typeInto("rename-geneset-modal", editText);
+  await clickOn(submitButton);
+}
+
+export async function deleteGeneset(genesetName) {
+  const targetId = `${genesetName}:delete-geneset`;
+
+  await clickOnUntil(`${genesetName}:see-actions`, async () => {
+    await expect(page).toMatchElement(getTestId(targetId));
+  });
+
+  await clickOn(targetId);
+
+  await assertGenesetDoesNotExist(genesetName);
+  await waitByClass("autosave-complete");
+}
+
+export async function assertGenesetDoesNotExist(genesetName) {
+  const result = await isElementPresent(
+    getTestId(`${genesetName}:geneset-name`)
+  );
+  await expect(result).toBe(false);
+}
+
+export async function assertGenesetExists(genesetName) {
+  const handle = await waitByID(`${genesetName}:geneset-name`);
+
+  const result = await handle.evaluate((node) =>
+    node.getAttribute("aria-label")
+  );
+
+  return expect(result).toBe(genesetName);
+}
+
+/*
+
+  GENE
+
+*/
+
+export async function addGeneToSet(genesetName, geneToAddToSet) {
+  const submitButton = `${genesetName}:submit-gene`;
+
+  await clickOn(`${genesetName}:add-new-gene-to-geneset`);
+  await typeInto("add-genes", geneToAddToSet);
+  await clickOn(submitButton);
+}
+
+export async function removeGene(geneSymbol) {
+  const targetId = `delete-from-geneset:${geneSymbol}`;
+
+  await clickOn(targetId);
+
+  await waitByClass("autosave-complete");
+}
+
+export async function assertGeneExistsInGeneset(geneSymbol) {
+  const handle = await waitByID(`${geneSymbol}:gene-label`);
+
+  const result = await handle.evaluate((node) =>
+    node.getAttribute("aria-label")
+  );
+
+  return expect(result).toBe(geneSymbol);
+}
+
+export async function assertGeneDoesNotExist(geneSymbol) {
+  const result = await isElementPresent(getTestId(`${geneSymbol}:gene-label`));
+
+  await expect(result).toBe(false);
+}
+
+export async function expandGene(geneSymbol) {
+  await clickOn(`maximize-${geneSymbol}`);
+}
+
+/*
+
+  CATEGORY
+
+*/
+
 export async function duplicateCategory(categoryName) {
   await clickOn("open-annotation-dialog");
 
@@ -238,7 +364,7 @@ export async function createLabel(categoryName, labelName) {
    * 4. You will see `123` is persisted in the input box
    * 5. Expected behavior is to get an empty input box
    */
-  await page.waitFor(500);
+  await page.waitForTimeout(500);
 
   await clickOn(`${categoryName}:see-actions`);
 
@@ -316,73 +442,4 @@ export async function assertCategoryDoesNotExist(categoryName) {
   await expect(result).toBe(false);
 }
 
-export async function login() {
-  const email = `cellxgene-smoke-test+${process.env.DEPLOYMENT_STAGE}@chanzuckerberg.com`;
-  const password = "Test1111";
-
-  await goToPage(appUrlBase);
-
-  await clickOn("log-in");
-
-  // (thuang): Auth0 form is unstable and unsafe for input until verified
-  await waitUntilFormFieldStable('[name="email"]');
-
-  await expect(page).toFillForm("form", {
-    email,
-    password,
-  });
-
-  await Promise.all([
-    page.waitForNavigation({ waitUntil: "networkidle0" }),
-    expect(page).toClick('[name="submit"]'),
-  ]);
-
-  expect(page.url()).toContain(appUrlBase);
-}
-
-export async function logout() {
-  await clickOnUntil("user-info", async () => {
-    await waitByID("log-out");
-    await Promise.all([
-      page.waitForNavigation({ waitUntil: "networkidle0" }),
-      clickOn("log-out"),
-    ]);
-  });
-
-  await waitByID("log-in");
-}
-
-async function waitUntilFormFieldStable(selector) {
-  const MAX_RETRY = 10;
-  const WAIT_FOR_MS = 200;
-
-  const EXPECTED_VALUE = "aaa";
-
-  let retry = 0;
-
-  while (retry < MAX_RETRY) {
-    try {
-      await expect(page).toFill(selector, EXPECTED_VALUE);
-
-      const fieldHandle = await expect(page).toMatchElement(selector);
-
-      const fieldValue = await page.evaluate(
-        (input) => input.value,
-        fieldHandle
-      );
-
-      expect(fieldValue).toBe(EXPECTED_VALUE);
-
-      break;
-    } catch (error) {
-      retry += 1;
-
-      await page.waitFor(WAIT_FOR_MS);
-    }
-  }
-
-  if (retry === MAX_RETRY) {
-    throw Error("clickOnUntil() assertion failed!");
-  }
-}
 /* eslint-enable no-await-in-loop -- await in loop is needed to emulate sequential user actions */

client/__tests__/e2e/config.js

@@ -6,3 +6,5 @@ export const appUrlBase =
 export const DATASET = "pbmc3k";
 export const isDev = jestEnv === ENV_DEFAULT.DEV;
 export const isDebug = jestEnv === ENV_DEFAULT.DEBUG;
+export const TEST_EMAIL = "user@example.com";
+export const TEST_PASSWORD = process.env.TEST_ACCOUNT_PASS ?? "";

client/__tests__/e2e/diffexpGeneSets.js

@@ -0,0 +1,105 @@
+export const diffexpPop1Genes = [
+  "CD79A",
+  "HLA-DRB1",
+  "HLA-DQA1",
+  "HLA-DPB1",
+  "HLA-DQB1",
+  "HLA-DPA1",
+  "MS4A1",
+  "LTB",
+  "CD79B",
+  "CD37",
+  "HLA-DMA",
+  "TCL1A",
+  "LINC00926",
+  "HLA-DMB",
+  "HVCN1",
+  "EAF2",
+  "FCRLA",
+  "IRF8",
+  "PKIG",
+  "P2RX5",
+  "HLA-DOB",
+  "SPIB",
+  "BLNK",
+  "SWAP70",
+  "PNOC",
+  "CD19",
+  "SMIM14",
+  "CD72",
+  "KIAA0125",
+  "IGLL5",
+  "ARHGAP24",
+  "COTL1",
+  "C16orf74",
+  "BTK",
+  "SNX29P2",
+  "ADAM28",
+  "FCGR2B",
+  "PLD4",
+  "PPP1R14A",
+  "MZB1",
+  "KIAA0040",
+  "PHACTR1",
+  "FCRL2",
+  "RIC3",
+  "P2RY10",
+  "SCPEP1",
+  "DRAM2",
+  "RP5-887A10.1",
+  "CD82",
+  "GPX1",
+];
+
+export const diffexpPop2Genes = [
+  "NKG7",
+  "GZMB",
+  "CTSW",
+  "PRF1",
+  "GNLY",
+  "GZMA",
+  "CST7",
+  "FGFBP2",
+  "SRGN",
+  "CD247",
+  "FCGR3A",
+  "TYROBP",
+  "FCER1G",
+  "ID2",
+  "SPON2",
+  "CCL4",
+  "CCL5",
+  "GZMH",
+  "GIMAP7",
+  "CLIC3",
+  "HOPX",
+  "XCL2",
+  "LGALS1",
+  "IGFBP7",
+  "AKR1C3",
+  "IL32",
+  "EFHD2",
+  "PRSS23",
+  "TTC38",
+  "ZAP70",
+  "S1PR5",
+  "SAMD3",
+  "GIMAP4",
+  "CCL3",
+  "ABI3",
+  "XCL1",
+  "S100A6",
+  "UBB",
+  "GPR56",
+  "PDIA3",
+  "S100A11",
+  "APOBEC3G",
+  "HAVCR2",
+  "PLEKHF1",
+  "LITAF",
+  "ARPC5L",
+  "PTGDR",
+  "PRMT2",
+  "GSTP1",
+  "FCRL6",
+];

client/__tests__/e2e/e2e.test.js

@@ -15,25 +15,16 @@ import {
   getOneElementInnerHTML,
   getTestId,
   goToPage,
-  typeInto,
   waitByID,
-  clickOnUntil,
 } from "./puppeteerUtils";
 
 import {
-  addGeneToSearch,
-  bulkAddGenes,
   calcDragCoordinates,
   clip,
   drag,
   getAllCategoriesAndCounts,
-  getAllHistograms,
   getCellSetCount,
-  runDiffExp,
   selectCategory,
-  subset,
-  login,
-  logout,
 } from "./cellxgeneActions";
 
 const data = datasets[DATASET];
@@ -149,47 +140,6 @@ describe("cell selection", () => {
   });
 });
 
-describe("gene entry", () => {
-  test("search for single gene", async () => {
-    await goToPage(appUrlBase);
-    await addGeneToSearch(data.genes.search);
-  });
-
-  test("bulk add genes", async () => {
-    await goToPage(appUrlBase);
-
-    const testGenes = data.genes.bulkadd;
-
-    await bulkAddGenes(testGenes);
-    const allHistograms = await getAllHistograms(
-      "histogram-user-gene",
-      testGenes
-    );
-
-    expect(allHistograms).toEqual(expect.arrayContaining(testGenes));
-    expect(allHistograms).toHaveLength(testGenes.length);
-  });
-});
-
-describe("differential expression", () => {
-  test("selects cells, saves them and performs diffexp", async () => {
-    await goToPage(appUrlBase);
-
-    await runDiffExp(data.diffexp.cellset1, data.diffexp.cellset2);
-
-    const allHistograms = await getAllHistograms(
-      "histogram-diffexp",
-      data.diffexp["gene-results"]
-    );
-
-    expect(allHistograms).toEqual(
-      expect.arrayContaining(data.diffexp["gene-results"])
-    );
-
-    expect(allHistograms).toHaveLength(data.diffexp["gene-results"].length);
-  });
-});
-
 describe("subset", () => {
   test("subset - cell count matches", async () => {
     await goToPage(appUrlBase);
@@ -236,94 +186,6 @@ describe("subset", () => {
     const cellCount = await getCellSetCount(1);
     expect(cellCount).toBe(data.subset.lasso.count);
   });
-
-  test("undo selection appends the top diff exp genes to user defined genes", async () => {
-    await goToPage(appUrlBase);
-
-    const userDefinedGenes = data.genes.bulkadd;
-    const diffExpGenes = data.diffexp["gene-results"];
-
-    await bulkAddGenes(userDefinedGenes);
-    const userDefinedHistograms = await getAllHistograms(
-      "histogram-user-gene",
-      userDefinedGenes
-    );
-
-    expect(userDefinedHistograms).toEqual(
-      expect.arrayContaining(userDefinedGenes)
-    );
-
-    await subset({ x1: 0.15, y1: 0.1, x2: 0.98, y2: 0.98 });
-    await runDiffExp(data.diffexp.cellset1, data.diffexp.cellset2);
-
-    const diffExpHistograms = await getAllHistograms(
-      "histogram-diffexp",
-      diffExpGenes
-    );
-
-    expect(diffExpHistograms).toEqual(expect.arrayContaining(diffExpGenes));
-
-    await clickOn("reset-subset-button");
-    const expected = [].concat(userDefinedGenes, diffExpGenes);
-    const userDefinedHistogramsAfterSubset = await getAllHistograms(
-      "histogram-user-gene",
-      expected
-    );
-
-    expect(userDefinedHistogramsAfterSubset).toEqual(
-      expect.arrayContaining(expected)
-    );
-  });
-
-  test("subset selection appends the top diff exp genes to user defined genes", async () => {
-    await goToPage(appUrlBase);
-
-    const userDefinedGenes = data.genes.bulkadd;
-    const diffExpGenes = data.diffexp["gene-results"];
-
-    await bulkAddGenes(userDefinedGenes);
-    const userDefinedHistograms = await getAllHistograms(
-      "histogram-user-gene",
-      userDefinedGenes
-    );
-
-    expect(userDefinedHistograms).toEqual(
-      expect.arrayContaining(userDefinedGenes)
-    );
-
-    await subset({ x1: 0.15, y1: 0.1, x2: 0.98, y2: 0.98 });
-    await runDiffExp(data.diffexp.cellset1, data.diffexp.cellset2);
-
-    const diffExpHistograms = await getAllHistograms(
-      "histogram-diffexp",
-      diffExpGenes
-    );
-
-    expect(diffExpHistograms).toEqual(expect.arrayContaining(diffExpGenes));
-
-    await subset({ x1: 0.16, y1: 0.11, x2: 0.97, y2: 0.97 });
-
-    const expected = [].concat(userDefinedGenes, diffExpGenes);
-    const userDefinedHistogramsAfterSubset = await getAllHistograms(
-      "histogram-user-gene",
-      expected
-    );
-
-    expect(userDefinedHistogramsAfterSubset).toEqual(
-      expect.arrayContaining(expected)
-    );
-  });
-});
-
-describe("scatter plot", () => {
-  test("scatter plot appears", async () => {
-    await goToPage(appUrlBase);
-
-    await bulkAddGenes(Object.values(data.scatter.genes));
-    await clickOn(`plot-x-${data.scatter.genes.x}`);
-    await clickOn(`plot-y-${data.scatter.genes.y}`);
-    await waitByID("scatterplot");
-  });
 });
 
 describe("clipping", () => {
@@ -340,30 +202,6 @@ describe("clipping", () => {
     const cellCount = await getCellSetCount(1);
     expect(cellCount).toBe(data.clip.count);
   });
-
-  test("clip gene", async () => {
-    await goToPage(appUrlBase);
-
-    await typeInto("gene-search", data.clip.gene);
-    await page.keyboard.press("Enter");
-
-    await page.waitForSelector(`[data-testid='histogram-${data.clip.gene}']`);
-
-    await clip(data.clip.min, data.clip.max);
-
-    const histBrushableAreaId = `histogram-${data.clip.gene}-plot-brushable-area`;
-
-    const coords = await calcDragCoordinates(
-      histBrushableAreaId,
-      data.clip["coordinates-as-percent"]
-    );
-
-    await drag(histBrushableAreaId, coords.start, coords.end);
-
-    const cellCount = await getCellSetCount(1);
-
-    expect(cellCount).toBe(data.clip["gene-cell-count"]);
-  });
 });
 
 // interact with UI elements just that they do not break
@@ -381,17 +219,6 @@ describe("ui elements don't error", () => {
     }
   });
 
-  test("color by for gene", async () => {
-    await goToPage(appUrlBase);
-
-    await typeInto("gene-search", data.genes.search);
-    await page.keyboard.press("Enter");
-    await page.waitForSelector(
-      `[data-testid='histogram-${data.genes.search}']`
-    );
-    await clickOn(`colorby-${data.genes.search}`);
-  });
-
   test("pan and zoom", async () => {
     await goToPage(appUrlBase);
 
@@ -521,34 +348,4 @@ test("lasso moves after pan", async () => {
 
   expect(panCount).toBe(initialCount);
 });
-
-const describeIfCalledByMakeFileTarget =
-  process.env.CXG_AUTH_TYPE?.toLowerCase() === "test"
-    ? describe
-    : describe.skip;
-
-describeIfCalledByMakeFileTarget("auth buttons", () => {
-  test("login then logout", async () => {
-    await goToPage(appUrlBase);
-    await clickOnUntil("log-in", async () => {
-      await page.waitForNavigation({ waitUntil: "networkidle0" });
-      await waitByID("user-info");
-    });
-    await logout();
-  });
-});
-
-const conditionalDescribe =
-  process.env.TEST_AUTH_INTEGRATION === "true" ? describe : describe.skip;
-
-conditionalDescribe("AuthN Integration", () => {
-  it("logs in", async () => {
-    await login();
-  });
-
-  it("logs out", async () => {
-    await login();
-    await logout();
-  });
-});
 /* eslint-enable no-await-in-loop -- await in loop is needed to emulate sequential user actions */

client/__tests__/e2e/e2eAnnotations.test.js

@@ -12,6 +12,8 @@ import {
   getTestId,
   getTestClass,
   getAllByClass,
+  clickOnUntil,
+  getOneElementInnerHTML,
 } from "./puppeteerUtils";
 
 import {
@@ -27,6 +29,21 @@ import {
   renameLabel,
   subset,
   duplicateCategory,
+  createGeneset,
+  deleteGeneset,
+  assertGenesetExists,
+  assertGenesetDoesNotExist,
+  getCellSetCount,
+  expandGeneset,
+  editGenesetName,
+  addGeneToSet,
+  assertGeneExistsInGeneset,
+  removeGene,
+  assertGeneDoesNotExist,
+  expandGene,
+  colorByGeneset,
+  assertColorLegendLabel,
+  colorByGene,
 } from "./cellxgeneActions";
 
 const data = datasets[DATASET];
@@ -34,12 +51,45 @@ const data = datasets[DATASET];
 const perTestCategoryName = "TEST-CATEGORY";
 const perTestLabelName = "TEST-LABEL";
 
+// geneset CRUD
+const genesetToDeleteName = "geneset_to_delete";
+const preExistingGenesetName = "fifth_dataset";
+const meanExpressionBrushGenesetName = "second_gene_set";
+const meanExpressionBrushCellsSelected = "557";
+const subsetMeanExpressionBrushCellsSelected = "452";
+
+// initial text, the text we type in, the result
+const editableGenesetName = "geneset_to_edit";
+const editText = "_111";
+const newGenesetName = "geneset_to_edit_111";
+
+// add gene to set
+const geneToAddToSet = "RER1";
+const setToAddGeneTo = "fill_this_geneset";
+
+// remove gene from set
+const geneToRemove = "SIK1";
+const setToRemoveFrom = "empty_this_geneset";
+
+// brush a gene
+const geneToBrushAndColorBy = "SIK1";
+const brushThisGeneGeneset = "brush_this_gene";
+const geneBrushedCellCount = "109";
+const subsetGeneBrushedCellCount = "96";
+
+const genesetDescriptionID =
+  "geneset-description-tooltip-fourth_gene_set: fourth description";
+const genesetDescriptionString = "fourth_gene_set: fourth description";
+const genesetToCheckForDescription = "fourth_gene_set";
+
 async function setup(config) {
   await goToPage(appUrlBase);
 
-  // setup the test fixtures
-  await createCategory(perTestCategoryName);
-  await createLabel(perTestCategoryName, perTestLabelName);
+  if (config.categoricalAnno) {
+    // setup the test fixtures
+    await createCategory(perTestCategoryName);
+    await createLabel(perTestCategoryName, perTestLabelName);
+  }
 
   if (config.withSubset) {
     await subset({ x1: 0.1, y1: 0.1, x2: 0.8, y2: 0.8 });
@@ -51,6 +101,200 @@ async function setup(config) {
 describe.each([
   { withSubset: true, tag: "subset" },
   { withSubset: false, tag: "whole" },
+])("geneSET crud operations and interactions", (config) => {
+  test("genesets load from csv", async () => {
+    await setup(config);
+
+    await assertGenesetExists(preExistingGenesetName);
+  });
+  test("brush on geneset mean", async () => {
+    await setup(config);
+
+    await expandGeneset(meanExpressionBrushGenesetName);
+
+    const histBrushableAreaId = `histogram-${meanExpressionBrushGenesetName}-plot-brushable-area`;
+
+    const coords = await calcDragCoordinates(histBrushableAreaId, {
+      x1: 0.25,
+      y1: 0.5,
+      x2: 0.55,
+      y2: 0.5,
+    });
+
+    await drag(histBrushableAreaId, coords.start, coords.end);
+
+    const cellCount = await getCellSetCount(1);
+    if (config.withSubset) {
+      expect(cellCount).toBe(subsetMeanExpressionBrushCellsSelected);
+    } else {
+      expect(cellCount).toBe(meanExpressionBrushCellsSelected);
+    }
+  });
+  test("color by mean expression", async () => {
+    await setup(config);
+
+    await colorByGeneset(meanExpressionBrushGenesetName);
+    await assertColorLegendLabel(meanExpressionBrushGenesetName);
+  });
+  test("diffexp", async () => {
+    if (config.withSubset) return;
+
+    await setup(config);
+
+    // set the two cell sets to b cells vs nk cells
+    await expandCategory(`louvain`);
+    await clickOn(`louvain:category-select`);
+    await clickOn(`categorical-value-select-louvain-B cells`);
+    await clickOn(`cellset-button-1`);
+    await clickOn(`categorical-value-select-louvain-B cells`);
+    await clickOn(`categorical-value-select-louvain-NK cells`);
+    await clickOn(`cellset-button-2`);
+
+    // run diffexp
+    await clickOn(`diffexp-button`);
+    await waitByClass("pop-1-geneset-expand");
+    await expect(page).toClick(getTestClass("pop-1-geneset-expand"));
+
+    await page.waitForFunction(
+      (selector) => !document.querySelector(selector),
+      {},
+      getTestClass("gene-loading-spinner")
+    );
+
+    let genesHTML = await getOneElementInnerHTML(
+      getTestClass("gene-set-genes")
+    );
+
+    expect(genesHTML).toMatchSnapshot();
+
+    await expect(page).toClick(getTestClass("pop-1-geneset-expand"));
+    await expect(page).toClick(getTestClass("pop-2-geneset-expand"));
+
+    await page.waitForFunction(
+      (selector) => !document.querySelector(selector),
+      {},
+      getTestClass("gene-loading-spinner")
+    );
+
+    genesHTML = await getOneElementInnerHTML(getTestClass("gene-set-genes"));
+
+    expect(genesHTML).toMatchSnapshot();
+  });
+  test("create a new geneset and undo/redo", async () => {
+    if (config.withSubset) return;
+
+    await setup(config);
+
+    const genesetName = `test-geneset-foo-123`;
+    await assertGenesetDoesNotExist(genesetName);
+    await createGeneset(genesetName);
+    /* note: as of June 2021, the aria label is in the truncate component which clones the element */
+    await assertGenesetExists(genesetName);
+    await clickOn("undo");
+    await assertGenesetDoesNotExist(genesetName);
+    await clickOn("redo");
+    await assertGenesetExists(genesetName);
+  });
+  test("edit geneset name and undo/redo", async () => {
+    await setup(config);
+
+    await editGenesetName(editableGenesetName, editText);
+    await assertGenesetExists(newGenesetName);
+    await clickOn("undo");
+    await assertGenesetExists(editableGenesetName);
+    await clickOn("redo");
+    await assertGenesetExists(newGenesetName);
+  });
+  test("delete a geneset and undo/redo", async () => {
+    if (config.withSubset) return;
+
+    await setup(config);
+
+    await deleteGeneset(genesetToDeleteName);
+    await clickOn("undo");
+    await assertGenesetExists(genesetToDeleteName);
+    await clickOn("redo");
+    await assertGenesetDoesNotExist(genesetToDeleteName);
+  });
+  test("geneset description", async () => {
+    if (config.withSubset) return;
+
+    await setup(config);
+
+    await clickOnUntil(
+      `${genesetToCheckForDescription}:geneset-expand`,
+      async () => {
+        expect(page).toMatchElement(getTestId(genesetDescriptionID), {
+          text: genesetDescriptionString,
+        });
+      }
+    );
+  });
+});
+
+describe.each([
+  { withSubset: true, tag: "subset" },
+  { withSubset: false, tag: "whole" },
+])("GENE crud operations and interactions", (config) => {
+  test("add a gene to geneset and undo/redo", async () => {
+    await setup(config);
+
+    await addGeneToSet(setToAddGeneTo, geneToAddToSet);
+    await expandGeneset(setToAddGeneTo);
+    await assertGeneExistsInGeneset(geneToAddToSet);
+    await clickOn("undo");
+    await assertGeneDoesNotExist(geneToAddToSet);
+    await clickOn("redo");
+    await assertGeneExistsInGeneset(geneToAddToSet);
+  });
+  test("expand gene and brush", async () => {
+    await setup(config);
+
+    await expandGeneset(brushThisGeneGeneset);
+    await expandGene(geneToBrushAndColorBy);
+    const histBrushableAreaId = `histogram-${geneToBrushAndColorBy}-plot-brushable-area`;
+
+    const coords = await calcDragCoordinates(histBrushableAreaId, {
+      x1: 0.25,
+      y1: 0.5,
+      x2: 0.55,
+      y2: 0.5,
+    });
+    await drag(histBrushableAreaId, coords.start, coords.end);
+    const cellCount = await getCellSetCount(1);
+    if (config.withSubset) {
+      expect(cellCount).toBe(subsetGeneBrushedCellCount);
+    } else {
+      expect(cellCount).toBe(geneBrushedCellCount);
+    }
+  });
+  test("color by gene in geneset", async () => {
+    await setup(config);
+
+    await expandGeneset(meanExpressionBrushGenesetName);
+
+    await colorByGene(geneToBrushAndColorBy);
+    await assertColorLegendLabel(geneToBrushAndColorBy);
+  });
+  test("delete gene from geneset and undo/redo", async () => {
+    // We've already deleted the gene
+    if (config.withSubset) return;
+
+    await setup(config);
+
+    await expandGeneset(setToRemoveFrom);
+    await removeGene(geneToRemove);
+    await assertGeneDoesNotExist(geneToRemove);
+    await clickOn("undo");
+    await assertGeneExistsInGeneset(geneToRemove);
+    await clickOn("redo");
+    await assertGeneDoesNotExist(geneToRemove);
+  });
+});
+
+describe.each([
+  { withSubset: true, tag: "subset", categoricalAnno: true },
+  { withSubset: false, tag: "whole", categoricalAnno: true },
 ])("annotations", (config) => {
   test("create a category", async () => {
     await setup(config);
@@ -287,11 +531,7 @@ describe.each([
     const labels = await getAllByClass("categorical-row");
 
     const result = await Promise.all(
-      labels.map((label) => {
-        return page.evaluate((element) => {
-          return element.outerHTML;
-        }, label);
-      })
+      labels.map((label) => page.evaluate((element) => element.outerHTML, label))
     );
 
     expect(result).toMatchSnapshot();

client/__tests__/e2e/e2eJestConfig.json

@@ -4,7 +4,7 @@
   "testMatch": ["**/__tests__/**/?(*.)(spec|test).js?(x)"],
   "setupFiles": ["../setupMissingGlobals.js"],
   "setupFilesAfterEnv": ["expect-puppeteer", "./puppeteer.setup.js"],
-  "globalSetup": "jest-environment-puppeteer/setup",
+  "globalSetup": "../globalSetup.js",
   "globalTeardown": "jest-environment-puppeteer/teardown",
   "testEnvironment": "./screenshot_env.js"
 }

client/__tests__/e2e/puppeteerUtils.js

@@ -12,7 +12,7 @@ export async function waitByID(testId, props = {}) {
 }
 
 export async function waitByClass(testClass, props = {}) {
-  await page.waitForSelector(`[data-testclass='${testClass}']`, props);
+  return page.waitForSelector(`[data-testclass='${testClass}']`, props);
 }
 
 export async function waitForAllByIds(testIds) {
@@ -32,7 +32,7 @@ export async function typeInto(testId, text) {
   const selector = getTestId(testId);
   // type ahead can be annoying if you don't pause before you type
   await page.click(selector);
-  await page.waitFor(200);
+  await page.waitForTimeout(200);
   await page.type(selector, text);
 }
 
@@ -42,7 +42,7 @@ export async function clearInputAndTypeInto(testId, text) {
   // only works for text without special characters
   // type ahead can be annoying if you don't pause before you type
   await page.click(selector);
-  await page.waitFor(200);
+  await page.waitForTimeout(200);
   // select all
   await page.click(selector, { clickCount: 3 });
   await page.keyboard.press("Backspace");
@@ -72,7 +72,7 @@ export async function clickOnUntil(testId, assert) {
     } catch (error) {
       retry += 1;
 
-      await page.waitFor(WAIT_FOR_MS);
+      await page.waitForTimeout(WAIT_FOR_MS);
     }
   }
 
@@ -100,12 +100,6 @@ export async function getElementCoordinates(testId) {
   });
 }
 
-async function clickTermsOfService() {
-  if (!(await isElementPresent(getTestId("tos-cookies-accept")))) return;
-
-  await clickOn("tos-cookies-accept");
-}
-
 async function nameNewAnnotation() {
   if (await isElementPresent(getTestId("annotation-dialog"))) {
     await typeInto("new-annotation-name", "ignoreE2E");
@@ -122,7 +116,6 @@ export async function goToPage(url) {
   });
 
   await nameNewAnnotation();
-  await clickTermsOfService();
 }
 
 export async function isElementPresent(selector, options) {

client/__tests__/e2e/test_config.yaml

@@ -1,33 +1,9 @@
 server:
   app:
     force_https: true
-
-    # By default, cellxgene will serve api requests from the same base url as the webpage.
-    # In general api_base_url and web_base_url will not need to be set.
-    # There are two reasons to set these parameters:
-    #  1. Oauth authentication is used; the oauth server will redirect back to the api_base_url after login,
-    #     which then redirects back to the web_base_url.  If the web_base_url is not set, it will default to
-    #     the api_base_url.  If oauth authentication is used, the api_base_url must be set.
-    #     For a local test (where the server runs on "http://localhost:<port>"), then the api_base_url may be
-    #     set to the string "local".
-    #  2. The cellxgene deploymnent is in an environment where the webpage and api have
-    #     different base urls.  In this case both api_base_url and web_base_url must be set.
-    #     It is up to the server admin to ensure that the networking is setup correctly for this environment.
-    api_base_url: http://localhost:5005
-    web_base_url: http://localhost:3000
-
-  authentication:
-    # The authentication types may be "none", "session", "oauth"
-    # none:  No authentication support, features like user_annotations must not be enabled.
-    # session:  A session based userid is automatically generated. (no params needed)
-    # oauth: oauth2 is used for authentication;  parameters are defined in params_oauth.
-    type: test
+    port: 5005
 
 dataset:
-  app:
-    about_legal_tos: null
-    about_legal_privacy: null
-
   presentation:
     max_categories: 1000
     custom_colors: true
@@ -38,10 +14,6 @@ dataset:
     local_file_csv:
       directory: null
       file: null
-    ontology:
-      enable: false
-      obo_location: null
 
   embeddings:
     names: []
-    enable_reembedding: false

client/__tests__/globalSetup.js

@@ -0,0 +1,5 @@
+const { setup } = require("jest-environment-puppeteer");
+
+module.exports = async () => {
+  await setup();
+};

client/__tests__/reducers/genesets.test.js

@@ -0,0 +1,524 @@
+import genesetsReducer from "../../src/reducers/genesets";
+
+describe("initial reducer state", () => {
+  test("some other action", () => {
+    expect(genesetsReducer(undefined, { type: "foo" })).toMatchObject({
+      initialized: false,
+      lastTid: undefined,
+      genesets: new Map(),
+    });
+  });
+});
+
+describe("geneset: initial load", () => {
+  test("missing JSON response", () => {
+    expect(() =>
+      genesetsReducer(undefined, {
+        type: "geneset: initial load",
+      })
+    ).toThrow("missing or malformed JSON response");
+  });
+
+  test("empty geneset", () => {
+    expect(
+      genesetsReducer(undefined, {
+        type: "geneset: initial load",
+        data: {
+          tid: 0,
+          genesets: [],
+        },
+      })
+    ).toMatchObject({
+      initialized: true,
+      lastTid: 0,
+      genesets: new Map(),
+    });
+  });
+
+  test("non-empty geneset", () => {
+    expect(
+      genesetsReducer(undefined, {
+        type: "geneset: initial load",
+        data: {
+          tid: 99,
+          genesets: [
+            {
+              geneset_name: "G1",
+              genes: [{ gene_symbol: "F5" }],
+            },
+            {
+              geneset_name: "G2",
+              geneset_description: "G2 desc",
+              genes: [{ gene_symbol: "F6" }],
+            },
+            {
+              geneset_name: "G3",
+              geneset_description: "G3 desc",
+              genes: [{ gene_symbol: "F7", gene_description: "gene desc" }],
+            },
+          ],
+        },
+      })
+    ).toMatchObject({
+      initialized: true,
+      lastTid: 99,
+      genesets: new Map([
+        [
+          "G1",
+          {
+            genesetName: "G1",
+            genesetDescription: "",
+            genes: new Map([["F5", { geneSymbol: "F5", geneDescription: "" }]]),
+          },
+        ],
+        [
+          "G2",
+          {
+            genesetName: "G2",
+            genesetDescription: "G2 desc",
+            genes: new Map([["F6", { geneSymbol: "F6", geneDescription: "" }]]),
+          },
+        ],
+        [
+          "G3",
+          {
+            genesetName: "G3",
+            genesetDescription: "G3 desc",
+            genes: new Map([
+              ["F7", { geneSymbol: "F7", geneDescription: "gene desc" }],
+            ]),
+          },
+        ],
+      ]),
+    });
+  });
+});
+
+describe("geneset: create", () => {
+  const initialState = genesetsReducer(undefined, {
+    type: "geneset: initial load",
+    data: {
+      tid: 0,
+      genesets: [],
+    },
+  });
+
+  test("simple create", () => {
+    expect(
+      genesetsReducer(initialState, {
+        type: "geneset: create",
+        genesetName: "a geneset",
+        genesetDescription: "",
+      })
+    ).toMatchObject({
+      ...initialState,
+      genesets: new Map([
+        [
+          "a geneset",
+          {
+            genesetName: "a geneset",
+            genesetDescription: "",
+            genes: new Map(),
+          },
+        ],
+      ]),
+    });
+  });
+
+  test("error - duplicate name", () => {
+    expect(() => {
+      genesetsReducer(
+        genesetsReducer(initialState, {
+          type: "geneset: create",
+          genesetName: "foo",
+          genesetDescription: "foo",
+        }),
+        {
+          type: "geneset: create",
+          genesetName: "foo",
+          genesetDescription: "bar",
+        }
+      );
+    }).toThrow("name already defined");
+  });
+
+  test("error - missing required action values", () => {
+    expect(() => {
+      genesetsReducer(initialState, {
+        type: "geneset: create",
+        genesetDescription: "foo",
+      });
+    }).toThrow();
+    expect(() => {
+      genesetsReducer(initialState, {
+        type: "geneset: create",
+        genesetName: "foo",
+      });
+    }).toThrow("name or description unspecified");
+  });
+});
+
+describe("geneset: delete", () => {
+  const initialState = genesetsReducer(undefined, {
+    type: "geneset: initial load",
+    data: {
+      tid: 0,
+      genesets: [],
+    },
+  });
+
+  test("simple delete", () => {
+    expect(
+      genesetsReducer(
+        genesetsReducer(initialState, {
+          type: "geneset: create",
+          genesetName: "foo",
+          genesetDescription: "foo",
+        }),
+        {
+          type: "geneset: delete",
+          genesetName: "foo",
+        }
+      )
+    ).toMatchObject({
+      initialized: true,
+      lastTid: 0,
+      genesets: new Map(),
+    });
+  });
+
+  test("error - missing name", () => {
+    expect(() => {
+      genesetsReducer(initialState, {
+        type: "geneset: delete",
+        genesetName: "foo",
+      });
+    }).toThrow("name does not exist");
+  });
+});
+
+describe("geneset: update", () => {
+  const initialState = genesetsReducer(undefined, {
+    type: "geneset: initial load",
+    data: {
+      tid: 0,
+      genesets: [],
+    },
+  });
+
+  test("simple update", () => {
+    expect(
+      genesetsReducer(
+        genesetsReducer(
+          genesetsReducer(initialState, {
+            type: "geneset: create",
+            genesetName: "foo1",
+            genesetDescription: "foo1",
+          }),
+          {
+            type: "geneset: create",
+            genesetName: "foo2",
+            genesetDescription: "foo2",
+          }
+        ),
+        {
+          type: "geneset: update",
+          genesetName: "foo1",
+          update: {
+            genesetName: "bar",
+            genesetDescription: "bar",
+          },
+        }
+      )
+    ).toMatchObject({
+      initialized: true,
+      lastTid: 0,
+      genesets: new Map([
+        [
+          "bar",
+          { genesetName: "bar", genesetDescription: "bar", genes: new Map() },
+        ],
+        [
+          "foo2",
+          { genesetName: "foo2", genesetDescription: "foo2", genes: new Map() },
+        ],
+      ]),
+    });
+  });
+
+  test("error - unknown name", () => {
+    expect(() => {
+      genesetsReducer(initialState, {
+        type: "geneset: update",
+        genesetName: "foo",
+        update: {
+          genesetName: "foo",
+          genesetDescription: "bar",
+        },
+      });
+    }).toThrow("name unspecified or does not exist");
+  });
+
+  test("error - duplicate name", () => {
+    expect(() => {
+      genesetsReducer(
+        genesetsReducer(initialState, {
+          type: "geneset: create",
+          genesetName: "foo",
+          genesetDescription: "foo",
+        }),
+        {
+          type: "geneset: update",
+          genesetName: "foo",
+          update: {
+            genesetName: "foo",
+            genesetDescription: "foo",
+          },
+        }
+      );
+    }).toThrow("update specified existing name and description");
+  });
+});
+
+describe("geneset: add genes", () => {
+  const initialState = genesetsReducer(
+    genesetsReducer(undefined, {
+      type: "geneset: initial load",
+      data: {
+        tid: 0,
+        genesets: [],
+      },
+    }),
+    {
+      type: "geneset: create",
+      genesetName: "test",
+      genesetDescription: "",
+    }
+  );
+
+  test("add a gene", () => {
+    expect(
+      genesetsReducer(initialState, {
+        type: "geneset: add genes",
+        genesetName: "test",
+        genes: [{ geneSymbol: "F5" }],
+      })
+    ).toMatchObject({
+      ...initialState,
+      genesets: new Map([
+        [
+          "test",
+          {
+            genesetName: "test",
+            genesetDescription: "",
+            genes: new Map([["F5", { geneSymbol: "F5", geneDescription: "" }]]),
+          },
+        ],
+      ]),
+    });
+
+    expect(
+      genesetsReducer(initialState, {
+        type: "geneset: add genes",
+        genesetName: "test",
+        genes: [
+          { geneSymbol: "F5", geneDescription: "desc" },
+          { geneSymbol: "SET1", geneDescription: "" },
+        ],
+      })
+    ).toMatchObject({
+      ...initialState,
+      genesets: new Map([
+        [
+          "test",
+          {
+            genesetName: "test",
+            genesetDescription: "",
+            genes: new Map([
+              ["F5", { geneSymbol: "F5", geneDescription: "desc" }],
+              ["SET1", { geneSymbol: "SET1", geneDescription: "" }],
+            ]),
+          },
+        ],
+      ]),
+    });
+  });
+
+  test("no such geneset error", () => {
+    expect(() => {
+      genesetsReducer(initialState, {
+        type: "geneset: add genes",
+        genesetName: "mumble",
+        genes: [],
+      });
+    }).toThrow("geneset name does not exist");
+  });
+});
+
+describe("geneset: delete genes", () => {
+  const initialState = genesetsReducer(
+    genesetsReducer(
+      genesetsReducer(undefined, {
+        type: "geneset: initial load",
+        data: {
+          tid: 0,
+          genesets: [],
+        },
+      }),
+      {
+        type: "geneset: create",
+        genesetName: "test",
+        genesetDescription: "",
+      }
+    ),
+    {
+      type: "geneset: add genes",
+      genesetName: "test",
+      genes: [{ geneSymbol: "F5" }],
+    }
+  );
+
+  test("simple", () => {
+    expect(
+      genesetsReducer(initialState, {
+        type: "geneset: delete genes",
+        genesetName: "test",
+        geneSymbols: ["F5"],
+      })
+    ).toMatchObject({
+      ...initialState,
+      genesets: new Map([
+        [
+          "test",
+          {
+            genesetName: "test",
+            genesetDescription: "",
+            genes: new Map(),
+          },
+        ],
+      ]),
+    });
+  });
+
+  test("no such geneset error", () => {
+    expect(() => {
+      genesetsReducer(initialState, {
+        type: "geneset: delete genes",
+        genesetName: "mumble",
+        geneSymbols: [],
+      });
+    }).toThrow("name does not exist");
+  });
+});
+
+describe("geneset: set gene description", () => {
+  const initialState = genesetsReducer(
+    genesetsReducer(
+      genesetsReducer(undefined, {
+        type: "geneset: initial load",
+        data: {
+          tid: 0,
+          genesets: [],
+        },
+      }),
+      {
+        type: "geneset: create",
+        genesetName: "test",
+        genesetDescription: "",
+      }
+    ),
+    {
+      type: "geneset: add genes",
+      genesetName: "test",
+      genes: [{ geneSymbol: "F5" }],
+    }
+  );
+
+  test("simple set", () => {
+    expect(
+      genesetsReducer(initialState, {
+        type: "geneset: set gene description",
+        genesetName: "test",
+        update: {
+          geneSymbol: "F5",
+          geneDescription: "mumble",
+        },
+      })
+    ).toMatchObject({
+      ...initialState,
+      genesets: new Map([
+        [
+          "test",
+          {
+            genesetName: "test",
+            genesetDescription: "",
+            genes: new Map([
+              ["F5", { geneSymbol: "F5", geneDescription: "mumble" }],
+            ]),
+          },
+        ],
+      ]),
+    });
+  });
+
+  test("no such geneset error", () => {
+    expect(() => {
+      genesetsReducer(initialState, {
+        type: "geneset: set gene description",
+        genesetName: "does not exist",
+        update: {
+          geneSymbol: "F5",
+          geneDescription: "mumble",
+        },
+      });
+    }).toThrow("geneset name does not exist");
+  });
+
+  test("no such gene error", () => {
+    expect(() => {
+      genesetsReducer(initialState, {
+        type: "geneset: set gene description",
+        genesetName: "test",
+        update: {
+          geneSymbol: "NO SUCH GENE",
+          geneDescription: "mumble",
+        },
+      });
+    }).toThrow("no such gene");
+  });
+});
+
+describe("geneset: set tid", () => {
+  test("simple set", () => {
+    expect(
+      genesetsReducer(undefined, {
+        type: "geneset: set tid",
+        tid: 1,
+      })
+    ).toMatchObject({ lastTid: 1 });
+  });
+
+  test("not a number error", () => {
+    expect(() => {
+      genesetsReducer(
+        { lastTid: 1 },
+        {
+          type: "geneset: set tid",
+          tid: "0",
+        }
+      );
+    }).toThrow("must be a positive integer");
+  });
+
+  test("decrement error", () => {
+    expect(() => {
+      genesetsReducer(
+        { lastTid: 1 },
+        {
+          type: "geneset: set tid",
+          tid: 0,
+        }
+      );
+    }).toThrow("may not be decremented");
+  });
+});

client/__tests__/reducers/genesetsUI.test.js

@@ -0,0 +1,75 @@
+import genesetsUIReducer from "../../src/reducers/genesetsUI";
+
+// Format: GeneSetsUI(state,action)
+
+const initialState = {
+  createGenesetModeActive: false,
+  isEditingGenesetName: false,
+  isAddingGenesToGeneset: false,
+};
+
+/* initial */
+describe("geneset UI states", () => {
+  test("initial state, some other action", () => {
+    expect(
+      genesetsUIReducer(undefined, {
+        type: "foo",
+      })
+    ).toMatchObject(initialState);
+  });
+  test("geneset: activate add new geneset mode", () => {
+    expect(
+      genesetsUIReducer(initialState, {
+        type: "geneset: activate add new geneset mode",
+      })
+    ).toMatchObject({
+      createGenesetModeActive: true,
+      isEditingGenesetName: false,
+      isAddingGenesToGeneset: false,
+    });
+  });
+  test("geneset: disable create geneset mode", () => {
+    expect(
+      genesetsUIReducer(undefined, { isEditingGenesetName: false })
+    ).toMatchObject(initialState);
+  });
+
+  test("activate add new genes mode", () => {
+    expect(
+      genesetsUIReducer(undefined, {
+        type: "geneset: activate add new genes mode",
+        geneset: "a geneset name",
+      })
+    ).toMatchObject({
+      createGenesetModeActive: false,
+      isEditingGenesetName: false,
+      isAddingGenesToGeneset: "a geneset name",
+    });
+  });
+  test("disable create geneset mode", () => {
+    expect(
+      genesetsUIReducer(undefined, {
+        type: "geneset: disable create geneset mode",
+      })
+    ).toMatchObject(initialState);
+  });
+  test("activate rename geneset mode", () => {
+    expect(
+      genesetsUIReducer(undefined, {
+        type: "geneset: activate rename geneset mode",
+        data: "a geneset name",
+      })
+    ).toMatchObject({
+      createGenesetModeActive: false,
+      isEditingGenesetName: "a geneset name",
+      isAddingGenesToGeneset: false,
+    });
+  });
+  test("disable rename geneset mode", () => {
+    expect(
+      genesetsUIReducer(undefined, {
+        type: "geneset: disable rename geneset mode",
+      })
+    ).toMatchObject(initialState);
+  });
+});

client/__tests__/reducers/undoable.test.js

@@ -23,9 +23,7 @@ describe("create", () => {
 describe("undo", () => {
   test("expected state modifications", () => {
     const initialState = { a: 0, b: 1000 };
-    const reducer = (state) => {
-      return { a: state.a + 1, b: state.b + 1 };
-    };
+    const reducer = (state) => ({ a: state.a + 1, b: state.b + 1 });
     const undoableReducer = undoable(reducer, ["a"]);
 
     const s1 = undoableReducer(initialState, { type: "test" });
@@ -43,9 +41,7 @@ describe("undo", () => {
 
 describe("redo", () => {
   const initialState = { a: 0, b: 1000 };
-  const reducer = (state) => {
-    return { a: state.a + 1, b: state.b + 1 };
-  };
+  const reducer = (state) => ({ a: state.a + 1, b: state.b + 1 });
   let UR;
 
   beforeEach(() => {

client/__tests__/util/actionHelpers.test.js

@@ -11,18 +11,14 @@ describe("rangeEncodeIndices", () => {
 
   test("sorted flag", () => {
     expect(rangeEncodeIndices([1, 9, 432], 10, true)).toMatchObject([
-      1,
-      9,
-      432,
+      1, 9, 432,
     ]);
     expect(rangeEncodeIndices([1, 9, 432], 10, false)).toMatchObject([
-      1,
-      9,
-      432,
+      1, 9, 432,
     ]);
-    expect(
-      rangeEncodeIndices([0, 1, 2, 3, 9, 10, 432], 2, true)
-    ).toMatchObject([[0, 3], [9, 10], 432]);
+    expect(rangeEncodeIndices([0, 1, 2, 3, 9, 10, 432], 2, true)).toMatchObject(
+      [[0, 3], [9, 10], 432]
+    );
     expect(
       rangeEncodeIndices([0, 1, 2, 3, 9, 10, 432], 2, false)
     ).toMatchObject([[0, 3], [9, 10], 432]);

client/__tests__/util/annoMatrix/annoMatrix.test.js

@@ -85,9 +85,11 @@ describe("AnnoMatrix", () => {
       fetch.once(serverMocks.responder);
       await expect(
         annoMatrix.fetch("X", {
-          field: "var",
-          column: annoMatrix.schema.annotations.var.index,
-          value: "TYMP",
+          where: {
+            field: "var",
+            column: annoMatrix.schema.annotations.var.index,
+            value: "TYMP",
+          },
         })
       ).resolves.toBeInstanceOf(Dataframe);
 
@@ -103,14 +105,18 @@ describe("AnnoMatrix", () => {
       await expect(
         annoMatrix.fetch("X", [
           {
-            field: "var",
-            column: varIndex,
-            value: "SUMO3",
+            where: {
+              field: "var",
+              column: varIndex,
+              value: "SUMO3",
+            },
           },
           {
-            field: "var",
-            column: varIndex,
-            value: "TYMP",
+            where: {
+              field: "var",
+              column: varIndex,
+              value: "TYMP",
+            },
           },
         ])
       ).resolves.toBeInstanceOf(Dataframe);

client/__tests__/util/annoMatrix/crossfilter.test.js

@@ -170,9 +170,11 @@ describe("AnnoMatrixCrossfilter", () => {
       const xfltr = await crossfilter.select(
         "X",
         {
-          field: "var",
-          column: varIndex,
-          value: "TYMP",
+          where: {
+            field: "var",
+            column: varIndex,
+            value: "TYMP",
+          },
         },
         {
           mode: "range",
@@ -186,9 +188,11 @@ describe("AnnoMatrixCrossfilter", () => {
       expect(xfltr.countSelected()).toEqual(501);
 
       const df = await annoMatrix.fetch("X", {
-        field: "var",
-        column: varIndex,
-        value: "TYMP",
+        where: {
+          field: "var",
+          column: varIndex,
+          value: "TYMP",
+        },
       });
       const values = df.icol(0).asArray();
       const selected = xfltr.allSelectedMask();

client/__tests__/util/annoMatrix/serverMocks/index.js

@@ -1,11 +1,4 @@
 export const baseDataURL = "https://a.fake.url/api/v0.2";
 
-window.CELLXGENE = {
-  API: {
-    prefix: baseDataURL,
-    version: "v0.2/",
-  },
-};
-
 export { schema } from "./schema";
 export * from "./routes";

client/__tests__/util/annoMatrix/whereCache.test.js

@@ -1,3 +1,4 @@
+import sha1 from "sha1";
 import {
   _whereCacheGet,
   _whereCacheCreate,
@@ -7,37 +8,82 @@ import {
 const schema = {};
 
 describe("whereCache", () => {
-  test("whereCacheGet - missing cache values", () => {
+  test("whereCacheGet - where query, missing cache values", () => {
     expect(
       _whereCacheGet({}, schema, "X", {
-        field: "var",
-        column: "foo",
-        value: "bar",
+        where: {
+          field: "var",
+          column: "foo",
+          value: "bar",
+        },
       })
     ).toEqual([undefined]);
     expect(
-      _whereCacheGet({ X: {} }, schema, "X", {
-        field: "var",
-        column: "foo",
-        value: "bar",
+      _whereCacheGet({}, schema, "X", {
+        summarize: {
+          field: "var",
+          column: "foo",
+          values: ["bar"],
+        },
       })
     ).toEqual([undefined]);
     expect(
-      _whereCacheGet({ X: { var: new Map() } }, schema, "X", {
-        field: "var",
-        column: "foo",
-        value: "bar",
+      _whereCacheGet({ where: { X: {} } }, schema, "X", {
+        where: {
+          field: "var",
+          column: "foo",
+          value: "bar",
+        },
+      })
+    ).toEqual([undefined]);
+    expect(
+      _whereCacheGet({ where: { X: { var: new Map() } } }, schema, "X", {
+        where: {
+          field: "var",
+          column: "foo",
+          value: "bar",
+        },
       })
     ).toEqual([undefined]);
     expect(
       _whereCacheGet(
-        { X: { var: new Map([["foo", new Map()]]) } },
+        { where: { X: { var: new Map([["foo", new Map()]]) } } },
         schema,
         "X",
         {
+          where: {
+            field: "var",
+            column: "foo",
+            value: "bar",
+          },
+        }
+      )
+    ).toEqual([undefined]);
+  });
+
+  test("whereCacheGet - summarize query, missing cache values", () => {
+    expect(
+      _whereCacheGet({}, schema, "X", {
+        summarize: {
+          method: "mean",
           field: "var",
           column: "foo",
-          value: "bar",
+          values: ["bar"],
+        },
+      })
+    ).toEqual([undefined]);
+    expect(
+      _whereCacheGet(
+        { summarize: { X: { mean: { var: new Map() } } } },
+        schema,
+        "X",
+        {
+          summarize: {
+            method: "mean",
+            field: "var",
+            column: "foo",
+            values: ["bar"],
+          },
         }
       )
     ).toEqual([undefined]);
@@ -45,140 +91,292 @@ describe("whereCache", () => {
 
   test("whereCacheGet - varied lookups", () => {
     const whereCache = {
-      X: {
-        var: new Map([
-          [
-            "foo",
-            new Map([
-              ["bar", [0]],
-              ["baz", [1, 2]],
+      where: {
+        X: {
+          var: new Map([
+            [
+              "foo",
+              new Map([
+                ["bar", [0]],
+                ["baz", [1, 2]],
+              ]),
+            ],
+          ]),
+        },
+      },
+      summarize: {
+        X: {
+          mean: {
+            var: new Map([
+              [
+                "foo",
+                new Map([
+                  [sha1("bar"), [0]],
+                  [sha1("baz"), [1, 2]],
+                ]),
+              ],
             ]),
-          ],
-        ]),
+          },
+        },
       },
     };
 
     expect(
       _whereCacheGet(whereCache, schema, "X", {
-        field: "var",
-        column: "foo",
-        value: "bar",
+        where: {
+          field: "var",
+          column: "foo",
+          value: "bar",
+        },
       })
     ).toEqual([0]);
     expect(
       _whereCacheGet(whereCache, schema, "X", {
-        field: "var",
-        column: "foo",
-        value: "baz",
+        summarize: {
+          method: "mean",
+          field: "var",
+          column: "foo",
+          values: ["bar"],
+        },
+      })
+    ).toEqual([0]);
+    expect(
+      _whereCacheGet(whereCache, schema, "X", {
+        where: {
+          field: "var",
+          column: "foo",
+          value: "baz",
+        },
+      })
+    ).toEqual([1, 2]);
+    expect(
+      _whereCacheGet(whereCache, schema, "X", {
+        summarize: {
+          method: "mean",
+          field: "var",
+          column: "foo",
+          values: ["baz"],
+        },
       })
     ).toEqual([1, 2]);
     expect(_whereCacheGet(whereCache, schema, "Y", {})).toEqual([undefined]);
     expect(
       _whereCacheGet(whereCache, schema, "X", {
-        field: "whoknows",
-        column: "whatever",
-        value: "snork",
+        where: {
+          field: "whoknows",
+          column: "whatever",
+          value: "snork",
+        },
       })
     ).toEqual([undefined]);
     expect(
       _whereCacheGet(whereCache, schema, "X", {
-        field: "var",
-        column: "whatever",
-        value: "snork",
+        where: {
+          field: "var",
+          column: "whatever",
+          value: "snork",
+        },
       })
     ).toEqual([undefined]);
     expect(
       _whereCacheGet(whereCache, schema, "X", {
-        field: "var",
-        column: "foo",
-        value: "snork",
+        where: {
+          field: "var",
+          column: "foo",
+          value: "snork",
+        },
       })
     ).toEqual([undefined]);
   });
 
-  test("whereCacheCreate", () => {
+  test("whereCacheCreate, where query", () => {
     const query = {
-      field: "queryField",
-      column: "queryColumn",
-      value: "queryValue",
+      where: {
+        field: "queryField",
+        column: "queryColumn",
+        value: "queryValue",
+      },
     };
     const wc = _whereCacheCreate(
       "field",
-      { field: "queryField", column: "queryColumn", value: "queryValue" },
+      {
+        where: {
+          field: "queryField",
+          column: "queryColumn",
+          value: "queryValue",
+        },
+      },
       [0, 1, 2]
     );
     expect(wc).toBeDefined();
     expect(wc).toEqual(
       expect.objectContaining({
-        field: {
-          queryField: expect.any(Map),
+        where: {
+          field: {
+            queryField: expect.any(Map),
+          },
         },
       })
     );
-    expect(wc.field.queryField.has("queryColumn")).toEqual(true);
-    expect(wc.field.queryField.get("queryColumn")).toBeInstanceOf(Map);
-    expect(wc.field.queryField.get("queryColumn").has("queryValue")).toEqual(
-      true
-    );
+    expect(wc.where.field.queryField.has("queryColumn")).toEqual(true);
+    expect(wc.where.field.queryField.get("queryColumn")).toBeInstanceOf(Map);
+    expect(
+      wc.where.field.queryField.get("queryColumn").has("queryValue")
+    ).toEqual(true);
     expect(_whereCacheGet(wc, schema, "field", query)).toEqual([0, 1, 2]);
   });
 
-  test("whereCacheMerge", () => {
+  test("whereCacheCreate, summarize query", () => {
+    const query = {
+      summarize: {
+        method: "method",
+        field: "queryField",
+        column: "queryColumn",
+        values: ["queryValue"],
+      },
+    };
+    const wc = _whereCacheCreate("field", query, [0, 1, 2]);
+    expect(_whereCacheGet(wc, schema, "field", query)).toEqual([0, 1, 2]);
+  });
+
+  test("whereCacheCreate, unknown query type", () => {
+    expect(_whereCacheCreate("field", { foobar: true }, [1])).toEqual({});
+  });
+
+  test("whereCacheMerge, where queries", () => {
     let wc;
 
     // remember, will mutate dst
     const src = _whereCacheCreate(
       "field",
-      { field: "queryField", column: "queryColumn", value: "foo" },
+      { where: { field: "queryField", column: "queryColumn", value: "foo" } },
       ["foo"]
     );
 
     const dst1 = _whereCacheCreate(
       "field",
-      { field: "queryField", column: "queryColumn", value: "bar" },
+      { where: { field: "queryField", column: "queryColumn", value: "bar" } },
       ["dst1"]
     );
     wc = _whereCacheMerge(dst1, src);
     expect(
       _whereCacheGet(wc, schema, "field", {
-        field: "queryField",
-        column: "queryColumn",
-        value: "foo",
+        where: {
+          field: "queryField",
+          column: "queryColumn",
+          value: "foo",
+        },
       })
     ).toEqual(["foo"]);
     expect(
       _whereCacheGet(wc, schema, "field", {
-        field: "queryField",
-        column: "queryColumn",
-        value: "bar",
+        where: {
+          field: "queryField",
+          column: "queryColumn",
+          value: "bar",
+        },
       })
     ).toEqual(["dst1"]);
 
     const dst2 = _whereCacheCreate(
       "field",
-      { field: "queryField", column: "queryColumn", value: "bar" },
+      { where: { field: "queryField", column: "queryColumn", value: "bar" } },
       ["dst2"]
     );
     wc = _whereCacheMerge(dst2, dst1, src);
     expect(
       _whereCacheGet(wc, schema, "field", {
-        field: "queryField",
-        column: "queryColumn",
-        value: "foo",
+        where: {
+          field: "queryField",
+          column: "queryColumn",
+          value: "foo",
+        },
       })
     ).toEqual(["foo"]);
     expect(
       _whereCacheGet(wc, schema, "field", {
-        field: "queryField",
-        column: "queryColumn",
-        value: "bar",
+        where: {
+          field: "queryField",
+          column: "queryColumn",
+          value: "bar",
+        },
       })
     ).toEqual(["dst1"]);
 
     wc = _whereCacheMerge({}, src);
     expect(wc).toEqual(src);
 
-    wc = _whereCacheMerge({ field: { queryField: new Map() } }, src);
+    wc = _whereCacheMerge({ where: { field: { queryField: new Map() } } }, src);
     expect(wc).toEqual(src);
   });
+
+  test("whereCacheMerge, mixed queries", () => {
+    const wc = _whereCacheMerge(
+      _whereCacheCreate(
+        "field",
+        {
+          where: {
+            field: "queryField",
+            column: "queryColumn",
+            value: "foo",
+          },
+        },
+        ["a"]
+      ),
+      _whereCacheCreate(
+        "field",
+        {
+          summarize: {
+            method: "mean",
+            field: "queryField",
+            column: "queryColumn",
+            values: ["foo", "bar", "baz"],
+          },
+        },
+        ["b"]
+      )
+    );
+
+    expect(
+      _whereCacheGet(wc, schema, "field", {
+        where: {
+          field: "queryField",
+          column: "queryColumn",
+          value: "foo",
+        },
+      })
+    ).toEqual(["a"]);
+
+    expect(
+      _whereCacheGet(wc, schema, "field", {
+        summarize: {
+          method: "mean",
+          field: "queryField",
+          column: "queryColumn",
+          values: ["foo", "bar", "baz"],
+        },
+      })
+    ).toEqual(["b"]);
+
+    expect(
+      _whereCacheGet(wc, schema, "field", {
+        where: {
+          field: "queryField",
+          column: "queryColumn",
+          value: "does-not-exist",
+        },
+      })
+    ).toEqual([undefined]);
+
+    expect(
+      _whereCacheGet(wc, schema, "field", {
+        summarize: {
+          method: "no-such-method",
+          field: "queryField",
+          column: "queryColumn",
+          values: ["does-not-exist"],
+        },
+      })
+    ).toEqual([undefined]);
+  });
 });

client/__tests__/util/centroid.test.js

@@ -1,11 +1,11 @@
-import _ from "lodash";
+import cloneDeep from "lodash.clonedeep";
 
 import calcCentroid from "../../src/util/centroid";
 import quantile from "../../src/util/quantile";
 import { matrixFBSToDataframe } from "../../src/util/stateManager/matrix";
 import * as REST from "./stateManager/sampleResponses";
 import { indexEntireSchema } from "../../src/util/stateManager/schemaHelpers";
-import { _normalizeCategoricalSchema } from "../../src/annoMatrix/schema";
+import { normalizeWritableCategoricalSchema } from "../../src/annoMatrix/normalize";
 
 describe("centroid", () => {
   let schema;
@@ -13,11 +13,11 @@ describe("centroid", () => {
   let obsLayout;
 
   beforeAll(() => {
-    schema = indexEntireSchema(_.cloneDeep(REST.schema.schema));
+    schema = indexEntireSchema(cloneDeep(REST.schema.schema));
     obsAnnotations = matrixFBSToDataframe(REST.annotationsObs);
     obsLayout = matrixFBSToDataframe(REST.layoutObs);
 
-    _normalizeCategoricalSchema(
+    normalizeWritableCategoricalSchema(
       schema.annotations.obsByName.field3,
       obsAnnotations.col("field3")
     );

client/__tests__/util/dataframe/dataframe.test.js

@@ -793,9 +793,7 @@ describe("dataframe factories", () => {
         [3, 3],
         [new Array(3).fill(0), new Array(3).fill(0), new Array(3).fill(0)]
       );
-      const dfB = dfA.mapColumns(() => {
-        return new Array(3).fill(1);
-      });
+      const dfB = dfA.mapColumns(() => new Array(3).fill(1));
       expect(dfA).not.toBe(dfB);
       expect(dfB.iat(0, 0)).toEqual(1);
       expect(dfB.iat(0, 1)).toEqual(1);

client/__tests__/util/dataframe/histogram.test.js

@@ -88,16 +88,7 @@ describe("Dataframe column histogram", () => {
     expect(df.col(1).histogram(5, [0, 100])).toEqual([5, 1, 0, 0, 2]);
     expect(df.col(0).histogram(2, [0, 10])).toEqual([2, 2]);
     expect(df.col(0).histogram(10, [0, 100])).toEqual([
-      3,
-      2,
-      1,
-      0,
-      0,
-      0,
-      0,
-      0,
-      0,
-      2,
+      3, 2, 1, 0, 0, 0, 0, 0, 0, 2,
     ]);
   });
 });

client/__tests__/util/quantile.test.js

@@ -19,11 +19,7 @@ describe("quantile", () => {
   test("multi q", () => {
     const arr = new Float32Array([9, 3, 5, 6, 0]);
     expect(quantile([0, 0.25, 0.5, 0.75, 1.0], arr)).toMatchObject([
-      0,
-      3,
-      5,
-      6,
-      9,
+      0, 3, 5, 6, 9,
     ]);
   });
 });

client/__tests__/util/stateManager/colorHelpers.test.js

@@ -0,0 +1,198 @@
+/* eslint-disable no-bitwise -- unsigned right shift better than Math.round */
+
+/*
+test color helpers
+*/
+import {
+  createColorTable,
+  loadUserColorConfig,
+} from "../../../src/util/stateManager/colorHelpers";
+import * as Dataframe from "../../../src/util/dataframe";
+
+describe("categorical color helpers", () => {
+  /*
+  Primary test constraint for categorical colors is that they are ordered/identified
+  by schema order, NOT by value.  Ie,
+
+    scale(schemaIndex) should match rgb[obsOffset]
+  */
+
+  const schema = indexSchema({
+    annotations: {
+      obs: {
+        columns: [
+          {
+            name: "name_0",
+            type: "string",
+            writable: false,
+          },
+          {
+            name: "continuousColumn",
+            type: "float32",
+            writable: false,
+          },
+          {
+            categories: [
+              "CD4 T cells",
+              "CD14+ Monocytes",
+              "B cells",
+              "CD8 T cells",
+              "NK cells",
+              "FCGR3A+ Monocytes",
+              "Dendritic cells",
+              "Megakaryocytes",
+            ],
+            name: "categoricalColumn",
+            type: "categorical",
+            writable: false,
+          },
+        ],
+        index: "name_0",
+      },
+      var: {
+        columns: [
+          {
+            name: "name_0",
+            type: "string",
+            writable: false,
+          },
+        ],
+        index: "name_0",
+      },
+    },
+    dataframe: {
+      nObs: 2638,
+      nVar: 1838,
+      type: "float32",
+    },
+    layout: {},
+  });
+
+  const catColCategories = schema.annotations.obs.columns[2].categories;
+  const obsDataframe = new Dataframe.Dataframe(
+    [schema.dataframe.nObs, 2],
+    [
+      new Float32Array(schema.dataframe.nObs).map(() => Math.random()),
+      new Array(schema.dataframe.nObs)
+        .fill("")
+        .map(
+          () =>
+            catColCategories[(Math.random() * catColCategories.length) >>> 0]
+        ),
+    ],
+    null,
+    new Dataframe.KeyIndex(["continuousColumn", "categoricalColumn"])
+  );
+
+  test("default category order", () => {
+    const ct = createColorTable(
+      "color by categorical metadata",
+      "categoricalColumn",
+      obsDataframe,
+      schema
+    );
+    expect(ct).toBeDefined();
+    const data = obsDataframe.col("categoricalColumn").asArray();
+    const cats = schema.annotations.obsByName.categoricalColumn.categories;
+    for (let i = 0; i < schema.dataframe.nObs; i += 1) {
+      expect(makeScale(ct.rgb[i])).toEqual(ct.scale(cats.indexOf(data[i])));
+    }
+  });
+
+  test("shuffle category order", () => {
+    const schemaClone = indexSchema(JSON.parse(JSON.stringify(schema)));
+    shuffle(schemaClone.annotations.obsByName.categoricalColumn.categories);
+    const ct = createColorTable(
+      "color by categorical metadata",
+      "categoricalColumn",
+      obsDataframe,
+      schemaClone
+    );
+    expect(ct).toBeDefined();
+    const data = obsDataframe.col("categoricalColumn").asArray();
+    const cats = schemaClone.annotations.obsByName.categoricalColumn.categories;
+    for (let i = 0; i < schemaClone.dataframe.nObs; i += 1) {
+      expect(makeScale(ct.rgb[i])).toEqual(ct.scale(cats.indexOf(data[i])));
+    }
+  });
+
+  test("user defined color order", () => {
+    const cats = schema.annotations.obsByName.categoricalColumn.categories;
+    const shuffleCats = shuffle(
+      Array.from(schema.annotations.obsByName.categoricalColumn.categories)
+    );
+    const userDefinedColorTable = {
+      categoricalColumn: shuffleCats.reduce((acc, label) => {
+        acc[label] = randRGBColor();
+        return acc;
+      }, {}),
+    };
+
+    const userColors = loadUserColorConfig(userDefinedColorTable);
+    expect(userColors).toBeDefined();
+
+    const ct = createColorTable(
+      "color by categorical metadata",
+      "categoricalColumn",
+      obsDataframe,
+      schema,
+      userColors
+    );
+    expect(ct).toBeDefined();
+    const data = obsDataframe.col("categoricalColumn").asArray();
+    for (let i = 0; i < schema.dataframe.nObs; i += 1) {
+      expect(makeScale(ct.rgb[i])).toEqual(
+        ct.scale(cats.indexOf(data[i])).toString()
+      );
+    }
+  });
+});
+
+/*
+TODO:
+1. mix up category order in schema to make sure it works with varied order
+2. user defined colors
+*/
+
+function indexSchema(schema) {
+  schema.annotations.obsByName = Object.fromEntries(
+    schema.annotations?.obs?.columns?.map((v) => [v.name, v]) ?? []
+  );
+  schema.annotations.varByName = Object.fromEntries(
+    schema.annotations?.var?.columns?.map((v) => [v.name, v]) ?? []
+  );
+  schema.layout.obsByName = Object.fromEntries(
+    schema.layout?.obs?.map((v) => [v.name, v]) ?? []
+  );
+  schema.layout.varByName = Object.fromEntries(
+    schema.layout?.var?.map((v) => [v.name, v]) ?? []
+  );
+
+  return schema;
+}
+
+function makeScale(rgb) {
+  // make a scale string from a rgb float triple
+  return `rgb(${(rgb[0] * 255) >>> 0}, ${(rgb[1] * 255) >>> 0}, ${
+    (rgb[2] * 256) >>> 0
+  })`;
+}
+
+function shuffle(array) {
+  for (let i = array.length - 1; i > 0; i -= 1) {
+    const j = (Math.random() * (i + 1)) >>> 0;
+    [array[i], array[j]] = [array[j], array[i]];
+  }
+  return array;
+}
+
+function randHexColor() {
+  const hex = ((Math.random() * 255) >>> 0).toString(16);
+  return `0${hex}`.slice(-2);
+}
+
+function randRGBColor() {
+  return `#${randHexColor()}${randHexColor()}${randHexColor()}`;
+}
+
+/* eslint-enable no-bitwise -- unsigned right shift better than Math.round */

client/__tests__/util/stateManager/controlsHelpers.test.js

@@ -1,38 +1,9 @@
 /*
 test controls helpers
 */
-import { subsetAndResetGeneLists } from "../../../src/util/stateManager/controlsHelpers";
-import * as globals from "../../../src/globals";
+// TODO #2227 test: improve test coverage on control helper functions
+// (`isSelectableCategoryName()`, `selectableCategoryNames()`, `createCategorySummaryFromDfCol()`, `createCategoricalSelection()`, )
 
 describe("controls helpers", () => {
-  test("subsetAndResetGeneLists", () => {
-    const geneList = [];
-    const genRandGene = () => Math.random().toString(36).substring(2, 6);
-
-    // build a unique set of genes
-    for (let i = 0; i < 150; i += 1) {
-      let randGene = genRandGene();
-      while (geneList.includes(randGene)) randGene = genRandGene();
-      geneList.push(randGene);
-    }
-    // insert duplicates
-    geneList[0] = "dupl";
-    geneList[20] = "dupl";
-    const state = {
-      userDefinedGenes: geneList.slice(0, 20),
-      diffexpGenes: geneList.slice(20),
-    };
-    const [newUserDefinedGenes, newDiffExpGenes] = subsetAndResetGeneLists(
-      state
-    );
-    const expectedNewUserDefinedGenes = [
-      ...geneList.slice(0, 20),
-      ...geneList.slice(21),
-    ].slice(0, globals.maxGenes);
-    expect(globals.maxUserDefinedGenes).toBeLessThan(globals.maxGenes);
-    expect(geneList.length).toBeGreaterThan(globals.maxGenes);
-    expect(newUserDefinedGenes).toHaveLength(globals.maxGenes);
-    expect(newUserDefinedGenes).toStrictEqual(expectedNewUserDefinedGenes);
-    expect(newDiffExpGenes).toStrictEqual([]);
-  });
+  test("placeholder", () => {});
 });

client/__tests__/util/stateManager/sampleResponses.js

@@ -1,3 +1,7 @@
+import every from "lodash.every";
+import map from "lodash.map";
+import isNumber from "lodash.isnumber";
+import zip from "lodash.zip";
 import _ from "lodash";
 import { flatbuffers } from "flatbuffers";
 import { NetEncoding } from "../../../src/util/stateManager/matrix_generated";
@@ -121,10 +125,10 @@ function encodeMatrix(columns, colIndex = undefined) {
   */
   const utf8Encoder = new TextEncoder("utf-8");
   const builder = new flatbuffers.Builder(1024);
-  const cols = _.map(columns, (carr) => {
+  const cols = map(columns, (carr) => {
     let uType;
     let tarr;
-    if (_.every(carr, _.isNumber)) {
+    if (every(carr, isNumber)) {
       uType = NetEncoding.TypedArray.Float32Array;
       tarr = encodeTypedArray(builder, uType, new Float32Array(carr));
     } else {
@@ -167,12 +171,12 @@ function encodeMatrix(columns, colIndex = undefined) {
 }
 
 const anAnnotationsObsFBSResponse = (() => {
-  const columns = _.zip(...anAnnotationsObsJSONResponse.data).slice(1);
+  const columns = zip(...anAnnotationsObsJSONResponse.data).slice(1);
   return encodeMatrix(columns, anAnnotationsObsJSONResponse.names);
 })();
 
 const anAnnotationsVarFBSResponse = (() => {
-  const columns = _.zip(...anAnnotationsVarJSONResponse.data).slice(1);
+  const columns = zip(...anAnnotationsVarJSONResponse.data).slice(1);
   return encodeMatrix(columns, anAnnotationsVarJSONResponse.names);
 })();
 

client/__tests__/util/typedCrossfilter/bitArray.test.js

@@ -1,5 +1,3 @@
-// jshint esversion: 6
-
 import BitArray from "../../../src/util/typedCrossfilter/bitArray";
 
 const defaultTestLength = 8;

client/__tests__/util/typedCrossfilter/crossfilter.test.js

@@ -1,4 +1,5 @@
-import _ from "lodash";
+import filter from "lodash.filter";
+import zip from "lodash.zip";
 
 import Crossfilter from "../../../src/util/typedCrossfilter";
 
@@ -251,12 +252,12 @@ describe("ImmutableTypedCrossfilter", () => {
     test.each([[[]], [[2]], [[2, 1]], [[9, 82]], [[0, 1]]])("exact: %p", (v) =>
       expect(
         p.select("quantity", { mode: "exact", values: v }).countSelected()
-      ).toEqual(_.filter(someData, (d) => v.includes(d.quantity)).length)
+      ).toEqual(filter(someData, (d) => v.includes(d.quantity)).length)
     );
     test("single value exact", () => {
       expect(
         p.select("quantity", { mode: "exact", values: 2 }).countSelected()
-      ).toEqual(_.filter(someData, (d) => d.quantity === 2).length);
+      ).toEqual(filter(someData, (d) => d.quantity === 2).length);
     });
     test.each([
       [0, 1],
@@ -267,7 +268,7 @@ describe("ImmutableTypedCrossfilter", () => {
       expect(
         p.select("quantity", { mode: "range", lo, hi }).countSelected()
       ).toEqual(
-        _.filter(someData, (d) => d.quantity >= lo && d.quantity < hi).length
+        filter(someData, (d) => d.quantity >= lo && d.quantity < hi).length
       )
     );
     test("bad mode", () => {
@@ -298,12 +299,12 @@ describe("ImmutableTypedCrossfilter", () => {
     ])("exact: %p", (v) =>
       expect(
         p.select("type", { mode: "exact", values: v }).countSelected()
-      ).toEqual(_.filter(someData, (d) => v.includes(d.type)).length)
+      ).toEqual(filter(someData, (d) => v.includes(d.type)).length)
     );
     test("single value exact", () => {
       expect(
         p.select("type", { mode: "exact", values: "tab" }).countSelected()
-      ).toEqual(_.filter(someData, (d) => d.type === "tab").length);
+      ).toEqual(filter(someData, (d) => d.type === "tab").length);
     });
     test("range", () => {
       expect(() => p.select("type", { mode: "range", lo: 0, hi: 9 })).toThrow(
@@ -341,7 +342,7 @@ describe("ImmutableTypedCrossfilter", () => {
           .select("coords", { mode: "within-rect", minX, minY, maxX, maxY })
           .allSelected()
       ).toEqual(
-        _.filter(someData, (d) => {
+        filter(someData, (d) => {
           const [x, y] = d.coords;
           return minX <= x && x < maxX && minY <= y && y < maxY;
         })
@@ -397,7 +398,7 @@ describe("ImmutableTypedCrossfilter", () => {
       expect(
         p.select("coords", { mode: "within-polygon", polygon }).allSelected()
       ).toEqual(
-        _.zip(someData, expected)
+        zip(someData, expected)
           .filter((x) => x[1])
           .map((x) => x[0])
       );

client/__tests__/util/typedCrossfilter/positiveInterval.test.js

@@ -1,5 +1,3 @@
-// jshint esversion: 6
-
 // const PositiveIntervals = require("../../src/util/typedCrossfilter/positiveIntervals");
 import PositiveIntervals from "../../../src/util/typedCrossfilter/positiveIntervals";
 
@@ -153,9 +151,9 @@ describe("intersection", () => {
       [1, 2],
       [6, 9],
     ]);
-    expect(
-      PositiveIntervals.intersection([[0, 2638]], [[1363, 2638]])
-    ).toEqual([[1363, 2638]]);
+    expect(PositiveIntervals.intersection([[0, 2638]], [[1363, 2638]])).toEqual(
+      [[1363, 2638]]
+    );
     expect(PositiveIntervals.intersection([[1, 2]], [[1, 2]])).toEqual([
       [1, 2],
     ]);

client/configuration/babel/babel.dev.js

@@ -16,6 +16,8 @@ module.exports = {
     "@babel/plugin-proposal-function-bind",
     ["@babel/plugin-proposal-decorators", { legacy: true }],
     ["@babel/plugin-proposal-class-properties", { loose: true }],
+    ["@babel/plugin-proposal-private-methods", { loose: true }],
+    ["@babel/plugin-proposal-private-property-in-object", { loose: true }],
     "@babel/plugin-proposal-export-namespace-from",
     "@babel/plugin-proposal-optional-chaining",
     "@babel/plugin-proposal-nullish-coalescing-operator",

client/configuration/babel/babel.prod.js

@@ -15,6 +15,8 @@ module.exports = {
     "@babel/plugin-proposal-function-bind",
     ["@babel/plugin-proposal-decorators", { legacy: true }],
     ["@babel/plugin-proposal-class-properties", { loose: true }],
+    ["@babel/plugin-proposal-private-methods", { loose: true }],
+    ["@babel/plugin-proposal-private-property-in-object", { loose: true }],
     "@babel/plugin-proposal-export-namespace-from",
     "@babel/plugin-transform-react-constant-elements",
     "@babel/plugin-transform-runtime",

client/configuration/eslint/eslint.js

@@ -1,24 +1,21 @@
 module.exports = {
   root: true,
-  parser: "babel-eslint",
+
   extends: [
     "airbnb",
     "plugin:eslint-comments/recommended",
     "plugin:@blueprintjs/recommended",
     "plugin:compat/recommended",
-    "plugin:prettier/recommended",
-    "prettier/react",
+    "plugin:jsx-a11y/recommended",
+    // (thuang) disable eslint formatting rules, so prettier can do its job
+    // Do not use `plugin:prettier/recommended` per doc below:
+    // https://prettier.io/docs/en/integrating-with-linters.html
+    "prettier",
   ],
   settings: {
-    polyfills: [
-      "TextDecoder",
-      "TextEncoder",
-      "fetch",
-      "Request",
-      "Response",
-      "Headers",
-      "AbortController",
-    ],
+    // AbortController is not supported in iOS Safari 10.3, Chrome 61
+    // Headers is not supported in iOS Safari 10.3
+    polyfills: ["Headers", "AbortController"],
   },
   env: { browser: true, commonjs: true, es6: true },
   globals: {
@@ -31,6 +28,7 @@ module.exports = {
     context: true,
     beforeEach: true,
   },
+  parser: "@babel/eslint-parser",
   parserOptions: {
     ecmaVersion: 2017,
     sourceType: "module",
@@ -38,6 +36,9 @@ module.exports = {
       jsx: true,
       generators: true,
     },
+    babelOptions: {
+      configFile: "./configuration/babel/babel.prod.js",
+    },
   },
   rules: {
     "react/jsx-no-target-blank": "off",

client/configuration/lint-staged/lint-staged.config.js

@@ -1,3 +1,4 @@
 module.exports = {
   "*.js": "eslint --fix",
+  "**/*": "prettier --write --ignore-unknown",
 };

client/configuration/webpack/obsoleteHTMLTemplate.html

@@ -56,13 +56,6 @@
         />
         <div>Chrome &gt; 60</div>
       </a>
-      <a href="https://www.apple.com/safari/" aria-label="Download Safari">
-        <img
-          src="https://cellxgene.cziscience.com/s3/cellxgene/static/images/safari.png"
-          style="width: 80px; height: 80px;"
-        />
-        <div>Safari ≥ 10.1</div>
-      </a>
       <a href="https://www.mozilla.com/firefox/" aria-label="Download Firefox">
         <img
           src="https://cellxgene.cziscience.com/s3/cellxgene/static/images/firefox.png"

client/configuration/webpack/webpack.config.dev.js

@@ -2,7 +2,6 @@ const path = require("path");
 const webpack = require("webpack");
 const HtmlWebpackPlugin = require("html-webpack-plugin");
 const FaviconsWebpackPlugin = require("favicons-webpack-plugin");
-const ScriptExtHtmlWebpackPlugin = require("script-ext-html-webpack-plugin");
 const MiniCssExtractPlugin = require("mini-css-extract-plugin");
 
 const { merge } = require("webpack-merge");
@@ -31,7 +30,7 @@ const devConfig = {
         test: /\.(jpg|png|gif|eot|svg|ttf|woff|woff2|otf)$/i,
         loader: "file-loader",
         include: [nodeModules, fonts],
-        query: {
+        options: {
           name: "static/assets/[name].[ext]",
           // (thuang): This is needed to make sure @font url path is '/static/assets/'
           publicPath: "/",
@@ -67,14 +66,16 @@ const devConfig = {
       __REACT_DEVTOOLS_GLOBAL_HOOK__: "({ isDisabled: true })",
     }),
     new webpack.DefinePlugin({
-      "process.env.CXG_SERVER_PORT": JSON.stringify(
-        process.env.CXG_SERVER_PORT
-      ),
-    }),
-    new ScriptExtHtmlWebpackPlugin({
-      async: "obsolete",
+      // webpack 5 no longer polyfills NodeJS modules, so fake the one we need
+      "process.env": JSON.stringify({
+        NODE_ENV: process.env.NODE_ENV || "development",
+        CXG_SERVER_PORT: process.env.CXG_SERVER_PORT || "5005",
+      }),
     }),
   ],
+  infrastructureLogging: {
+    level: "warn",
+  },
 };
 
 module.exports = merge(sharedConfig, devConfig);

client/configuration/webpack/webpack.config.prod.js

@@ -1,9 +1,8 @@
 const path = require("path");
+const webpack = require("webpack");
 const HtmlWebpackPlugin = require("html-webpack-plugin");
 const { CleanWebpackPlugin } = require("clean-webpack-plugin");
 const TerserJSPlugin = require("terser-webpack-plugin");
-const CleanCss = require("clean-css");
-const OptimizeCSSAssetsPlugin = require("optimize-css-assets-webpack-plugin");
 const FaviconsWebpackPlugin = require("favicons-webpack-plugin");
 const MiniCssExtractPlugin = require("mini-css-extract-plugin");
 
@@ -26,12 +25,7 @@ const prodConfig = {
   },
   optimization: {
     minimize: true,
-    minimizer: [
-      new TerserJSPlugin({}),
-      new OptimizeCSSAssetsPlugin({
-        cssProcessor: CleanCss,
-      }),
-    ],
+    minimizer: [new TerserJSPlugin({})],
   },
   devtool: "source-map",
   module: {
@@ -45,7 +39,7 @@ const prodConfig = {
         test: /\.(jpg|png|gif|eot|svg|ttf|woff|woff2|otf)$/i,
         loader: "file-loader",
         include: [nodeModules, fonts],
-        query: {
+        options: {
           name: "static/assets/[name]-[contenthash].[ext]",
           // (thuang): This is needed to make sure @font url path is '../static/assets/'
           publicPath: "static/",
@@ -86,6 +80,12 @@ const prodConfig = {
     new CspHashPlugin({
       filename: "csp-hashes.json",
     }),
+    new webpack.DefinePlugin({
+      // webpack 5 no longer polyfills NodeJS modules, so fake the one we need
+      "process.env": JSON.stringify({
+        NODE_ENV: "production",
+      }),
+    }),
   ],
   performance: {
     maxEntrypointSize: 2000000,

client/configuration/webpack/webpack.config.shared.js

@@ -1,26 +1,16 @@
 const path = require("path");
-const fs = require("fs");
 const MiniCssExtractPlugin = require("mini-css-extract-plugin");
-const ObsoleteWebpackPlugin = require("obsolete-webpack-plugin");
-const ScriptExtHtmlWebpackPlugin = require("script-ext-html-webpack-plugin");
+// eslint-disable-next-line @blueprintjs/classes-constants -- incorrect match
 
 const src = path.resolve("src");
 const nodeModules = path.resolve("node_modules");
 
 const publicPath = "";
 
-const rawObsoleteHTMLTemplate = fs.readFileSync(
-  `${__dirname}/obsoleteHTMLTemplate.html`,
-  "utf8"
-);
-
-const obsoleteHTMLTemplate = rawObsoleteHTMLTemplate.replace(/'/g, '"');
-
 module.exports = {
   entry: [
     "core-js",
     "regenerator-runtime/runtime",
-    "fastestsmallesttextencoderdecoder",
     "whatwg-fetch",
     "abort-controller/polyfill",
     "./src/index",
@@ -41,7 +31,7 @@ module.exports = {
             loader: "css-loader",
             options: {
               modules: {
-                localIdentName: "[name]__[local]___[hash:base64:5]",
+                localIdentName: "[name]__[local]___[contenthash:base64:5]",
               },
               importLoaders: 1,
             },
@@ -61,14 +51,4 @@ module.exports = {
       },
     ],
   },
-  plugins: [
-    new ObsoleteWebpackPlugin({
-      name: "obsolete",
-      template: obsoleteHTMLTemplate,
-      promptOnNonTargetBrowser: false,
-    }),
-    new ScriptExtHtmlWebpackPlugin({
-      async: "obsolete",
-    }),
-  ],
 };

client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cellxgene",
-  "version": "0.16.0",
+  "version": "1.0.0",
   "license": "MIT",
   "description": "cellxgene is a web application for the interactive exploration of single cell sequence data.",
   "repository": "https://github.com/chanzuckerberg/cellxgene",
@@ -10,11 +10,11 @@
     "dev": "npm run build -- configuration/webpack/webpack.config.dev.js",
     "e2e": "jest --config __tests__/e2e/e2eJestConfig.json e2e/e2e.test.js",
     "e2e-annotations": "jest --config __tests__/e2e/e2eJestConfig.json e2e/e2eAnnotations.test.js",
-    "e2e-prod": "CXG_URL_BASE='https://cellxgene.cziscience.com/d/pbmc3k.cxg/' jest --config __tests__/e2e/e2eJestConfig.json e2e/e2e.test.js",
     "fmt": "eslint --fix src __tests__",
     "lint": "eslint --fix src __tests__",
     "prod": "npm run build -- configuration/webpack/webpack.config.prod.js",
-    "test": "jest --testPathIgnorePatterns e2e"
+    "test": "jest --testPathIgnorePatterns e2e",
+    "prepare": "cd .. && husky install client/.husky"
   },
   "engineStrict": true,
   "engines": {
@@ -31,112 +31,119 @@
   },
   "browserslist": [
     "Chrome > 60",
-    "Safari >= 10.1",
     "iOS >= 10.3",
     "Firefox >= 60",
     "Edge >= 79",
-    "not Explorer > 0"
+    "not Explorer > 0",
+    "not Safari > 0"
   ],
   "dependencies": {
-    "@blueprintjs/core": "^3.30.0",
+    "@babel/eslint-parser": "^7.15.0",
+    "@blueprintjs/core": "^3.44.0",
     "@blueprintjs/icons": "^3.19.0",
-    "@blueprintjs/select": "^3.13.5",
+    "@blueprintjs/popover2": "^0.11.2",
+    "@blueprintjs/select": "^3.16.0",
     "abort-controller": "^3.0.0",
-    "core-js": "^3.6.5",
+    "core-js": "^3.16.3",
     "d3": "^4.10.0",
     "d3-scale-chromatic": "^1.5.0",
-    "fastestsmallesttextencoderdecoder": "^1.0.22",
     "flatbuffers": "^1.11.0",
     "fuzzysort": "^1.1.4",
     "gl-mat4": "^1.2.0",
     "gl-matrix": "^3.3.0",
     "gl-vec3": "^1.1.3",
     "is-number": "^7.0.0",
-    "lodash": "^4.17.20",
+    "lodash.clonedeep": "^4.5.0",
+    "lodash.difference": "^4.5.0",
+    "lodash.foreach": "^4.5.0",
+    "lodash.pull": "^4.1.0",
+    "lodash.sortby": "^4.7.0",
+    "lodash.uniq": "^4.5.0",
     "memoize-one": "^5.1.1",
-    "react": "^16.13.1",
+    "pako": "^2.0.3",
+    "react": "^17.0.2",
     "react-async": "^10.0.1",
-    "react-dom": "^16.13.1",
+    "react-dom": "^17.0.2",
     "react-flip-toolkit": "^7.0.12",
-    "react-helmet": "^5.2.1",
-    "react-icons": "^3.10.0",
+    "react-helmet": "^6.1.0",
+    "react-icons": "^4.2.0",
     "react-redux": "^7.2.0",
     "redux": "^4.0.5",
     "redux-thunk": "^2.3.0",
     "regenerator-runtime": "^0.13.7",
-    "regl": "^1.6.1",
-    "script-ext-html-webpack-plugin": "^2.1.4",
+    "regl": "^2.1.0",
+    "sha1": "^1.1.1",
     "tinyqueue": "^2.0.3",
-    "webpack-merge": "^5.0.9",
     "whatwg-fetch": "^3.2.0"
   },
   "devDependencies": {
-    "@babel/core": "^7.10.5",
+    "@babel/core": "^7.13.16",
     "@babel/plugin-proposal-class-properties": "^7.10.4",
-    "@babel/plugin-proposal-decorators": "^7.10.5",
+    "@babel/plugin-proposal-decorators": "^7.13.15",
     "@babel/plugin-proposal-export-namespace-from": "^7.10.4",
     "@babel/plugin-proposal-function-bind": "^7.10.5",
     "@babel/plugin-proposal-nullish-coalescing-operator": "^7.10.4",
     "@babel/plugin-proposal-optional-chaining": "^7.10.4",
-    "@babel/plugin-transform-react-constant-elements": "^7.10.4",
-    "@babel/plugin-transform-runtime": "^7.10.5",
-    "@babel/preset-env": "^7.10.4",
-    "@babel/preset-react": "^7.10.4",
-    "@babel/register": "^7.10.5",
-    "@babel/runtime": "^7.10.5",
+    "@babel/plugin-proposal-private-methods": "^7.18.6",
+    "@babel/plugin-proposal-private-property-in-object": "^7.21.11",
+    "@babel/plugin-transform-react-constant-elements": "^7.13.13",
+    "@babel/plugin-transform-runtime": "^7.13.15",
+    "@babel/preset-env": "^7.13.15",
+    "@babel/preset-react": "^7.13.13",
+    "@babel/register": "^7.13.16",
+    "@babel/runtime": "^7.13.16",
     "@blueprintjs/eslint-plugin": "^0.3.0",
-    "@sentry/webpack-plugin": "^1.12.0",
-    "babel-eslint": "^10.1.0",
+    "@sentry/webpack-plugin": "^1.15.0",
     "babel-jest": "^26.1.0",
     "babel-loader": "^8.1.0",
-    "babel-preset-modern-browsers": "^14.2.1",
+    "babel-preset-modern-browsers": "^15.0.2",
     "chalk": "^4.1.0",
-    "cheerio": "^1.0.0-rc.3",
-    "clean-css": "^4.2.3",
-    "clean-webpack-plugin": "^3.0.0",
+    "cheerio": "^1.0.0-rc.6",
+    "clean-css": "^5.1.2",
+    "clean-webpack-plugin": "^4.0.0-alpha.0",
     "codecov": "^3.7.1",
-    "connect-history-api-fallback": "^1.6.0",
-    "css-loader": "^3.6.0",
-    "eslint": "^7.4.0",
+    "css-loader": "^5.2.4",
+    "eslint": "^7.24.0",
     "eslint-config-airbnb": "^18.2.0",
-    "eslint-config-prettier": "^6.11.0",
-    "eslint-loader": "^3.0.4",
+    "eslint-config-prettier": "^8.2.0",
     "eslint-plugin-compat": "^3.8.0",
     "eslint-plugin-eslint-comments": "^3.2.0",
     "eslint-plugin-filenames": "^1.3.2",
-    "eslint-plugin-import": "^2.22.0",
-    "eslint-plugin-jest": "^23.18.0",
+    "eslint-plugin-import": "^2.24.2",
+    "eslint-plugin-jest": "^24.3.5",
     "eslint-plugin-jsx-a11y": "^6.3.1",
-    "eslint-plugin-prettier": "^3.1.4",
-    "eslint-plugin-react": "^7.20.3",
+    "eslint-plugin-react": "^7.23.2",
     "eslint-plugin-react-hooks": "^4.0.8",
-    "expect-puppeteer": "^4.4.0",
+    "expect-puppeteer": "^5.0.0",
     "express": "^4.17.1",
-    "favicons-webpack-plugin": "^3.0.1",
+    "favicons": "^6.2.2",
+    "favicons-webpack-plugin": "^5.0.2",
     "file-loader": "^6.0.0",
-    "html-webpack-plugin": "^4.3.0",
-    "husky": "^4.2.5",
-    "jest": "^26.1.0",
-    "jest-circus": "^26.1.0",
-    "jest-environment-puppeteer": "^4.4.0",
+    "html-webpack-plugin": "^5.3.1",
+    "husky": "^7.0.2",
+    "jest": "^27.0.6",
+    "jest-circus": "^27.0.6",
+    "jest-environment-puppeteer": "^5.0.1",
     "jest-fetch-mock": "^3.0.3",
-    "jest-puppeteer": "^4.4.0",
+    "jest-puppeteer": "^5.0.1",
     "json-loader": "^0.5.7",
     "lint-staged": "^10.2.11",
-    "mini-css-extract-plugin": "^0.9.0",
-    "obsolete-webpack-plugin": "^0.5.6",
-    "optimize-css-assets-webpack-plugin": "^5.0.3",
+    "lodash": "^4.17.21",
+    "lodash.every": "^4.6.0",
+    "lodash.filter": "^4.6.0",
+    "lodash.isnumber": "^3.0.3",
+    "lodash.map": "^4.6.0",
+    "lodash.zip": "^4.2.0",
+    "mini-css-extract-plugin": "^1.5.0",
     "prettier": "^2.0.5",
-    "puppeteer": "^3.3.0",
+    "puppeteer": "^8.0.0",
     "rimraf": "^3.0.2",
     "serve-favicon": "^2.5.0",
-    "style-loader": "^1.2.1",
-    "sw-precache-webpack-plugin": "^1.0.0",
-    "terser-webpack-plugin": "^3.0.7",
-    "url-loader": "^4.1.0",
-    "webpack": "^4.43.0",
-    "webpack-cli": "^3.3.12",
-    "webpack-dev-middleware": "^3.7.2"
+    "terser-webpack-plugin": "^5.1.1",
+    "webpack": "^5.34.0",
+    "webpack-cli": "^4.6.0",
+    "webpack-dev-middleware": "^4.1.0",
+    "webpack-merge": "^5.0.9"
   },
   "jest": {
     "testMatch": [
@@ -169,6 +176,18 @@
               "loose": true
             }
           ],
+          [
+            "@babel/plugin-proposal-private-methods",
+            {
+              "loose": true
+            }
+          ],
+          [
+            "@babel/plugin-proposal-private-property-in-object",
+            {
+              "loose": true
+            }
+          ],
           "@babel/plugin-proposal-export-namespace-from",
           "@babel/plugin-transform-react-constant-elements",
           "@babel/plugin-transform-runtime",
@@ -177,10 +196,5 @@
         ]
       }
     }
-  },
-  "husky": {
-    "hooks": {
-      "pre-commit": "lint-staged --config \"./configuration/lint-staged/lint-staged.config.js\""
-    }
   }
 }

client/server/development.js

@@ -32,7 +32,6 @@ const app = express();
 
 app.use(
   devMiddleware(compiler, {
-    logLevel: "warn",
     publicPath: config.output.publicPath,
     index: true,
   })
@@ -40,14 +39,6 @@ app.use(
 
 app.use(favicon("./favicon.png"));
 
-app.get("/login", async (req, res) => {
-  try {
-    res.redirect(`${API.prefix}login?dataset=http://localhost:${CLIENT_PORT}`);
-  } catch (err) {
-    console.error(err);
-  }
-});
-
 app.get("/logout", async (req, res) => {
   try {
     res.redirect(`${API.prefix}logout?dataset=http://localhost:${CLIENT_PORT}`);

client/server/utils.js

@@ -8,7 +8,9 @@ function isLikelyASyntaxError(message) {
   return message.indexOf(friendlySyntaxErrorLabel) !== -1;
 }
 
-function formatMessage(message) {
+function formatMessage(messageObject) {
+  let { message, details } = messageObject;
+  if (details) message = message + ": " + details;
   return message
     .replace("Module build failed: SyntaxError:", friendlySyntaxErrorLabel)
     .replace(

client/src/actions/annotation.js

@@ -1,287 +1,260 @@
 /*
 Action creators for user annotation
 */
-import _ from "lodash";
+import difference from "lodash.difference";
+import pako from "pako";
 import * as globals from "../globals";
 import { MatrixFBS, AnnotationsHelpers } from "../util/stateManager";
 
 const { isUserAnnotation } = AnnotationsHelpers;
 
-export const annotationCreateCategoryAction = (
-  newCategoryName,
-  categoryToDuplicate
-) => async (dispatch, getState) => {
-  /*
+export const annotationCreateCategoryAction =
+  (newCategoryName, categoryToDuplicate) => async (dispatch, getState) => {
+    /*
   Add a new user-created category to the obs annotations.
 
   Arguments:
     newCategoryName - string name for the category.
     categoryToDuplicate - obs category to use for initial values, or null.
   */
-  const {
-    annoMatrix: prevAnnoMatrix,
-    obsCrossfilter: prevObsCrossfilter,
-  } = getState();
-  if (!prevAnnoMatrix || !prevObsCrossfilter) return;
-  const { schema } = prevAnnoMatrix;
+    const { annoMatrix: prevAnnoMatrix, obsCrossfilter: prevObsCrossfilter } =
+      getState();
+    if (!prevAnnoMatrix || !prevObsCrossfilter) return;
+    const { schema } = prevAnnoMatrix;
 
-  /* name must be a string,  non-zero length */
-  if (typeof newCategoryName !== "string" || newCategoryName.length === 0)
-    throw new Error("user annotations require string name");
+    /* name must be a string,  non-zero length */
+    if (typeof newCategoryName !== "string" || newCategoryName.length === 0)
+      throw new Error("user annotations require string name");
 
-  /* ensure the name isn't already in use! */
-  if (schema.annotations.obsByName[newCategoryName])
-    throw new Error("name collision on annotation category create");
+    /* ensure the name isn't already in use! */
+    if (schema.annotations.obsByName[newCategoryName])
+      throw new Error("name collision on annotation category create");
 
-  let initialValue;
-  let newSchema;
-  let ctor;
-  if (categoryToDuplicate) {
-    /* if we are duplicating a category, retrieve it */
-    const catDupSchema = schema.annotations.obsByName[categoryToDuplicate];
-    const catDupType = catDupSchema?.type;
-    if (catDupType !== "string" && catDupType !== "categorical")
-      throw new Error("categoryToDuplicate does not exist or has invalid type");
+    let initialValue;
+    let newSchema;
+    let ctor;
+    if (categoryToDuplicate) {
+      /* if we are duplicating a category, retrieve it */
+      const catDupSchema = schema.annotations.obsByName[categoryToDuplicate];
+      const catDupType = catDupSchema?.type;
+      if (catDupType !== "string" && catDupType !== "categorical")
+        throw new Error(
+          "categoryToDuplicate does not exist or has invalid type"
+        );
 
-    const catToDupDf = await prevAnnoMatrix
-      .base()
-      .fetch("obs", categoryToDuplicate);
-    const col = catToDupDf.col(categoryToDuplicate);
-    initialValue = col.asArray();
-    const { categories } = col.summarizeCategorical();
-    // all user-created annotations must have the unassigned category
-    if (!categories.includes(globals.unassignedCategoryLabel)) {
-      categories.push(globals.unassignedCategoryLabel);
+      const catToDupDf = await prevAnnoMatrix
+        .base()
+        .fetch("obs", categoryToDuplicate);
+      const col = catToDupDf.col(categoryToDuplicate);
+      initialValue = col.asArray();
+      const { categories } = col.summarizeCategorical();
+      // all user-created annotations must have the unassigned category
+      if (!categories.includes(globals.unassignedCategoryLabel)) {
+        categories.push(globals.unassignedCategoryLabel);
+      }
+      ctor = initialValue.constructor;
+      newSchema = {
+        ...catDupSchema,
+        name: newCategoryName,
+        categories,
+        writable: true,
+      };
+    } else {
+      /* else assign to the standard default value */
+      initialValue = globals.unassignedCategoryLabel;
+      ctor = Array;
+      newSchema = {
+        name: newCategoryName,
+        type: "categorical",
+        categories: [globals.unassignedCategoryLabel],
+        writable: true,
+      };
     }
-    ctor = initialValue.constructor;
-    newSchema = {
-      ...catDupSchema,
-      name: newCategoryName,
-      categories,
-      writable: true,
-    };
-  } else {
-    /* else assign to the standard default value */
-    initialValue = globals.unassignedCategoryLabel;
-    ctor = Array;
-    newSchema = {
-      name: newCategoryName,
-      type: "categorical",
-      categories: [globals.unassignedCategoryLabel],
-      writable: true,
-    };
-  }
 
-  const obsCrossfilter = prevObsCrossfilter.addObsColumn(
-    newSchema,
-    ctor,
-    initialValue
-  );
+    const obsCrossfilter = prevObsCrossfilter.addObsColumn(
+      newSchema,
+      ctor,
+      initialValue
+    );
 
-  dispatch({
-    type: "annotation: create category",
-    data: newCategoryName,
-    categoryToDuplicate,
-    annoMatrix: obsCrossfilter.annoMatrix,
-    obsCrossfilter,
-  });
-};
+    dispatch({
+      type: "annotation: create category",
+      data: newCategoryName,
+      categoryToDuplicate,
+      annoMatrix: obsCrossfilter.annoMatrix,
+      obsCrossfilter,
+    });
+  };
 
-export const annotationRenameCategoryAction = (
-  oldCategoryName,
-  newCategoryName
-) => (dispatch, getState) => {
-  /*
+export const annotationRenameCategoryAction =
+  (oldCategoryName, newCategoryName) => (dispatch, getState) => {
+    /*
   Rename a user-created annotation category
   */
-  const {
-    annoMatrix: prevAnnoMatrix,
-    obsCrossfilter: prevObsCrossfilter,
-  } = getState();
-  if (!prevAnnoMatrix || !prevObsCrossfilter) return;
-  if (!isUserAnnotation(prevAnnoMatrix, oldCategoryName))
-    throw new Error("not a user annotation");
+    const { annoMatrix: prevAnnoMatrix, obsCrossfilter: prevObsCrossfilter } =
+      getState();
+    if (!prevAnnoMatrix || !prevObsCrossfilter) return;
+    if (!isUserAnnotation(prevAnnoMatrix, oldCategoryName))
+      throw new Error("not a user annotation");
 
-  /* name must be a string,  non-zero length */
-  if (typeof newCategoryName !== "string" || newCategoryName.length === 0)
-    throw new Error("user annotations require string name");
+    /* name must be a string,  non-zero length */
+    if (typeof newCategoryName !== "string" || newCategoryName.length === 0)
+      throw new Error("user annotations require string name");
 
-  if (oldCategoryName === newCategoryName) return;
+    if (oldCategoryName === newCategoryName) return;
 
-  const obsCrossfilter = prevObsCrossfilter.renameObsColumn(
-    oldCategoryName,
-    newCategoryName
-  );
+    const obsCrossfilter = prevObsCrossfilter.renameObsColumn(
+      oldCategoryName,
+      newCategoryName
+    );
 
-  dispatch({
-    type: "annotation: category edited",
-    annoMatrix: obsCrossfilter.annoMatrix,
-    obsCrossfilter,
-    metadataField: oldCategoryName,
-    newCategoryText: newCategoryName,
-    data: newCategoryName,
-  });
-};
+    dispatch({
+      type: "annotation: category edited",
+      annoMatrix: obsCrossfilter.annoMatrix,
+      obsCrossfilter,
+      metadataField: oldCategoryName,
+      newCategoryText: newCategoryName,
+      data: newCategoryName,
+    });
+  };
 
-export const annotationDeleteCategoryAction = (categoryName) => (
-  dispatch,
-  getState
-) => {
-  /*
+export const annotationDeleteCategoryAction =
+  (categoryName) => (dispatch, getState) => {
+    /*
   Delete a user-created category
   */
-  const {
-    annoMatrix: prevAnnoMatrix,
-    obsCrossfilter: prevObsCrossfilter,
-  } = getState();
-  if (!prevAnnoMatrix || !prevObsCrossfilter) return;
-  if (!isUserAnnotation(prevAnnoMatrix, categoryName))
-    throw new Error("not a user annotation");
+    const { annoMatrix: prevAnnoMatrix, obsCrossfilter: prevObsCrossfilter } =
+      getState();
+    if (!prevAnnoMatrix || !prevObsCrossfilter) return;
+    if (!isUserAnnotation(prevAnnoMatrix, categoryName))
+      throw new Error("not a user annotation");
 
-  const obsCrossfilter = prevObsCrossfilter.dropObsColumn(categoryName);
-  dispatch({
-    type: "annotation: delete category",
-    annoMatrix: obsCrossfilter.annoMatrix,
-    obsCrossfilter,
-    metadataField: categoryName,
-  });
-};
+    const obsCrossfilter = prevObsCrossfilter.dropObsColumn(categoryName);
+    dispatch({
+      type: "annotation: delete category",
+      annoMatrix: obsCrossfilter.annoMatrix,
+      obsCrossfilter,
+      metadataField: categoryName,
+    });
+  };
 
-export const annotationCreateLabelInCategory = (
-  categoryName,
-  labelName,
-  assignSelected
-) => async (dispatch, getState) => {
-  /*
-  Add a new label to a user-defined category.  If assignSelected is true, assign 
+export const annotationCreateLabelInCategory =
+  (categoryName, labelName, assignSelected) => async (dispatch, getState) => {
+    /*
+  Add a new label to a user-defined category.  If assignSelected is true, assign
   the label to all currently selected cells.
   */
-  const {
-    annoMatrix: prevAnnoMatrix,
-    obsCrossfilter: prevObsCrossfilter,
-  } = getState();
-  if (!prevAnnoMatrix || !prevObsCrossfilter) return;
-  if (!isUserAnnotation(prevAnnoMatrix, categoryName))
-    throw new Error("not a user annotation");
+    const { annoMatrix: prevAnnoMatrix, obsCrossfilter: prevObsCrossfilter } =
+      getState();
+    if (!prevAnnoMatrix || !prevObsCrossfilter) return;
+    if (!isUserAnnotation(prevAnnoMatrix, categoryName))
+      throw new Error("not a user annotation");
 
-  let obsCrossfilter = prevObsCrossfilter.addObsAnnoCategory(
-    categoryName,
-    labelName
-  );
-  if (assignSelected) {
-    obsCrossfilter = await obsCrossfilter.setObsColumnValues(
+    let obsCrossfilter = prevObsCrossfilter.addObsAnnoCategory(
+      categoryName,
+      labelName
+    );
+    if (assignSelected) {
+      obsCrossfilter = await obsCrossfilter.setObsColumnValues(
+        categoryName,
+        prevObsCrossfilter.allSelectedLabels(),
+        labelName
+      );
+    }
+
+    dispatch({
+      type: "annotation: add new label to category",
+      annoMatrix: obsCrossfilter.annoMatrix,
+      obsCrossfilter,
+      metadataField: categoryName,
+      newLabelText: labelName,
+      assignSelectedCells: assignSelected,
+    });
+  };
+
+export const annotationDeleteLabelFromCategory =
+  (categoryName, labelName) => async (dispatch, getState) => {
+    /*
+  delete a label from a user-defined category
+  */
+    const { annoMatrix: prevAnnoMatrix, obsCrossfilter: prevObsCrossfilter } =
+      getState();
+    if (!prevAnnoMatrix || !prevObsCrossfilter) return;
+    if (!isUserAnnotation(prevAnnoMatrix, categoryName))
+      throw new Error("not a user annotation");
+
+    const obsCrossfilter = await prevObsCrossfilter.removeObsAnnoCategory(
+      categoryName,
+      labelName,
+      globals.unassignedCategoryLabel
+    );
+
+    dispatch({
+      type: "annotation: delete label",
+      metadataField: categoryName,
+      label: labelName,
+      annoMatrix: obsCrossfilter.annoMatrix,
+      obsCrossfilter,
+    });
+  };
+
+export const annotationRenameLabelInCategory =
+  (categoryName, oldLabelName, newLabelName) => async (dispatch, getState) => {
+    /*
+  label name change
+  */
+    const { annoMatrix: prevAnnoMatrix, obsCrossfilter: prevObsCrossfilter } =
+      getState();
+    if (!prevAnnoMatrix || !prevObsCrossfilter) return;
+    if (!isUserAnnotation(prevAnnoMatrix, categoryName))
+      throw new Error("not a user annotation");
+
+    let obsCrossfilter = await prevObsCrossfilter.resetObsColumnValues(
+      categoryName,
+      oldLabelName,
+      newLabelName
+    );
+    obsCrossfilter = await obsCrossfilter.removeObsAnnoCategory(
+      categoryName,
+      oldLabelName,
+      globals.unassignedCategoryLabel
+    );
+
+    dispatch({
+      type: "annotation: label edited",
+      editedLabel: newLabelName,
+      metadataField: categoryName,
+      label: oldLabelName,
+      annoMatrix: obsCrossfilter.annoMatrix,
+      obsCrossfilter,
+    });
+  };
+
+export const annotationLabelCurrentSelection =
+  (categoryName, labelName) => async (dispatch, getState) => {
+    /*
+  set the label on all currently selected
+  */
+    const { annoMatrix: prevAnnoMatrix, obsCrossfilter: prevObsCrossfilter } =
+      getState();
+    if (!prevAnnoMatrix || !prevObsCrossfilter) return;
+    if (!isUserAnnotation(prevAnnoMatrix, categoryName))
+      throw new Error("not a user annotation");
+
+    const obsCrossfilter = await prevObsCrossfilter.setObsColumnValues(
       categoryName,
       prevObsCrossfilter.allSelectedLabels(),
       labelName
     );
-  }
 
-  dispatch({
-    type: "annotation: add new label to category",
-    annoMatrix: obsCrossfilter.annoMatrix,
-    obsCrossfilter,
-    metadataField: categoryName,
-    newLabelText: labelName,
-    assignSelectedCells: assignSelected,
-  });
-};
-
-export const annotationDeleteLabelFromCategory = (
-  categoryName,
-  labelName
-) => async (dispatch, getState) => {
-  /*
-  delete a label from a user-defined category
-  */
-  const {
-    annoMatrix: prevAnnoMatrix,
-    obsCrossfilter: prevObsCrossfilter,
-  } = getState();
-  if (!prevAnnoMatrix || !prevObsCrossfilter) return;
-  if (!isUserAnnotation(prevAnnoMatrix, categoryName))
-    throw new Error("not a user annotation");
-
-  const obsCrossfilter = await prevObsCrossfilter.removeObsAnnoCategory(
-    categoryName,
-    labelName,
-    globals.unassignedCategoryLabel
-  );
-
-  dispatch({
-    type: "annotation: delete label",
-    metadataField: categoryName,
-    label: labelName,
-    annoMatrix: obsCrossfilter.annoMatrix,
-    obsCrossfilter,
-  });
-};
-
-export const annotationRenameLabelInCategory = (
-  categoryName,
-  oldLabelName,
-  newLabelName
-) => async (dispatch, getState) => {
-  /*
-  label name change
-  */
-  const {
-    annoMatrix: prevAnnoMatrix,
-    obsCrossfilter: prevObsCrossfilter,
-  } = getState();
-  if (!prevAnnoMatrix || !prevObsCrossfilter) return;
-  if (!isUserAnnotation(prevAnnoMatrix, categoryName))
-    throw new Error("not a user annotation");
-
-  let obsCrossfilter = await prevObsCrossfilter.resetObsColumnValues(
-    categoryName,
-    oldLabelName,
-    newLabelName
-  );
-  obsCrossfilter = await obsCrossfilter.removeObsAnnoCategory(
-    categoryName,
-    oldLabelName,
-    globals.unassignedCategoryLabel
-  );
-
-  dispatch({
-    type: "annotation: label edited",
-    editedLabel: newLabelName,
-    metadataField: categoryName,
-    label: oldLabelName,
-    annoMatrix: obsCrossfilter.annoMatrix,
-    obsCrossfilter,
-  });
-};
-
-export const annotationLabelCurrentSelection = (
-  categoryName,
-  labelName
-) => async (dispatch, getState) => {
-  /*
-  set the label on all currently selected
-  */
-  const {
-    annoMatrix: prevAnnoMatrix,
-    obsCrossfilter: prevObsCrossfilter,
-  } = getState();
-  if (!prevAnnoMatrix || !prevObsCrossfilter) return;
-  if (!isUserAnnotation(prevAnnoMatrix, categoryName))
-    throw new Error("not a user annotation");
-
-  const obsCrossfilter = await prevObsCrossfilter.setObsColumnValues(
-    categoryName,
-    prevObsCrossfilter.allSelectedLabels(),
-    labelName
-  );
-
-  dispatch({
-    type: "annotation: label current cell selection",
-    metadataField: categoryName,
-    label: labelName,
-    obsCrossfilter,
-    annoMatrix: obsCrossfilter.annoMatrix,
-  });
-};
+    dispatch({
+      type: "annotation: label current cell selection",
+      metadataField: categoryName,
+      label: labelName,
+      obsCrossfilter,
+      annoMatrix: obsCrossfilter.annoMatrix,
+    });
+  };
 
 function writableAnnotations(annoMatrix) {
   return annoMatrix.schema.annotations.obs.columns
@@ -307,7 +280,7 @@ export const needToSaveObsAnnotations = (annoMatrix, lastSavedAnnoMatrix) => {
 
   // if the schema has changed, we need to save
   const currentWritable = writableAnnotations(annoMatrix);
-  if (_.difference(currentWritable, writableAnnotations(lastSavedAnnoMatrix))) {
+  if (difference(currentWritable, writableAnnotations(lastSavedAnnoMatrix))) {
     return true;
   }
 
@@ -347,6 +320,7 @@ export const saveObsAnnotationsAction = () => async (dispatch, getState) => {
 
   const df = await annoMatrix.fetch("obs", writableAnnotations(annoMatrix));
   const matrix = MatrixFBS.encodeMatrixFBS(df);
+  const compressedMatrix = pako.deflate(matrix);
   try {
     const queryString =
       !dataCollectionNameIsReadOnly && !!dataCollectionName
@@ -358,7 +332,7 @@ export const saveObsAnnotationsAction = () => async (dispatch, getState) => {
       `${globals.API.prefix}${globals.API.version}annotations/obs${queryString}`,
       {
         method: "PUT",
-        body: matrix,
+        body: compressedMatrix,
         headers: new Headers({
           "Content-Type": "application/octet-stream",
         }),
@@ -385,3 +359,97 @@ export const saveObsAnnotationsAction = () => async (dispatch, getState) => {
     });
   }
 };
+
+export const saveGenesetsAction = () => async (dispatch, getState) => {
+  const state = getState();
+
+  // bail if gene sets not available, or in readonly mode.
+  const { config } = state;
+  const { lastTid, genesets } = state.genesets;
+
+  const genesetsAreAvailable =
+    config?.parameters?.annotations_genesets ?? false;
+  const genesetsReadonly =
+    config?.parameters?.annotations_genesets_readonly ?? true;
+  if (!genesetsAreAvailable || genesetsReadonly) {
+    // our non-save was completed!
+    return dispatch({
+      type: "autosave: genesets complete",
+      lastSavedGenesets: genesets,
+    });
+  }
+
+  dispatch({
+    type: "autosave: genesets started",
+  });
+
+  /* Create the JSON OTA data structure */
+  const tid = (lastTid ?? 0) + 1;
+  const genesetsOTA = [];
+  for (const [name, gs] of genesets) {
+    const genes = [];
+    for (const g of gs.genes.values()) {
+      genes.push({
+        gene_symbol: g.geneSymbol,
+        gene_description: g.geneDescription,
+      });
+    }
+    genesetsOTA.push({
+      geneset_name: name,
+      geneset_description: gs.genesetDescription,
+      genes,
+    });
+  }
+  const ota = {
+    tid,
+    genesets: genesetsOTA,
+  };
+
+  /* Save to server */
+  try {
+    const { dataCollectionNameIsReadOnly, dataCollectionName } =
+      state.annotations;
+    const queryString =
+      !dataCollectionNameIsReadOnly && !!dataCollectionName
+        ? `?annotation-collection-name=${encodeURIComponent(
+            dataCollectionName
+          )}`
+        : "";
+
+    const res = await fetch(
+      `${globals.API.prefix}${globals.API.version}genesets${queryString}`,
+      {
+        method: "PUT",
+        headers: new Headers({
+          Accept: "application/json",
+          "Content-Type": "application/json",
+        }),
+        body: JSON.stringify(ota),
+        credentials: "include",
+      }
+    );
+    if (!res.ok) {
+      return dispatch({
+        type: "autosave: genesets error",
+        message: `HTTP error ${res.status} - ${res.statusText}`,
+        res,
+      });
+    }
+    return Promise.all([
+      dispatch({
+        type: "autosave: genesets complete",
+        lastSavedGenesets: genesets,
+      }),
+      dispatch({
+        type: "geneset: set tid",
+        tid,
+      }),
+    ]);
+  } catch (error) {
+    return dispatch({
+      type: "autosave: genesets error",
+      message: error.toString(),
+      error,
+    });
+  }
+};

client/src/actions/embedding.js

@@ -11,7 +11,7 @@ export async function _switchEmbedding(
   newEmbeddingName
 ) {
   /*
-  DRY helper used by this and reembedding action creators
+  DRY helper used by embedding action creators
   */
   const base = prevAnnoMatrix.base();
   const embeddingDf = await base.fetch("emb", newEmbeddingName);
@@ -25,27 +25,23 @@ export async function _switchEmbedding(
   return [annoMatrix, obsCrossfilter];
 }
 
-export const layoutChoiceAction = (newLayoutChoice) => async (
-  dispatch,
-  getState
-) => {
-  /*
+export const layoutChoiceAction =
+  (newLayoutChoice) => async (dispatch, getState) => {
+    /*
   On layout choice, make sure we have selected all on the previous layout, AND the new
   layout.
   */
-  const {
-    annoMatrix: prevAnnoMatrix,
-    obsCrossfilter: prevCrossfilter,
-  } = getState();
-  const [annoMatrix, obsCrossfilter] = await _switchEmbedding(
-    prevAnnoMatrix,
-    prevCrossfilter,
-    newLayoutChoice
-  );
-  dispatch({
-    type: "set layout choice",
-    layoutChoice: newLayoutChoice,
-    obsCrossfilter,
-    annoMatrix,
-  });
-};
+    const { annoMatrix: prevAnnoMatrix, obsCrossfilter: prevCrossfilter } =
+      getState();
+    const [annoMatrix, obsCrossfilter] = await _switchEmbedding(
+      prevAnnoMatrix,
+      prevCrossfilter,
+      newLayoutChoice
+    );
+    dispatch({
+      type: "set layout choice",
+      layoutChoice: newLayoutChoice,
+      obsCrossfilter,
+      annoMatrix,
+    });
+  };

client/src/actions/geneset.js

@@ -0,0 +1,149 @@
+import { postUserErrorToast } from "../components/framework/toasters";
+/*
+Action creators for gene sets
+
+Primarily used to keep the crossfilter and underlying data in sync with the UI.
+
+The behavior manifest in these action creators:
+
+    Delete a gene set, will
+      * drop index & clear selection state on the gene set summary
+      * drop index & clear selection state of each gene in the geneset
+
+    Delete a gene from a gene set, will:
+      * drop index & clear selection state on the gene set summary
+      * drop index & clear selection state on the gene
+
+    Add a gene to a gene set, will:
+      * drop index & clear selection state on the gene set summary
+      * will NOT touch the selection state for the gene
+
+Note that crossfilter indices are lazy created, as needed.
+*/
+
+export const genesetDelete = (genesetName) => (dispatch, getState) => {
+  const state = getState();
+  const { genesets } = state;
+  const gs = genesets?.genesets?.get(genesetName) ?? {};
+  const geneSymbols = Array.from(gs.genes.keys());
+  const obsCrossfilter = dropGeneset(dispatch, state, genesetName, geneSymbols);
+  if (genesetName === state.colors.colorAccessor) {
+    dispatch({
+      type: "reset colorscale",
+    });
+  }
+  dispatch({
+    type: "geneset: delete",
+    genesetName,
+    obsCrossfilter,
+    annoMatrix: obsCrossfilter.annoMatrix,
+  });
+};
+
+export const genesetAddGenes =
+  (genesetName, genes) => async (dispatch, getState) => {
+    const state = getState();
+    const { obsCrossfilter: prevObsCrossfilter, annoMatrix } = state;
+    const { schema } = annoMatrix;
+    const varIndex = schema.annotations.var.index;
+    const df = await annoMatrix.fetch("var", varIndex);
+    const geneNames = df.col(varIndex).asArray();
+    genes = genes.reduce((acc, gene) => {
+      if (geneNames.indexOf(gene.geneSymbol) === -1) {
+        postUserErrorToast(
+          `${gene.geneSymbol} doesn't appear to be a valid gene name.`
+        );
+      } else acc.push(gene);
+      return acc;
+    }, []);
+
+    const obsCrossfilter = dropGenesetSummaryDimension(
+      prevObsCrossfilter,
+      state,
+      genesetName
+    );
+    dispatch({
+      type: "continuous metadata histogram cancel",
+      continuousNamespace: { isGeneSetSummary: true },
+      selection: genesetName,
+    });
+    return dispatch({
+      type: "geneset: add genes",
+      genesetName,
+      genes,
+      obsCrossfilter,
+      annoMatrix: obsCrossfilter.annoMatrix,
+    });
+  };
+
+export const genesetDeleteGenes =
+  (genesetName, geneSymbols) => (dispatch, getState) => {
+    const state = getState();
+    const obsCrossfilter = dropGeneset(
+      dispatch,
+      state,
+      genesetName,
+      geneSymbols
+    );
+    return dispatch({
+      type: "geneset: delete genes",
+      genesetName,
+      geneSymbols,
+      obsCrossfilter,
+      annoMatrix: obsCrossfilter.annoMatrix,
+    });
+  };
+
+/*
+Private
+*/
+
+function dropGenesetSummaryDimension(obsCrossfilter, state, genesetName) {
+  const { annoMatrix, genesets } = state;
+  const varIndex = annoMatrix.schema.annotations?.var?.index;
+  const gs = genesets?.genesets?.get(genesetName) ?? {};
+  const genes = Array.from(gs.genes.keys());
+  const query = {
+    summarize: {
+      method: "mean",
+      field: "var",
+      column: varIndex,
+      values: genes,
+    },
+  };
+  return obsCrossfilter.dropDimension("X", query);
+}
+
+function dropGeneDimension(obsCrossfilter, state, gene) {
+  const { annoMatrix } = state;
+  const varIndex = annoMatrix.schema.annotations?.var?.index;
+  const query = {
+    where: {
+      field: "var",
+      column: varIndex,
+      value: gene,
+    },
+  };
+  return obsCrossfilter.dropDimension("X", query);
+}
+
+function dropGeneset(dispatch, state, genesetName, geneSymbols) {
+  const { obsCrossfilter: prevObsCrossfilter } = state;
+  const obsCrossfilter = geneSymbols.reduce(
+    (crossfilter, gene) => dropGeneDimension(crossfilter, state, gene),
+    dropGenesetSummaryDimension(prevObsCrossfilter, state, genesetName)
+  );
+  dispatch({
+    type: "continuous metadata histogram cancel",
+    continuousNamespace: { isGeneSetSummary: true },
+    selection: genesetName,
+  });
+  geneSymbols.forEach((g) =>
+    dispatch({
+      type: "continuous metadata histogram cancel",
+      continuousNamespace: { isUserDefined: true },
+      selection: g,
+    })
+  );
+  return obsCrossfilter;
+}

client/src/actions/index.js

@@ -5,14 +5,23 @@ import {
   doJsonRequest,
   dispatchNetworkErrorMessageToUser,
 } from "../util/actionHelpers";
-import {
-  requestReembed /* , reembedResetWorldToUniverse -- disabled temporarily, TODO issue #1606 */,
-} from "./reembed";
 import { loadUserColorConfig } from "../util/stateManager/colorHelpers";
 import * as selnActions from "./selection";
 import * as annoActions from "./annotation";
+import * as spatialActions from "./spatial";
 import * as viewActions from "./viewStack";
 import * as embActions from "./embedding";
+import * as genesetActions from "./geneset";
+
+function setGlobalConfig(config) {
+  /**
+   * Set any global run-time config not _exclusively_ managed by the config reducer.
+   * This should only set fields defined in globals.globalConfig.
+   */
+  globals.globalConfig.maxCategoricalOptionsToDisplay =
+    config?.parameters?.["max-category-items"] ??
+    globals.globalConfig.maxCategoricalOptionsToDisplay;
+}
 
 /*
 return promise fetching user-configured colors
@@ -33,6 +42,9 @@ async function schemaFetch() {
 async function configFetch(dispatch) {
   return fetchJson("config").then((response) => {
     const config = { ...globals.configDefaults, ...response.config };
+
+    setGlobalConfig(config);
+
     dispatch({
       type: "configuration load complete",
       config,
@@ -41,15 +53,25 @@ async function configFetch(dispatch) {
   });
 }
 
-async function userInfoFetch(dispatch) {
-  return fetchJson("userinfo").then((response) => {
-    const { userinfo: userInfo } = response || {};
-    dispatch({
-      type: "userInfo load complete",
-      userInfo,
+async function genesetsFetch(dispatch, config) {
+  /* request genesets ONLY if the backend supports the feature */
+  const defaultResponse = {
+    genesets: [],
+    tid: 0,
+  };
+  if (config?.parameters?.annotations_genesets ?? false) {
+    fetchJson("genesets").then((response) => {
+      dispatch({
+        type: "geneset: initial load",
+        data: response ?? defaultResponse,
+      });
     });
-    return userInfo;
-  });
+  } else {
+    dispatch({
+      type: "geneset: initial load",
+      data: defaultResponse,
+    });
+  }
 }
 
 function prefetchEmbeddings(annoMatrix) {
@@ -73,9 +95,10 @@ const doInitialDataLoad = () =>
         configFetch(dispatch),
         schemaFetch(dispatch),
         userColorsFetchAndLoad(dispatch),
-        userInfoFetch(dispatch),
       ]);
 
+      genesetsFetch(dispatch, config);
+
       const baseDataUrl = `${globals.API.prefix}${globals.API.version}`;
       const annoMatrix = new AnnoMatrixLoader(baseDataUrl, schema.schema);
       const obsCrossfilter = new AnnoMatrixObsCrossfilter(annoMatrix);
@@ -88,7 +111,7 @@ const doInitialDataLoad = () =>
       });
       dispatch({ type: "initial data load complete" });
 
-      const defaultEmbedding = config?.parameters?.["default_embedding"];
+      const defaultEmbedding = config?.parameters?.default_embedding;
       const layoutSchema = schema?.schema?.layout?.obs ?? [];
       if (
         defaultEmbedding &&
@@ -138,72 +161,74 @@ const dispatchDiffExpErrors = (dispatch, response) => {
   }
 };
 
-const requestDifferentialExpression = (set1, set2, num_genes = 10) => async (
-  dispatch,
-  getState
-) => {
-  dispatch({ type: "request differential expression started" });
-  try {
-    /*
+const requestDifferentialExpression =
+  (set1, set2, num_genes = 50) =>
+  async (dispatch, getState) => {
+    dispatch({ type: "request differential expression started" });
+    try {
+      /*
     Steps:
     1. get the most differentially expressed genes
     2. get expression data for each
     */
-    const { annoMatrix } = getState();
-    const varIndexName = annoMatrix.schema.annotations.var.index;
+      const { annoMatrix } = getState();
+      const varIndexName = annoMatrix.schema.annotations.var.index;
 
-    // Legal values are null, Array or TypedArray.  Null is initial state.
-    if (!set1) set1 = [];
-    if (!set2) set2 = [];
+      // Legal values are null, Array or TypedArray.  Null is initial state.
+      if (!set1) set1 = [];
+      if (!set2) set2 = [];
 
-    // These lines ensure that we convert any TypedArray to an Array.
-    // This is necessary because JSON.stringify() does some very strange
-    // things with TypedArrays (they are marshalled to JSON objects, rather
-    // than being marshalled as a JSON array).
-    set1 = Array.isArray(set1) ? set1 : Array.from(set1);
-    set2 = Array.isArray(set2) ? set2 : Array.from(set2);
+      // These lines ensure that we convert any TypedArray to an Array.
+      // This is necessary because JSON.stringify() does some very strange
+      // things with TypedArrays (they are marshalled to JSON objects, rather
+      // than being marshalled as a JSON array).
+      set1 = Array.isArray(set1) ? set1 : Array.from(set1);
+      set2 = Array.isArray(set2) ? set2 : Array.from(set2);
 
-    const res = await fetch(
-      `${globals.API.prefix}${globals.API.version}diffexp/obs`,
-      {
-        method: "POST",
-        headers: new Headers({
-          Accept: "application/json",
-          "Content-Type": "application/json",
-        }),
-        body: JSON.stringify({
-          mode: "topN",
-          count: num_genes,
-          set1: { filter: { obs: { index: set1 } } },
-          set2: { filter: { obs: { index: set2 } } },
-        }),
-        credentials: "include",
+      const res = await fetch(
+        `${globals.API.prefix}${globals.API.version}diffexp/obs`,
+        {
+          method: "POST",
+          headers: new Headers({
+            Accept: "application/json",
+            "Content-Type": "application/json",
+          }),
+          body: JSON.stringify({
+            mode: "topN",
+            count: num_genes,
+            set1: { filter: { obs: { index: set1 } } },
+            set2: { filter: { obs: { index: set2 } } },
+          }),
+          credentials: "include",
+        }
+      );
+
+      if (!res.ok || res.headers.get("Content-Type") !== "application/json") {
+        return dispatchDiffExpErrors(dispatch, res);
       }
-    );
 
-    if (!res.ok || res.headers.get("Content-Type") !== "application/json") {
-      return dispatchDiffExpErrors(dispatch, res);
+      const response = await res.json();
+      const varIndex = await annoMatrix.fetch("var", varIndexName);
+      const diffexpLists = { negative: [], positive: [] };
+      for (const polarity of Object.keys(diffexpLists)) {
+        diffexpLists[polarity] = response[polarity].map((v) => [
+          varIndex.at(v[0], varIndexName),
+          ...v.slice(1),
+        ]);
+      }
+
+      /* then send the success case action through */
+      return dispatch({
+        type: "request differential expression success",
+        data: diffexpLists,
+      });
+    } catch (error) {
+      return dispatch({
+        type: "request differential expression error",
+        error,
+      });
     }
-
-    const response = await res.json();
-    const varIndex = await annoMatrix.fetch("var", varIndexName);
-    const data = response.map((v) => [
-      varIndex.at(v[0], varIndexName),
-      ...v.slice(1),
-    ]);
-
-    /* then send the success case action through */
-    return dispatch({
-      type: "request differential expression success",
-      data,
-    });
-  } catch (error) {
-    return dispatch({
-      type: "request differential expression error",
-      error,
-    });
-  }
-};
+  };
 
 function fetchJson(pathAndQuery) {
   return doJsonRequest(
@@ -216,7 +241,6 @@ export default {
   requestDifferentialExpression,
   requestSingleGeneExpressionCountsForColoringPOST,
   requestUserDefinedGene,
-  requestReembed,
   selectContinuousMetadataAction: selnActions.selectContinuousMetadataAction,
   selectCategoricalMetadataAction: selnActions.selectCategoricalMetadataAction,
   selectCategoricalAllMetadataAction:
@@ -242,7 +266,12 @@ export default {
   annotationRenameLabelInCategory: annoActions.annotationRenameLabelInCategory,
   annotationLabelCurrentSelection: annoActions.annotationLabelCurrentSelection,
   saveObsAnnotationsAction: annoActions.saveObsAnnotationsAction,
+  saveGenesetsAction: annoActions.saveGenesetsAction,
   needToSaveObsAnnotations: annoActions.needToSaveObsAnnotations,
   layoutChoiceAction: embActions.layoutChoiceAction,
   setCellSetFromSelection: selnActions.setCellSetFromSelection,
+  genesetDelete: genesetActions.genesetDelete,
+  genesetAddGenes: genesetActions.genesetAddGenes,
+  genesetDeleteGenes: genesetActions.genesetDeleteGenes,
+  requestSpatialMetadata: spatialActions.requestSpatialMetadata,
 };

client/src/actions/reembed.js

@@ -1,109 +0,0 @@
-import { API } from "../globals";
-import {
-  postNetworkErrorToast,
-  postAsyncSuccessToast,
-  postAsyncFailureToast,
-} from "../components/framework/toasters";
-import { _switchEmbedding } from "./embedding";
-
-function abortableFetch(request, opts, timeout = 0) {
-  const controller = new AbortController();
-  const { signal } = controller;
-
-  return {
-    abort: () => controller.abort(),
-    isAborted: () => signal.aborted,
-    ready: () => {
-      if (timeout) {
-        setTimeout(() => controller.abort(), timeout);
-      }
-      return fetch(request, { ...opts, signal });
-    },
-  };
-}
-
-async function doReembedFetch(dispatch, getState) {
-  const state = getState();
-  let cells = state.annoMatrix.rowIndex.labels();
-
-  // These lines ensure that we convert any TypedArray to an Array.
-  // This is necessary because JSON.stringify() does some very strange
-  // things with TypedArrays (they are marshalled to JSON objects, rather
-  // than being marshalled as a JSON array).
-  cells = Array.isArray(cells) ? cells : Array.from(cells);
-
-  const af = abortableFetch(
-    `${API.prefix}${API.version}layout/obs`,
-    {
-      method: "PUT",
-      headers: new Headers({
-        Accept: "application/octet-stream",
-        "Content-Type": "application/json",
-      }),
-      body: JSON.stringify({
-        method: "umap",
-        filter: { obs: { index: cells } },
-      }),
-      credentials: "include",
-    },
-    60000 // 1 minute timeout
-  );
-  dispatch({
-    type: "reembed: request start",
-    abortableFetch: af,
-  });
-  const res = await af.ready();
-
-  if (res.ok && res.headers.get("Content-Type").includes("application/json")) {
-    return res;
-  }
-
-  // else an error
-  let msg = `Unexpected HTTP response ${res.status}, ${res.statusText}`;
-  const body = await res.text();
-  if (body && body.length > 0) {
-    msg = `${msg} -- ${body}`;
-  }
-  throw new Error(msg);
-}
-
-/*
-functions below are dispatch-able
-*/
-export function requestReembed() {
-  return async (dispatch, getState) => {
-    try {
-      const res = await doReembedFetch(dispatch, getState);
-      const schema = await res.json();
-      dispatch({
-        type: "reembed: request completed",
-      });
-
-      const { annoMatrix: prevAnnoMatrix, obsCrossfilter: prevCrossfilter } = getState();
-      const base = prevAnnoMatrix.base().addEmbedding(schema);
-      const [annoMatrix, obsCrossfilter] = await _switchEmbedding(
-        base,
-        prevCrossfilter,
-        schema.name
-      );
-      dispatch({
-        type: "reembed: add reembedding",
-        schema,
-        annoMatrix,
-        obsCrossfilter,
-      });
-
-      postAsyncSuccessToast("Re-embedding has completed.");
-    } catch (error) {
-      dispatch({
-        type: "reembed: request aborted",
-      });
-      if (error.name === "AbortError") {
-        postAsyncFailureToast("Re-embedding calculation was aborted.");
-      } else {
-        postNetworkErrorToast(`Re-embedding: ${error.message}`);
-      }
-      console.log("Reembed exception:", error, error.name, error.message);
-    }
-  };
-}

client/src/actions/selection.js

@@ -1,104 +1,101 @@
 /*
 Action creators for selection 
 */
-export const selectContinuousMetadataAction = (
-  type,
-  query,
-  range,
-  oldProps = {}
-) => async (dispatch, getState) => {
-  const { obsCrossfilter: prevObsCrossfilter } = getState();
+export const selectContinuousMetadataAction =
+  (type, query, range, oldProps = {}) =>
+  async (dispatch, getState) => {
+    const { obsCrossfilter: prevObsCrossfilter } = getState();
 
-  const selection = range
-    ? {
-        mode: "range",
-        lo: range[0],
-        hi: range[1],
-        inclusive: true, // [lo, hi] incluisve selection
-      }
-    : { mode: "all" };
+    const selection = range
+      ? {
+          mode: "range",
+          lo: range[0],
+          hi: range[1],
+          inclusive: true, // [lo, hi] incluisve selection
+        }
+      : { mode: "all" };
 
-  const obsCrossfilter = await prevObsCrossfilter.select(...query, selection);
+    const obsCrossfilter = await prevObsCrossfilter.select(...query, selection);
 
-  dispatch({
-    type,
-    obsCrossfilter,
-    range,
-    ...oldProps,
-  });
-};
-
-export const selectCategoricalMetadataAction = (
-  type, // action type
-  metadataField, // annotation category name
-  labels,
-  label, // the label being selected/deselected
-  isSelected, // bool
-  oldProps = {}
-) => async (dispatch, getState) => {
-  const {
-    obsCrossfilter: prevObsCrossfilter,
-    categoricalSelection,
-  } = getState();
-
-  const labelSelectionState = new Map(categoricalSelection[metadataField]);
-  labels.forEach(
-    (l) => labelSelectionState.has(l) || labelSelectionState.set(l, true)
-  );
-  labelSelectionState.set(label, isSelected);
-
-  const values = Array.from(labelSelectionState.keys()).filter((k) =>
-    labelSelectionState.get(k)
-  );
-  const selection = {
-    mode: "exact",
-    values,
+    dispatch({
+      type,
+      obsCrossfilter,
+      range,
+      ...oldProps,
+    });
   };
-  const obsCrossfilter = await prevObsCrossfilter.select(
-    "obs",
-    metadataField,
-    selection
-  );
 
-  dispatch({
-    type,
-    obsCrossfilter,
-    metadataField,
-    labelSelectionState,
-    ...oldProps,
-  });
-};
+export const selectCategoricalMetadataAction =
+  (
+    type, // action type
+    metadataField, // annotation category name
+    labels,
+    label, // the label being selected/deselected
+    isSelected, // bool
+    oldProps = {}
+  ) =>
+  async (dispatch, getState) => {
+    const { obsCrossfilter: prevObsCrossfilter, categoricalSelection } =
+      getState();
 
-export const selectCategoricalAllMetadataAction = (
-  type, // action type
-  metadataField, // annotation category name
-  labels,
-  isSelected, // bool, select all or none
-  oldProps = {}
-) => async (dispatch, getState) => {
-  const {
-    obsCrossfilter: prevObsCrossfilter,
-    categoricalSelection,
-  } = getState();
+    const labelSelectionState = new Map(categoricalSelection[metadataField]);
+    labels.forEach(
+      (l) => labelSelectionState.has(l) || labelSelectionState.set(l, true)
+    );
+    labelSelectionState.set(label, isSelected);
 
-  const labelSelectionState = new Map(categoricalSelection[metadataField]);
-  labels.forEach((label) => labelSelectionState.set(label, isSelected));
+    const values = Array.from(labelSelectionState.keys()).filter((k) =>
+      labelSelectionState.get(k)
+    );
+    const selection = {
+      mode: "exact",
+      values,
+    };
+    const obsCrossfilter = await prevObsCrossfilter.select(
+      "obs",
+      metadataField,
+      selection
+    );
 
-  const selection = { mode: isSelected ? "all" : "none" };
-  const obsCrossfilter = await prevObsCrossfilter.select(
-    "obs",
-    metadataField,
-    selection
-  );
+    dispatch({
+      type,
+      obsCrossfilter,
+      metadataField,
+      labelSelectionState,
+      ...oldProps,
+    });
+  };
 
-  dispatch({
-    type,
-    obsCrossfilter,
-    metadataField,
-    labelSelectionState,
-    ...oldProps,
-  });
-};
+export const selectCategoricalAllMetadataAction =
+  (
+    type, // action type
+    metadataField, // annotation category name
+    labels,
+    isSelected, // bool, select all or none
+    oldProps = {}
+  ) =>
+  async (dispatch, getState) => {
+    const { obsCrossfilter: prevObsCrossfilter, categoricalSelection } =
+      getState();
+
+    const labelSelectionState = new Map(categoricalSelection[metadataField]);
+    labels.forEach((label) => labelSelectionState.set(label, isSelected));
+
+    const selection = { mode: isSelected ? "all" : "none" };
+    const obsCrossfilter = await prevObsCrossfilter.select(
+      "obs",
+      metadataField,
+      selection
+    );
+
+    dispatch({
+      type,
+      obsCrossfilter,
+      metadataField,
+      labelSelectionState,
+      ...oldProps,
+    });
+  };
 
 /**
  ** Graph selection-related actions
@@ -108,25 +105,23 @@ export const graphBrushStartAction = () =>
   /* no change to crossfilter until a change fires */
   ({ type: "graph brush start" });
 
-const _graphBrushWithinRectAction = (type, embName, brushCoords) => async (
-  dispatch,
-  getState
-) => {
-  const { obsCrossfilter: prevObsCrossfilter } = getState();
+const _graphBrushWithinRectAction =
+  (type, embName, brushCoords) => async (dispatch, getState) => {
+    const { obsCrossfilter: prevObsCrossfilter } = getState();
 
-  const selection = { mode: "within-rect", ...brushCoords };
-  const obsCrossfilter = await prevObsCrossfilter.select(
-    "emb",
-    embName,
-    selection
-  );
+    const selection = { mode: "within-rect", ...brushCoords };
+    const obsCrossfilter = await prevObsCrossfilter.select(
+      "emb",
+      embName,
+      selection
+    );
 
-  dispatch({
-    type,
-    obsCrossfilter,
-    brushCoords,
-  });
-};
+    dispatch({
+      type,
+      obsCrossfilter,
+      brushCoords,
+    });
+  };
 
 const _graphAllAction = (type, embName) => async (dispatch, getState) => {
   const { obsCrossfilter: prevObsCrossfilter } = getState();
@@ -162,28 +157,26 @@ export const graphLassoCancelAction = (embName) =>
 export const graphLassoDeselectAction = (embName) =>
   _graphAllAction("graph lasso cancel", embName);
 
-export const graphLassoEndAction = (embName, polygon) => async (
-  dispatch,
-  getState
-) => {
-  const { obsCrossfilter: prevObsCrossfilter } = getState();
+export const graphLassoEndAction =
+  (embName, polygon) => async (dispatch, getState) => {
+    const { obsCrossfilter: prevObsCrossfilter } = getState();
 
-  const selection = {
-    mode: "within-polygon",
-    polygon,
+    const selection = {
+      mode: "within-polygon",
+      polygon,
+    };
+    const obsCrossfilter = await prevObsCrossfilter.select(
+      "emb",
+      embName,
+      selection
+    );
+
+    dispatch({
+      type: "graph lasso end",
+      obsCrossfilter,
+      polygon,
+    });
   };
-  const obsCrossfilter = await prevObsCrossfilter.select(
-    "emb",
-    embName,
-    selection
-  );
-
-  dispatch({
-    type: "graph lasso end",
-    obsCrossfilter,
-    polygon,
-  });
-};
 
 /*
 Differential expression set selection

client/src/actions/spatial.js

@@ -0,0 +1,35 @@
+import * as globals from "../globals";
+
+export const requestSpatialMetadata = () => async (dispatch) => {
+  dispatch({ type: "request spatial metadata started" });
+  try {
+    const res = await fetch(
+      `${globals.API.prefix}${globals.API.version}spatial/meta`,
+      {
+        method: "GET",
+        headers: new Headers({
+          Accept: "application/json",
+          "Content-Type": "application/json",
+        }),
+        credentials: "include",
+      }
+    );
+
+    if (!res.ok || res.headers.get("Content-Type") !== "application/json") {
+      return null; // TODO need a dispatch //dispatchDiffExpErrors(dispatch, res);
+    }
+
+    const response = await res.json();
+
+    /* then send the success case action through */
+    return dispatch({
+      type: "request spatial metadata success",
+      data: response,
+    });
+  } catch (error) {
+    return dispatch({
+      type: "request spatial metadata error",
+      error,
+    });
+  }
+};

client/src/actions/viewStack.js

@@ -42,10 +42,8 @@ export const subsetAction = () => (dispatch, getState) => {
   By convention, a clip view is ALWAYS the top view, so if present, pop
   off and re-apply
   */
-  const {
-    annoMatrix: prevAnnoMatrix,
-    obsCrossfilter: prevObsCrossfilter,
-  } = getState();
+  const { annoMatrix: prevAnnoMatrix, obsCrossfilter: prevObsCrossfilter } =
+    getState();
   const annoMatrix = _userSubsetAnnoMatrix(
     prevAnnoMatrix,
     prevObsCrossfilter.allSelectedMask()

client/src/annoMatrix/annoMatrix.js

@@ -12,6 +12,7 @@ import {
 import { indexEntireSchema } from "../util/stateManager/schemaHelpers";
 import { _whereCacheGet, _whereCacheMerge } from "./whereCache";
 import _shallowClone from "./clone";
+import { _queryValidate, _queryCacheKey } from "./query";
 
 const _dataframeCache = dataframeMemo(128);
 
@@ -181,7 +182,7 @@ export default class AnnoMatrix {
 		Returns a Promise for the query result, which will resolve to a dataframe.
 
 		Field must be one of the matrix fields: 'obs', 'var', 'X', 'emb'.  Value
-		represents the underlying object upon which the query is occuring.
+		represents the underlying object upon which the query is occurring.
 
 		Query is one of:
 			* a string, representing a single column name from the field, eg,
@@ -197,12 +198,6 @@ export default class AnnoMatrix {
 		field/column, similar to a join.  Currently only supported on the var
 		dimension, allowing query of X columns by var value (eg, gene name)
 
-    The query filter	is a single value filter:
-			{ "field name": [
-				{name: "column name", values: [ list of values ]}
-			]}
-		One and only one value filter is allowed in a value query.
-
 		Examples:
 
     1. Fetch the "n_genes" column the "obs":
@@ -220,7 +215,9 @@ export default class AnnoMatrix {
        value "TYMP" in the var index.
 
 			fetch("X", { 
-				where: {field: "var", column: this.schema.annotations.var.index, value: "TYMP"}
+				where: {
+          field: "var", column: this.schema.annotations.var.index, value: "TYMP"
+        }
 			})
 
       In AnnData & Pandas DataFrame API, this is equivalent to:
@@ -410,6 +407,14 @@ export default class AnnoMatrix {
     _subclassResponsibility();
   }
 
+  getCacheKeys(field, query) {
+    /*
+    Return cache keys for columns associated with this query.  May return
+    [unknown] if no keys are known (ie, nothing is or was cached).
+    */
+    return _whereCacheGet(this._whereCache, this.schema, field, query);
+  }
+
   /**
    ** Private interfaces below.
    **/
@@ -427,6 +432,7 @@ export default class AnnoMatrix {
   async _fetch(field, q) {
     if (!AnnoMatrix.fields().includes(field)) return undefined;
     const queries = Array.isArray(q) ? q : [q];
+    queries.forEach(_queryValidate);
 
     /* find cached columns we need, and GC the rest */
     const cachedColumns = this._resolveCachedQueries(field, queries);
@@ -507,7 +513,7 @@ export default class AnnoMatrix {
     * obs, var and emb do not grow without bounds, and are needed constantly 
       for rendering.  
       a) There is no upside to GC'ing these in the base (loader)
-      b) The undo/redo cache can hold a large number in views, which is worht GC'ing
+      b) The undo/redo cache can hold a large number in views, which is worth GC'ing
     * X is often much larger than memory, and the UI allows add/del from
       this.  Most of the GC potential is here in both the base and views.
 
@@ -522,7 +528,7 @@ export default class AnnoMatrix {
   as much of the cache is pinned by that data structure.
   */
   _gcField(field, isHot, pinnedColumns) {
-    const maxColumns = isHot ? 256 : 10; // maybe to aggessive?
+    const maxColumns = isHot ? 256 : 10; // maybe to aggressive?
 
     const cache = this._cache[field];
     if (cache.colIndex.size() < maxColumns) return; // trivial rejection
@@ -590,23 +596,18 @@ export default class AnnoMatrix {
     called each time a query is performed, allowing the gc to update any bookkeeping
     information.  Currently, this is just a simple last-fetched timestamp, stored
     in a Map.
-
-    Map objects preserve order of insertion. This is leveraged as a cheap way to
-    do LRU, by removing and re-inserting keys.  IMPORTANT: the cleanup code assumes
-    the map insertion order is least-recently-used first.
     */
     const cols = dataframe.colIndex.labels();
     const { _gcInfo } = this;
     const now = Date.now();
     cols.forEach((c) => {
-      // gcInfo.delete(c);
       _gcInfo.set(_columnCacheKey(field, c), now);
     });
   }
 
   /**
-  Cloning sublcass protocol - we rely in cloning to preserve immutable
-  symantics while not causing races or other side effects in internal
+  Cloning subclass protocol - we rely in cloning to preserve immutable
+  semantics while not causing races or other side effects in internal
   cache management.
 
   Subclasses must override _cloneDeeper() if they have state which requires 
@@ -639,15 +640,6 @@ export default class AnnoMatrix {
 /*
 private utility functions below
 */
-
-function _queryCacheKey(field, query) {
-  if (typeof query === "object") {
-    const { field: queryField, column: queryColumn, value: queryValue } = query;
-    return `${field}/${queryField}/${queryColumn}/${queryValue}`;
-  }
-  return `${field}/${query}`;
-}
-
 function _columnCacheKey(field, column) {
   return `${field}/${column}`;
 }

client/src/annoMatrix/crossfilter.js

@@ -123,6 +123,24 @@ export default class AnnoMatrixObsCrossfilter {
     return new AnnoMatrixObsCrossfilter(annoMatrix, this.obsCrossfilter);
   }
 
+  /**
+   * Drop the crossfilter dimension. Do not change the annoMatrix. Useful when we
+   * want to stop trackin the selection state, but aren't sure we want to blow the
+   * annomatrix cache.
+   */
+  dropDimension(field, query) {
+    const { annoMatrix } = this;
+    let { obsCrossfilter } = this;
+    const keys = annoMatrix
+      .getCacheKeys(field, query)
+      .filter((k) => k !== undefined);
+    const dimName = _dimensionName(field, keys);
+    if (obsCrossfilter.hasDimension(dimName)) {
+      obsCrossfilter = obsCrossfilter.delDimension(dimName);
+    }
+    return new AnnoMatrixObsCrossfilter(annoMatrix, obsCrossfilter);
+  }
+
   /**
   Selection state - API is identical to ImmutableTypedCrossfilter, as these
   are just wrappers to lazy create indices.
@@ -164,9 +182,7 @@ export default class AnnoMatrixObsCrossfilter {
 		*/
     const { annoMatrix } = this;
     const currentDims = this.obsCrossfilter.dimensionNames();
-    const obsCrossfilter = currentDims.reduce((xfltr, dim) => {
-      return xfltr.select(dim, { mode: "all" });
-    }, this.obsCrossfilter);
+    const obsCrossfilter = currentDims.reduce((xfltr, dim) => xfltr.select(dim, { mode: "all" }), this.obsCrossfilter);
     return new AnnoMatrixObsCrossfilter(annoMatrix, obsCrossfilter);
   }
 

client/src/annoMatrix/fetchHelpers.js

@@ -1,4 +1,4 @@
-export { doBinaryRequest } from "../util/actionHelpers";
+export { doBinaryRequest, doFetch } from "../util/actionHelpers";
 
 /* double URI encode - needed for query-param filters */
 export function _dubEncURIComp(s) {
@@ -19,9 +19,7 @@ export function _fetchResult(promise) {
     }
   );
 
-  res.status = () => {
-    return _status;
-  };
+  res.status = () => _status;
 
   return res;
 }

client/src/annoMatrix/loader.js

@@ -1,6 +1,6 @@
-import { doBinaryRequest, _dubEncURIComp } from "./fetchHelpers";
+import { doBinaryRequest, doFetch } from "./fetchHelpers";
 import { matrixFBSToDataframe } from "../util/stateManager/matrix";
-import { _getColumnSchema, _normalizeCategoricalSchema } from "./schema";
+import { _getColumnSchema } from "./schema";
 import {
   addObsAnnoColumn,
   removeObsAnnoColumn,
@@ -12,6 +12,17 @@ import { isArrayOrTypedArray } from "../util/typeHelpers";
 import { _whereCacheCreate } from "./whereCache";
 import AnnoMatrix from "./annoMatrix";
 import PromiseLimit from "../util/promiseLimit";
+import {
+  _expectSimpleQuery,
+  _expectComplexQuery,
+  _urlEncodeLabelQuery,
+  _urlEncodeComplexQuery,
+  _hashStringValues,
+} from "./query";
+import {
+  normalizeResponse,
+  normalizeWritableCategoricalSchema,
+} from "./normalize";
 
 const promiseThrottle = new PromiseLimit(5);
 
@@ -115,7 +126,7 @@ export default class AnnoMatrixLoader extends AnnoMatrix {
       data = new Ctor(this.nObs).fill(value);
     }
     newAnnoMatrix._cache.obs = this._cache.obs.withCol(colName, data);
-    _normalizeCategoricalSchema(
+    normalizeWritableCategoricalSchema(
       colSchema,
       newAnnoMatrix._cache.obs.col(colName)
     );
@@ -223,27 +234,23 @@ export default class AnnoMatrixLoader extends AnnoMatrix {
     /*
     _doLoad - evaluates the query against the field. Returns:
       * whereCache update: column query map mapping the query to the column labels
-      * Dataframe containing the new colums (one per dimension)
+      * Dataframe containing the new columns (one per dimension)
     */
-    let urlQuery;
-    let urlBase;
+    let doRequest;
     let priority = 10; // default fetch priority
 
     switch (field) {
       case "obs":
       case "var": {
-        urlBase = `${this.baseURL}annotations/${field}`;
-        urlQuery = _encodeQuery("annotation-name", query);
+        doRequest = _obsOrVarLoader(this.baseURL, field, query);
         break;
       }
       case "X": {
-        urlBase = `${this.baseURL}data/var`;
-        urlQuery = _encodeQuery(undefined, query);
+        doRequest = _XLoader(this.baseURL, field, query);
         break;
       }
       case "emb": {
-        urlBase = `${this.baseURL}layout/obs`;
-        urlQuery = _encodeQuery("layout-name", query);
+        doRequest = _embLoader(this.baseURL, field, query);
         priority = 0; // high prio load for embeddings
         break;
       }
@@ -251,13 +258,8 @@ export default class AnnoMatrixLoader extends AnnoMatrix {
         throw new Error("Unknown field name");
     }
 
-    const url = `${urlBase}?${urlQuery}`;
-    const buffer = await promiseThrottle.priorityAdd(
-      priority,
-      doBinaryRequest,
-      url
-    );
-    const result = matrixFBSToDataframe(buffer);
+    const buffer = await promiseThrottle.priorityAdd(priority, doRequest);
+    let result = matrixFBSToDataframe(buffer);
     if (!result || result.isEmpty()) throw Error("Unknown field/col");
 
     const whereCacheUpdate = _whereCacheCreate(
@@ -266,13 +268,7 @@ export default class AnnoMatrixLoader extends AnnoMatrix {
       result.colIndex.labels()
     );
 
-    if (field === "obs") {
-      /* cough, cough - see comment on method */
-      _normalizeCategoricalSchema(
-        this.schema.annotations.obsByName[query],
-        result.col(query)
-      );
-    }
+    result = normalizeResponse(field, query, this.schema, result);
 
     return [whereCacheUpdate, result];
   }
@@ -282,17 +278,6 @@ export default class AnnoMatrixLoader extends AnnoMatrix {
 Utility functions below
 */
 
-function _encodeQuery(colKey, q) {
-  if (typeof q === "object") {
-    const { field: queryField, column: queryColumn, value: queryValue } = q;
-    return `${_dubEncURIComp(queryField)}:${_dubEncURIComp(
-      queryColumn
-    )}=${_dubEncURIComp(queryValue)}`;
-  }
-  if (!colKey) throw new Error("Unsupported query by name");
-  return `${colKey}=${encodeURIComponent(q)}`;
-}
-
 function _writableCheck(colSchema) {
   if (!colSchema?.writable) {
     throw new Error("Unknown or readonly obs column");
@@ -305,3 +290,57 @@ function _writableCategoryTypeCheck(colSchema) {
     throw new Error("column must be categorical");
   }
 }
+
+function _embLoader(baseURL, _field, query) {
+  _expectSimpleQuery(query);
+
+  const urlBase = `${baseURL}layout/obs`;
+  const urlQuery = _urlEncodeLabelQuery("layout-name", query);
+  const url = `${urlBase}?${urlQuery}`;
+  return () => doBinaryRequest(url);
+}
+
+function _obsOrVarLoader(baseURL, field, query) {
+  _expectSimpleQuery(query);
+
+  const urlBase = `${baseURL}annotations/${field}`;
+  const urlQuery = _urlEncodeLabelQuery("annotation-name", query);
+  const url = `${urlBase}?${urlQuery}`;
+  return () => doBinaryRequest(url);
+}
+
+function _XLoader(baseURL, field, query) {
+  _expectComplexQuery(query);
+
+  if (query.where) {
+    const urlBase = `${baseURL}data/var`;
+    const urlQuery = _urlEncodeComplexQuery(query);
+    const url = `${urlBase}?${urlQuery}`;
+    return () => doBinaryRequest(url);
+  }
+
+  if (query.summarize) {
+    const urlBase = `${baseURL}summarize/var`;
+    const urlQuery = _urlEncodeComplexQuery(query);
+
+    if (urlBase.length + urlQuery.length < 2000) {
+      const url = `${urlBase}?${urlQuery}`;
+      return () => doBinaryRequest(url);
+    }
+
+    const url = `${urlBase}?key=${_hashStringValues([urlQuery])}`;
+    return async () => {
+      const res = await doFetch(url, {
+        method: "POST",
+        body: urlQuery,
+        headers: new Headers({
+          Accept: "application/octet-stream",
+          "Content-Type": "application/x-www-form-urlencoded",
+        }),
+      });
+      return res.arrayBuffer();
+    };
+  }
+
+  throw new Error("Unknown query structure");
+}

client/src/annoMatrix/normalize.js

@@ -0,0 +1,150 @@
+import { _getColumnSchema, _isIndex } from "./schema";
+import catLabelSort from "../util/catLabelSort";
+import {
+  unassignedCategoryLabel,
+  overflowCategoryLabel,
+  globalConfig,
+} from "../globals";
+
+export function normalizeResponse(field, query, schema, response) {
+  /**
+   * There are a number of assumptions in the front-end about data typing and data
+   * characteristics. This routine will normalize a server response dataframe
+   * to match front-end expectations and UI conventions. This includes cast/transform
+   * of the data and schema updates.
+   *
+   * This consolidates all assumptions into one location, for ease of update.
+   *
+   * Currently, this includes normalization for obs/var columns only:
+   *
+   * - Dataframe columns in var/obs that are declared type: boolean may be sent by
+   *   the server in a variety of formats (eg uint8, etc).  Cast to JS Array[boolean]
+   *
+   * - "Categorical" columns may not have all categories represented in the server-provided
+   *   schema (for valid reasons, eg, floating point rounding differences).  For all
+   *   types we treat as categorical in the UI (string, boolean, categorical), update
+   *   the schema to contain all categories as a convenience.
+   *
+   * - "Categorical" columns (ie, string, boolean, categorical) may contain an excess
+   *   of category values (aka labels).  Consolidate any excess into an "all other"
+   *   category.
+   */
+
+  // currently no data or schema normalization necessary for X or emb
+  if (field !== "obs" && field !== "var") return response;
+
+  const colLabels = response.colIndex.labels();
+  for (const colLabel of colLabels) {
+    const colSchema = _getColumnSchema(schema, field, colLabel);
+    const isIndex = _isIndex(schema, field, colLabel);
+    const { type, writable } = colSchema;
+
+    // Boolean data -- cast entire array to Array[bool]
+    if (type === "boolean") {
+      response = castColumnToBoolean(response, colLabel);
+    }
+
+    // Types that are categorical in UI (string, boolean, categorical) OR are writable
+    // are introspected to ensure the schema `categories` field and data values match,
+    // and that we do not have an excess of category values (for non-writable columns)
+    const isEnumType =
+      type === "boolean" ||
+      type === "string" ||
+      type === "categorical" ||
+      writable;
+    if (!isIndex && isEnumType) {
+      response = normalizeCategorical(response, colLabel, colSchema);
+    }
+  }
+  return response;
+}
+
+function castColumnToBoolean(df, label) {
+  const colData = df.col(label).asArray();
+  const newColData = new Array(colData.length);
+  for (let i = 0; i < colData.length; i += 1) newColData[i] = !!colData[i];
+  df = df.replaceColData(label, newColData);
+  return df;
+}
+
+export function normalizeWritableCategoricalSchema(colSchema, col) {
+  /*
+  Ensure all enum writable / categorical schema have a categories array, that
+  the categories array contains all unique values in the data array, AND that
+  the array is UI sorted.
+  */
+  const categorySet = new Set(
+    col.summarizeCategorical().categories.concat(colSchema.categories ?? [])
+  );
+  if (!categorySet.has(unassignedCategoryLabel)) {
+    categorySet.add(unassignedCategoryLabel);
+  }
+  colSchema.categories = catLabelSort(true, Array.from(categorySet));
+  return colSchema;
+}
+
+export function normalizeCategorical(df, colLabel, colSchema) {
+  /*
+  If writable, ensure schema matches data and we have an unassigned label
+  If not writable, ensure schema matches data and that we consolidate labels in excess
+  of "top N" into an overflow labels.
+  */
+  const { writable } = colSchema;
+  const col = df.col(colLabel);
+
+  if (writable) {
+    // writable (aka user) annotations
+    normalizeWritableCategoricalSchema(colSchema, col);
+    return df;
+  }
+
+  // else read-only, categorical columns
+  const TopN = globalConfig.maxCategoricalOptionsToDisplay;
+
+  // consolidate all categories from data and schema into a single list
+  const colDataSummary = col.summarizeCategorical();
+  const allCategories = new Set(
+    colDataSummary.categories.concat(colSchema.categories ?? [])
+  );
+
+  // if no overflow, just UI sort schema categories and return
+  if (allCategories.size <= TopN) {
+    colSchema.categories = catLabelSort(writable, [...allCategories.keys()]);
+    return df;
+  }
+
+  // Otherwise, pick top N categories by count and rewrite data
+
+  // choose unique overflow category label
+  let overflowCatName = `${colLabel}${overflowCategoryLabel}`;
+  while (allCategories.has(overflowCatName)) {
+    overflowCatName += "_";
+  }
+
+  // pick top N category labels and add overflow label
+  const topNCategories = new Set(
+    [...colDataSummary.categoryCounts.keys()].slice(0, TopN)
+  );
+  topNCategories.add(overflowCatName);
+
+  // rewrite data - consolidate all excess labels into overflow label
+  const newColData = Array.from(col.asArray());
+  for (let i = 0; i < newColData.length; i += 1) {
+    if (!topNCategories.has(newColData[i])) {
+      newColData[i] = overflowCatName;
+    }
+  }
+
+  // replace data in dataframe
+  df = df.replaceColData(colLabel, newColData);
+
+  // Update schema with categories, in UI sort order. Ensure overflow label is at end
+  // of list for display purposes.
+  const revisedCategories = df.col(colLabel).summarizeCategorical().categories;
+  revisedCategories.push(
+    revisedCategories.splice(revisedCategories.indexOf(overflowCatName), 1)[0]
+  );
+  colSchema.categories = catLabelSort(writable, revisedCategories);
+
+  return df;
+}

client/src/annoMatrix/query.js

@@ -0,0 +1,126 @@
+import sha1 from "sha1";
+import { _dubEncURIComp } from "./fetchHelpers";
+
+/**
+ * Query utilities, mostly for debugging support and validation.
+ */
+
+/**
+ * Normalize & error check the query.
+ * @param {object | string} query - the query
+ * @returns {object | string} - the normalized query
+ */
+export function _queryValidate(query) {
+  if (typeof query !== "object") return query;
+
+  if (query.where && query.summarize)
+    throw new Error("query may not specify both where and summarize");
+  if (query.where) {
+    const {
+      field: queryField,
+      column: queryColumn,
+      value: queryValue,
+    } = query.where;
+    if (!queryField || !queryColumn || !queryValue)
+      throw new Error("Incomplete where query");
+    return query;
+  }
+  if (query.summarize) {
+    const {
+      field: queryField,
+      column: queryColumn,
+      values: queryValues,
+    } = query.summarize;
+    if (!queryField || !queryColumn || !queryValues)
+      throw new Error("Incomplete where query");
+    if (!Array.isArray(queryValues))
+      throw new Error("Summarize query values must be an array");
+    return query;
+  }
+  throw new Error("query must specify one of where or summarize");
+}
+
+export function _expectSimpleQuery(query) {
+  if (typeof query === "object") throw new Error("expected simple query");
+}
+
+export function _expectComplexQuery(query) {
+  if (typeof query !== "object") throw new Error("expected complex query");
+}
+
+/**
+ * Generate a unique key which can be used to reference this query.
+ *
+ * @param {string} field
+ * @param {string|object} query
+ * @returns the key
+ */
+export function _queryCacheKey(field, query) {
+  if (typeof query === "object") {
+    // complex query
+    if (query.where) {
+      const {
+        field: queryField,
+        column: queryColumn,
+        value: queryValue,
+      } = query.where;
+      return `${field}/${queryField}/${queryColumn}/${queryValue}`;
+    }
+    if (query.summarize) {
+      const {
+        method,
+        field: queryField,
+        column: queryColumn,
+        values: queryValues,
+      } = query.summarize;
+      return `${field}/${method}/${queryField}/${queryColumn}/${queryValues.join(
+        ","
+      )}`;
+    }
+    throw new Error("Unrecognized complex query type");
+  }
+
+  // simple query
+  return `${field}/${query}`;
+}
+
+function _urlEncodeWhereQuery(q) {
+  const { field: queryField, column: queryColumn, value: queryValue } = q;
+  return `${_dubEncURIComp(queryField)}:${_dubEncURIComp(
+    queryColumn
+  )}=${_dubEncURIComp(queryValue)}`;
+}
+
+function _urlEncodeSummarizeQuery(q) {
+  const { method, field, column, values } = q;
+  const filter = values
+    .map((value) => _urlEncodeWhereQuery({ field, column, value }))
+    .join("&");
+  return `method=${method}&${filter}`;
+}
+
+export function _urlEncodeComplexQuery(q) {
+  if (typeof q === "object") {
+    if (q.where) {
+      return _urlEncodeWhereQuery(q.where);
+    }
+    if (q.summarize) {
+      return _urlEncodeSummarizeQuery(q.summarize);
+    }
+  }
+  throw new Error("Unrecognized complex query type");
+}
+
+export function _urlEncodeLabelQuery(colKey, q) {
+  if (!colKey) throw new Error("Unsupported query by name");
+  if (typeof q !== "string") throw new Error("Query must be a simple label.");
+  return `${colKey}=${encodeURIComponent(q)}`;
+}
+
+/**
+ * Generate the column key the server will send us for this query.
+ */
+export function _hashStringValues(arrayOfString) {
+  const hash = sha1(arrayOfString.join(""));
+  return hash;
+}

client/src/annoMatrix/schema.js

@@ -1,9 +1,6 @@
 /*
 Private helper functions related to schema
 */
-import catLabelSort from "../util/catLabelSort";
-import { unassignedCategoryLabel } from "../globals";
-
 export function _getColumnSchema(schema, field, col) {
   /* look up the column definition */
   switch (field) {
@@ -26,10 +23,15 @@ export function _getColumnSchema(schema, field, col) {
   }
 }
 
+export function _isIndex(schema, field, col) {
+  const index = schema.annotations?.[field].index;
+  return index && index === col;
+}
+
 export function _getColumnDimensionNames(schema, field, col) {
   /*
 		field/col may be an alias for multiple columns. Currently used to map ND 
-		values to 1D dataframe columns for embeddings/layout. Signfied by the presence
+		values to 1D dataframe columns for embeddings/layout. Signified by the presence
 		of the "dims" value in the schema.
 		*/
   const colSchema = _getColumnSchema(schema, field, col);
@@ -63,26 +65,3 @@ export function _isContinuousType(schema) {
   const { type } = schema;
   return !(type === "string" || type === "boolean" || type === "categorical");
 }
-
-export function _normalizeCategoricalSchema(colSchema, col) {
-  const { type, writable } = colSchema;
-  if (
-    type === "string" ||
-    type === "boolean" ||
-    type === "categorical" ||
-    writable
-  ) {
-    const categorySet = new Set(
-      col.summarizeCategorical().categories.concat(colSchema.categories ?? [])
-    );
-    if (writable && !categorySet.has(unassignedCategoryLabel)) {
-      categorySet.add(unassignedCategoryLabel);
-    }
-    colSchema.categories = Array.from(categorySet);
-  }
-
-  if (colSchema.categories) {
-    colSchema.categories = catLabelSort(writable, colSchema.categories);
-  }
-  return colSchema;
-}

client/src/annoMatrix/whereCache.js

@@ -1,27 +1,55 @@
 /*
 Private support functions.
 
-Support for a "where" query, eg,
+This implements a query resolver cache, mapping a query onto the column labels
+resolved by that query. These labels are then used to manage the acutal data cache,
+which stores data by the resolved label.
 
-	{ where: { field: "var", column: "gene", value: "FOXP2" }}
+There are three query forms:
+  * primitive (string, number) - which is just reference the column label of same value
+  * where query (object) - eg, { where: { field: "var", column: "gene", value: "FOXP2" }}
+  * summary query (object) - eg, { summarize: { method: "mean", field: "var", column: "gene", values: ["FOXP2", "GNE", "F5"]}}
 
-These evaluate to a given column label.
+These queries all resolve to one or more column labels on a field.  This
+cache maintains a record of this, allowing direct access to the data caches
+without a server round-trip.
 
-The "where cache" is a map that saves evaluated queries and points 
-to the column label they resolve to.
+The data structure for where queries, the following query against X as an example:
+  { where: { field: "var", column: "column_label_in_var", value: "value_in_var_column" } }
+results in the following cached entry:
+{
+  where: {
+    X: {
+      var: Map(
+        column_label_in_var => Map(
+          value_in_var_column => [column_label_in_X, ...]
+        )
+      )
+    }
+  },
+  summarize: {},
+}
 
-Data structure, using X as the example field being queried, and var as
-the index.
-
-{ 
-	X: {
-		var: Map(
-			column_label_in_var => Map(value_in_var_column => [column_label_in_X, ...])
-		)
-	}
+And for summarize queries, for the following summary on X:
+  { summarize: { method: "mean", field: "var", column: "gene", values: ["G1", "G2"]}}
+creates a cache entry of:
+{
+  where: {},
+  summarize: {
+    X: {
+      mean: {
+        var: Map(
+          "gene" => Map(
+            "G1,G2" => [summary_column_label, ...]
+          )
+        )
+      }
+    },
+  },
 }
 */
 import { _getColumnDimensionNames } from "./schema";
+import { _hashStringValues } from "./query";
 
 export function _whereCacheGet(whereCache, schema, field, query) {
   /* 
@@ -31,20 +59,30 @@ export function _whereCacheGet(whereCache, schema, field, query) {
 	*/
 
   if (typeof query === "object") {
-    const { field: queryField, column: queryColumn, value: queryValue } = query;
-
-    const columnMap = whereCache?.[field]?.[queryField];
-    if (columnMap === undefined) return [undefined];
-
-    const valueMap = columnMap.get(queryColumn);
-    if (valueMap === undefined) return [undefined];
-
-    const columnLabels = valueMap.get(queryValue);
-    return columnLabels === undefined ? [undefined] : columnLabels;
+    if (query.where) {
+      const {
+        field: queryField,
+        column: queryColumn,
+        value: queryValue,
+      } = query.where;
+      const columnMap = whereCache?.where?.[field]?.[queryField];
+      return columnMap?.get(queryColumn)?.get(queryValue) ?? [undefined];
+    }
+    if (query.summarize) {
+      const {
+        method,
+        field: queryField,
+        column: queryColumn,
+        values: queryValues,
+      } = query.summarize;
+      const columnMap = whereCache?.summarize?.[field]?.[method]?.[queryField];
+      const queryValueHash = _hashStringValues(queryValues);
+      return columnMap?.get(queryColumn)?.get(queryValueHash) ?? [undefined];
+    }
+    return [undefined];
   }
 
-  const colDims = _getColumnDimensionNames(schema, field, query);
-  return colDims === undefined ? [undefined] : colDims;
+  return _getColumnDimensionNames(schema, field, query) ?? [undefined];
 }
 
 export function _whereCacheCreate(field, query, columnLabels) {
@@ -53,40 +91,86 @@ export function _whereCacheCreate(field, query, columnLabels) {
 	*/
   if (typeof query !== "object") return null;
 
-  const { field: queryField, column: queryColumn, value: queryValue } = query;
-  const whereCache = {
-    [field]: {
-      [queryField]: new Map([
-        [queryColumn, new Map([[queryValue, columnLabels]])],
-      ]),
-    },
-  };
-  return whereCache;
+  if (query.where) {
+    const {
+      field: queryField,
+      column: queryColumn,
+      value: queryValue,
+    } = query.where;
+    return {
+      where: {
+        [field]: {
+          [queryField]: new Map([
+            [queryColumn, new Map([[queryValue, columnLabels]])],
+          ]),
+        },
+      },
+    };
+  }
+  if (query.summarize) {
+    const {
+      method,
+      field: queryField,
+      column: queryColumn,
+      values: queryValues,
+    } = query.summarize;
+    const queryValueHash = _hashStringValues(queryValues);
+    return {
+      summarize: {
+        [field]: {
+          [method]: {
+            [queryField]: new Map([
+              [queryColumn, new Map([[queryValueHash, columnLabels]])],
+            ]),
+          },
+        },
+      },
+    };
+  }
+  // oops, not sure what that query is!
+  return {};
+}
+
+function __mergeQueries(dst, src) {
+  for (const [queryField, columnMap] of Object.entries(src)) {
+    dst[queryField] = dst[queryField] || new Map();
+    for (const [queryColumn, valueMap] of columnMap) {
+      if (!dst[queryField].has(queryColumn))
+        dst[queryField].set(queryColumn, new Map());
+      for (const [queryValue, columnLabels] of valueMap) {
+        dst[queryField].get(queryColumn).set(queryValue, columnLabels);
+      }
+    }
+  }
 }
 
 function __whereCacheMerge(dst, src) {
   /*
 	merge src into dst (modifies dst)
 	*/
-  if (!dst) dst = {};
   if (!src || typeof src !== "object") return dst;
-  Object.entries(src).forEach(([field, query]) => {
-    if (!Object.prototype.hasOwnProperty.call(dst, field)) dst[field] = {};
-    Object.entries(query).forEach(([queryField, columnMap]) => {
-      if (!Object.prototype.hasOwnProperty.call(dst[field], queryField))
-        dst[field][queryField] = new Map();
-      columnMap.forEach((valueMap, queryColumn) => {
-        if (!dst[field][queryField].has(queryColumn))
-          dst[field][queryField].set(queryColumn, new Map());
-        valueMap.forEach((columnLabels, queryValue) => {
-          dst[field][queryField].get(queryColumn).set(queryValue, columnLabels);
-        });
-      });
-    });
-  });
+
+  if (src.where) {
+    dst.where = dst.where || {};
+    for (const [field, query] of Object.entries(src.where)) {
+      dst.where[field] = dst.where[field] || {};
+      __mergeQueries(dst.where[field], query);
+    }
+  }
+  if (src.summarize) {
+    dst.summarize = dst.summarize || {};
+    for (const [field, method] of Object.entries(src.summarize)) {
+      dst.summarize[field] = dst.summarize[field] || {};
+      for (const [methodName, query] of Object.entries(method)) {
+        dst.summarize[field][methodName] =
+          dst.summarize[field][methodName] || {};
+        __mergeQueries(dst.summarize[field][methodName], query);
+      }
+    }
+  }
   return dst;
 }
 
 export function _whereCacheMerge(...caches) {
-  return caches.reduce((dst, src) => __whereCacheMerge(dst, src), {});
+  return caches.reduce(__whereCacheMerge, {});
 }

client/src/components/annoDialog.js

@@ -18,6 +18,8 @@ class AnnoDialog extends React.PureComponent {
       validationError,
       annoSelect,
       annoInput,
+      secondaryInstructions,
+      secondaryInput,
       handleCancel,
       handleSubmit,
       primaryButtonText,
@@ -40,12 +42,19 @@ class AnnoDialog extends React.PureComponent {
               <p
                 style={{
                   marginTop: 7,
-                  visibility: validationError ? "visible" : "hidden",
+                  visibility: errorMessage !== "" ? "visible" : "hidden",
                   color: Colors.ORANGE3,
                 }}
               >
                 {errorMessage}
               </p>
+              {/* we might rename, secondary button and secondary input are not related */}
+              {secondaryInstructions && (
+                <p style={{ marginTop: secondaryInstructions ? 20 : 0 }}>
+                  {secondaryInstructions}
+                </p>
+              )}
+              {secondaryInput || null}
             </div>
             {annoSelect || null}
           </div>
@@ -54,6 +63,7 @@ class AnnoDialog extends React.PureComponent {
               <Tooltip content={cancelTooltipContent}>
                 <Button onClick={handleCancel}>Cancel</Button>
               </Tooltip>
+              {/* we might rename, secondary button and secondary input are not related */}
               {handleSecondaryButtonSubmit && secondaryButtonText ? (
                 <Button
                   onClick={handleSecondaryButtonSubmit}

client/src/components/app.js

@@ -1,4 +1,3 @@
-// jshint esversion: 6
 import React from "react";
 import Helmet from "react-helmet";
 import { connect } from "react-redux";
@@ -12,7 +11,6 @@ import Graph from "./graph/graph";
 import MenuBar from "./menubar";
 import Autosave from "./autosave";
 import Embedding from "./embedding";
-import TermsOfServicePrompt from "./termsPrompt";
 
 import actions from "../actions";
 
@@ -25,6 +23,8 @@ class App extends React.Component {
   componentDidMount() {
     const { dispatch } = this.props;
 
+    dispatch(actions.requestSpatialMetadata());
+
     /* listen for url changes, fire one when we start the app up */
     window.addEventListener("popstate", this._onURLChanged);
     this._onURLChanged();
@@ -76,7 +76,6 @@ class App extends React.Component {
                 <MenuBar />
                 <Embedding />
                 <Autosave />
-                <TermsOfServicePrompt />
                 <Legend viewportRef={viewportRef} />
                 <Graph key={graphRenderCounter} viewportRef={viewportRef} />
               </>

client/src/components/autosave/filenameDialog.js

@@ -14,9 +14,10 @@ import {
 @connect((state) => ({
   idhash: state.config?.parameters?.["annotations-user-data-idhash"] ?? null,
   annotations: state.annotations,
-  auth: state.config?.authentication,
-  userInfo: state.userInfo,
   writableCategoriesEnabled: state.config?.parameters?.annotations ?? false,
+  writableGenesetsEnabled: !(
+    state.config?.parameters?.annotations_genesets_readonly ?? true
+  ),
 }))
 class FilenameDialog extends React.Component {
   constructor(props) {
@@ -95,20 +96,19 @@ class FilenameDialog extends React.Component {
   render() {
     const {
       writableCategoriesEnabled,
+      writableGenesetsEnabled,
       annotations,
       idhash,
-      userInfo,
     } = this.props;
     const { filenameText } = this.state;
 
-    return writableCategoriesEnabled &&
+    return (writableCategoriesEnabled || writableGenesetsEnabled) &&
       annotations.promptForFilename &&
       !annotations.dataCollectionNameIsReadOnly &&
-      !annotations.dataCollectionName &&
-      userInfo.is_authenticated ? (
+      !annotations.dataCollectionName ? (
       <Dialog
         icon="tag"
-        title="Annotations Collection"
+        title="User Generated Data Directory"
         isOpen={!annotations.dataCollectionName}
         onClose={this.dismissFilenameDialog}
       >
@@ -120,7 +120,7 @@ class FilenameDialog extends React.Component {
         >
           <div className={Classes.DIALOG_BODY} data-testid="annotation-dialog">
             <div style={{ marginBottom: 20 }}>
-              <p>Name your annotations collection:</p>
+              <p>Name your user generated data directory:</p>
               <InputGroup
                 autoFocus
                 value={filenameText}
@@ -145,9 +145,15 @@ class FilenameDialog extends React.Component {
             </div>
             <div>
               <p>
-                Your annotations are stored in this file:
+                {"Your annotations are stored in this file: "}
                 <Code>
-                  {filenameText}-{idhash}.csv
+                  {filenameText}-cell-labels-{idhash}.csv
+                </Code>
+              </p>
+              <p>
+                {"Your gene sets are stored in this file: "}
+                <Code>
+                  {filenameText}-gene-sets-{idhash}.csv
                 </Code>
               </p>
               <p style={{ fontStyle: "italic" }}>
@@ -167,7 +173,7 @@ class FilenameDialog extends React.Component {
                 type="submit"
                 data-testid="submit-annotation"
               >
-                Create annotations collection
+                Create user generated data directory
               </Button>
             </div>
           </div>

client/src/components/autosave/index.js

@@ -5,11 +5,18 @@ import FilenameDialog from "./filenameDialog";
 
 @connect((state) => ({
   annotations: state.annotations,
-  saveInProgress: state.autosave?.saveInProgress ?? false,
+  obsAnnotationSaveInProgress:
+    state.autosave?.obsAnnotationSaveInProgress ?? false,
+  genesetSaveInProgress: state.autosave?.genesetSaveInProgress ?? false,
   error: state.autosave?.error,
   writableCategoriesEnabled: state.config?.parameters?.annotations ?? false,
+  writableGenesetsEnabled: !(
+    state.config?.parameters?.annotations_genesets_readonly ?? true
+  ),
   annoMatrix: state.annoMatrix,
+  genesets: state.genesets,
   lastSavedAnnoMatrix: state.autosave?.lastSavedAnnoMatrix,
+  lastSavedGenesets: state.autosave?.lastSavedGenesets,
 }))
 class Autosave extends React.Component {
   constructor(props) {
@@ -20,11 +27,11 @@ class Autosave extends React.Component {
   }
 
   componentDidMount() {
-    const { writableCategoriesEnabled } = this.props;
+    const { writableCategoriesEnabled, writableGenesetsEnabled } = this.props;
 
     let { timer } = this.state;
     if (timer) clearInterval(timer);
-    if (writableCategoriesEnabled) {
+    if (writableCategoriesEnabled || writableGenesetsEnabled) {
       timer = setInterval(this.tick, 2500);
     } else {
       timer = null;
@@ -38,18 +45,37 @@ class Autosave extends React.Component {
   }
 
   tick = () => {
-    const { dispatch, saveInProgress } = this.props;
-    if (this.needToSave() && !saveInProgress) {
+    const { dispatch, obsAnnotationSaveInProgress, genesetSaveInProgress } =
+      this.props;
+    if (!obsAnnotationSaveInProgress && this.needToSaveObsAnnotations()) {
       dispatch(actions.saveObsAnnotationsAction());
     }
+    if (!genesetSaveInProgress && this.needToSaveGenesets()) {
+      dispatch(actions.saveGenesetsAction());
+    }
   };
 
-  needToSave = () => {
-    /* return true if we need to save, false if we don't */
+  needToSaveObsAnnotations = () => {
+    /* return true if we need to save obs cell labels, false if we don't */
     const { annoMatrix, lastSavedAnnoMatrix } = this.props;
     return actions.needToSaveObsAnnotations(annoMatrix, lastSavedAnnoMatrix);
   };
 
+  needToSaveGenesets = () => {
+    /* return true if we need to save gene ses, false if we do not */
+    const { genesets, lastSavedGenesets } = this.props;
+    return genesets.initialized && genesets.genesets !== lastSavedGenesets;
+  };
+
+  needToSave() {
+    return this.needToSaveGenesets() || this.needToSaveObsAnnotations();
+  }
+
+  saveInProgress() {
+    const { obsAnnotationSaveInProgress, genesetSaveInProgress } = this.props;
+    return obsAnnotationSaveInProgress || genesetSaveInProgress;
+  }
+
   statusMessage() {
     const { error } = this.props;
     if (error) {
@@ -61,12 +87,12 @@ class Autosave extends React.Component {
   render() {
     const {
       writableCategoriesEnabled,
-      saveInProgress,
+      writableGenesetsEnabled,
       lastSavedAnnoMatrix,
     } = this.props;
     const initialDataLoadComplete = lastSavedAnnoMatrix;
 
-    if (!writableCategoriesEnabled) return null;
+    if (!writableCategoriesEnabled && !writableGenesetsEnabled) return null;
 
     return (
       <div
@@ -74,7 +100,7 @@ class Autosave extends React.Component {
         data-testclass={
           !initialDataLoadComplete
             ? "autosave-init"
-            : this.needToSave() || saveInProgress
+            : this.saveInProgress() || this.needToSave()
             ? "autosave-incomplete"
             : "autosave-complete"
         }

client/src/components/brushableHistogram/error.js

@@ -0,0 +1,15 @@
+import React from "react";
+import * as globals from "../../globals";
+
+const ErrorLoading = ({ displayName, zebra }) => (
+    <div
+      style={{
+        backgroundColor: zebra ? globals.lightestGrey : "white",
+        fontStyle: "italic",
+      }}
+    >
+      <span>{`Failure loading ${displayName}`}</span>
+    </div>
+  );
+
+export default ErrorLoading;

client/src/components/brushableHistogram/footer.js

@@ -0,0 +1,63 @@
+import React from "react";
+
+const HistogramFooter = React.memo(
+  ({
+    displayName,
+    hideRanges,
+    rangeMin,
+    rangeMax,
+    rangeColorMin,
+    rangeColorMax,
+    isObs,
+    isGeneSetSummary,
+  }) => 
+    /*
+    Footer of each histogram.  Will render range and title.
+
+    Required props:
+      * displayName - the displayName, aka "n_genes", "FOXP2", etc.
+      * hideRanges - true/false, enables/disable rendering of ranges
+      * range - length two array, [min, max], containing the range values to display
+      * rangeColor - length two array, [mincolor, maxcolor], each a CSS color
+    */
+     (
+      <div>
+        <div
+          style={{
+            display: "flex",
+            justifyContent: hideRanges ? "center" : "space-between",
+          }}
+        >
+          <span
+            style={{
+              color: rangeColorMin,
+              display: hideRanges ? "none" : "block",
+            }}
+          >
+            min {rangeMin.toPrecision(4)}
+          </span>
+          <span
+            data-testclass="brushable-histogram-field-name"
+            style={{ fontStyle: "italic" }}
+          >
+            {isObs && displayName}
+            {isGeneSetSummary && "gene set mean expression"}
+          </span>
+          <div style={{ display: hideRanges ? "block" : "none" }}>
+            : {rangeMin}
+          </div>
+          <span
+            style={{
+              color: rangeColorMax,
+              display: hideRanges ? "none" : "block",
+            }}
+          >
+            max {rangeMax.toPrecision(4)}
+          </span>
+        </div>
+      </div>
+    )
+  
+);
+
+export default HistogramFooter;

client/src/components/brushableHistogram/header.js

@@ -0,0 +1,100 @@
+import React, { useCallback } from "react";
+import { Button, ButtonGroup, Tooltip, Icon } from "@blueprintjs/core";
+import { IconNames } from "@blueprintjs/icons";
+import * as globals from "../../globals";
+
+const HistogramHeader = React.memo(
+  ({
+    fieldId,
+    isColorBy,
+    onColorByClick,
+    onRemoveClick,
+    isScatterPlotX,
+    isScatterPlotY,
+    onScatterPlotXClick,
+    onScatterPlotYClick,
+    isObs,
+  }) => {
+    /*
+        Render the toolbar for the histogram.  Props:
+          * fieldId - field identifier, used for various IDs
+          * isColorBy - true/false, is this the current color-by
+          * onColorByClick - color-by click handler
+          * onRemoveClick - optional handler for remove.  Button will not render if not defined.
+          * isScatterPlotX - optional, true/false if currently the X scatterplot field
+          * isScatterPlotY - optional, true/false if currently the Y scatterplot field
+          * onScatterPlotXClick - optional, handler for scatterPlot X button.
+          * onScatterPlotYClick - optional, handler for scatterPlot X button.
+  
+        Scatterplot controls will not render if either handler unspecified.
+      */
+
+    const memoizedColorByCallback = useCallback(
+      () => onColorByClick(fieldId, isObs),
+      [fieldId, isObs]
+    );
+
+    return (
+      <div
+        style={{
+          display: "flex",
+          justifyContent: "flex-end",
+          paddingBottom: "8px",
+        }}
+      >
+        {onScatterPlotXClick && onScatterPlotYClick ? (
+          <span>
+            <Icon icon={IconNames.SCATTER_PLOT} style={{ marginRight: 7 }} />
+            <ButtonGroup style={{ marginRight: 7 }}>
+              <Button
+                data-testid={`plot-x-${fieldId}`}
+                onClick={onScatterPlotXClick}
+                active={isScatterPlotX}
+                intent={isScatterPlotX ? "primary" : "none"}
+              >
+                plot x
+              </Button>
+              <Button
+                data-testid={`plot-y-${fieldId}`}
+                onClick={onScatterPlotYClick}
+                active={isScatterPlotY}
+                intent={isScatterPlotY ? "primary" : "none"}
+              >
+                plot y
+              </Button>
+            </ButtonGroup>
+          </span>
+        ) : null}
+        {onRemoveClick ? (
+          <Button
+            minimal
+            onClick={onRemoveClick}
+            style={{
+              color: globals.blue,
+              cursor: "pointer",
+              marginLeft: 7,
+            }}
+          >
+            remove
+          </Button>
+        ) : null}
+        <Tooltip
+          content="Use as color scale"
+          position="bottom"
+          hoverOpenDelay={globals.tooltipHoverOpenDelay}
+        >
+          <Button
+            onClick={memoizedColorByCallback}
+            active={isColorBy}
+            intent={isColorBy ? "primary" : "none"}
+            data-testclass="colorby"
+            data-testid={`colorby-${fieldId}`}
+            icon="tint"
+          />
+        </Tooltip>
+      </div>
+    );
+  }
+);
+
+export default HistogramHeader;

client/src/components/brushableHistogram/histogram.js

@@ -0,0 +1,191 @@
+import React, { useEffect, useRef, useState } from "react";
+import { interpolateCool } from "d3-scale-chromatic";
+import * as d3 from "d3";
+
+import maybeScientific from "../../util/maybeScientific";
+import clamp from "../../util/clamp";
+
+const Histogram = ({
+  field,
+  fieldForId,
+  display,
+  histogram,
+  width,
+  height,
+  onBrush,
+  onBrushEnd,
+  margin,
+  isColorBy,
+  selectionRange,
+  mini,
+}) => {
+  const svgRef = useRef(null);
+  const [brush, setBrush] = useState(null);
+
+  useEffect(() => {
+    /*
+      Create the d3 histogram
+      */
+    //  This is just a constant that's flipped by parent's `mini` boolean
+    const {
+      LEFT: marginLeft,
+      RIGHT: marginRight,
+      BOTTOM: marginBottom,
+      TOP: marginTop,
+    } = margin;
+    const { x, y, bins, binStart, binEnd, binWidth } = histogram;
+    const svg = d3.select(svgRef.current);
+    const binPadding = mini ? 0 : -1;
+    const defaultBarColor = mini ? "black" : "#bbb";
+
+    /* Remove everything */
+    svg.selectAll("*").remove();
+
+    /* Set margins within the SVG */
+    const container = svg
+      .attr("width", width + marginLeft + marginRight)
+      .attr("height", height + marginTop + marginBottom)
+      .append("g")
+      .attr("class", "histogram-container")
+      .attr("transform", `translate(${marginLeft},${marginTop})`);
+
+    const colorScale = d3
+      .scaleSequential(interpolateCool)
+      .domain([0, bins.length]);
+
+    const histogramScale = d3
+      .scaleLinear()
+      .domain(x.domain())
+      .range([
+        colorScale.domain()[1],
+        colorScale.domain()[0],
+      ]); /* we flip this to make colors dark if high in the color scale */
+
+    if (binWidth > 0) {
+      /* BINS */
+      container
+        .insert("g", "*")
+        .selectAll("rect")
+        .data(bins)
+        .enter()
+        .append("rect")
+        .attr("x", (d, i) => x(binStart(i)) + 1)
+        .attr("y", (d) => y(d))
+        .attr("width", (d, i) => x(binEnd(i)) - x(binStart(i)) - binPadding)
+        .attr("height", (d) => y(0) - y(d))
+        .style(
+          "fill",
+          isColorBy
+            ? (d, i) => colorScale(histogramScale(binStart(i)))
+            : defaultBarColor
+        );
+    }
+
+    if (!mini) {
+      // BRUSH
+      // Note the brushable area is bounded by the data on three sides, but goes down to cover the x-axis
+      const brushX = d3
+        .brushX()
+        .extent([
+          [x.range()[0], y.range()[1]],
+          [x.range()[1], marginTop + height + marginBottom],
+        ])
+        /*
+        emit start so that the Undoable history can save an undo point
+        upon drag start, and ignore the subsequent intermediate drag events.
+        */
+        .on("start", onBrush(field, x.invert, "start"))
+        .on("brush", onBrush(field, x.invert, "brush"))
+        .on("end", onBrushEnd(field, x.invert));
+
+      const brushXselection = container
+        .insert("g")
+        .attr("class", "brush")
+        .attr("data-testid", `${svgRef.current.dataset.testid}-brushable-area`)
+        .call(brushX);
+
+      /* X AXIS */
+      container
+        .insert("g")
+        .attr("class", "axis axis--x")
+        .attr("transform", `translate(0,${marginTop + height})`)
+        .call(
+          d3
+            .axisBottom(x)
+            .ticks(4)
+            .tickFormat(d3.format(maybeScientific(x)))
+        );
+
+      /* Y AXIS */
+      container
+        .insert("g")
+        .attr("class", "axis axis--y")
+        .attr("transform", `translate(${marginLeft + width},0)`)
+        .call(
+          d3
+            .axisRight(y)
+            .ticks(3)
+            .tickFormat(
+              d3.format(
+                y.domain().some((n) => Math.abs(n) >= 10000) ? ".0e" : ","
+              )
+            )
+        );
+
+      /* axis style */
+      svg.selectAll(".axis text").style("fill", "rgb(80,80,80)");
+      svg.selectAll(".axis path").style("stroke", "rgb(230,230,230)");
+      svg.selectAll(".axis line").style("stroke", "rgb(230,230,230)");
+
+      setBrush({ brushX, brushXselection });
+    }
+  }, [histogram, isColorBy]);
+
+  useEffect(() => {
+    /*
+      paint/update selection brush
+      */
+    if (!brush) return;
+    const { brushX, brushXselection } = brush;
+    const selection = d3.brushSelection(brushXselection.node());
+    if (!selectionRange && selection) {
+      /* no active selection - clear brush */
+      brushXselection.call(brushX.move, null);
+    } else if (selectionRange) {
+      const { x, domain } = histogram;
+      const [min, max] = domain;
+      const x0 = x(clamp(selectionRange[0], [min, max]));
+      const x1 = x(clamp(selectionRange[1], [min, max]));
+      if (!selection) {
+        /* there is an active selection, but no brush - set the brush */
+        brushXselection.call(brushX.move, [x0, x1]);
+      } else {
+        /* there is an active selection and a brush - make sure they match */
+        const moveDeltaThreshold = 1;
+        const dX0 = Math.abs(x0 - selection[0]);
+        const dX1 = Math.abs(x1 - selection[1]);
+        /*
+          only update the brush if it is grossly incorrect,
+          as defined by the moveDeltaThreshold
+          */
+        if (dX0 > moveDeltaThreshold || dX1 > moveDeltaThreshold) {
+          brushXselection.call(brushX.move, [x0, x1]);
+        }
+      }
+    }
+  }, [brush, selectionRange]);
+
+  return (
+    <svg
+      style={{ display }}
+      width={width}
+      height={height}
+      id={`histogram_${fieldForId}_svg`}
+      data-testclass="histogram-plot"
+      data-testid={`histogram-${field}-plot`}
+      ref={svgRef}
+    />
+  );
+};
+
+export default Histogram;

client/src/components/brushableHistogram/index.js

@@ -1,447 +1,38 @@
-/*
-https://bl.ocks.org/mbostock/4341954
-https://bl.ocks.org/mbostock/34f08d5e11952a80609169b7917d4172
-https://bl.ocks.org/SpaceActuary/2f004899ea1b2bd78d6f1dbb2febf771
-https://bl.ocks.org/mbostock/3019563
-*/
-import React, { useEffect, useRef, useState, useCallback } from "react";
-import { Button, ButtonGroup, Icon, Tooltip } from "@blueprintjs/core";
-import { connect } from "react-redux";
+import React from "react";
+import { connect, shallowEqual } from "react-redux";
 import * as d3 from "d3";
-import { interpolateCool } from "d3-scale-chromatic";
 import Async from "react-async";
 import memoize from "memoize-one";
-import { IconNames } from "@blueprintjs/icons";
 import * as globals from "../../globals";
 import actions from "../../actions";
-import { histogramContinuous } from "../../util/dataframe/histogram";
 import { makeContinuousDimensionName } from "../../util/nameCreators";
-import significantDigits from "../../util/significantDigits";
+import HistogramHeader from "./header";
+import Histogram from "./histogram";
+import HistogramFooter from "./footer";
+import StillLoading from "./loading";
+import ErrorLoading from "./error";
 
-function clamp(val, rng) {
-  return Math.max(Math.min(val, rng[1]), rng[0]);
-}
-
-function maybeScientific(x) {
-  let format = ",";
-  const _ticks = x.ticks(4);
-
-  if (x.domain().some((n) => Math.abs(n) >= 10000)) {
-    /*
-      heuristic: if the last tick d3 wants to render has one significant
-      digit ie., 2000, render 2e+3, but if it's anything else ie., 42000000 render
-      4.20e+n
-    */
-    format = significantDigits(_ticks[_ticks.length - 1]) === 1 ? ".0e" : ".2e";
-  }
-
-  return format;
-}
-
-const StillLoading = ({ zebra, displayName }) => {
-  /*
-  Render a loading indicator for the field.
-  */
-  return (
-    <div
-      style={{
-        padding: globals.leftSidebarSectionPadding,
-        backgroundColor: zebra ? globals.lightestGrey : "white",
-      }}
-    >
-      <div
-        style={{
-          display: "flex",
-          justifyContent: "space-between",
-          justifyItems: "center",
-          alignItems: "center",
-        }}
-      >
-        <div style={{ minWidth: 30 }} />
-        <div style={{ display: "flex", alignSelf: "center" }}>
-          <span style={{ fontStyle: "italic" }}>{displayName}</span>
-        </div>
-        <div
-          style={{
-            display: "flex",
-            justifyContent: "flex-end",
-          }}
-        >
-          <Button minimal loading intent="primary" />
-        </div>
-      </div>
-    </div>
-  );
+const MARGIN = {
+  LEFT: 10, // Space for 0 tick label on X axis
+  RIGHT: 54, // space for Y axis & labels
+  BOTTOM: 25, // space for X axis & labels
+  TOP: 3,
 };
-
-const ErrorLoading = ({ displayName, error, zebra }) => {
-  console.log(error); // log to console as this is unexpected
-  return (
-    <div
-      style={{
-        padding: globals.leftSidebarSectionPadding,
-        backgroundColor: zebra ? globals.lightestGrey : "white",
-      }}
-    >
-      <span>{`Failure loading ${displayName}`}</span>
-    </div>
-  );
-};
-
-const HistogramFooter = React.memo(
-  ({
-    displayName,
-    hideRanges,
-    rangeMin,
-    rangeMax,
-    rangeColorMin,
-    rangeColorMax,
-    logFoldChange,
-    pvalAdj,
-  }) => {
-    /*
-  Footer of each histogram.  Will render range, title, and optionally
-  differential expression info.
-
-  Required props:
-    * displayName - the displayName, aka "n_genes", "FOXP2", etc.
-    * hideRanges - true/false, enables/disable rendering of ranges
-    * range - length two array, [min, max], containing the range values to display
-    * rangeColor - length two array, [mincolor, maxcolor], each a CSS color
-    * logFoldChange - lfc to display, optional.
-    * pValue - pValue to display, optional.
-  */
-    return (
-      <div>
-        <div
-          style={{
-            display: "flex",
-            justifyContent: hideRanges ? "center" : "space-between",
-          }}
-        >
-          <span
-            style={{
-              color: rangeColorMin,
-              display: hideRanges ? "none" : "block",
-            }}
-          >
-            min {rangeMin.toPrecision(4)}
-          </span>
-          <span
-            data-testclass="brushable-histogram-field-name"
-            style={{ fontStyle: "italic" }}
-          >
-            {displayName}
-          </span>
-          <div style={{ display: hideRanges ? "block" : "none" }}>
-            : {rangeMin}
-          </div>
-          <span
-            style={{
-              color: rangeColorMax,
-              display: hideRanges ? "none" : "block",
-            }}
-          >
-            max {rangeMax.toPrecision(4)}
-          </span>
-        </div>
-
-        {logFoldChange && pvalAdj ? (
-          <div
-            style={{
-              display: "flex",
-              justifyContent: "center",
-              alignItems: "baseline",
-            }}
-          >
-            <span>
-              <strong>log fold change:</strong>
-              {` ${logFoldChange.toPrecision(4)}`}
-            </span>
-            <span
-              style={{
-                marginLeft: 7,
-                padding: 2,
-              }}
-            >
-              <strong>p-value (adj):</strong>
-              {pvalAdj < 0.0001 ? " < 0.0001" : ` ${pvalAdj.toFixed(4)}`}
-            </span>
-          </div>
-        ) : null}
-      </div>
-    );
-  }
-);
-
-const HistogramHeader = React.memo(
-  ({
-    fieldId,
-    isColorBy,
-    onColorByClick,
-    onRemoveClick,
-    isScatterPlotX,
-    isScatterPlotY,
-    onScatterPlotXClick,
-    onScatterPlotYClick,
-    isObs,
-  }) => {
-    /*
-      Render the toolbar for the histogram.  Props:
-        * fieldId - field identifier, used for various IDs
-        * isColorBy - true/false, is this the current color-by
-        * onColorByClick - color-by click handler
-        * onRemoveClick - optional handler for remove.  Button will not render if not defined.
-        * isScatterPlotX - optional, true/false if currently the X scatterplot field
-        * isScatterPlotY - optional, true/false if currently the Y scatterplot field
-        * onScatterPlotXClick - optional, handler for scatterPlot X button.
-        * onScatterPlotYClick - optional, handler for scatterPlot X button.
-
-      Scatterplot controls will not render if either handler unspecified.
-    */
-
-    const memoizedColorByCallback = useCallback(
-      () => onColorByClick(fieldId, isObs),
-      [fieldId, isObs]
-    );
-
-    return (
-      <div
-        style={{
-          display: "flex",
-          justifyContent: "flex-end",
-          paddingBottom: "8px",
-        }}
-      >
-        {onScatterPlotXClick && onScatterPlotYClick ? (
-          <span>
-            <Icon icon={IconNames.SCATTER_PLOT} style={{ marginRight: 7 }} />
-            <ButtonGroup style={{ marginRight: 7 }}>
-              <Button
-                data-testid={`plot-x-${fieldId}`}
-                onClick={onScatterPlotXClick}
-                active={isScatterPlotX}
-                intent={isScatterPlotX ? "primary" : "none"}
-              >
-                plot x
-              </Button>
-              <Button
-                data-testid={`plot-y-${fieldId}`}
-                onClick={onScatterPlotYClick}
-                active={isScatterPlotY}
-                intent={isScatterPlotY ? "primary" : "none"}
-              >
-                plot y
-              </Button>
-            </ButtonGroup>
-          </span>
-        ) : null}
-        {onRemoveClick ? (
-          <Button
-            minimal
-            onClick={onRemoveClick}
-            style={{
-              color: globals.blue,
-              cursor: "pointer",
-              marginLeft: 7,
-            }}
-          >
-            remove
-          </Button>
-        ) : null}
-        <Tooltip
-          content="Use as color scale"
-          position="bottom"
-          hoverOpenDelay={globals.tooltipHoverOpenDelay}
-        >
-          <Button
-            onClick={memoizedColorByCallback}
-            active={isColorBy}
-            intent={isColorBy ? "primary" : "none"}
-            data-testclass="colorby"
-            data-testid={`colorby-${fieldId}`}
-            icon="tint"
-          />
-        </Tooltip>
-      </div>
-    );
-  }
-);
-
-const Histogram = ({
-  field,
-  fieldForId,
-  display,
-  histogram,
-  width,
-  height,
-  onBrush,
-  onBrushEnd,
-  margin,
-  isColorBy,
-  selectionRange,
-}) => {
-  const svgRef = useRef(null);
-  const [brush, setBrush] = useState(null);
-
-  useEffect(() => {
-    /*
-    Create the d3 histogram
-    */
-    const { marginLeft, marginRight, marginBottom, marginTop } = margin;
-    const { x, y, bins, binStart, binEnd, binWidth } = histogram;
-    const svg = d3.select(svgRef.current);
-
-    /* Remove everything */
-    svg.selectAll("*").remove();
-
-    /* Set margins within the SVG */
-    const container = svg
-      .attr("width", width + marginLeft + marginRight)
-      .attr("height", height + marginTop + marginBottom)
-      .append("g")
-      .attr("class", "histogram-container")
-      .attr("transform", `translate(${marginLeft},${marginTop})`);
-
-    const colorScale = d3
-      .scaleSequential(interpolateCool)
-      .domain([0, bins.length]);
-
-    const histogramScale = d3
-      .scaleLinear()
-      .domain(x.domain())
-      .range([
-        colorScale.domain()[1],
-        colorScale.domain()[0],
-      ]); /* we flip this to make colors dark if high in the color scale */
-
-    if (binWidth > 0) {
-      /* BINS */
-      container
-        .insert("g", "*")
-        .selectAll("rect")
-        .data(bins)
-        .enter()
-        .append("rect")
-        .attr("x", (d, i) => x(binStart(i)) + 1)
-        .attr("y", (d) => y(d))
-        .attr("width", (d, i) => x(binEnd(i)) - x(binStart(i)) - 1)
-        .attr("height", (d) => y(0) - y(d))
-        .style(
-          "fill",
-          isColorBy ? (d, i) => colorScale(histogramScale(binStart(i))) : "#bbb"
-        );
-    }
-
-    // BRUSH
-    // Note the brushable area is bounded by the data on three sides, but goes down to cover the x-axis
-    const brushX = d3
-      .brushX()
-      .extent([
-        [x.range()[0], y.range()[1]],
-        [x.range()[1], marginTop + height + marginBottom],
-      ])
-      /*
-      emit start so that the Undoable history can save an undo point
-      upon drag start, and ignore the subsequent intermediate drag events.
-      */
-      .on("start", onBrush(field, x.invert, "start"))
-      .on("brush", onBrush(field, x.invert, "brush"))
-      .on("end", onBrushEnd(field, x.invert));
-
-    const brushXselection = container
-      .insert("g")
-      .attr("class", "brush")
-      .attr("data-testid", `${svgRef.current.dataset.testid}-brushable-area`)
-      .call(brushX);
-
-    /* X AXIS */
-    container
-      .insert("g")
-      .attr("class", "axis axis--x")
-      .attr("transform", `translate(0,${marginTop + height})`)
-      .call(
-        d3
-          .axisBottom(x)
-          .ticks(4)
-          .tickFormat(d3.format(maybeScientific(x)))
-      );
-
-    /* Y AXIS */
-    container
-      .insert("g")
-      .attr("class", "axis axis--y")
-      .attr("transform", `translate(${marginLeft + width},0)`)
-      .call(
-        d3
-          .axisRight(y)
-          .ticks(3)
-          .tickFormat(
-            d3.format(
-              y.domain().some((n) => Math.abs(n) >= 10000) ? ".0e" : ","
-            )
-          )
-      );
-
-    /* axis style */
-    svg.selectAll(".axis text").style("fill", "rgb(80,80,80)");
-    svg.selectAll(".axis path").style("stroke", "rgb(230,230,230)");
-    svg.selectAll(".axis line").style("stroke", "rgb(230,230,230)");
-
-    setBrush({ brushX, brushXselection });
-  }, [histogram, isColorBy]);
-
-  useEffect(() => {
-    /*
-    paint/update selection brush
-    */
-    if (!brush) return;
-    const { brushX, brushXselection } = brush;
-    const selection = d3.brushSelection(brushXselection.node());
-    if (!selectionRange && selection) {
-      /* no active selection - clear brush */
-      brushXselection.call(brushX.move, null);
-    } else if (selectionRange) {
-      const { x, domain } = histogram;
-      const [min, max] = domain;
-      const x0 = x(clamp(selectionRange[0], [min, max]));
-      const x1 = x(clamp(selectionRange[1], [min, max]));
-      if (!selection) {
-        /* there is an active selection, but no brush - set the brush */
-        brushXselection.call(brushX.move, [x0, x1]);
-      } else {
-        /* there is an active selection and a brush - make sure they match */
-        const moveDeltaThreshold = 1;
-        const dX0 = Math.abs(x0 - selection[0]);
-        const dX1 = Math.abs(x1 - selection[1]);
-        /*
-        only update the brush if it is grossly incorrect,
-        as defined by the moveDeltaThreshold
-        */
-        if (dX0 > moveDeltaThreshold || dX1 > moveDeltaThreshold) {
-          brushXselection.call(brushX.move, [x0, x1]);
-        }
-      }
-    }
-  }, [brush, selectionRange]);
-
-  return (
-    <svg
-      style={{ display }}
-      width={width}
-      height={height}
-      id={`histogram_${fieldForId}_svg`}
-      data-testclass="histogram-plot"
-      data-testid={`histogram-${field}-plot`}
-      ref={svgRef}
-    />
-  );
+const WIDTH = 340 - MARGIN.LEFT - MARGIN.RIGHT;
+const HEIGHT = 135 - MARGIN.TOP - MARGIN.BOTTOM;
+const MARGIN_MINI = {
+  LEFT: 0, // Space for 0 tick label on X axis
+  RIGHT: 0, // space for Y axis & labels
+  BOTTOM: 0, // space for X axis & labels
+  TOP: 0,
 };
+const WIDTH_MINI = 120 - MARGIN_MINI.LEFT - MARGIN_MINI.RIGHT;
+const HEIGHT_MINI = 15 - MARGIN_MINI.TOP - MARGIN_MINI.BOTTOM;
 
 @connect((state, ownProps) => {
-  const { isObs, isUserDefined, isDiffExp, field } = ownProps;
+  const { isObs, isUserDefined, isGeneSetSummary, field } = ownProps;
   const myName = makeContinuousDimensionName(
-    { isObs, isUserDefined, isDiffExp },
+    { isObs, isUserDefined, isGeneSetSummary },
     field
   );
   return {
@@ -450,10 +41,13 @@ const Histogram = ({
     isScatterplotYYaccessor: state.controls.scatterplotYYaccessor === field,
     continuousSelectionRange: state.continuousSelection[myName],
     isColorAccessor: state.colors.colorAccessor === field,
-    singleContinuousValues: state.singleContinuousValue.singleContinuousValues,
   };
 })
 class HistogramBrush extends React.PureComponent {
+  static watchAsync(props, prevProps) {
+    return !shallowEqual(props.watchProps, prevProps.watchProps);
+  }
+
   /* memoized closure to prevent HistogramHeader unecessary repaint */
   handleColorAction = memoize((dispatch) => (field, isObs) => {
     if (isObs) {
@@ -466,27 +60,11 @@ class HistogramBrush extends React.PureComponent {
     }
   });
 
-  constructor(props) {
-    super(props);
-
-    const marginLeft = 10; // Space for 0 tick label on X axis
-    const marginRight = 54; // space for Y axis & labels
-    const marginBottom = 25; // space for X axis & labels
-    const marginTop = 3;
-    this.margin = {
-      marginLeft,
-      marginRight,
-      marginBottom,
-      marginTop,
-    };
-    this.width = 340 - marginLeft - marginRight;
-    this.height = 135 - marginTop - marginBottom;
-  }
-
   onBrush = (selection, x, eventType) => {
     const type = `continuous metadata histogram ${eventType}`;
     return () => {
-      const { dispatch, field, isObs, isUserDefined, isDiffExp } = this.props;
+      const { dispatch, field, isObs, isUserDefined, isGeneSetSummary } =
+        this.props;
 
       // ignore programmatically generated events
       if (!d3.event.sourceEvent) return;
@@ -502,7 +80,7 @@ class HistogramBrush extends React.PureComponent {
         continuousNamespace: {
           isObs,
           isUserDefined,
-          isDiffExp,
+          isGeneSetSummary,
         },
       };
       dispatch(
@@ -511,9 +89,9 @@ class HistogramBrush extends React.PureComponent {
     };
   };
 
-  onBrushEnd = (selection, x) => {
-    return () => {
-      const { dispatch, field, isObs, isUserDefined, isDiffExp } = this.props;
+  onBrushEnd = (selection, x) => () => {
+      const { dispatch, field, isObs, isUserDefined, isGeneSetSummary } =
+        this.props;
       const minAllowedBrushSize = 10;
       const smallAmountToAvoidInfiniteLoop = 0.1;
 
@@ -552,14 +130,13 @@ class HistogramBrush extends React.PureComponent {
         continuousNamespace: {
           isObs,
           isUserDefined,
-          isDiffExp,
+          isGeneSetSummary,
         },
       };
       dispatch(
         actions.selectContinuousMetadataAction(type, query, range, otherProps)
       );
     };
-  };
 
   handleSetGeneAsScatterplotX = () => {
     const { dispatch, field } = this.props;
@@ -609,44 +186,19 @@ class HistogramBrush extends React.PureComponent {
   };
 
   fetchAsyncProps = async () => {
-    const { annoMatrix, field, dispatch, singleContinuousValues } = this.props;
+    const { annoMatrix, width } = this.props;
+
     const { isClipped } = annoMatrix;
-    if (singleContinuousValues.has(field)) {
-      return {
-        histogram: undefined,
-        range: undefined,
-        unclippedRange: undefined,
-        unclippedRangeColor: globals.blue,
-        isSingleValue: true,
-        OK2Render: false,
-      };
-    }
+
     const query = this.createQuery();
     const df = await annoMatrix.fetch(...query);
     const column = df.icol(0);
 
+    // if we are clipped, fetch both our value and our unclipped value,
+    // as we need the absolute min/max range, not just the clipped min/max.
     const summary = column.summarize();
     const range = [summary.min, summary.max];
 
-    if (summary.min === summary.max && !isClipped) {
-      dispatch({
-        type: "add single continuous value",
-        field,
-        value: summary.min,
-      });
-      return {
-        histogram: undefined,
-        range,
-        unclippedRange: range,
-        unclippedRangeColor: globals.blue,
-        isSingleValue: true,
-        OK2Render: false,
-      };
-    }
-
-    const isSingleValue = summary.min === summary.max;
-    // if we are clipped, fetch both our value and our unclipped value,
-    // as we need the absolute min/max range, not just the clipped min/max.
     let unclippedRange = [...range];
     if (isClipped) {
       const parent = await annoMatrix.viewOf.fetch(...query);
@@ -665,11 +217,18 @@ class HistogramBrush extends React.PureComponent {
 
     const histogram = this.calcHistogramCache(
       column,
-      this.margin,
-      this.width,
-      this.height
+      MARGIN,
+      width || WIDTH,
+      HEIGHT
+    );
+    const miniHistogram = this.calcHistogramCache(
+      column,
+      MARGIN_MINI,
+      width || WIDTH_MINI,
+      HEIGHT_MINI
     );
 
+    const isSingleValue = summary.min === summary.max;
     const nonFiniteExtent =
       summary.min === undefined ||
       summary.max === undefined ||
@@ -680,6 +239,7 @@ class HistogramBrush extends React.PureComponent {
 
     return {
       histogram,
+      miniHistogram,
       range,
       unclippedRange,
       unclippedRangeColor,
@@ -689,27 +249,33 @@ class HistogramBrush extends React.PureComponent {
   };
 
   // eslint-disable-next-line class-methods-use-this -- instance method allows for memoization per annotation
-  calcHistogramCache(col, margin, width, height) {
+  calcHistogramCache(col, newMargin, newWidth, newHeight) {
     /*
      recalculate expensive stuff, notably bins, summaries, etc.
     */
-    const histogramCache = {};
-    const summary = col.summarize();
+    const histogramCache = {}; /* maybe change this so that it computes ... */
+    const summary =
+      col.summarize(); /* this is memoized, so it's free the second time you call it */
     const { min: domainMin, max: domainMax } = summary;
     const numBins = 40;
-    const { marginTop, marginLeft } = margin;
+    const { TOP: topMargin, LEFT: leftMargin } =
+      newMargin; /* changes with mini */
 
-    histogramCache.domain = [domainMin, domainMax];
+    histogramCache.domain = [
+      domainMin,
+      domainMax,
+    ]; /* doesn't change with mini */
 
     histogramCache.x = d3
       .scaleLinear()
       .domain([domainMin, domainMax])
-      .range([marginLeft, marginLeft + width]);
+      .range([leftMargin, leftMargin + newWidth]);
 
-    histogramCache.bins = histogramContinuous(col, numBins, [
+    histogramCache.bins = col.histogram(numBins, [
       domainMin,
       domainMax,
-    ]);
+    ]); /* memoized */
+
     histogramCache.binWidth = (domainMax - domainMin) / numBins;
 
     histogramCache.binStart = (i) => domainMin + i * histogramCache.binWidth;
@@ -721,25 +287,43 @@ class HistogramBrush extends React.PureComponent {
     histogramCache.y = d3
       .scaleLinear()
       .domain([0, yMax])
-      .range([marginTop + height, marginTop]);
+      .range([topMargin + newHeight, topMargin]);
 
     return histogramCache;
   }
 
   createQuery() {
-    const { isObs, field, annoMatrix } = this.props;
+    const { isObs, isGeneSetSummary, field, setGenes, annoMatrix } = this.props;
     const { schema } = annoMatrix;
     if (isObs) {
       return ["obs", field];
     }
     const varIndex = schema?.annotations?.var?.index;
     if (!varIndex) return null;
+
+    if (isGeneSetSummary) {
+      return [
+        "X",
+        {
+          summarize: {
+            method: "mean",
+            field: "var",
+            column: varIndex,
+            values: [...setGenes.keys()],
+          },
+        },
+      ];
+    }
+
+    // else, we assume it is a gene expression
     return [
       "X",
       {
-        field: "var",
-        column: varIndex,
-        value: field,
+        where: {
+          field: "var",
+          column: varIndex,
+          value: field,
+        },
       },
     ];
   }
@@ -751,20 +335,34 @@ class HistogramBrush extends React.PureComponent {
       field,
       isColorAccessor,
       isUserDefined,
-      isDiffExp,
-      logFoldChange,
-      pvalAdj,
+      isGeneSetSummary,
       isScatterplotXXaccessor,
       isScatterplotYYaccessor,
       zebra,
       continuousSelectionRange,
       isObs,
+      mini,
+      setGenes,
     } = this.props;
+
+    let { width } = this.props;
+    if (!width) {
+      width = mini ? WIDTH_MINI : WIDTH;
+    }
+
     const fieldForId = field.replace(/\s/g, "_");
-    const showScatterPlot = isDiffExp || isUserDefined;
+    const showScatterPlot = isUserDefined;
+
+    let testClass = "histogram-continuous-metadata";
+    if (isUserDefined) testClass = "histogram-user-gene";
+    else if (isGeneSetSummary) testClass = "histogram-gene-set-summary";
 
     return (
-      <Async watch={annoMatrix} promiseFn={this.fetchAsyncProps}>
+      <Async
+        watchFn={HistogramBrush.watchAsync}
+        promiseFn={this.fetchAsyncProps}
+        watchProps={{ annoMatrix, setGenes }}
+      >
         <Async.Pending initial>
           <StillLoading displayName={field} zebra={zebra} />
         </Async.Pending>
@@ -779,56 +377,57 @@ class HistogramBrush extends React.PureComponent {
               <div
                 id={`histogram_${fieldForId}`}
                 data-testid={`histogram-${field}`}
-                data-testclass={
-                  isDiffExp
-                    ? "histogram-diffexp"
-                    : isUserDefined
-                    ? "histogram-user-gene"
-                    : "histogram-continuous-metadata"
-                }
+                data-testclass={testClass}
                 style={{
-                  padding: globals.leftSidebarSectionPadding,
+                  padding: mini ? 0 : globals.leftSidebarSectionPadding,
                   backgroundColor: zebra ? globals.lightestGrey : "white",
                 }}
               >
-                <HistogramHeader
-                  fieldId={field}
-                  isColorBy={isColorAccessor}
-                  isObs={isObs}
-                  onColorByClick={this.handleColorAction(dispatch)}
-                  onRemoveClick={isUserDefined ? this.removeHistogram : null}
-                  isScatterPlotX={isScatterplotXXaccessor}
-                  isScatterPlotY={isScatterplotYYaccessor}
-                  onScatterPlotXClick={
-                    showScatterPlot ? this.handleSetGeneAsScatterplotX : null
-                  }
-                  onScatterPlotYClick={
-                    showScatterPlot ? this.handleSetGeneAsScatterplotY : null
-                  }
-                />
+                {!mini && isObs ? (
+                  <HistogramHeader
+                    fieldId={field}
+                    isColorBy={isColorAccessor}
+                    isObs={isObs}
+                    onColorByClick={this.handleColorAction(dispatch)}
+                    onRemoveClick={isUserDefined ? this.removeHistogram : null}
+                    isScatterPlotX={isScatterplotXXaccessor}
+                    isScatterPlotY={isScatterplotYYaccessor}
+                    onScatterPlotXClick={
+                      showScatterPlot ? this.handleSetGeneAsScatterplotX : null
+                    }
+                    onScatterPlotYClick={
+                      showScatterPlot ? this.handleSetGeneAsScatterplotY : null
+                    }
+                  />
+                ) : null}
                 <Histogram
                   field={field}
                   fieldForId={fieldForId}
                   display={asyncProps.isSingleValue ? "none" : "block"}
-                  histogram={asyncProps.histogram}
-                  width={this.width}
-                  height={this.height}
+                  histogram={
+                    mini ? asyncProps.miniHistogram : asyncProps.histogram
+                  }
+                  width={width}
+                  height={mini ? HEIGHT_MINI : HEIGHT}
                   onBrush={this.onBrush}
                   onBrushEnd={this.onBrushEnd}
-                  margin={this.margin}
+                  margin={mini ? MARGIN_MINI : MARGIN}
                   isColorBy={isColorAccessor}
                   selectionRange={continuousSelectionRange}
+                  mini={mini}
                 />
-                <HistogramFooter
-                  displayName={field}
-                  hideRanges={asyncProps.isSingleValue}
-                  rangeMin={asyncProps.unclippedRange[0]}
-                  rangeMax={asyncProps.unclippedRange[1]}
-                  rangeColorMin={asyncProps.unclippedRangeColor[0]}
-                  rangeColorMax={asyncProps.unclippedRangeColor[1]}
-                  logFoldChange={logFoldChange}
-                  pvalAdj={pvalAdj}
-                />
+                {!mini && (
+                  <HistogramFooter
+                    isGeneSetSummary={isGeneSetSummary}
+                    isObs={isObs}
+                    displayName={field}
+                    hideRanges={asyncProps.isSingleValue}
+                    rangeMin={asyncProps.unclippedRange[0]}
+                    rangeMax={asyncProps.unclippedRange[1]}
+                    rangeColorMin={asyncProps.unclippedRangeColor[0]}
+                    rangeColorMax={asyncProps.unclippedRangeColor[1]}
+                  />
+                )}
               </div>
             ) : null
           }

client/src/components/brushableHistogram/loading.js

@@ -0,0 +1,43 @@
+import React from "react";
+import { Button } from "@blueprintjs/core";
+
+import * as globals from "../../globals";
+
+const StillLoading = ({ zebra, displayName }) => 
+  /*
+    Render a loading indicator for the field.
+    */
+   (
+    <div
+      data-testclass="gene-loading-spinner"
+      style={{
+        padding: globals.leftSidebarSectionPadding,
+        backgroundColor: zebra ? globals.lightestGrey : "white",
+      }}
+    >
+      <div
+        style={{
+          display: "flex",
+          justifyContent: "space-between",
+          justifyItems: "center",
+          alignItems: "center",
+        }}
+      >
+        <div style={{ minWidth: 30 }} />
+        <div style={{ display: "flex", alignSelf: "center" }}>
+          <span style={{ fontStyle: "italic" }}>{displayName}</span>
+        </div>
+        <div
+          style={{
+            display: "flex",
+            justifyContent: "flex-end",
+          }}
+        >
+          <Button minimal loading intent="primary" />
+        </div>
+      </div>
+    </div>
+  )
+;
+
+export default StillLoading;

client/src/components/categorical/annoSelect.js

@@ -26,16 +26,14 @@ class DuplicateCategorySelect extends React.PureComponent {
             [] /* this is a placeholder, could be  a subcomponent to avoid this */
           }
           filterable={false}
-          itemRenderer={(d, { handleClick }) => {
-            return (
+          itemRenderer={(d, { handleClick }) => (
               <MenuItem
                 data-testclass="duplicate-category-dropdown-option"
                 onClick={handleClick}
                 key={d}
                 text={d}
               />
-            );
-          }}
+            )}
           noResults={<MenuItem disabled text="No results." />}
           onItemSelect={(d) => {
             handleModalDuplicateCategorySelection(d);

client/src/components/categorical/category/annoDialogAddLabel.js

@@ -1,14 +1,13 @@
 import React from "react";
 import { connect } from "react-redux";
-import AnnoDialog from "../annoDialog";
-import LabelInput from "../labelInput";
+import AnnoDialog from "../../annoDialog";
+import LabelInput from "../../labelInput";
 import { labelPrompt, isLabelErroneous } from "../labelUtil";
 import actions from "../../../actions";
 
 @connect((state) => ({
   annotations: state.annotations,
   schema: state.annoMatrix?.schema,
-  ontology: state.ontology,
   obsCrossfilter: state.obsCrossfilter,
 }))
 class Category extends React.PureComponent {
@@ -57,13 +56,11 @@ class Category extends React.PureComponent {
   };
 
   labelNameError = (name) => {
-    const { metadataField, ontology, schema } = this.props;
-    return isLabelErroneous(name, metadataField, ontology, schema);
+    const { metadataField, schema } = this.props;
+    return isLabelErroneous(name, metadataField, schema);
   };
 
-  instruction = (label) => {
-    return labelPrompt(this.labelNameError(label), "New, unique label", ":");
-  };
+  instruction = (label) => labelPrompt(this.labelNameError(label), "New, unique label", ":");
 
   handleChangeOrSelect = (label) => {
     this.setState({ newLabelText: label });
@@ -71,8 +68,7 @@ class Category extends React.PureComponent {
 
   render() {
     const { newLabelText } = this.state;
-    const { metadataField, annotations, ontology, obsCrossfilter } = this.props;
-    const ontologyEnabled = ontology?.enabled ?? false;
+    const { metadataField, annotations, obsCrossfilter } = this.props;
 
     return (
       <>
@@ -97,7 +93,7 @@ class Category extends React.PureComponent {
           handleCancel={this.disableAddNewLabelMode}
           annoInput={
             <LabelInput
-              labelSuggestions={ontologyEnabled ? ontology.terms : null}
+              labelSuggestions={null}
               onChange={this.handleChangeOrSelect}
               onSelect={this.handleChangeOrSelect}
               inputProps={{

client/src/components/categorical/category/annoDialogEditCategoryName.js

@@ -1,7 +1,7 @@
 import React from "react";
 import { connect } from "react-redux";
-import AnnoDialog from "../annoDialog";
-import LabelInput from "../labelInput";
+import AnnoDialog from "../../annoDialog";
+import LabelInput from "../../labelInput";
 import { labelPrompt } from "../labelUtil";
 
 import { AnnotationsHelpers } from "../../../util/stateManager";
@@ -10,7 +10,6 @@ import actions from "../../../actions";
 @connect((state) => ({
   annotations: state.annotations,
   schema: state.annoMatrix?.schema,
-  ontology: state.ontology,
 }))
 class AnnoDialogEditCategoryName extends React.PureComponent {
   constructor(props) {
@@ -90,13 +89,11 @@ class AnnoDialogEditCategoryName extends React.PureComponent {
     return false;
   };
 
-  instruction = (name) => {
-    return labelPrompt(
+  instruction = (name) => labelPrompt(
       this.editedCategoryNameError(name),
       "New, unique category name",
       ":"
     );
-  };
 
   allCategoryNames() {
     const { schema } = this.props;
@@ -105,8 +102,7 @@ class AnnoDialogEditCategoryName extends React.PureComponent {
 
   render() {
     const { newCategoryText } = this.state;
-    const { metadataField, annotations, ontology } = this.props;
-    const ontologyEnabled = ontology?.enabled ?? false;
+    const { metadataField, annotations } = this.props;
 
     return (
       <>
@@ -132,7 +128,7 @@ class AnnoDialogEditCategoryName extends React.PureComponent {
           annoInput={
             <LabelInput
               label={newCategoryText}
-              labelSuggestions={ontologyEnabled ? ontology.terms : null}
+              labelSuggestions={null}
               onChange={this.handleChangeOrSelect}
               onSelect={this.handleChangeOrSelect}
               inputProps={{

client/src/components/categorical/category/annoMenuCategory.js

@@ -9,7 +9,9 @@ import {
   Tooltip,
   Icon,
   PopoverInteractionKind,
+  Intent,
 } from "@blueprintjs/core";
+import { IconNames } from "@blueprintjs/icons";
 
 import * as globals from "../../../globals";
 import actions from "../../../actions";
@@ -89,8 +91,8 @@ class AnnoMenuCategory extends React.PureComponent {
                     text={editText}
                   />
                   <MenuItem
-                    icon="delete"
-                    intent="danger"
+                    icon={IconNames.TRASH}
+                    intent={Intent.DANGER}
                     data-testclass="handleDeleteCategory"
                     data-testid={`${metadataField}:delete-category`}
                     onClick={this.handleDeleteCategory}

client/src/components/categorical/category/index.js

@@ -44,6 +44,7 @@ const LABEL_WIDTH_ANNO = LABEL_WIDTH - ANNO_BUTTON_WIDTH;
     schema,
     crossfilter: state.obsCrossfilter,
     isUserAnno,
+    genesets: state.genesets.genesets,
   };
 })
 class Category extends React.PureComponent {
@@ -142,9 +143,15 @@ class Category extends React.PureComponent {
     */
     const { schema } = annoMatrix;
     const { colorAccessor, colorMode } = colors;
+    const { genesets } = this.props;
     let colorDataPromise = Promise.resolve(null);
     if (colorAccessor) {
-      const query = createColorQuery(colorMode, colorAccessor, schema);
+      const query = createColorQuery(
+        colorMode,
+        colorAccessor,
+        schema,
+        genesets
+      );
       if (query) colorDataPromise = annoMatrix.fetch(...query);
     }
     const [categoryData, colorData] = await Promise.all([
@@ -251,14 +258,12 @@ class Category extends React.PureComponent {
                 isColorAccessor,
                 handleCategoryToggleAllClick,
               } = asyncProps;
-              const isTruncated = !!categorySummary?.isTruncated;
               const selectionState = this.getSelectionState(categorySummary);
               return (
                 <CategoryRender
                   metadataField={metadataField}
                   checkboxID={checkboxID}
                   isUserAnno={isUserAnno}
-                  isTruncated={isTruncated}
                   isExpanded={isExpanded}
                   isColorAccessor={isColorAccessor}
                   selectionState={selectionState}
@@ -283,57 +288,54 @@ class Category extends React.PureComponent {
 
 export default Category;
 
-const StillLoading = ({ metadataField, checkboxID }) => {
+const StillLoading = ({ metadataField, checkboxID }) => (
   /*
   We are still loading this category, so render a "busy" signal.
   */
-  return (
+  <div
+    style={{
+      maxWidth: globals.maxControlsWidth,
+    }}
+  >
     <div
       style={{
-        maxWidth: globals.maxControlsWidth,
+        display: "flex",
+        justifyContent: "space-between",
+        alignItems: "baseline",
       }}
     >
       <div
         style={{
           display: "flex",
-          justifyContent: "space-between",
-          alignItems: "baseline",
+          justifyContent: "flex-start",
+          alignItems: "flex-start",
         }}
       >
-        <div
-          style={{
-            display: "flex",
-            justifyContent: "flex-start",
-            alignItems: "flex-start",
-          }}
+        <label
+          htmlFor={checkboxID}
+          className={`${Classes.CONTROL} ${Classes.CHECKBOX}`}
         >
-          <label
-            htmlFor={checkboxID}
-            className={`${Classes.CONTROL} ${Classes.CHECKBOX}`}
+          <input disabled id={checkboxID} checked type="checkbox" />
+          <span className={Classes.CONTROL_INDICATOR} />
+        </label>
+        <Truncate>
+          <span
+            style={{
+              cursor: "pointer",
+              display: "inline-block",
+              width: LABEL_WIDTH,
+            }}
           >
-            <input disabled id={checkboxID} checked type="checkbox" />
-            <span className={Classes.CONTROL_INDICATOR} />
-          </label>
-          <Truncate>
-            <span
-              style={{
-                cursor: "pointer",
-                display: "inline-block",
-                width: LABEL_WIDTH,
-              }}
-            >
-              {metadataField}
-            </span>
-          </Truncate>
-        </div>
-        <div>
-          <Button minimal loading intent="primary" />
-        </div>
+            {metadataField}
+          </span>
+        </Truncate>
+      </div>
+      <div>
+        <Button minimal loading intent="primary" />
       </div>
     </div>
-  );
-};
-
+  </div>
+);
 const ErrorLoading = ({ metadataField, error }) => {
   console.error(error); // log error to console as it is unexpected.
   return (
@@ -357,7 +359,6 @@ const CategoryHeader = React.memo(
     metadataField,
     checkboxID,
     isUserAnno,
-    isTruncated,
     isColorAccessor,
     isExpanded,
     selectionState,
@@ -416,6 +417,7 @@ const CategoryHeader = React.memo(
                   maxWidth: isUserAnno ? LABEL_WIDTH_ANNO : LABEL_WIDTH,
                 }}
                 data-testid={`${metadataField}:category-label`}
+                tabIndex="-1"
               >
                 {metadataField}
               </span>
@@ -433,8 +435,8 @@ const CategoryHeader = React.memo(
             )}
           </span>
         </div>
-        {<AnnoDialogEditCategoryName metadataField={metadataField} />}
-        {<AnnoDialogAddLabel metadataField={metadataField} />}
+        <AnnoDialogEditCategoryName metadataField={metadataField} />
+        <AnnoDialogAddLabel metadataField={metadataField} />
         <div>
           <AnnoMenu
             metadataField={metadataField}
@@ -445,11 +447,7 @@ const CategoryHeader = React.memo(
           />
 
           <Tooltip
-            content={
-              isTruncated
-                ? `Coloring by ${metadataField} is disabled, as it exceeds the limit of ${globals.maxCategoricalOptionsToDisplay} labels`
-                : "Use as color scale"
-            }
+            content="Use as color scale"
             position={Position.LEFT}
             usePortal
             hoverOpenDelay={globals.tooltipHoverOpenDelay}
@@ -464,7 +462,6 @@ const CategoryHeader = React.memo(
               onClick={onColorChangeClick}
               active={isColorAccessor}
               intent={isColorAccessor ? "primary" : "none"}
-              disabled={isTruncated}
               icon="tint"
             />
           </Tooltip>
@@ -479,7 +476,6 @@ const CategoryRender = React.memo(
     metadataField,
     checkboxID,
     isUserAnno,
-    isTruncated,
     isColorAccessor,
     isExpanded,
     selectionState,
@@ -498,12 +494,23 @@ const CategoryRender = React.memo(
     */
     const { numCategoryValues } = categorySummary;
     const isSingularValue = !isUserAnno && numCategoryValues === 1;
-
     if (isSingularValue) {
       /*
       Entire category has a single value, special case.
       */
-      return null;
+      const theOneValue = categorySummary.categoryValues[0];
+      return (
+        <div style={{ marginBottom: 10, marginTop: 4 }}>
+          <Truncate>
+            <span style={{ maxWidth: 150, fontWeight: 700 }}>
+              {metadataField}
+            </span>
+          </Truncate>
+          <Truncate>
+            <span style={{ maxWidth: 150 }}>{`: ${theOneValue}`}</span>
+          </Truncate>
+        </div>
+      );
     }
 
     /*
@@ -528,7 +535,6 @@ const CategoryRender = React.memo(
             metadataField={metadataField}
             checkboxID={checkboxID}
             isUserAnno={isUserAnno}
-            isTruncated={isTruncated}
             isExpanded={isExpanded}
             isColorAccessor={isColorAccessor}
             selectionState={selectionState}
@@ -554,11 +560,6 @@ const CategoryRender = React.memo(
             ) : null
           }
         </div>
-        <div>
-          {isExpanded && isTruncated ? (
-            <p style={{ paddingLeft: 15 }}>... truncated list ...</p>
-          ) : null}
-        </div>
       </div>
     );
   }

client/src/components/categorical/index.js

@@ -1,21 +1,18 @@
-// jshint esversion: 6
 import React from "react";
 import { AnchorButton, Tooltip, Position } from "@blueprintjs/core";
 import { connect } from "react-redux";
 import * as globals from "../../globals";
 import Category from "./category";
 import { AnnotationsHelpers, ControlsHelpers } from "../../util/stateManager";
-import AnnoDialog from "./annoDialog";
+import AnnoDialog from "../annoDialog";
 import AnnoSelect from "./annoSelect";
-import LabelInput from "./labelInput";
+import LabelInput from "../labelInput";
 import { labelPrompt } from "./labelUtil";
 import actions from "../../actions";
 
 @connect((state) => ({
   writableCategoriesEnabled: state.config?.parameters?.annotations ?? false,
   schema: state.annoMatrix?.schema,
-  ontology: state.ontology,
-  userInfo: state.userInfo,
 }))
 class Categories extends React.Component {
   constructor(props) {
@@ -100,13 +97,8 @@ class Categories extends React.Component {
     this.setState({ newCategoryText: name });
   };
 
-  instruction = (name) => {
-    return labelPrompt(
-      this.categoryNameError(name),
-      "New, unique category name",
-      ":"
-    );
-  };
+  instruction = (name) =>
+    labelPrompt(this.categoryNameError(name), "New, unique category name", ":");
 
   onExpansionChange = (catName) => {
     const { expandedCats } = this.state;
@@ -128,17 +120,10 @@ class Categories extends React.Component {
       newCategoryText,
       expandedCats,
     } = this.state;
-    const {
-      writableCategoriesEnabled,
-      schema,
-      ontology,
-      userInfo,
-    } = this.props;
-    const ontologyEnabled = ontology?.enabled ?? false;
+    const { writableCategoriesEnabled, schema } = this.props;
     /* all names, sorted in display order.  Will be rendered in this order */
-    const allCategoryNames = ControlsHelpers.selectableCategoryNames(
-      schema
-    ).sort();
+    const allCategoryNames =
+      ControlsHelpers.selectableCategoryNames(schema).sort();
 
     return (
       <div
@@ -159,7 +144,7 @@ class Categories extends React.Component {
           handleCancel={this.handleDisableAnnoMode}
           annoInput={
             <LabelInput
-              labelSuggestions={ontologyEnabled ? ontology.terms : null}
+              labelSuggestions={null}
               onChange={this.handleChange}
               onSelect={this.handleSelect}
               inputProps={{
@@ -182,20 +167,43 @@ class Categories extends React.Component {
           }
         />
 
+        {writableCategoriesEnabled ? (
+          <div style={{ marginBottom: 10 }}>
+            <Tooltip
+              content="Create a new category"
+              position={Position.RIGHT}
+              boundary="viewport"
+              hoverOpenDelay={globals.tooltipHoverOpenDelay}
+              modifiers={{
+                preventOverflow: { enabled: false },
+                hide: { enabled: false },
+              }}
+            >
+              <AnchorButton
+                type="button"
+                data-testid="open-annotation-dialog"
+                onClick={this.handleEnableAnnoMode}
+                intent="primary"
+              >
+                Create new <strong>category</strong>
+              </AnchorButton>
+            </Tooltip>
+          </div>
+        ) : null}
+
         {/* READ ONLY CATEGORICAL FIELDS */}
         {/* this is duplicative but flat, could be abstracted */}
-        {allCategoryNames.map((catName) =>
-          !schema.annotations.obsByName[catName].writable &&
-          (schema.annotations.obsByName[catName].categories?.length > 1 ||
-            !schema.annotations.obsByName[catName].categories) ? (
-            <Category
-              key={catName}
-              metadataField={catName}
-              onExpansionChange={this.onExpansionChange}
-              isExpanded={expandedCats.has(catName)}
-              createAnnoModeActive={createAnnoModeActive}
-            />
-          ) : null
+        {allCategoryNames.map(
+          (catName) =>
+            !schema.annotations.obsByName[catName].writable && (
+              <Category
+                key={catName}
+                metadataField={catName}
+                onExpansionChange={this.onExpansionChange}
+                isExpanded={expandedCats.has(catName)}
+                createAnnoModeActive={createAnnoModeActive}
+              />
+            )
         )}
         {/* WRITEABLE FIELDS */}
         {allCategoryNames.map((catName) =>
@@ -209,33 +217,6 @@ class Categories extends React.Component {
             />
           ) : null
         )}
-
-        {writableCategoriesEnabled ? (
-          <Tooltip
-            content={
-              userInfo.is_authenticated
-                ? "Create a new category"
-                : "You must be logged in to create new categorical fields"
-            }
-            position={Position.RIGHT}
-            boundary="viewport"
-            hoverOpenDelay={globals.tooltipHoverOpenDelay}
-            modifiers={{
-              preventOverflow: { enabled: false },
-              hide: { enabled: false },
-            }}
-          >
-            <AnchorButton
-              type="button"
-              data-testid="open-annotation-dialog"
-              onClick={this.handleEnableAnnoMode}
-              intent="primary"
-              disabled={!userInfo.is_authenticated}
-            >
-              Create new category
-            </AnchorButton>
-          </Tooltip>
-        ) : null}
       </div>
     );
   }

client/src/components/categorical/labelUtil.js

@@ -3,7 +3,7 @@ import { Colors } from "@blueprintjs/core";
 
 import { AnnotationsHelpers } from "../../util/stateManager";
 
-export function isLabelErroneous(label, metadataField, ontology, schema) {
+export function isLabelErroneous(label, metadataField, schema) {
   /*
     return false if this is a LEGAL/acceptable category name or NULL/empty string,
     or return an error type.
@@ -12,10 +12,9 @@ export function isLabelErroneous(label, metadataField, ontology, schema) {
   /* allow empty string */
   if (label === "") return false;
 
-  /* check for label syntax errors, but allow terms in ontology */
-  const termInOntology = ontology?.termSet.has(label) ?? false;
+  /* check for label syntax errors */
   const error = AnnotationsHelpers.annotationNameIsErroneous(label);
-  if (error && !termInOntology) return error;
+  if (error) return error;
 
   /* disallow duplicates */
   const { obsByName } = schema.annotations;

client/src/components/categorical/value/index.js

@@ -14,8 +14,8 @@ import {
 } from "@blueprintjs/core";
 import * as globals from "../../../globals";
 import styles from "../categorical.css";
-import AnnoDialog from "../annoDialog";
-import LabelInput from "../labelInput";
+import AnnoDialog from "../../annoDialog";
+import LabelInput from "../../labelInput";
 import Truncate from "../../util/truncate";
 
 import { AnnotationsHelpers } from "../../../util/stateManager";
@@ -25,8 +25,8 @@ import MiniHistogram from "../../miniHistogram";
 import MiniStackedBar from "../../miniStackedBar";
 import { CategoryCrossfilterContext } from "../categoryContext";
 
-const VALUE_HEIGHT = 11;
-const CHART_WIDTH = 100;
+const STACKED_BAR_HEIGHT = 11;
+const STACKED_BAR_WIDTH = 100;
 
 /* this is defined outside of the class so we can use it in connect() */
 function _currentLabelAsString(ownProps) {
@@ -50,7 +50,6 @@ function _currentLabelAsString(ownProps) {
   return {
     annotations: state.annotations,
     schema: state.annoMatrix?.schema,
-    ontology: state.ontology,
     isDilated,
     isSelected,
     label,
@@ -118,14 +117,12 @@ class CategoryValue extends React.Component {
   };
 
   labelNameError = (name) => {
-    const { metadataField, ontology, schema } = this.props;
+    const { metadataField, schema } = this.props;
     if (name === this.currentLabelAsString()) return false;
-    return isLabelErroneous(name, metadataField, ontology, schema);
+    return isLabelErroneous(name, metadataField, schema);
   };
 
-  instruction = (label) => {
-    return labelPrompt(this.labelNameError(label), "New, unique label", ":");
-  };
+  instruction = (label) => labelPrompt(this.labelNameError(label), "New, unique label", ":");
 
   activateEditLabelMode = () => {
     const { dispatch, metadataField, categoryIndex, label } = this.props;
@@ -151,12 +148,8 @@ class CategoryValue extends React.Component {
   };
 
   toggleOff = () => {
-    const {
-      dispatch,
-      metadataField,
-      categoryIndex,
-      categorySummary,
-    } = this.props;
+    const { dispatch, metadataField, categoryIndex, categorySummary } =
+      this.props;
     const label = categorySummary.categoryValues[categoryIndex];
     dispatch(
       actions.selectCategoricalMetadataAction(
@@ -174,7 +167,7 @@ class CategoryValue extends React.Component {
     Checks to see if at least one of the following changed:
     * world state
     * the color accessor (what is currently being colored by)
-    * if this catagorical value's selection status has changed
+    * if this categorical value's selection status has changed
     * the crossfilter (ie, global selection state)
 
     If and only if true, update the component
@@ -201,6 +194,13 @@ class CategoryValue extends React.Component {
     const newCount = newCategorySummary.categoryValueCounts[newCategoryIndex];
     const countChanged = count !== newCount;
 
+    // If the user edits an annotation that is currently colored-by, colors may be re-assigned.
+    // This test is conservative - it may cause re-rendering of entire category (all labels)
+    // if any one changes, but only for the currently colored-by category.
+    const colorMightHaveChanged =
+      nextProps.colorAccessor === nextProps.metadataField &&
+      props.categorySummary !== nextProps.categorySummary;
+
     return (
       labelChanged ||
       valueSelectionChange ||
@@ -208,17 +208,14 @@ class CategoryValue extends React.Component {
       annotationsChange ||
       editingLabel ||
       dilationChange ||
-      countChanged
+      countChanged ||
+      colorMightHaveChanged
     );
   };
 
   toggleOn = () => {
-    const {
-      dispatch,
-      metadataField,
-      categoryIndex,
-      categorySummary,
-    } = this.props;
+    const { dispatch, metadataField, categoryIndex, categorySummary } =
+      this.props;
     const label = categorySummary.categoryValues[categoryIndex];
     dispatch(
       actions.selectCategoricalMetadataAction(
@@ -402,7 +399,7 @@ class CategoryValue extends React.Component {
         label,
         colorTable,
         schema,
-        CHART_WIDTH
+        STACKED_BAR_WIDTH
       ) ?? {};
 
     if (!domainValues || !scale || !domain || !occupancy) {
@@ -418,8 +415,8 @@ class CategoryValue extends React.Component {
           domain,
           occupancy,
         }}
-        height={VALUE_HEIGHT}
-        width={CHART_WIDTH}
+        height={STACKED_BAR_HEIGHT}
+        width={STACKED_BAR_WIDTH}
       />
     );
   };
@@ -452,8 +449,8 @@ class CategoryValue extends React.Component {
         colorAccessor,
         colorData,
         label,
-        CHART_WIDTH,
-        VALUE_HEIGHT
+        STACKED_BAR_WIDTH,
+        STACKED_BAR_HEIGHT
       ) ?? {}; // if createHistogramBins returns empty object assign null to deconstructed
 
     if (!xScale || !yScale || !bins) return null;
@@ -468,8 +465,8 @@ class CategoryValue extends React.Component {
         }}
         obsOrVarContinuousFieldDisplayName={colorAccessor}
         domainLabel={label}
-        height={VALUE_HEIGHT}
-        width={CHART_WIDTH}
+        height={STACKED_BAR_HEIGHT}
+        width={STACKED_BAR_WIDTH}
       />
     );
   };
@@ -482,14 +479,12 @@ class CategoryValue extends React.Component {
       colorTable,
       isUserAnno,
       annotations,
-      ontology,
       isDilated,
       isSelected,
       categorySummary,
       label,
     } = this.props;
     const colorScale = colorTable?.scale;
-    const ontologyEnabled = ontology?.enabled ?? false;
 
     const { editedLabelText } = this.state;
 
@@ -526,7 +521,7 @@ class CategoryValue extends React.Component {
       colorAccessor && !isColorBy
         ? globals.leftSidebarWidth -
           otherElementsWidth -
-          CHART_WIDTH -
+          STACKED_BAR_WIDTH -
           CHART_MARGIN
         : globals.leftSidebarWidth - otherElementsWidth;
 
@@ -582,6 +577,7 @@ class CategoryValue extends React.Component {
               <span
                 data-testid={`categorical-value-${metadataField}-${displayString}`}
                 data-testclass="categorical-value"
+                tabIndex="-1"
                 style={{
                   width: labelWidth,
                   color:
@@ -625,7 +621,7 @@ class CategoryValue extends React.Component {
                   annoInput={
                     <LabelInput
                       label={editedLabelText}
-                      labelSuggestions={ontologyEnabled ? ontology.terms : null}
+                      labelSuggestions={null}
                       onChange={this.handleTextChange}
                       onSelect={this.handleTextChange}
                       inputProps={{
@@ -669,8 +665,8 @@ class CategoryValue extends React.Component {
               display={isColorBy && categoryValueIndices ? "auto" : "none"}
               style={{
                 marginLeft: 5,
-                width: VALUE_HEIGHT,
-                height: VALUE_HEIGHT,
+                width: 15,
+                height: 15,
                 backgroundColor:
                   isColorBy && categoryValueIndices
                     ? colorScale(categoryValueIndices.get(label))
@@ -731,7 +727,7 @@ class CategoryValue extends React.Component {
                       ) : null}
                       {displayString !== globals.unassignedCategoryLabel ? (
                         <MenuItem
-                          icon="delete"
+                          icon="trash"
                           intent="danger"
                           data-testclass="handleDeleteValue"
                           data-testid={`${metadataField}:${displayString}:delete-label`}

client/src/components/categorical/value/occupancy.js

@@ -1,4 +1,3 @@
-// jshint esversion: 6
 import React from "react";
 import { connect } from "react-redux";
 import * as d3 from "d3";
@@ -23,12 +22,8 @@ class Occupancy extends React.PureComponent {
       createHistogram fetches the continous data in relation to the cells releveant to the catagory value.
       It then seperates that data into 50 bins for drawing the mini-histogram
     */
-    const {
-      metadataField,
-      categoryData,
-      colorData,
-      categoryValue,
-    } = this.props;
+    const { metadataField, categoryData, colorData, categoryValue } =
+      this.props;
 
     if (!this.canvas) return;
 

client/src/components/continuous/setupParallelCoordinates.js

@@ -3,7 +3,6 @@
       Setup SVG & Canvas elements
 ******************************************
 ******************************************/
-// jshint esversion: 6
 import * as d3 from "d3";
 
 const setupParallelCoordinates = (width, height, margin) => {

client/src/components/continuous/util.js

@@ -1,5 +1,4 @@
-// jshint esversion: 6
-import _ from "lodash";
+import each from "lodash.foreach";
 import * as d3 from "d3";
 
 const paddingRight = 120;
@@ -15,7 +14,7 @@ export const devicePixelRatio = window.devicePixelRatio || 1;
 
 export const createDimensions = (data) => {
   const newArr = [];
-  _.each(data, (value, key) => {
+  each(data, (value, key) => {
     if (value.range) {
       newArr.push({
         key /* room for confusion: lodash calls this key, it's also the name of the property parallel coords code is looking for */,

client/src/components/continuousLegend/index.js

@@ -1,4 +1,3 @@
-// jshint esversion: 6
 import React from "react";
 import { connect } from "react-redux";
 import * as d3 from "d3";
@@ -10,21 +9,20 @@ import {
 } from "../../util/stateManager/colorHelpers";
 
 // create continuous color legend
-// http://bl.ocks.org/syntagmatic/e8ccca52559796be775553b467593a9f
-const continuous = (selectorId, colorscale, colorAccessor) => {
-  const legendheight = 200;
-  const legendwidth = 80;
+const continuous = (selectorId, colorScale, colorAccessor) => {
+  const legendHeight = 200;
+  const legendWidth = 80;
   const margin = { top: 10, right: 60, bottom: 10, left: 2 };
 
   const canvas = d3
     .select(selectorId)
-    .style("height", `${legendheight}px`)
-    .style("width", `${legendwidth}px`)
+    .style("height", `${legendHeight}px`)
+    .style("width", `${legendWidth}px`)
     .append("canvas")
-    .attr("height", legendheight - margin.top - margin.bottom)
+    .attr("height", legendHeight - margin.top - margin.bottom)
     .attr("width", 1)
-    .style("height", `${legendheight - margin.top - margin.bottom}px`)
-    .style("width", `${legendwidth - margin.left - margin.right}px`)
+    .style("height", `${legendHeight - margin.top - margin.bottom}px`)
+    .style("width", `${legendWidth - margin.left - margin.right}px`)
     .style("position", "absolute")
     .style("top", `${margin.top + 1}px`)
     .style("left", `${margin.left + 1}px`)
@@ -37,18 +35,18 @@ const continuous = (selectorId, colorscale, colorAccessor) => {
 
   const ctx = canvas.getContext("2d");
 
-  const legendscale = d3
+  const legendScale = d3
     .scaleLinear()
-    .range([1, legendheight - margin.top - margin.bottom])
+    .range([1, legendHeight - margin.top - margin.bottom])
     .domain([
-      colorscale.domain()[1],
-      colorscale.domain()[0],
+      colorScale.domain()[1],
+      colorScale.domain()[0],
     ]); /* we flip this to make viridis colors dark if high in the color scale */
 
   // image data hackery based on http://bl.ocks.org/mbostock/048d21cf747371b11884f75ad896e5a5
-  const image = ctx.createImageData(1, legendheight);
-  d3.range(legendheight).forEach((i) => {
-    const c = d3.rgb(colorscale(legendscale.invert(i)));
+  const image = ctx.createImageData(1, legendHeight);
+  d3.range(legendHeight).forEach((i) => {
+    const c = d3.rgb(colorScale(legendScale.invert(i)));
     image.data[4 * i] = c.r;
     image.data[4 * i + 1] = c.g;
     image.data[4 * i + 2] = c.b;
@@ -66,20 +64,20 @@ const continuous = (selectorId, colorscale, colorAccessor) => {
   });
   */
 
-  const legendaxis = d3
-    .axisRight(legendscale)
+  const legendAxis = d3
+    .axisRight(legendScale)
     .ticks(6)
     .tickFormat(
       d3.format(
-        legendscale.domain().some((n) => Math.abs(n) >= 10000) ? ".0e" : ","
+        legendScale.domain().some((n) => Math.abs(n) >= 10000) ? ".0e" : ","
       )
     );
 
   const svg = d3
     .select(selectorId)
     .append("svg")
-    .attr("height", `${legendheight}px`)
-    .attr("width", `${legendwidth}px`)
+    .attr("height", `${legendHeight}px`)
+    .attr("width", `${legendWidth}px`)
     .style("position", "absolute")
     .style("left", "0px")
     .style("top", "0px");
@@ -89,17 +87,19 @@ const continuous = (selectorId, colorscale, colorAccessor) => {
     .attr("class", "axis")
     .attr(
       "transform",
-      `translate(${legendwidth - margin.left - margin.right + 3},${margin.top})`
+      `translate(${legendWidth - margin.left - margin.right + 3},${margin.top})`
     )
-    .call(legendaxis);
+    .call(legendAxis);
 
   // text label for the y axis
   svg
     .append("text")
     .attr("transform", "rotate(-90)")
     .attr("y", 2)
-    .attr("x", 0 - legendheight / 2)
+    .attr("x", 0 - legendHeight / 2)
     .attr("dy", "1em")
+    .attr("data-testid", "continuous_legend_color_by_label")
+    .attr("aria-label", colorAccessor)
     .style("text-anchor", "middle")
     .style("fill", "white")
     .text(colorAccessor);
@@ -108,33 +108,24 @@ const continuous = (selectorId, colorscale, colorAccessor) => {
 @connect((state) => ({
   annoMatrix: state.annoMatrix,
   colors: state.colors,
+  genesets: state.genesets.genesets,
 }))
 class ContinuousLegend extends React.Component {
-  constructor(props) {
-    super(props);
-    this.ref = null;
-    this.state = {
-      colorAccessor: null,
-      colorScale: null,
-    };
-  }
-
-  componentDidMount() {
-    this.updateState(null);
-  }
-
-  componentDidUpdate(prevProps) {
-    this.updateState(prevProps);
-  }
-
-  async updateState(prevProps) {
-    const { annoMatrix, colors } = this.props;
+  async componentDidUpdate(prevProps) {
+    const { annoMatrix, colors, genesets } = this.props;
     if (!colors || !annoMatrix) return;
 
     if (colors !== prevProps?.colors || annoMatrix !== prevProps?.annoMatrix) {
       const { schema } = annoMatrix;
       const { colorMode, colorAccessor, userColors } = colors;
-      const colorQuery = createColorQuery(colorMode, colorAccessor, schema);
+
+      const colorQuery = createColorQuery(
+        colorMode,
+        colorAccessor,
+        schema,
+        genesets
+      );
+
       const colorDf = colorQuery ? await annoMatrix.fetch(...colorQuery) : null;
       const colorTable = createColorTable(
         colorMode,
@@ -161,35 +152,19 @@ class ContinuousLegend extends React.Component {
           );
         }
       }
-
-      this.setState({
-        colorAccessor,
-        colorScale: colorTable.scale,
-      });
     }
   }
 
   render() {
-    const { colorAccessor, colorScale } = this.state;
-    if (
-      colorScale?.domain &&
-      colorScale.domain()[1] === colorScale.domain()[0]
-    ) {
-      /* it's a single value, not a distribution, min max are the same */
-      return null;
-    }
     return (
       <div
         id="continuous_legend"
-        ref={(ref) => {
-          this.ref = ref;
-        }}
         style={{
-          display: colorAccessor ? "inherit" : "none",
           position: "absolute",
           left: 8,
           top: 35,
           zIndex: 1,
+          pointerEvents: "none",
         }}
       />
     );

client/src/components/embedding/index.js

@@ -15,13 +15,12 @@ import * as globals from "../../globals";
 import actions from "../../actions";
 import { getDiscreteCellEmbeddingRowIndex } from "../../util/stateManager/viewStackHelpers";
 
-@connect((state) => {
-  return {
-    layoutChoice: state.layoutChoice, // TODO: really should clean up naming, s/layout/embedding/g
-    schema: state.annoMatrix?.schema,
-    crossfilter: state.obsCrossfilter,
-  };
-})
+@connect((state) => ({
+  imageUnderlay: state.imageUnderlay,
+  layoutChoice: state.layoutChoice, // TODO: really should clean up naming, s/layout/embedding/g
+  schema: state.annoMatrix?.schema,
+  crossfilter: state.obsCrossfilter,
+}))
 class Embedding extends React.PureComponent {
   constructor(props) {
     super(props);
@@ -29,8 +28,18 @@ class Embedding extends React.PureComponent {
   }
 
   handleLayoutChoiceChange = (e) => {
-    const { dispatch } = this.props;
+    const { dispatch, imageUnderlay } = this.props;
     dispatch(actions.layoutChoiceAction(e.currentTarget.value));
+
+    // if we just switched off spatial, if the image is on, turn it off
+    if (
+      imageUnderlay.isActive &&
+      e.target.value !== globals.spatialEmbeddingKeyword
+    ) {
+      dispatch({
+        type: "toggle image underlay",
+      });
+    }
   };
 
   render() {

client/src/components/framework/container.js

@@ -1,4 +1,3 @@
-// jshint esversion: 6
 import React from "react";
 
 function Container(props) {

client/src/components/framework/layout.js

@@ -1,4 +1,3 @@
-// jshint esversion: 6
 import React from "react";
 import * as globals from "../../globals";
 

client/src/components/framework/toasters.js

@@ -1,31 +1,33 @@
 import { Position, Toaster, Intent } from "@blueprintjs/core";
 
 /** Singleton toaster instance. Create separate instances for different options. */
-
-const ToastTopCenter = Toaster.create({
-  className: "recipe-toaster",
-  position: Position.TOP,
-  maxToasts: 4,
-});
+let ToastTopCenter = null;
+if (typeof document !== "undefined") {
+  ToastTopCenter = Toaster.create({
+    className: "recipe-toaster",
+    position: Position.TOP,
+    maxToasts: 4,
+  });
+}
 
 /*
 A "user" error - eg, bad input
 */
 export const postUserErrorToast = (message) =>
-  ToastTopCenter.show({ message, intent: Intent.WARNING });
+  ToastTopCenter?.show({ message, intent: Intent.WARNING });
 
 /*
 A toast the user must dismiss manually, because they need to act on its information,
 ie., 8 bulk add genes out of 40 were bad. Manually see which ones and fix.
 */
 export const keepAroundErrorToast = (message) =>
-  ToastTopCenter.show({ message, timeout: 0, intent: Intent.WARNING });
+  ToastTopCenter?.show({ message, timeout: 0, intent: Intent.WARNING });
 
 /*
 a hard network error
 */
 export const postNetworkErrorToast = (message, key = undefined) =>
-  ToastTopCenter.show(
+  ToastTopCenter?.show(
     {
       message,
       timeout: 30000,
@@ -38,14 +40,14 @@ export const postNetworkErrorToast = (message, key = undefined) =>
 Async message to user
 */
 export const postAsyncSuccessToast = (message) =>
-  ToastTopCenter.show({
+  ToastTopCenter?.show({
     message,
     timeout: 10000,
     intent: Intent.SUCCESS,
   });
 
 export const postAsyncFailureToast = (message) =>
-  ToastTopCenter.show({
+  ToastTopCenter?.show({
     message,
     timeout: 10000,
     intent: Intent.WARNING,