Bump version: 0.16.8-rc.0 → 0.16.8

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 1.1.1
+current_version = 0.16.8
 commit = True
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(?:-(?P<prerel>rc)\.(?P<prerelversion>\d+))?
 serialize = 

.github/ISSUE_TEMPLATE/tech-issue.md

@@ -1,23 +0,0 @@
----
-name: Tech Issue
-about: Engineering-specific technical work that is not product-specific. Engineering team "owns" these issues.
-title: ""
-labels: tech
-assignees: ""
----
-
-## Motivation
-
-Why is this work important to engineers?
-
-## Definition of Done
-
-What should the end result look like? What will have been changed?
-
-## Tasks
-
-Detail the specific tasks that can be used to accomplish the desired changes.
-If detailed steps cannot be provided at this time, please file a [Tech Proposal](https://docs.google.com/document/d/1o2vuvl-kXwRJN1nBoPzJS_MAQgDGYnjmPZWa4qRDi-I/edit#heading=h.7dvzhm7gqc3v) instead.
-
-- [ ]
-- [ ]

.github/workflows/compatibility_tests.yml

@@ -16,108 +16,93 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v1
         with:
           python-version: ${{ matrix.python-version }}
       - name: Build docker image
         run: docker build .
 
-  matrix-compatibility-test:
+  cellxgene-main-with-python-and-anndata-versions:
-    name: cxg:${{ matrix.cellxgene_build }} os:${{ matrix.os }} py:${{ matrix.python-version }} anndata:${{ matrix.anndata_version || 'latest' }}
+    name: python versions x anndata versions
-    runs-on: ${{ matrix.os }}
+    runs-on: ubuntu-latest
+    continue-on-error: true
     strategy:
-      fail-fast: false
       matrix:
-        # note: The `macos-latest` is latest Catalina version, and not Big Sur. So we explicitly ask for Big Sur (`macos-11`)
+        python-version: [3.6, 3.7] # As of Oct 2020 Anndata is not compatible with 3.8
-        os: [ubuntu-latest, macos-latest, macos-11]
+        anndata-version: [0.7.0, 0.7.1, 0.7.2, 0.7.3, 0.7.4, 0.7.5]
-        python-version: [3.6, 3.7, 3.8, 3.9]
+        test-suite: [smoke-test, smoke-test-annotations]
-        cellxgene_build: [main, latest]
-        exclude:
-          # 3.6 no longer avail on Big Sur (`macos-11`)
-          - os: macos-11
-            python-version: 3.6
-          # no pypi build exists for macos+py3.9 and source install fails to
-          # install `tables` py pkg (a `scanpy` dependency), so we test py3.9
-          # only on ubuntu
-          - os: macos-11
-            python-version: 3.9
-          - os: macos-latest
-            python-version: 3.9
-        # add anndata pinned version test for subset of matrix configurations,
-        # in order to reduce matrix cross-product explosion
-        include:
-          - python-version: 3.8
-            cellxgene_build: latest
-            # TODO: dynamically use the literal version in requirements.txt,
-            # to avoid having to update this in manually in the future
-            # TODO: Do not bother running this if anndata latest version
-            # matches this pinned version, to avoid a redundant test
-            anndata_version: '==0.7.6'
     steps:
     - uses: actions/checkout@v2
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v1
       with:
         python-version: ${{ matrix.python-version }}
-    - name: Cache env vars
+    - name: Install dependencies
-      run: echo "PIP_CACHE=`python -m pip cache dir`" >> $GITHUB_ENV
-    - name: Cache env vars (MacOS)
-      if: startsWith(matrix.os, 'macos')
-      run: echo "BREW_CACHE=`brew --cache`" >> $GITHUB_ENV
-    # FIXME: Only working for Linux
-    - name: Python cache
-      uses: actions/cache@v1
-      with:
-        path: ${{ env.PIP_CACHE }}
-        key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}
-        restore-keys: |
-          ${{ runner.os }}-pip-
-    - name: Node cache
-      uses: actions/cache@v1
-      with:
-        path: ~/.npm
-        key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
-        restore-keys: |
-          ${{ runner.os }}-node-
-    - name: Brew cache (MacOS)
-      if: startsWith(matrix.os, 'macos')
-      uses: actions/cache@v1
-      with:
-        path: ${{ env.BREW_CACHE }}
-        key: ${{ runner.os }}-brew-
-    - name: Install dependencies (Ubuntu Linux)
-      if: startsWith(matrix.os, 'ubuntu')
-      run: |
-        sudo apt-get update
-        sudo apt-get install -y libhdf5-serial-dev
-    - name: Install dependencies (MacOS)
-      if: startsWith(matrix.os, 'macos')
-      run: brew install hdf5
-    - name: Install cellxgene from `main` branch
-      if: matrix.cellxgene_build == 'main'
       run: |
+        # 1. only install the dev requirements on top of what is in the cellxgene pip package
+        sudo apt-get update && sudo apt-get install -y libhdf5-serial-dev
+        sed -i 's/-r requirements.txt//' server/requirements-dev.txt
         pip install -r server/requirements-dev.txt
+        # 2. install cellxgene
         make pydist install-dist
-    - name: Install cellxgene from latest release (pypi.org)
+        # 3. install anndata
-      if: matrix.cellxgene_build == 'latest'
+        pip install anndata==${{ matrix.anndata-version }}
-      run: |
+        # workaround for anndata 0.6.22.post1 bug
-        pip install --upgrade cellxgene
+        [[ "0.6.22.post1" = "${{ matrix.anndata-version }}" ]] && pip install h5py==2.9.0 || true
-        # install the additional dev requirements on top of what is in the
+    - name: Tests
-        # cellxgene pip package, which are needed for testing, but otherwise
+      run: make unit-test ${{ matrix.test-suite }}
-        # keep same pip pkg versions as in the cxg release
+
-        sed -i'' -e 's/-r requirements.txt//' server/requirements-dev.txt
+  cellxgene-release-with-anndata-master:
-        pip install -r server/requirements-dev.txt
+    name: cellxgene release with anndata master
-    - name: Install anndata version per matrix variable
+    runs-on: ubuntu-latest
-      run: pip install anndata${{ matrix.anndata_version }}
+    strategy:
-    - name: Install node
+      matrix:
-      run: make dev-env-client
+        test-suite: [smoke-test, smoke-test-annotations]
-    # Run different types of test separately, to facilitate troubleshooting
+    steps:
-    - name: Unit Tests - client
+      - uses: actions/checkout@v2
-      run: make unit-test-client
+      - name: Set up Python 3.7
-    - name: Unit Tests - server
+        uses: actions/setup-python@v1
-      run: make unit-test-server
+        with:
-    - name: Smoke Tests
+          python-version: 3.7
-      run: make smoke-test
+      - name: Checkout
-    # FIXME: Fails intermittently. See https://app.zenhub.com/workspaces/single-cell-5e2a191dad828d52cc78b028/issues/chanzuckerberg/cellxgene/2415
+        uses: actions/checkout@v2
-    # - name: Smoke Tests with Annotations
+        with:
-    #   run: make smoke-test-annotations
+          path: cellxgene
+      - name: Install dependencies
+        run: |
+          cd cellxgene
+          # 1. only install the dev requirements on top of what is in the cellxgene pip package
+          make dev-env-client
+          sed -i 's/-r requirements.txt//' server/requirements-dev.txt
+          pip install -r server/requirements-dev.txt
+          # 2. install cellxgene
+          pip install --upgrade cellxgene
+          # 3. install anndata
+          pip install git+https://github.com/theislab/anndata
+      - name: Tests
+        run: cd cellxgene && make unit-test ${{ matrix.test-suite }}
+
+  cellxgene-main-with-anndata-master:
+    name: cellxgene main with anndata master
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        test-suite: [smoke-test, smoke-test-annotations]
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python 3.7
+        uses: actions/setup-python@v1
+        with:
+          python-version: 3.7
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          path: cellxgene
+      - name: Install dependencies
+        run: |
+          cd cellxgene
+          sed -i -E 's/^anndata[>=]=[0-9]+.[0-9]+.[0-9]+$/anndata/g' server/requirements.txt
+          make pydist install-dist dev-env
+          pip install git+https://github.com/theislab/anndata
+      - name: Tests
+        run: cd cellxgene && make unit-test ${{ matrix.test-suite }}

.github/workflows/deploy.yml

@@ -0,0 +1,13 @@
+name: Deploy via single cell infra repo
+
+on:
+  push:
+    branches: main
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - name: repository dispatch
+        run: |
+          curl -XPOST -u czi-sci-single-cell-eng:${{secrets.SCI_GITHUB_TOKEN}} -H "Accept: application/vnd.github.everest-preview+json"  -H "Content-Type: application/json" https://api.github.com/repos/chanzuckerberg/single-cell-infra/dispatches --data '{"event_type": "cellxgene-hook"}'

.github/workflows/lint-pr-commit-message.yml

@@ -1,19 +0,0 @@
-
-name: "Lint PR commit message"
-
-on:
-  pull_request_target:
-    types:
-      - opened
-      - edited
-      - synchronize
-
-jobs:
-  main:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: amannn/action-semantic-pull-request@v3.4.1
-        with:
-          validateSingleCommit: true
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/push_tests.yml

@@ -18,7 +18,7 @@ jobs:
       - run: |
           git fetch --depth=1 origin +${{github.base_ref}}
       - name: Set up Python 3.7
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v1
         with:
           python-version: 3.7
       - name: Node cache
@@ -40,18 +40,16 @@ jobs:
       - name: Lint src with eslint
         working-directory: ./client
         run: |
-          npx eslint src __tests__
+          make lint
 
   unit-test:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: Set up Python 3.7 (pyenv)  # pyenv needed for mlflow in cli annotate tests
+      - name: Set up Python 3.7
-        uses: gabrielfalcao/pyenv-action@v9
+        uses: actions/setup-python@v1
         with:
-          default: 3.7
+          python-version: 3.7
-          command: pip install -U pip  # upgrade pip after installing python
-      - run: pip install virtualenv  # virtualenv needed for mlflow in cli annotate tests
       - name: Python cache
         uses: actions/cache@v1
         with:
@@ -70,8 +68,8 @@ jobs:
         run: make pydist install-dist dev-env-server
       - name: Unit tests
         run: |
-          make unit-test-server unit-test-client
+          make unit-test
-          bash <(curl -s https://codecov.io/bash) -y .codecov.yml -k server -cF server,python,unitTest
+          bash <(curl -s https://codecov.io/bash) -y .codecov.yml -k server -cF backend,python,unitTest
           cd client && ./node_modules/codecov/bin/codecov --yml=../.codecov.yml --root=../ --gcov-root=../ -C -F frontend,javascript,unitTest
 
   smoke-tests:
@@ -80,7 +78,7 @@ jobs:
     steps:
       - uses: actions/checkout@v2
       - name: Set up Python 3.7
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v1
         with:
           python-version: 3.7
       - name: Python cache
@@ -104,33 +102,32 @@ jobs:
           cd client && make smoke-test
           ./node_modules/codecov/bin/codecov --yml=../.codecov.yml --root=../ --gcov-root=../ -C -F frontend,javascript,smokeTest
 
-  # TODO: reinstate: https://github.com/chanzuckerberg/cellxgene/issues/2544
+  smoke-tests-annotations:
-  # smoke-tests-annotations:
+    runs-on: ubuntu-latest
-  #   runs-on: ubuntu-latest
+    timeout-minutes: 20
-  #   timeout-minutes: 20
+    steps:
-  #   steps:
+      - uses: actions/checkout@v2
-  #     - uses: actions/checkout@v2
+      - name: Set up Python 3.7
-  #     - name: Set up Python 3.7
+        uses: actions/setup-python@v1
-  #       uses: actions/setup-python@v4
+        with:
-  #       with:
+          python-version: 3.7
-  #         python-version: 3.7
+      - name: Python cache
-  #     - name: Python cache
+        uses: actions/cache@v1
-  #       uses: actions/cache@v1
+        with:
-  #       with:
+          path: ~/.cache/pip
-  #         path: ~/.cache/pip
+          key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}
-  #         key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}
+          restore-keys: |
-  #         restore-keys: |
+            ${{ runner.os }}-pip-
-  #           ${{ runner.os }}-pip-
+      - name: Node cache
-  #     - name: Node cache
+        uses: actions/cache@v1
-  #       uses: actions/cache@v1
+        with:
-  #       with:
+          path: ~/.npm
-  #         path: ~/.npm
+          key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
-  #         key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
+          restore-keys: |
-  #         restore-keys: |
+            ${{ runner.os }}-node-
-  #           ${{ runner.os }}-node-
+      - name: Install dependencies
-  #     - name: Install dependencies
+        run: make pydist install-dist
-  #       run: make pydist install-dist
+      - name: Smoke tests (with annotations feature)
-  #     - name: Smoke tests (with annotations feature)
+        run: |
-  #       run: |
+          cd client && make smoke-test-annotations
-  #         cd client && make smoke-test-annotations
+          ./node_modules/codecov/bin/codecov --yml=../.codecov.yml --root=../ --gcov-root=../ -C -F frontend,javascript,smokeTestAnnotations
-  #         ./node_modules/codecov/bin/codecov --yml=../.codecov.yml --root=../ --gcov-root=../ -C -F frontend,javascript,smokeTestAnnotations

.github/workflows/scale-test.yml

@@ -0,0 +1,30 @@
+name: "Scale test cellxgene APIs for initial loading"
+
+on:
+  schedule:
+    - cron: "0 0 * * Sun"
+
+jobs:
+  locust-build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python 3.7
+        uses: actions/setup-python@v1
+        with:
+          python-version: 3.7
+      - name: Install dependencies
+        run: |
+          pip install -r server/test/locust/requirements-locust.txt
+      - name: Dev Scale Test
+        run: |
+          locust -f server/test/locust/locustfile.py --headless -u 30 -r 10 --host https://api.cellxgene.dev.single-cell.czi.technology/cellxgene/e/ --run-time 5m 2>&1 | tee locust_dev_stats.txt
+      - name: Slack success webhook
+        env:
+          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }}
+        run: |
+          DEV_STATS=$(tail -n 61 locust_dev_stats.txt)
+          DEV_MSG="\`\`\`CELLXGENE EXPLORER DEV SCALE TEST RESULTS: ${DEV_STATS}\`\`\`"
+          curl -X POST -H 'Content-type: application/json' --data "{'text':'${DEV_MSG}'}" $SLACK_WEBHOOK
+
+

.gitignore

@@ -15,7 +15,7 @@ dist/
 *.egg-info
 
 # Environments
-venv*/
+venv/
 cellxgene/
 
 # client build
@@ -54,6 +54,3 @@ client/.eslintcache
 
 # E2E Testing
 ignoreE2E*
-
-# annotate subcmd
-.models_cache

CONTRIBUTING.md

@@ -1,3 +1,3 @@
 We warmly welcome contributions from the community!
 
-Whether you want to contribute ideas, requests, documentation, or code, you can get started by visiting our [contribution guide](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/contribute.md).
+Whether you want to contribute ideas, requests, documentation, or code, you can get started by visiting our [contribution guide](https://chanzuckerberg.github.io/cellxgene/posts/contribute).

LICENSE.txt

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2017-2023 Chan Zuckerberg Initiative
+Copyright (c) 2017-2020 Chan Zuckerberg Initiative
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of
 this software and associated documentation files (the "Software"), to deal in

MANIFEST.in

@@ -3,6 +3,5 @@ recursive-include server/common/web/static *
 
 include server/requirements.txt
 include server/requirements-prepare.txt
-include server/requirements-annotate.txt
 include server/converters/schema/hgnc_complete_set.txt.gz
 include server/converters/schema/schema_definitions/*

Makefile

@@ -16,13 +16,8 @@ clean: clean-lite clean-server clean-client
 clean-lite:
 	rm -rf $(CLEANFILES)
 
-.PHONY: clean-client
+clean-%:
-clean-client:
+	cd $(*) && $(MAKE) clean
-	cd client && $(MAKE) clean
-
-.PHONY: clean-server
-clean-server:
-	cd server && $(MAKE) clean
 
 
 # BUILDING PACKAGE
@@ -33,20 +28,20 @@ build-client:
 
 .PHONY: build
 build: clean build-client
-	git ls-files server/ | cpio -pdm $(BUILDDIR)
+	git ls-files server/ | grep -v 'server/test/' | cpio -pdm $(BUILDDIR)
 	cp -r client/build/  $(CLIENTBUILD)
 	$(call copy_client_assets,$(CLIENTBUILD),$(SERVERBUILD))
 	cp MANIFEST.in README.md setup.cfg setup.py $(BUILDDIR)
 
 # If you are actively developing in the server folder use this, dirties the source tree
 .PHONY: build-for-server-dev
-build-for-server-dev: clean-server build-client copy-client-assets
+build-for-server-dev: clean-server build-client
+	$(call copy_client_assets,client/build,server)
 
 .PHONY: copy-client-assets
 copy-client-assets:
 	$(call copy_client_assets,client/build,server)
 
-
 # TESTING
 .PHONY: test
 test: unit-test smoke-test
@@ -54,22 +49,8 @@ test: unit-test smoke-test
 .PHONY: unit-test
 unit-test: unit-test-server unit-test-client
 
-.PHONY: test-server
+unit-test-%:
-test-server: unit-test-server smoke-test
+	cd $(*) && $(MAKE) unit-test
-
-.PHONY: unit-test-client
-unit-test-client:
-	cd client && $(MAKE) unit-test
-
-.PHONY: unit-test-server
-unit-test-server:
-	PYTHONWARNINGS=ignore:ResourceWarning coverage run \
-		--source=server \
-		--omit=.coverage,venv \
-		-m unittest discover \
-		--start-directory test/unit \
-		--verbose; test_result=$$?; \
-	exit $$test_result \
 
 .PHONY: smoke-test
 smoke-test:
@@ -79,9 +60,14 @@ smoke-test:
 smoke-test-annotations:
 	cd client && $(MAKE) smoke-test-annotations
 
+.PHONY: test-db
+test-db:
+	cd server && $(MAKE) test-db
+
+
 # FORMATTING CODE
 
-.PHONY: fmt
+.PHOHY: fmt
 fmt: fmt-client fmt-py
 
 .PHONY: fmt-client
@@ -95,10 +81,10 @@ fmt-py:
 .PHONY: lint
 lint: lint-server lint-client
 
-
 .PHONY: lint-server
 lint-server: fmt-py
-	flake8 server --per-file-ignores='test/fixtures/dataset_config_outline.py:F821 test/fixtures/server_config_outline.py:F821 test/performance/scale_test_annotations.py:E501'
+	flake8 server --per-file-ignores='server/test/fixtures/dataset_config_outline.py:F821 server/test/fixtures/server_config_outline.py:F821 server/test/performance/scale_test_annotations.py:E501'
+
 
 .PHONY: lint-client
 lint-client:
@@ -111,34 +97,29 @@ pydist: build
 	cd $(BUILDDIR); python setup.py sdist -d ../dist
 	@echo "done"
 
-# RELEASE HELPERS
 
-# Set PART=[major, minor, patch] as param to make bump.
+# RELEASE HELPERS
-# This will create a release candidate. (i.e. 0.16.1 -> 0.16.2-rc.0 for a patch bump)
-.PHONY: bump-version
-bump-version:
-	bumpversion --config-file .bumpversion.cfg $(PART)
 
 # Create new version to commit to main
 .PHONY: create-release-candidate
-create-release-candidate: bump-version clean-lite gen-package-lock
+create-release-candidate: dev-env bump-version clean-lite gen-package-lock
 	@echo "Version bumped part:$(PART) and client built. Ready to commit and push"
 
 # Bump the release candidate version if needed (i.e. the previous release candidate had errors).
 .PHONY: recreate-release-candidate
-recreate-release-candidate: bump-release-candidate clean-lite gen-package-lock
+recreate-release-candidate: dev-env bump-release-candidate clean-lite gen-package-lock
 	@echo "Version bumped part:$(PART) and client built. Ready to commit and push"
 
 # Build dist and release to Test PyPI
 .PHONY: release-candidate-to-test-pypi
-release-candidate-to-test-pypi: pydist twine
+release-candidate-to-test-pypi: dev-env pydist twine
 	@echo "Dist built and uploaded to test.pypi.org"
 	@echo "Test the install:"
 	@echo "    make install-release-test"
 
 # Build final dist (gets rid of the rc tag) and release final candidate to TestPyPI
 .PHONY: release-final-to-test-pypi
-release-final-to-test-pypi: bump-release clean-lite gen-package-lock pydist twine
+release-final-to-test-pypi: dev-env bump-release clean-lite gen-package-lock pydist twine
 	@echo "Final release dist built and uploaded to test.pypi.org"
 	@echo "Test the install:"
 	@echo "    make install-release-test"
@@ -148,9 +129,9 @@ release-final: twine-prod
 	@echo "Release uploaded to pypi.org"
 
 # DANGER: releases directly to prod
-# use this if you accidentally burned a test release version number,
+# use this if you accidently burned a test release version number,
 .PHONY: release-directly-to-prod
-release-directly-to-prod: pydist twine-prod
+release-directly-to-prod: dev-env pydist twine-prod
 	@echo "Dist built and uploaded to pypi.org"
 	@echo "Test the install:"
 	@echo "    make install-release"
@@ -166,6 +147,12 @@ dev-env-client:
 dev-env-server:
 	pip install -r server/requirements-dev.txt
 
+# Set PART=[major, minor, patch] as param to make bump.
+# This will create a release candidate. (i.e. 0.16.1 -> 0.16.2-rc.0 for a patch bump)
+.PHONY: bump-version
+bump-version:
+	bumpversion --config-file .bumpversion.cfg $(PART)
+
 # Increments the release candidate version (i.e. 0.16.2-rc.1 -> 0.16.2-rc.2)
 .PHONY: bump-release-candidate
 bump-release-candidate:
@@ -200,7 +187,7 @@ install-dev: uninstall
 # install from test.pypi to test your release
 .PHONY: install-release-test
 install-release-test: uninstall
-	pip install --no-cache-dir --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple cellxgene==$(VERSION)
+	pip install --no-cache-dir --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple cellxgene
 	@echo "Installed cellxgene from test.pypi.org, now run and smoke test"
 
 # install from pypi to test your release

README.md

@@ -7,27 +7,27 @@ _an interactive explorer for single-cell transcriptomics data_
 [![Compatibility Tests](https://github.com/chanzuckerberg/cellxgene/workflows/Compatibility%20Tests/badge.svg)](https://github.com/chanzuckerberg/cellxgene/actions?query=workflow%3A%22Compatibility+Tests%22)
 ![Code Coverage](https://codecov.io/gh/chanzuckerberg/cellxgene/branch/main/graph/badge.svg)
 
-CZ CELLxGENE Annotate (pronounced "cell-by-gene") is an interactive data explorer for single-cell datasets, such as those coming from the [Human Cell Atlas](https://humancellatlas.org). Leveraging modern web development techniques to enable fast visualizations of at least 1 million cells, we hope to enable biologists and computational researchers to explore their data.
+cellxgene (pronounced "cell-by-gene") is an interactive data explorer for single-cell transcriptomics datasets, such as those coming from the [Human Cell Atlas](https://humancellatlas.org). Leveraging modern web development techniques to enable fast visualizations of at least 1 million cells, we hope to enable biologists and computational researchers to explore their data.
 
-Whether you need to visualize one thousand cells or one million, CELLxGENE Annotate helps you gain insight into your single-cell data.
+Whether you need to visualize one thousand cells or one million, cellxgene helps you gain insight into your single-cell data.
 
 <img src="https://github.com/chanzuckerberg/cellxgene/raw/main/docs/images/crossfilter.gif" width="350" height="200" hspace="30"><img src="https://github.com/chanzuckerberg/cellxgene/raw/main/docs/images/category-breakdown.gif" width="350" height="200" hspace="30">
 
 # Getting started
 
-### The comprehensive guide to CZ CELLxGENE Annotate
+### The comprehensive guide to cellxgene
 
-[The CZ CELLxGENE Annotate documentation is your one-stop-shop for information about CELLxGENE Annotate](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/README.md)! You may be particularly interested in:
+[The cellxgene documentation is your one-stop-shop for information about cellxgene](https://chanzuckerberg.github.io/cellxgene/)! You may be particularly interested in:
 
-- Seeing [what Annotate can do](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/explore-data/explorer-tutorials.md)
+- Seeing [what cellxgene can do](https://chanzuckerberg.github.io/cellxgene/posts/gallery)
-- Learning more about Annotate [installation](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/desktop/install.md) and [usage](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/desktop/quick-start.md#quick-start-1)
+- Learning more about cellxgene [installation](https://chanzuckerberg.github.io/cellxgene/posts/install) and [usage](https://chanzuckerberg.github.io/cellxgene/posts/launch)
-- [Preparing your own data](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/desktop/data-reqs.md) for use in Annotate
+- [Preparing your own data](https://chanzuckerberg.github.io/cellxgene/posts/prepare) for use in cellxgene
-- Checking out [our roadmap](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/roadmap.md) for future development
+- Checking out [our roadmap](https://chanzuckerberg.github.io/cellxgene/posts/roadmap) for future development
-- [Contributing](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/contribute.md) to Annotate
+- [Contributing](https://chanzuckerberg.github.io/cellxgene/posts/contribute) to cellxgene
 
 ### Quick start
 
-To install CELLxGENE Annotate you need Python 3.6+. We recommend [installing Annotate into a conda or virtual environment.](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/desktop/install.md)
+To install cellxgene you need Python 3.6+. We recommend [installing cellxgene into a conda or virtual environment.](https://chanzuckerberg.github.io/cellxgene/posts/install)
 
 Install the package.
 
@@ -35,23 +35,24 @@ Install the package.
 pip install cellxgene
 ```
 
-Launch Annotate with an example [anndata](https://anndata.readthedocs.io/en/latest/) file
+Launch cellxgene with an example [anndata](https://anndata.readthedocs.io/en/latest/) file
 
 ```bash
 cellxgene launch https://cellxgene-example-data.czi.technology/pbmc3k.h5ad
 ```
 
-To explore more datasets already formatted for Annotate, check out the [Demo data](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/desktop/quick-start.md#example-datasets) or
+To explore more datasets already formatted for cellxgene, check out the [Demo data](https://chanzuckerberg.github.io/cellxgene/posts/demo-data) or
-see [Preparing your data](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/desktop/data-reqs.md) to learn more about formatting your own
+see [Preparing your data](https://chanzuckerberg.github.io/cellxgene/posts/prepare) to learn more about formatting your own
-data for CELLxGENE Annotate.
+data for cellxgene.
 
 ### Supported browsers
 
-CELLxGENE Annotate currently supports the following browsers:
+cellxgene currently supports the following browsers:
 
 - Google Chrome 61+
 - Edge 15+
 - Firefox 60+
+- Safari 10.1+
 
 Please [file an issue](https://github.com/chanzuckerberg/cellxgene/issues/new/choose) if you would like us to add support for an unsupported browser.
 
@@ -62,11 +63,11 @@ For questions, suggestions, or accolades, [join the `#cellxgene-users` channel o
 
 For any errors, [report bugs on Github](https://github.com/chanzuckerberg/cellxgene/issues).
 
-# Developing with CZ CELLxGENE Annotate
+# Developing with cellxgene
 
 ### Contributing
 
-We warmly welcome contributions from the community! Please see our [contributing guide](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/contribute.md) and don't hesitate to open an issue or send a pull request to improve CELLxGENE Annotate. Please see the [dev_docs](https://github.com/chanzuckerberg/cellxgene/tree/main/dev_docs) for pull request suggestions, unit test details, local documentation preview, and other development specifics. 
+We warmly welcome contributions from the community! Please see our [contributing guide](https://chanzuckerberg.github.io/cellxgene/posts/contribute) and don't hesitate to open an issue or send a pull request to improve cellxgene. Please see the [dev_docs](https://github.com/chanzuckerberg/cellxgene/tree/main/dev_docs) for pull request suggestions, unit test details, local documentation preview, and other development specifics. 
 
 This project adheres to the Contributor Covenant [code of conduct](https://github.com/chanzuckerberg/.github/blob/master/CODE_OF_CONDUCT.md). By participating, you are expected to uphold this code. Please report unacceptable behavior to opensource@chanzuckerberg.com.
 
@@ -77,11 +78,11 @@ As such, we encourage other scientific tool builders in academia or industry to
 this project. All code is freely available for reuse under the [MIT license](https://opensource.org/licenses/MIT).
 
 
-Before extending CELLxGENE Annotate, we encourage you to reach out to us with ideas or questions. It might be possible that an 
+Before extending cellxgene, we encourage you to reach out to us with ideas or questions. It might be possible that an 
 extension could be directly contributed, which would make it available for a wider audience, or that it's on our 
-[roadmap](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/roadmap.md) and under active development. 
+[roadmap](./docs/posts/roadmap.md) and under active development. 
 
-See the [CELLxGENE extensions](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/community-extensions.md) section of our documentation for examples of community use and CELLxGENE extensions. 
+See the [cellxgene extensions](./docs/posts/extensions.md) section of our documentation for examples of community use and cellxgene extensions. 
 
 ### Security
 

client/.husky/pre-commit

@@ -1,5 +0,0 @@
-#!/bin/sh
-. "$(dirname "$0")/_/husky.sh"
-
-cd client
-npx --no-install lint-staged --config "./configuration/lint-staged/lint-staged.config.js"

client/Makefile

@@ -1,13 +1,9 @@
 include ../common.mk
 
-ANNOTATIONS := $(if $(ANNOTATIONS),$(ANNOTATIONS),../test/fixtures/pbmc3k-annotations.csv)
+ANNOTATIONS := $(if $(ANNOTATIONS),$(ANNOTATIONS),../server/test/fixtures/pbmc3k-annotations.csv)
-GENE_SETS := $(if $(GENE_SETS),$(GENE_SETS),../test/fixtures/pbmc3k-genesets.csv)
 ANNOTATIONS_FILENAME := $(shell basename $(ANNOTATIONS))
-GENE_SETS_FILENAME := $(shell basename $(GENE_SETS))
-
-CXG_CONFIG := $(if $(CXG_CONFIG),$(CXG_CONFIG),./__tests__/e2e/test_config.yaml)
-
 
+CXG_CONFIG := $(if $(CXG_CONFIG), $(CXG_CONFIG), ./__tests__/e2e/test_config.yaml)
 
 # Packaging
 .PHONY: clean
@@ -39,16 +35,15 @@ smoke-test:
 	start_server_and_test \
 		'CXG_OPTIONS="--config-file $(CXG_CONFIG)" $(MAKE) start-server' \
 		$(CXG_SERVER_PORT) \
-		'CXG_URL_BASE="http://localhost:$(CXG_SERVER_PORT)" npm run e2e -- --verbose false'
+		'CXG_URL_BASE="http://localhost:$(CXG_SERVER_PORT)" CXG_AUTH_TYPE="test" npm run e2e -- --verbose false'
 
 # start an instance of cellxgene and run the end-to-end annotations tests
 .PHONY: smoke-test-annotations
 smoke-test-annotations:
 	$(eval TMP_DIR := $(shell  mktemp -d /tmp/cellxgene_XXXXXX))
 	cp $(ANNOTATIONS) $(TMP_DIR)/ && \
-	cp $(GENE_SETS) $(TMP_DIR)/ && \
 	start_server_and_test \
-		'CXG_OPTIONS="--annotations-file $(TMP_DIR)/$(ANNOTATIONS_FILENAME) --gene-sets-file $(TMP_DIR)/$(GENE_SETS_FILENAME)" $(MAKE) start-server' \
+		'CXG_OPTIONS="--annotations-file $(TMP_DIR)/$(ANNOTATIONS_FILENAME)" $(MAKE) start-server' \
 		$(CXG_SERVER_PORT) \
 		'CXG_URL_BASE="http://localhost:$(CXG_SERVER_PORT)" npm run e2e-annotations -- --verbose false'
 	rm -rf $(TMP_DIR)

client/__tests__/e2e/__snapshots__/e2e.test.js.snap

@@ -2,4 +2,4 @@
 
 exports[`did launch page launched 1`] = `"<span style=\\"max-width: 155px; display: flex; overflow: hidden; justify-content: flex-start; width: 100%; padding: 0px;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">pbm</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">c3k</span><span style=\\"position: absolute; right: 0px; color: inherit;\\">c3k</span></span></span>"`;
 
-exports[`metadata loads categories and values from dataset appear 1`] = `"<div style=\\"display: flex; justify-content: space-between; align-items: baseline;\\"><div style=\\"display: flex; justify-content: flex-start; align-items: flex-start;\\"><label class=\\"bp3-control bp3-checkbox\\" for=\\"category-select-louvain\\"><input id=\\"category-select-louvain\\" data-testclass=\\"category-select\\" data-testid=\\"louvain:category-select\\" type=\\"checkbox\\" checked=\\"\\"><span class=\\"bp3-control-indicator\\"></span></label><span role=\\"menuitem\\" tabindex=\\"0\\" data-testclass=\\"category-expand\\" data-testid=\\"louvain:category-expand\\" style=\\"cursor: pointer;\\"><span aria-haspopup=\\"true\\" class=\\"bp3-popover2-target\\"><span data-testid=\\"louvain:category-label\\" tabindex=\\"-1\\" aria-label=\\"louvain\\" class=\\"\\" style=\\"max-width: 265px;\\"><span style=\\"max-width: 265px; display: flex; overflow: hidden; justify-content: flex-start; width: 100%; padding: 0px;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">lou</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">vain</span><span style=\\"position: absolute; right: 0px; color: inherit;\\">vain</span></span></span></span></span><svg stroke=\\"currentColor\\" fill=\\"currentColor\\" stroke-width=\\"0\\" viewBox=\\"0 0 320 512\\" data-testclass=\\"category-expand-is-not-expanded\\" height=\\"1em\\" width=\\"1em\\" xmlns=\\"http://www.w3.org/2000/svg\\" style=\\"font-size: 10px; margin-left: 5px;\\"><path d=\\"M285.476 272.971L91.132 467.314c-9.373 9.373-24.569 9.373-33.941 0l-22.667-22.667c-9.357-9.357-9.375-24.522-.04-33.901L188.505 256 34.484 101.255c-9.335-9.379-9.317-24.544.04-33.901l22.667-22.667c9.373-9.373 24.569-9.373 33.941 0L285.475 239.03c9.373 9.372 9.373 24.568.001 33.941z\\"></path></svg></span></div><div><span class=\\"bp3-popover-wrapper\\"><span aria-haspopup=\\"true\\" class=\\"bp3-popover-target\\"><a role=\\"button\\" data-testclass=\\"colorby\\" data-testid=\\"colorby-louvain\\" class=\\"bp3-button\\" tabindex=\\"0\\"><span icon=\\"tint\\" aria-hidden=\\"true\\" tabindex=\\"0\\" class=\\"bp3-icon bp3-icon-tint\\"><svg data-icon=\\"tint\\" width=\\"16\\" height=\\"16\\" viewBox=\\"0 0 16 16\\"><path d=\\"M7.88 1s-4.9 6.28-4.9 8.9c.01 2.82 2.34 5.1 4.99 5.1 2.65-.01 5.03-2.3 5.03-5.13C12.99 7.17 7.88 1 7.88 1z\\" fill-rule=\\"evenodd\\"></path></svg></span></a></span></span></div></div><div style=\\"margin-left: 26px;\\"></div>"`;
+exports[`metadata loads categories and values from dataset appear 1`] = `"<div style=\\"display: flex; justify-content: space-between; align-items: baseline;\\"><div style=\\"display: flex; justify-content: flex-start; align-items: flex-start;\\"><label class=\\"bp3-control bp3-checkbox\\" for=\\"category-select-louvain\\"><input id=\\"category-select-louvain\\" data-testclass=\\"category-select\\" data-testid=\\"louvain:category-select\\" type=\\"checkbox\\" checked=\\"\\"><span class=\\"bp3-control-indicator\\"></span></label><span role=\\"menuitem\\" tabindex=\\"0\\" data-testclass=\\"category-expand\\" data-testid=\\"louvain:category-expand\\" style=\\"cursor: pointer;\\"><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\" style=\\"max-width: 265px;\\"><span data-testid=\\"louvain:category-label\\" aria-label=\\"louvain\\" class=\\"\\" tabindex=\\"0\\" style=\\"max-width: 265px;\\"><span style=\\"max-width: 265px; display: flex; overflow: hidden; justify-content: flex-start; width: 100%; padding: 0px;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">lou</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">vain</span><span style=\\"position: absolute; right: 0px; color: inherit;\\">vain</span></span></span></span></span></span><svg stroke=\\"currentColor\\" fill=\\"currentColor\\" stroke-width=\\"0\\" viewBox=\\"0 0 320 512\\" data-testclass=\\"category-expand-is-not-expanded\\" height=\\"1em\\" width=\\"1em\\" xmlns=\\"http://www.w3.org/2000/svg\\" style=\\"font-size: 10px; margin-left: 5px;\\"><path d=\\"M285.476 272.971L91.132 467.314c-9.373 9.373-24.569 9.373-33.941 0l-22.667-22.667c-9.357-9.357-9.375-24.522-.04-33.901L188.505 256 34.484 101.255c-9.335-9.379-9.317-24.544.04-33.901l22.667-22.667c9.373-9.373 24.569-9.373 33.941 0L285.475 239.03c9.373 9.372 9.373 24.568.001 33.941z\\"></path></svg></span></div><div><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\"><a role=\\"button\\" data-testclass=\\"colorby\\" data-testid=\\"colorby-louvain\\" class=\\"bp3-button\\" tabindex=\\"0\\"><span icon=\\"tint\\" class=\\"bp3-icon bp3-icon-tint\\"><svg data-icon=\\"tint\\" width=\\"16\\" height=\\"16\\" viewBox=\\"0 0 16 16\\"><desc>tint</desc><path d=\\"M7.88 1s-4.9 6.28-4.9 8.9c.01 2.82 2.34 5.1 4.99 5.1 2.65-.01 5.03-2.3 5.03-5.13C12.99 7.17 7.88 1 7.88 1z\\" fill-rule=\\"evenodd\\"></path></svg></span></a></span></span></div></div><div style=\\"margin-left: 26px;\\"></div><div></div>"`;

client/__tests__/e2e/cellxgeneActions.js

@@ -13,8 +13,11 @@ import {
   getTestClass,
   getTestId,
   isElementPresent,
+  goToPage,
 } from "./puppeteerUtils";
 
+import { appUrlBase } from "./config";
+
 export async function drag(testId, start, end, lasso = false) {
   const layout = await waitByID(testId);
   const elBox = await layout.boxModel();
@@ -58,9 +61,11 @@ export async function getAllHistograms(testclass, testIds) {
   const allHistograms = await getAllByClass(testclass);
 
   const testIDs = await Promise.all(
-    allHistograms.map((hist) =>
+    allHistograms.map((hist) => {
-      page.evaluate((elem) => elem.dataset.testid, hist)
+      return page.evaluate((elem) => {
-    )
+        return elem.dataset.testid;
+      }, hist);
+    })
   );
 
   return testIDs.map((id) => id.replace(/^histogram-/, ""));
@@ -171,137 +176,6 @@ export async function createCategory(categoryName) {
   await clickOn("submit-category");
 }
 
-/*
-
-  GENESET
-
-*/
-
-export async function colorByGeneset(genesetName) {
-  await clickOn(`${genesetName}:colorby-entire-geneset`);
-}
-
-export async function colorByGene(gene) {
-  await clickOn(`colorby-${gene}`);
-}
-
-export async function assertColorLegendLabel(label) {
-  const handle = await waitByID("continuous_legend_color_by_label");
-
-  const result = await handle.evaluate((node) =>
-    node.getAttribute("aria-label")
-  );
-
-  return expect(result).toBe(label);
-}
-
-export async function expandGeneset(genesetName) {
-  const expand = await waitByID(`${genesetName}:geneset-expand`);
-  const notExpanded = await expand.$(
-    "[data-testclass='geneset-expand-is-not-expanded']"
-  );
-  if (notExpanded) await clickOn(`${genesetName}:geneset-expand`);
-}
-
-export async function createGeneset(genesetName) {
-  await clickOnUntil("open-create-geneset-dialog", async () => {
-    await expect(page).toMatchElement(getTestId("create-geneset-input"));
-  });
-
-  await typeInto("create-geneset-input", genesetName);
-  await clickOn("submit-geneset");
-  await waitByClass("autosave-complete");
-}
-
-export async function editGenesetName(genesetName, editText) {
-  const editButton = `${genesetName}:edit-genesetName-mode`;
-  const submitButton = `${genesetName}:submit-geneset`;
-  await clickOnUntil(`${genesetName}:see-actions`, async () => {
-    await expect(page).toMatchElement(getTestId(editButton));
-  });
-  await clickOn(editButton);
-  await typeInto("rename-geneset-modal", editText);
-  await clickOn(submitButton);
-}
-
-export async function deleteGeneset(genesetName) {
-  const targetId = `${genesetName}:delete-geneset`;
-
-  await clickOnUntil(`${genesetName}:see-actions`, async () => {
-    await expect(page).toMatchElement(getTestId(targetId));
-  });
-
-  await clickOn(targetId);
-
-  await assertGenesetDoesNotExist(genesetName);
-  await waitByClass("autosave-complete");
-}
-
-export async function assertGenesetDoesNotExist(genesetName) {
-  const result = await isElementPresent(
-    getTestId(`${genesetName}:geneset-name`)
-  );
-  await expect(result).toBe(false);
-}
-
-export async function assertGenesetExists(genesetName) {
-  const handle = await waitByID(`${genesetName}:geneset-name`);
-
-  const result = await handle.evaluate((node) =>
-    node.getAttribute("aria-label")
-  );
-
-  return expect(result).toBe(genesetName);
-}
-
-/*
-
-  GENE
-
-*/
-
-export async function addGeneToSet(genesetName, geneToAddToSet) {
-  const submitButton = `${genesetName}:submit-gene`;
-
-  await clickOn(`${genesetName}:add-new-gene-to-geneset`);
-  await typeInto("add-genes", geneToAddToSet);
-  await clickOn(submitButton);
-}
-
-export async function removeGene(geneSymbol) {
-  const targetId = `delete-from-geneset:${geneSymbol}`;
-
-  await clickOn(targetId);
-
-  await waitByClass("autosave-complete");
-}
-
-export async function assertGeneExistsInGeneset(geneSymbol) {
-  const handle = await waitByID(`${geneSymbol}:gene-label`);
-
-  const result = await handle.evaluate((node) =>
-    node.getAttribute("aria-label")
-  );
-
-  return expect(result).toBe(geneSymbol);
-}
-
-export async function assertGeneDoesNotExist(geneSymbol) {
-  const result = await isElementPresent(getTestId(`${geneSymbol}:gene-label`));
-
-  await expect(result).toBe(false);
-}
-
-export async function expandGene(geneSymbol) {
-  await clickOn(`maximize-${geneSymbol}`);
-}
-
-/*
-
-  CATEGORY
-
-*/
-
 export async function duplicateCategory(categoryName) {
   await clickOn("open-annotation-dialog");
 
@@ -364,7 +238,7 @@ export async function createLabel(categoryName, labelName) {
    * 4. You will see `123` is persisted in the input box
    * 5. Expected behavior is to get an empty input box
    */
-  await page.waitForTimeout(500);
+  await page.waitFor(500);
 
   await clickOn(`${categoryName}:see-actions`);
 
@@ -442,4 +316,73 @@ export async function assertCategoryDoesNotExist(categoryName) {
   await expect(result).toBe(false);
 }
 
+export async function login() {
+  const email = `cellxgene-smoke-test+${process.env.DEPLOYMENT_STAGE}@chanzuckerberg.com`;
+  const password = "Test1111";
+
+  await goToPage(appUrlBase);
+
+  await clickOn("log-in");
+
+  // (thuang): Auth0 form is unstable and unsafe for input until verified
+  await waitUntilFormFieldStable('[name="email"]');
+
+  await expect(page).toFillForm("form", {
+    email,
+    password,
+  });
+
+  await Promise.all([
+    page.waitForNavigation({ waitUntil: "networkidle0" }),
+    expect(page).toClick('[name="submit"]'),
+  ]);
+
+  expect(page.url()).toContain(appUrlBase);
+}
+
+export async function logout() {
+  await clickOnUntil("user-info", async () => {
+    await waitByID("log-out");
+    await Promise.all([
+      page.waitForNavigation({ waitUntil: "networkidle0" }),
+      clickOn("log-out"),
+    ]);
+  });
+
+  await waitByID("log-in");
+}
+
+async function waitUntilFormFieldStable(selector) {
+  const MAX_RETRY = 10;
+  const WAIT_FOR_MS = 200;
+
+  const EXPECTED_VALUE = "aaa";
+
+  let retry = 0;
+
+  while (retry < MAX_RETRY) {
+    try {
+      await expect(page).toFill(selector, EXPECTED_VALUE);
+
+      const fieldHandle = await expect(page).toMatchElement(selector);
+
+      const fieldValue = await page.evaluate(
+        (input) => input.value,
+        fieldHandle
+      );
+
+      expect(fieldValue).toBe(EXPECTED_VALUE);
+
+      break;
+    } catch (error) {
+      retry += 1;
+
+      await page.waitFor(WAIT_FOR_MS);
+    }
+  }
+
+  if (retry === MAX_RETRY) {
+    throw Error("clickOnUntil() assertion failed!");
+  }
+}
 /* eslint-enable no-await-in-loop -- await in loop is needed to emulate sequential user actions */

client/__tests__/e2e/config.js

@@ -6,5 +6,3 @@ export const appUrlBase =
 export const DATASET = "pbmc3k";
 export const isDev = jestEnv === ENV_DEFAULT.DEV;
 export const isDebug = jestEnv === ENV_DEFAULT.DEBUG;
-export const TEST_EMAIL = "user@example.com";
-export const TEST_PASSWORD = process.env.TEST_ACCOUNT_PASS ?? "";

client/__tests__/e2e/diffexpGeneSets.js

@@ -1,105 +0,0 @@
-export const diffexpPop1Genes = [
-  "CD79A",
-  "HLA-DRB1",
-  "HLA-DQA1",
-  "HLA-DPB1",
-  "HLA-DQB1",
-  "HLA-DPA1",
-  "MS4A1",
-  "LTB",
-  "CD79B",
-  "CD37",
-  "HLA-DMA",
-  "TCL1A",
-  "LINC00926",
-  "HLA-DMB",
-  "HVCN1",
-  "EAF2",
-  "FCRLA",
-  "IRF8",
-  "PKIG",
-  "P2RX5",
-  "HLA-DOB",
-  "SPIB",
-  "BLNK",
-  "SWAP70",
-  "PNOC",
-  "CD19",
-  "SMIM14",
-  "CD72",
-  "KIAA0125",
-  "IGLL5",
-  "ARHGAP24",
-  "COTL1",
-  "C16orf74",
-  "BTK",
-  "SNX29P2",
-  "ADAM28",
-  "FCGR2B",
-  "PLD4",
-  "PPP1R14A",
-  "MZB1",
-  "KIAA0040",
-  "PHACTR1",
-  "FCRL2",
-  "RIC3",
-  "P2RY10",
-  "SCPEP1",
-  "DRAM2",
-  "RP5-887A10.1",
-  "CD82",
-  "GPX1",
-];
-
-export const diffexpPop2Genes = [
-  "NKG7",
-  "GZMB",
-  "CTSW",
-  "PRF1",
-  "GNLY",
-  "GZMA",
-  "CST7",
-  "FGFBP2",
-  "SRGN",
-  "CD247",
-  "FCGR3A",
-  "TYROBP",
-  "FCER1G",
-  "ID2",
-  "SPON2",
-  "CCL4",
-  "CCL5",
-  "GZMH",
-  "GIMAP7",
-  "CLIC3",
-  "HOPX",
-  "XCL2",
-  "LGALS1",
-  "IGFBP7",
-  "AKR1C3",
-  "IL32",
-  "EFHD2",
-  "PRSS23",
-  "TTC38",
-  "ZAP70",
-  "S1PR5",
-  "SAMD3",
-  "GIMAP4",
-  "CCL3",
-  "ABI3",
-  "XCL1",
-  "S100A6",
-  "UBB",
-  "GPR56",
-  "PDIA3",
-  "S100A11",
-  "APOBEC3G",
-  "HAVCR2",
-  "PLEKHF1",
-  "LITAF",
-  "ARPC5L",
-  "PTGDR",
-  "PRMT2",
-  "GSTP1",
-  "FCRL6",
-];

client/__tests__/e2e/e2e.test.js

@@ -15,16 +15,25 @@ import {
   getOneElementInnerHTML,
   getTestId,
   goToPage,
+  typeInto,
   waitByID,
+  clickOnUntil,
 } from "./puppeteerUtils";
 
 import {
+  addGeneToSearch,
+  bulkAddGenes,
   calcDragCoordinates,
   clip,
   drag,
   getAllCategoriesAndCounts,
+  getAllHistograms,
   getCellSetCount,
+  runDiffExp,
   selectCategory,
+  subset,
+  login,
+  logout,
 } from "./cellxgeneActions";
 
 const data = datasets[DATASET];
@@ -140,6 +149,47 @@ describe("cell selection", () => {
   });
 });
 
+describe("gene entry", () => {
+  test("search for single gene", async () => {
+    await goToPage(appUrlBase);
+    await addGeneToSearch(data.genes.search);
+  });
+
+  test("bulk add genes", async () => {
+    await goToPage(appUrlBase);
+
+    const testGenes = data.genes.bulkadd;
+
+    await bulkAddGenes(testGenes);
+    const allHistograms = await getAllHistograms(
+      "histogram-user-gene",
+      testGenes
+    );
+
+    expect(allHistograms).toEqual(expect.arrayContaining(testGenes));
+    expect(allHistograms).toHaveLength(testGenes.length);
+  });
+});
+
+describe("differential expression", () => {
+  test("selects cells, saves them and performs diffexp", async () => {
+    await goToPage(appUrlBase);
+
+    await runDiffExp(data.diffexp.cellset1, data.diffexp.cellset2);
+
+    const allHistograms = await getAllHistograms(
+      "histogram-diffexp",
+      data.diffexp["gene-results"]
+    );
+
+    expect(allHistograms).toEqual(
+      expect.arrayContaining(data.diffexp["gene-results"])
+    );
+
+    expect(allHistograms).toHaveLength(data.diffexp["gene-results"].length);
+  });
+});
+
 describe("subset", () => {
   test("subset - cell count matches", async () => {
     await goToPage(appUrlBase);
@@ -186,6 +236,94 @@ describe("subset", () => {
     const cellCount = await getCellSetCount(1);
     expect(cellCount).toBe(data.subset.lasso.count);
   });
+
+  test("undo selection appends the top diff exp genes to user defined genes", async () => {
+    await goToPage(appUrlBase);
+
+    const userDefinedGenes = data.genes.bulkadd;
+    const diffExpGenes = data.diffexp["gene-results"];
+
+    await bulkAddGenes(userDefinedGenes);
+    const userDefinedHistograms = await getAllHistograms(
+      "histogram-user-gene",
+      userDefinedGenes
+    );
+
+    expect(userDefinedHistograms).toEqual(
+      expect.arrayContaining(userDefinedGenes)
+    );
+
+    await subset({ x1: 0.15, y1: 0.1, x2: 0.98, y2: 0.98 });
+    await runDiffExp(data.diffexp.cellset1, data.diffexp.cellset2);
+
+    const diffExpHistograms = await getAllHistograms(
+      "histogram-diffexp",
+      diffExpGenes
+    );
+
+    expect(diffExpHistograms).toEqual(expect.arrayContaining(diffExpGenes));
+
+    await clickOn("reset-subset-button");
+    const expected = [].concat(userDefinedGenes, diffExpGenes);
+    const userDefinedHistogramsAfterSubset = await getAllHistograms(
+      "histogram-user-gene",
+      expected
+    );
+
+    expect(userDefinedHistogramsAfterSubset).toEqual(
+      expect.arrayContaining(expected)
+    );
+  });
+
+  test("subset selection appends the top diff exp genes to user defined genes", async () => {
+    await goToPage(appUrlBase);
+
+    const userDefinedGenes = data.genes.bulkadd;
+    const diffExpGenes = data.diffexp["gene-results"];
+
+    await bulkAddGenes(userDefinedGenes);
+    const userDefinedHistograms = await getAllHistograms(
+      "histogram-user-gene",
+      userDefinedGenes
+    );
+
+    expect(userDefinedHistograms).toEqual(
+      expect.arrayContaining(userDefinedGenes)
+    );
+
+    await subset({ x1: 0.15, y1: 0.1, x2: 0.98, y2: 0.98 });
+    await runDiffExp(data.diffexp.cellset1, data.diffexp.cellset2);
+
+    const diffExpHistograms = await getAllHistograms(
+      "histogram-diffexp",
+      diffExpGenes
+    );
+
+    expect(diffExpHistograms).toEqual(expect.arrayContaining(diffExpGenes));
+
+    await subset({ x1: 0.16, y1: 0.11, x2: 0.97, y2: 0.97 });
+
+    const expected = [].concat(userDefinedGenes, diffExpGenes);
+    const userDefinedHistogramsAfterSubset = await getAllHistograms(
+      "histogram-user-gene",
+      expected
+    );
+
+    expect(userDefinedHistogramsAfterSubset).toEqual(
+      expect.arrayContaining(expected)
+    );
+  });
+});
+
+describe("scatter plot", () => {
+  test("scatter plot appears", async () => {
+    await goToPage(appUrlBase);
+
+    await bulkAddGenes(Object.values(data.scatter.genes));
+    await clickOn(`plot-x-${data.scatter.genes.x}`);
+    await clickOn(`plot-y-${data.scatter.genes.y}`);
+    await waitByID("scatterplot");
+  });
 });
 
 describe("clipping", () => {
@@ -202,6 +340,30 @@ describe("clipping", () => {
     const cellCount = await getCellSetCount(1);
     expect(cellCount).toBe(data.clip.count);
   });
+
+  test("clip gene", async () => {
+    await goToPage(appUrlBase);
+
+    await typeInto("gene-search", data.clip.gene);
+    await page.keyboard.press("Enter");
+
+    await page.waitForSelector(`[data-testid='histogram-${data.clip.gene}']`);
+
+    await clip(data.clip.min, data.clip.max);
+
+    const histBrushableAreaId = `histogram-${data.clip.gene}-plot-brushable-area`;
+
+    const coords = await calcDragCoordinates(
+      histBrushableAreaId,
+      data.clip["coordinates-as-percent"]
+    );
+
+    await drag(histBrushableAreaId, coords.start, coords.end);
+
+    const cellCount = await getCellSetCount(1);
+
+    expect(cellCount).toBe(data.clip["gene-cell-count"]);
+  });
 });
 
 // interact with UI elements just that they do not break
@@ -219,6 +381,17 @@ describe("ui elements don't error", () => {
     }
   });
 
+  test("color by for gene", async () => {
+    await goToPage(appUrlBase);
+
+    await typeInto("gene-search", data.genes.search);
+    await page.keyboard.press("Enter");
+    await page.waitForSelector(
+      `[data-testid='histogram-${data.genes.search}']`
+    );
+    await clickOn(`colorby-${data.genes.search}`);
+  });
+
   test("pan and zoom", async () => {
     await goToPage(appUrlBase);
 
@@ -348,4 +521,34 @@ test("lasso moves after pan", async () => {
 
   expect(panCount).toBe(initialCount);
 });
+
+const describeIfCalledByMakeFileTarget =
+  process.env.CXG_AUTH_TYPE?.toLowerCase() === "test"
+    ? describe
+    : describe.skip;
+
+describeIfCalledByMakeFileTarget("auth buttons", () => {
+  test("login then logout", async () => {
+    await goToPage(appUrlBase);
+    await clickOnUntil("log-in", async () => {
+      await page.waitForNavigation({ waitUntil: "networkidle0" });
+      await waitByID("user-info");
+    });
+    await logout();
+  });
+});
+
+const conditionalDescribe =
+  process.env.TEST_AUTH_INTEGRATION === "true" ? describe : describe.skip;
+
+conditionalDescribe("AuthN Integration", () => {
+  it("logs in", async () => {
+    await login();
+  });
+
+  it("logs out", async () => {
+    await login();
+    await logout();
+  });
+});
 /* eslint-enable no-await-in-loop -- await in loop is needed to emulate sequential user actions */

client/__tests__/e2e/e2eAnnotations.test.js

@@ -12,8 +12,6 @@ import {
   getTestId,
   getTestClass,
   getAllByClass,
-  clickOnUntil,
-  getOneElementInnerHTML,
 } from "./puppeteerUtils";
 
 import {
@@ -29,21 +27,6 @@ import {
   renameLabel,
   subset,
   duplicateCategory,
-  createGeneset,
-  deleteGeneset,
-  assertGenesetExists,
-  assertGenesetDoesNotExist,
-  getCellSetCount,
-  expandGeneset,
-  editGenesetName,
-  addGeneToSet,
-  assertGeneExistsInGeneset,
-  removeGene,
-  assertGeneDoesNotExist,
-  expandGene,
-  colorByGeneset,
-  assertColorLegendLabel,
-  colorByGene,
 } from "./cellxgeneActions";
 
 const data = datasets[DATASET];
@@ -51,45 +34,12 @@ const data = datasets[DATASET];
 const perTestCategoryName = "TEST-CATEGORY";
 const perTestLabelName = "TEST-LABEL";
 
-// geneset CRUD
-const genesetToDeleteName = "geneset_to_delete";
-const preExistingGenesetName = "fifth_dataset";
-const meanExpressionBrushGenesetName = "second_gene_set";
-const meanExpressionBrushCellsSelected = "557";
-const subsetMeanExpressionBrushCellsSelected = "452";
-
-// initial text, the text we type in, the result
-const editableGenesetName = "geneset_to_edit";
-const editText = "_111";
-const newGenesetName = "geneset_to_edit_111";
-
-// add gene to set
-const geneToAddToSet = "RER1";
-const setToAddGeneTo = "fill_this_geneset";
-
-// remove gene from set
-const geneToRemove = "SIK1";
-const setToRemoveFrom = "empty_this_geneset";
-
-// brush a gene
-const geneToBrushAndColorBy = "SIK1";
-const brushThisGeneGeneset = "brush_this_gene";
-const geneBrushedCellCount = "109";
-const subsetGeneBrushedCellCount = "96";
-
-const genesetDescriptionID =
-  "geneset-description-tooltip-fourth_gene_set: fourth description";
-const genesetDescriptionString = "fourth_gene_set: fourth description";
-const genesetToCheckForDescription = "fourth_gene_set";
-
 async function setup(config) {
   await goToPage(appUrlBase);
 
-  if (config.categoricalAnno) {
+  // setup the test fixtures
-    // setup the test fixtures
+  await createCategory(perTestCategoryName);
-    await createCategory(perTestCategoryName);
+  await createLabel(perTestCategoryName, perTestLabelName);
-    await createLabel(perTestCategoryName, perTestLabelName);
-  }
 
   if (config.withSubset) {
     await subset({ x1: 0.1, y1: 0.1, x2: 0.8, y2: 0.8 });
@@ -101,200 +51,6 @@ async function setup(config) {
 describe.each([
   { withSubset: true, tag: "subset" },
   { withSubset: false, tag: "whole" },
-])("geneSET crud operations and interactions", (config) => {
-  test("genesets load from csv", async () => {
-    await setup(config);
-
-    await assertGenesetExists(preExistingGenesetName);
-  });
-  test("brush on geneset mean", async () => {
-    await setup(config);
-
-    await expandGeneset(meanExpressionBrushGenesetName);
-
-    const histBrushableAreaId = `histogram-${meanExpressionBrushGenesetName}-plot-brushable-area`;
-
-    const coords = await calcDragCoordinates(histBrushableAreaId, {
-      x1: 0.25,
-      y1: 0.5,
-      x2: 0.55,
-      y2: 0.5,
-    });
-
-    await drag(histBrushableAreaId, coords.start, coords.end);
-
-    const cellCount = await getCellSetCount(1);
-    if (config.withSubset) {
-      expect(cellCount).toBe(subsetMeanExpressionBrushCellsSelected);
-    } else {
-      expect(cellCount).toBe(meanExpressionBrushCellsSelected);
-    }
-  });
-  test("color by mean expression", async () => {
-    await setup(config);
-
-    await colorByGeneset(meanExpressionBrushGenesetName);
-    await assertColorLegendLabel(meanExpressionBrushGenesetName);
-  });
-  test("diffexp", async () => {
-    if (config.withSubset) return;
-
-    await setup(config);
-
-    // set the two cell sets to b cells vs nk cells
-    await expandCategory(`louvain`);
-    await clickOn(`louvain:category-select`);
-    await clickOn(`categorical-value-select-louvain-B cells`);
-    await clickOn(`cellset-button-1`);
-    await clickOn(`categorical-value-select-louvain-B cells`);
-    await clickOn(`categorical-value-select-louvain-NK cells`);
-    await clickOn(`cellset-button-2`);
-
-    // run diffexp
-    await clickOn(`diffexp-button`);
-    await waitByClass("pop-1-geneset-expand");
-    await expect(page).toClick(getTestClass("pop-1-geneset-expand"));
-
-    await page.waitForFunction(
-      (selector) => !document.querySelector(selector),
-      {},
-      getTestClass("gene-loading-spinner")
-    );
-
-    let genesHTML = await getOneElementInnerHTML(
-      getTestClass("gene-set-genes")
-    );
-
-    expect(genesHTML).toMatchSnapshot();
-
-    await expect(page).toClick(getTestClass("pop-1-geneset-expand"));
-    await expect(page).toClick(getTestClass("pop-2-geneset-expand"));
-
-    await page.waitForFunction(
-      (selector) => !document.querySelector(selector),
-      {},
-      getTestClass("gene-loading-spinner")
-    );
-
-    genesHTML = await getOneElementInnerHTML(getTestClass("gene-set-genes"));
-
-    expect(genesHTML).toMatchSnapshot();
-  });
-  test("create a new geneset and undo/redo", async () => {
-    if (config.withSubset) return;
-
-    await setup(config);
-
-    const genesetName = `test-geneset-foo-123`;
-    await assertGenesetDoesNotExist(genesetName);
-    await createGeneset(genesetName);
-    /* note: as of June 2021, the aria label is in the truncate component which clones the element */
-    await assertGenesetExists(genesetName);
-    await clickOn("undo");
-    await assertGenesetDoesNotExist(genesetName);
-    await clickOn("redo");
-    await assertGenesetExists(genesetName);
-  });
-  test("edit geneset name and undo/redo", async () => {
-    await setup(config);
-
-    await editGenesetName(editableGenesetName, editText);
-    await assertGenesetExists(newGenesetName);
-    await clickOn("undo");
-    await assertGenesetExists(editableGenesetName);
-    await clickOn("redo");
-    await assertGenesetExists(newGenesetName);
-  });
-  test("delete a geneset and undo/redo", async () => {
-    if (config.withSubset) return;
-
-    await setup(config);
-
-    await deleteGeneset(genesetToDeleteName);
-    await clickOn("undo");
-    await assertGenesetExists(genesetToDeleteName);
-    await clickOn("redo");
-    await assertGenesetDoesNotExist(genesetToDeleteName);
-  });
-  test("geneset description", async () => {
-    if (config.withSubset) return;
-
-    await setup(config);
-
-    await clickOnUntil(
-      `${genesetToCheckForDescription}:geneset-expand`,
-      async () => {
-        expect(page).toMatchElement(getTestId(genesetDescriptionID), {
-          text: genesetDescriptionString,
-        });
-      }
-    );
-  });
-});
-
-describe.each([
-  { withSubset: true, tag: "subset" },
-  { withSubset: false, tag: "whole" },
-])("GENE crud operations and interactions", (config) => {
-  test("add a gene to geneset and undo/redo", async () => {
-    await setup(config);
-
-    await addGeneToSet(setToAddGeneTo, geneToAddToSet);
-    await expandGeneset(setToAddGeneTo);
-    await assertGeneExistsInGeneset(geneToAddToSet);
-    await clickOn("undo");
-    await assertGeneDoesNotExist(geneToAddToSet);
-    await clickOn("redo");
-    await assertGeneExistsInGeneset(geneToAddToSet);
-  });
-  test("expand gene and brush", async () => {
-    await setup(config);
-
-    await expandGeneset(brushThisGeneGeneset);
-    await expandGene(geneToBrushAndColorBy);
-    const histBrushableAreaId = `histogram-${geneToBrushAndColorBy}-plot-brushable-area`;
-
-    const coords = await calcDragCoordinates(histBrushableAreaId, {
-      x1: 0.25,
-      y1: 0.5,
-      x2: 0.55,
-      y2: 0.5,
-    });
-    await drag(histBrushableAreaId, coords.start, coords.end);
-    const cellCount = await getCellSetCount(1);
-    if (config.withSubset) {
-      expect(cellCount).toBe(subsetGeneBrushedCellCount);
-    } else {
-      expect(cellCount).toBe(geneBrushedCellCount);
-    }
-  });
-  test("color by gene in geneset", async () => {
-    await setup(config);
-
-    await expandGeneset(meanExpressionBrushGenesetName);
-
-    await colorByGene(geneToBrushAndColorBy);
-    await assertColorLegendLabel(geneToBrushAndColorBy);
-  });
-  test("delete gene from geneset and undo/redo", async () => {
-    // We've already deleted the gene
-    if (config.withSubset) return;
-
-    await setup(config);
-
-    await expandGeneset(setToRemoveFrom);
-    await removeGene(geneToRemove);
-    await assertGeneDoesNotExist(geneToRemove);
-    await clickOn("undo");
-    await assertGeneExistsInGeneset(geneToRemove);
-    await clickOn("redo");
-    await assertGeneDoesNotExist(geneToRemove);
-  });
-});
-
-describe.each([
-  { withSubset: true, tag: "subset", categoricalAnno: true },
-  { withSubset: false, tag: "whole", categoricalAnno: true },
 ])("annotations", (config) => {
   test("create a category", async () => {
     await setup(config);
@@ -531,7 +287,11 @@ describe.each([
     const labels = await getAllByClass("categorical-row");
 
     const result = await Promise.all(
-      labels.map((label) => page.evaluate((element) => element.outerHTML, label))
+      labels.map((label) => {
+        return page.evaluate((element) => {
+          return element.outerHTML;
+        }, label);
+      })
     );
 
     expect(result).toMatchSnapshot();

client/__tests__/e2e/e2eJestConfig.json

@@ -4,7 +4,7 @@
   "testMatch": ["**/__tests__/**/?(*.)(spec|test).js?(x)"],
   "setupFiles": ["../setupMissingGlobals.js"],
   "setupFilesAfterEnv": ["expect-puppeteer", "./puppeteer.setup.js"],
-  "globalSetup": "../globalSetup.js",
+  "globalSetup": "jest-environment-puppeteer/setup",
   "globalTeardown": "jest-environment-puppeteer/teardown",
   "testEnvironment": "./screenshot_env.js"
 }

client/__tests__/e2e/puppeteerUtils.js

@@ -12,7 +12,7 @@ export async function waitByID(testId, props = {}) {
 }
 
 export async function waitByClass(testClass, props = {}) {
-  return page.waitForSelector(`[data-testclass='${testClass}']`, props);
+  await page.waitForSelector(`[data-testclass='${testClass}']`, props);
 }
 
 export async function waitForAllByIds(testIds) {
@@ -32,7 +32,7 @@ export async function typeInto(testId, text) {
   const selector = getTestId(testId);
   // type ahead can be annoying if you don't pause before you type
   await page.click(selector);
-  await page.waitForTimeout(200);
+  await page.waitFor(200);
   await page.type(selector, text);
 }
 
@@ -42,7 +42,7 @@ export async function clearInputAndTypeInto(testId, text) {
   // only works for text without special characters
   // type ahead can be annoying if you don't pause before you type
   await page.click(selector);
-  await page.waitForTimeout(200);
+  await page.waitFor(200);
   // select all
   await page.click(selector, { clickCount: 3 });
   await page.keyboard.press("Backspace");
@@ -72,7 +72,7 @@ export async function clickOnUntil(testId, assert) {
     } catch (error) {
       retry += 1;
 
-      await page.waitForTimeout(WAIT_FOR_MS);
+      await page.waitFor(WAIT_FOR_MS);
     }
   }
 
@@ -100,6 +100,12 @@ export async function getElementCoordinates(testId) {
   });
 }
 
+async function clickTermsOfService() {
+  if (!(await isElementPresent(getTestId("tos-cookies-accept")))) return;
+
+  await clickOn("tos-cookies-accept");
+}
+
 async function nameNewAnnotation() {
   if (await isElementPresent(getTestId("annotation-dialog"))) {
     await typeInto("new-annotation-name", "ignoreE2E");
@@ -116,6 +122,7 @@ export async function goToPage(url) {
   });
 
   await nameNewAnnotation();
+  await clickTermsOfService();
 }
 
 export async function isElementPresent(selector, options) {

client/__tests__/e2e/test_config.yaml

@@ -1,9 +1,33 @@
 server:
   app:
     force_https: true
-    port: 5005
+
+    # By default, cellxgene will serve api requests from the same base url as the webpage.
+    # In general api_base_url and web_base_url will not need to be set.
+    # There are two reasons to set these parameters:
+    #  1. Oauth authentication is used; the oauth server will redirect back to the api_base_url after login,
+    #     which then redirects back to the web_base_url.  If the web_base_url is not set, it will default to
+    #     the api_base_url.  If oauth authentication is used, the api_base_url must be set.
+    #     For a local test (where the server runs on "http://localhost:<port>"), then the api_base_url may be
+    #     set to the string "local".
+    #  2. The cellxgene deploymnent is in an environment where the webpage and api have
+    #     different base urls.  In this case both api_base_url and web_base_url must be set.
+    #     It is up to the server admin to ensure that the networking is setup correctly for this environment.
+    api_base_url: http://localhost:5005
+    web_base_url: http://localhost:3000
+
+  authentication:
+    # The authentication types may be "none", "session", "oauth"
+    # none:  No authentication support, features like user_annotations must not be enabled.
+    # session:  A session based userid is automatically generated. (no params needed)
+    # oauth: oauth2 is used for authentication;  parameters are defined in params_oauth.
+    type: test
 
 dataset:
+  app:
+    about_legal_tos: null
+    about_legal_privacy: null
+
   presentation:
     max_categories: 1000
     custom_colors: true
@@ -14,6 +38,10 @@ dataset:
     local_file_csv:
       directory: null
       file: null
+    ontology:
+      enable: false
+      obo_location: null
 
   embeddings:
     names: []
+    enable_reembedding: false

client/__tests__/globalSetup.js

@@ -1,5 +0,0 @@
-const { setup } = require("jest-environment-puppeteer");
-
-module.exports = async () => {
-  await setup();
-};

client/__tests__/reducers/genesets.test.js

@@ -1,524 +0,0 @@
-import genesetsReducer from "../../src/reducers/genesets";
-
-describe("initial reducer state", () => {
-  test("some other action", () => {
-    expect(genesetsReducer(undefined, { type: "foo" })).toMatchObject({
-      initialized: false,
-      lastTid: undefined,
-      genesets: new Map(),
-    });
-  });
-});
-
-describe("geneset: initial load", () => {
-  test("missing JSON response", () => {
-    expect(() =>
-      genesetsReducer(undefined, {
-        type: "geneset: initial load",
-      })
-    ).toThrow("missing or malformed JSON response");
-  });
-
-  test("empty geneset", () => {
-    expect(
-      genesetsReducer(undefined, {
-        type: "geneset: initial load",
-        data: {
-          tid: 0,
-          genesets: [],
-        },
-      })
-    ).toMatchObject({
-      initialized: true,
-      lastTid: 0,
-      genesets: new Map(),
-    });
-  });
-
-  test("non-empty geneset", () => {
-    expect(
-      genesetsReducer(undefined, {
-        type: "geneset: initial load",
-        data: {
-          tid: 99,
-          genesets: [
-            {
-              geneset_name: "G1",
-              genes: [{ gene_symbol: "F5" }],
-            },
-            {
-              geneset_name: "G2",
-              geneset_description: "G2 desc",
-              genes: [{ gene_symbol: "F6" }],
-            },
-            {
-              geneset_name: "G3",
-              geneset_description: "G3 desc",
-              genes: [{ gene_symbol: "F7", gene_description: "gene desc" }],
-            },
-          ],
-        },
-      })
-    ).toMatchObject({
-      initialized: true,
-      lastTid: 99,
-      genesets: new Map([
-        [
-          "G1",
-          {
-            genesetName: "G1",
-            genesetDescription: "",
-            genes: new Map([["F5", { geneSymbol: "F5", geneDescription: "" }]]),
-          },
-        ],
-        [
-          "G2",
-          {
-            genesetName: "G2",
-            genesetDescription: "G2 desc",
-            genes: new Map([["F6", { geneSymbol: "F6", geneDescription: "" }]]),
-          },
-        ],
-        [
-          "G3",
-          {
-            genesetName: "G3",
-            genesetDescription: "G3 desc",
-            genes: new Map([
-              ["F7", { geneSymbol: "F7", geneDescription: "gene desc" }],
-            ]),
-          },
-        ],
-      ]),
-    });
-  });
-});
-
-describe("geneset: create", () => {
-  const initialState = genesetsReducer(undefined, {
-    type: "geneset: initial load",
-    data: {
-      tid: 0,
-      genesets: [],
-    },
-  });
-
-  test("simple create", () => {
-    expect(
-      genesetsReducer(initialState, {
-        type: "geneset: create",
-        genesetName: "a geneset",
-        genesetDescription: "",
-      })
-    ).toMatchObject({
-      ...initialState,
-      genesets: new Map([
-        [
-          "a geneset",
-          {
-            genesetName: "a geneset",
-            genesetDescription: "",
-            genes: new Map(),
-          },
-        ],
-      ]),
-    });
-  });
-
-  test("error - duplicate name", () => {
-    expect(() => {
-      genesetsReducer(
-        genesetsReducer(initialState, {
-          type: "geneset: create",
-          genesetName: "foo",
-          genesetDescription: "foo",
-        }),
-        {
-          type: "geneset: create",
-          genesetName: "foo",
-          genesetDescription: "bar",
-        }
-      );
-    }).toThrow("name already defined");
-  });
-
-  test("error - missing required action values", () => {
-    expect(() => {
-      genesetsReducer(initialState, {
-        type: "geneset: create",
-        genesetDescription: "foo",
-      });
-    }).toThrow();
-    expect(() => {
-      genesetsReducer(initialState, {
-        type: "geneset: create",
-        genesetName: "foo",
-      });
-    }).toThrow("name or description unspecified");
-  });
-});
-
-describe("geneset: delete", () => {
-  const initialState = genesetsReducer(undefined, {
-    type: "geneset: initial load",
-    data: {
-      tid: 0,
-      genesets: [],
-    },
-  });
-
-  test("simple delete", () => {
-    expect(
-      genesetsReducer(
-        genesetsReducer(initialState, {
-          type: "geneset: create",
-          genesetName: "foo",
-          genesetDescription: "foo",
-        }),
-        {
-          type: "geneset: delete",
-          genesetName: "foo",
-        }
-      )
-    ).toMatchObject({
-      initialized: true,
-      lastTid: 0,
-      genesets: new Map(),
-    });
-  });
-
-  test("error - missing name", () => {
-    expect(() => {
-      genesetsReducer(initialState, {
-        type: "geneset: delete",
-        genesetName: "foo",
-      });
-    }).toThrow("name does not exist");
-  });
-});
-
-describe("geneset: update", () => {
-  const initialState = genesetsReducer(undefined, {
-    type: "geneset: initial load",
-    data: {
-      tid: 0,
-      genesets: [],
-    },
-  });
-
-  test("simple update", () => {
-    expect(
-      genesetsReducer(
-        genesetsReducer(
-          genesetsReducer(initialState, {
-            type: "geneset: create",
-            genesetName: "foo1",
-            genesetDescription: "foo1",
-          }),
-          {
-            type: "geneset: create",
-            genesetName: "foo2",
-            genesetDescription: "foo2",
-          }
-        ),
-        {
-          type: "geneset: update",
-          genesetName: "foo1",
-          update: {
-            genesetName: "bar",
-            genesetDescription: "bar",
-          },
-        }
-      )
-    ).toMatchObject({
-      initialized: true,
-      lastTid: 0,
-      genesets: new Map([
-        [
-          "bar",
-          { genesetName: "bar", genesetDescription: "bar", genes: new Map() },
-        ],
-        [
-          "foo2",
-          { genesetName: "foo2", genesetDescription: "foo2", genes: new Map() },
-        ],
-      ]),
-    });
-  });
-
-  test("error - unknown name", () => {
-    expect(() => {
-      genesetsReducer(initialState, {
-        type: "geneset: update",
-        genesetName: "foo",
-        update: {
-          genesetName: "foo",
-          genesetDescription: "bar",
-        },
-      });
-    }).toThrow("name unspecified or does not exist");
-  });
-
-  test("error - duplicate name", () => {
-    expect(() => {
-      genesetsReducer(
-        genesetsReducer(initialState, {
-          type: "geneset: create",
-          genesetName: "foo",
-          genesetDescription: "foo",
-        }),
-        {
-          type: "geneset: update",
-          genesetName: "foo",
-          update: {
-            genesetName: "foo",
-            genesetDescription: "foo",
-          },
-        }
-      );
-    }).toThrow("update specified existing name and description");
-  });
-});
-
-describe("geneset: add genes", () => {
-  const initialState = genesetsReducer(
-    genesetsReducer(undefined, {
-      type: "geneset: initial load",
-      data: {
-        tid: 0,
-        genesets: [],
-      },
-    }),
-    {
-      type: "geneset: create",
-      genesetName: "test",
-      genesetDescription: "",
-    }
-  );
-
-  test("add a gene", () => {
-    expect(
-      genesetsReducer(initialState, {
-        type: "geneset: add genes",
-        genesetName: "test",
-        genes: [{ geneSymbol: "F5" }],
-      })
-    ).toMatchObject({
-      ...initialState,
-      genesets: new Map([
-        [
-          "test",
-          {
-            genesetName: "test",
-            genesetDescription: "",
-            genes: new Map([["F5", { geneSymbol: "F5", geneDescription: "" }]]),
-          },
-        ],
-      ]),
-    });
-
-    expect(
-      genesetsReducer(initialState, {
-        type: "geneset: add genes",
-        genesetName: "test",
-        genes: [
-          { geneSymbol: "F5", geneDescription: "desc" },
-          { geneSymbol: "SET1", geneDescription: "" },
-        ],
-      })
-    ).toMatchObject({
-      ...initialState,
-      genesets: new Map([
-        [
-          "test",
-          {
-            genesetName: "test",
-            genesetDescription: "",
-            genes: new Map([
-              ["F5", { geneSymbol: "F5", geneDescription: "desc" }],
-              ["SET1", { geneSymbol: "SET1", geneDescription: "" }],
-            ]),
-          },
-        ],
-      ]),
-    });
-  });
-
-  test("no such geneset error", () => {
-    expect(() => {
-      genesetsReducer(initialState, {
-        type: "geneset: add genes",
-        genesetName: "mumble",
-        genes: [],
-      });
-    }).toThrow("geneset name does not exist");
-  });
-});
-
-describe("geneset: delete genes", () => {
-  const initialState = genesetsReducer(
-    genesetsReducer(
-      genesetsReducer(undefined, {
-        type: "geneset: initial load",
-        data: {
-          tid: 0,
-          genesets: [],
-        },
-      }),
-      {
-        type: "geneset: create",
-        genesetName: "test",
-        genesetDescription: "",
-      }
-    ),
-    {
-      type: "geneset: add genes",
-      genesetName: "test",
-      genes: [{ geneSymbol: "F5" }],
-    }
-  );
-
-  test("simple", () => {
-    expect(
-      genesetsReducer(initialState, {
-        type: "geneset: delete genes",
-        genesetName: "test",
-        geneSymbols: ["F5"],
-      })
-    ).toMatchObject({
-      ...initialState,
-      genesets: new Map([
-        [
-          "test",
-          {
-            genesetName: "test",
-            genesetDescription: "",
-            genes: new Map(),
-          },
-        ],
-      ]),
-    });
-  });
-
-  test("no such geneset error", () => {
-    expect(() => {
-      genesetsReducer(initialState, {
-        type: "geneset: delete genes",
-        genesetName: "mumble",
-        geneSymbols: [],
-      });
-    }).toThrow("name does not exist");
-  });
-});
-
-describe("geneset: set gene description", () => {
-  const initialState = genesetsReducer(
-    genesetsReducer(
-      genesetsReducer(undefined, {
-        type: "geneset: initial load",
-        data: {
-          tid: 0,
-          genesets: [],
-        },
-      }),
-      {
-        type: "geneset: create",
-        genesetName: "test",
-        genesetDescription: "",
-      }
-    ),
-    {
-      type: "geneset: add genes",
-      genesetName: "test",
-      genes: [{ geneSymbol: "F5" }],
-    }
-  );
-
-  test("simple set", () => {
-    expect(
-      genesetsReducer(initialState, {
-        type: "geneset: set gene description",
-        genesetName: "test",
-        update: {
-          geneSymbol: "F5",
-          geneDescription: "mumble",
-        },
-      })
-    ).toMatchObject({
-      ...initialState,
-      genesets: new Map([
-        [
-          "test",
-          {
-            genesetName: "test",
-            genesetDescription: "",
-            genes: new Map([
-              ["F5", { geneSymbol: "F5", geneDescription: "mumble" }],
-            ]),
-          },
-        ],
-      ]),
-    });
-  });
-
-  test("no such geneset error", () => {
-    expect(() => {
-      genesetsReducer(initialState, {
-        type: "geneset: set gene description",
-        genesetName: "does not exist",
-        update: {
-          geneSymbol: "F5",
-          geneDescription: "mumble",
-        },
-      });
-    }).toThrow("geneset name does not exist");
-  });
-
-  test("no such gene error", () => {
-    expect(() => {
-      genesetsReducer(initialState, {
-        type: "geneset: set gene description",
-        genesetName: "test",
-        update: {
-          geneSymbol: "NO SUCH GENE",
-          geneDescription: "mumble",
-        },
-      });
-    }).toThrow("no such gene");
-  });
-});
-
-describe("geneset: set tid", () => {
-  test("simple set", () => {
-    expect(
-      genesetsReducer(undefined, {
-        type: "geneset: set tid",
-        tid: 1,
-      })
-    ).toMatchObject({ lastTid: 1 });
-  });
-
-  test("not a number error", () => {
-    expect(() => {
-      genesetsReducer(
-        { lastTid: 1 },
-        {
-          type: "geneset: set tid",
-          tid: "0",
-        }
-      );
-    }).toThrow("must be a positive integer");
-  });
-
-  test("decrement error", () => {
-    expect(() => {
-      genesetsReducer(
-        { lastTid: 1 },
-        {
-          type: "geneset: set tid",
-          tid: 0,
-        }
-      );
-    }).toThrow("may not be decremented");
-  });
-});

client/__tests__/reducers/genesetsUI.test.js

@@ -1,75 +0,0 @@
-import genesetsUIReducer from "../../src/reducers/genesetsUI";
-
-// Format: GeneSetsUI(state,action)
-
-const initialState = {
-  createGenesetModeActive: false,
-  isEditingGenesetName: false,
-  isAddingGenesToGeneset: false,
-};
-
-/* initial */
-describe("geneset UI states", () => {
-  test("initial state, some other action", () => {
-    expect(
-      genesetsUIReducer(undefined, {
-        type: "foo",
-      })
-    ).toMatchObject(initialState);
-  });
-  test("geneset: activate add new geneset mode", () => {
-    expect(
-      genesetsUIReducer(initialState, {
-        type: "geneset: activate add new geneset mode",
-      })
-    ).toMatchObject({
-      createGenesetModeActive: true,
-      isEditingGenesetName: false,
-      isAddingGenesToGeneset: false,
-    });
-  });
-  test("geneset: disable create geneset mode", () => {
-    expect(
-      genesetsUIReducer(undefined, { isEditingGenesetName: false })
-    ).toMatchObject(initialState);
-  });
-
-  test("activate add new genes mode", () => {
-    expect(
-      genesetsUIReducer(undefined, {
-        type: "geneset: activate add new genes mode",
-        geneset: "a geneset name",
-      })
-    ).toMatchObject({
-      createGenesetModeActive: false,
-      isEditingGenesetName: false,
-      isAddingGenesToGeneset: "a geneset name",
-    });
-  });
-  test("disable create geneset mode", () => {
-    expect(
-      genesetsUIReducer(undefined, {
-        type: "geneset: disable create geneset mode",
-      })
-    ).toMatchObject(initialState);
-  });
-  test("activate rename geneset mode", () => {
-    expect(
-      genesetsUIReducer(undefined, {
-        type: "geneset: activate rename geneset mode",
-        data: "a geneset name",
-      })
-    ).toMatchObject({
-      createGenesetModeActive: false,
-      isEditingGenesetName: "a geneset name",
-      isAddingGenesToGeneset: false,
-    });
-  });
-  test("disable rename geneset mode", () => {
-    expect(
-      genesetsUIReducer(undefined, {
-        type: "geneset: disable rename geneset mode",
-      })
-    ).toMatchObject(initialState);
-  });
-});

client/__tests__/reducers/undoable.test.js

@@ -23,7 +23,9 @@ describe("create", () => {
 describe("undo", () => {
   test("expected state modifications", () => {
     const initialState = { a: 0, b: 1000 };
-    const reducer = (state) => ({ a: state.a + 1, b: state.b + 1 });
+    const reducer = (state) => {
+      return { a: state.a + 1, b: state.b + 1 };
+    };
     const undoableReducer = undoable(reducer, ["a"]);
 
     const s1 = undoableReducer(initialState, { type: "test" });
@@ -41,7 +43,9 @@ describe("undo", () => {
 
 describe("redo", () => {
   const initialState = { a: 0, b: 1000 };
-  const reducer = (state) => ({ a: state.a + 1, b: state.b + 1 });
+  const reducer = (state) => {
+    return { a: state.a + 1, b: state.b + 1 };
+  };
   let UR;
 
   beforeEach(() => {

client/__tests__/util/actionHelpers.test.js

@@ -11,14 +11,18 @@ describe("rangeEncodeIndices", () => {
 
   test("sorted flag", () => {
     expect(rangeEncodeIndices([1, 9, 432], 10, true)).toMatchObject([
-      1, 9, 432,
+      1,
+      9,
+      432,
     ]);
     expect(rangeEncodeIndices([1, 9, 432], 10, false)).toMatchObject([
-      1, 9, 432,
+      1,
+      9,
+      432,
     ]);
-    expect(rangeEncodeIndices([0, 1, 2, 3, 9, 10, 432], 2, true)).toMatchObject(
+    expect(
-      [[0, 3], [9, 10], 432]
+      rangeEncodeIndices([0, 1, 2, 3, 9, 10, 432], 2, true)
-    );
+    ).toMatchObject([[0, 3], [9, 10], 432]);
     expect(
       rangeEncodeIndices([0, 1, 2, 3, 9, 10, 432], 2, false)
     ).toMatchObject([[0, 3], [9, 10], 432]);

client/__tests__/util/annoMatrix/annoMatrix.test.js

@@ -85,11 +85,9 @@ describe("AnnoMatrix", () => {
       fetch.once(serverMocks.responder);
       await expect(
         annoMatrix.fetch("X", {
-          where: {
+          field: "var",
-            field: "var",
+          column: annoMatrix.schema.annotations.var.index,
-            column: annoMatrix.schema.annotations.var.index,
+          value: "TYMP",
-            value: "TYMP",
-          },
         })
       ).resolves.toBeInstanceOf(Dataframe);
 
@@ -105,18 +103,14 @@ describe("AnnoMatrix", () => {
       await expect(
         annoMatrix.fetch("X", [
           {
-            where: {
+            field: "var",
-              field: "var",
+            column: varIndex,
-              column: varIndex,
+            value: "SUMO3",
-              value: "SUMO3",
-            },
           },
           {
-            where: {
+            field: "var",
-              field: "var",
+            column: varIndex,
-              column: varIndex,
+            value: "TYMP",
-              value: "TYMP",
-            },
           },
         ])
       ).resolves.toBeInstanceOf(Dataframe);

client/__tests__/util/annoMatrix/crossfilter.test.js

@@ -170,11 +170,9 @@ describe("AnnoMatrixCrossfilter", () => {
       const xfltr = await crossfilter.select(
         "X",
         {
-          where: {
+          field: "var",
-            field: "var",
+          column: varIndex,
-            column: varIndex,
+          value: "TYMP",
-            value: "TYMP",
-          },
         },
         {
           mode: "range",
@@ -188,11 +186,9 @@ describe("AnnoMatrixCrossfilter", () => {
       expect(xfltr.countSelected()).toEqual(501);
 
       const df = await annoMatrix.fetch("X", {
-        where: {
+        field: "var",
-          field: "var",
+        column: varIndex,
-          column: varIndex,
+        value: "TYMP",
-          value: "TYMP",
-        },
       });
       const values = df.icol(0).asArray();
       const selected = xfltr.allSelectedMask();

client/__tests__/util/annoMatrix/serverMocks/index.js

@@ -1,4 +1,11 @@
 export const baseDataURL = "https://a.fake.url/api/v0.2";
 
+window.CELLXGENE = {
+  API: {
+    prefix: baseDataURL,
+    version: "v0.2/",
+  },
+};
+
 export { schema } from "./schema";
 export * from "./routes";

client/__tests__/util/annoMatrix/whereCache.test.js

@@ -1,4 +1,3 @@
-import sha1 from "sha1";
 import {
   _whereCacheGet,
   _whereCacheCreate,
@@ -8,82 +7,37 @@ import {
 const schema = {};
 
 describe("whereCache", () => {
-  test("whereCacheGet - where query, missing cache values", () => {
+  test("whereCacheGet - missing cache values", () => {
     expect(
       _whereCacheGet({}, schema, "X", {
-        where: {
+        field: "var",
-          field: "var",
+        column: "foo",
-          column: "foo",
+        value: "bar",
-          value: "bar",
-        },
       })
     ).toEqual([undefined]);
     expect(
-      _whereCacheGet({}, schema, "X", {
+      _whereCacheGet({ X: {} }, schema, "X", {
-        summarize: {
+        field: "var",
-          field: "var",
+        column: "foo",
-          column: "foo",
+        value: "bar",
-          values: ["bar"],
-        },
       })
     ).toEqual([undefined]);
     expect(
-      _whereCacheGet({ where: { X: {} } }, schema, "X", {
+      _whereCacheGet({ X: { var: new Map() } }, schema, "X", {
-        where: {
+        field: "var",
-          field: "var",
+        column: "foo",
-          column: "foo",
+        value: "bar",
-          value: "bar",
-        },
-      })
-    ).toEqual([undefined]);
-    expect(
-      _whereCacheGet({ where: { X: { var: new Map() } } }, schema, "X", {
-        where: {
-          field: "var",
-          column: "foo",
-          value: "bar",
-        },
       })
     ).toEqual([undefined]);
     expect(
       _whereCacheGet(
-        { where: { X: { var: new Map([["foo", new Map()]]) } } },
+        { X: { var: new Map([["foo", new Map()]]) } },
         schema,
         "X",
         {
-          where: {
-            field: "var",
-            column: "foo",
-            value: "bar",
-          },
-        }
-      )
-    ).toEqual([undefined]);
-  });
-
-  test("whereCacheGet - summarize query, missing cache values", () => {
-    expect(
-      _whereCacheGet({}, schema, "X", {
-        summarize: {
-          method: "mean",
           field: "var",
           column: "foo",
-          values: ["bar"],
+          value: "bar",
-        },
-      })
-    ).toEqual([undefined]);
-    expect(
-      _whereCacheGet(
-        { summarize: { X: { mean: { var: new Map() } } } },
-        schema,
-        "X",
-        {
-          summarize: {
-            method: "mean",
-            field: "var",
-            column: "foo",
-            values: ["bar"],
-          },
         }
       )
     ).toEqual([undefined]);
@@ -91,292 +45,140 @@ describe("whereCache", () => {
 
   test("whereCacheGet - varied lookups", () => {
     const whereCache = {
-      where: {
+      X: {
-        X: {
+        var: new Map([
-          var: new Map([
+          [
-            [
+            "foo",
-              "foo",
+            new Map([
-              new Map([
+              ["bar", [0]],
-                ["bar", [0]],
+              ["baz", [1, 2]],
-                ["baz", [1, 2]],
-              ]),
-            ],
-          ]),
-        },
-      },
-      summarize: {
-        X: {
-          mean: {
-            var: new Map([
-              [
-                "foo",
-                new Map([
-                  [sha1("bar"), [0]],
-                  [sha1("baz"), [1, 2]],
-                ]),
-              ],
             ]),
-          },
+          ],
-        },
+        ]),
       },
     };
 
     expect(
       _whereCacheGet(whereCache, schema, "X", {
-        where: {
+        field: "var",
-          field: "var",
+        column: "foo",
-          column: "foo",
+        value: "bar",
-          value: "bar",
-        },
       })
     ).toEqual([0]);
     expect(
       _whereCacheGet(whereCache, schema, "X", {
-        summarize: {
+        field: "var",
-          method: "mean",
+        column: "foo",
-          field: "var",
+        value: "baz",
-          column: "foo",
-          values: ["bar"],
-        },
-      })
-    ).toEqual([0]);
-    expect(
-      _whereCacheGet(whereCache, schema, "X", {
-        where: {
-          field: "var",
-          column: "foo",
-          value: "baz",
-        },
-      })
-    ).toEqual([1, 2]);
-    expect(
-      _whereCacheGet(whereCache, schema, "X", {
-        summarize: {
-          method: "mean",
-          field: "var",
-          column: "foo",
-          values: ["baz"],
-        },
       })
     ).toEqual([1, 2]);
     expect(_whereCacheGet(whereCache, schema, "Y", {})).toEqual([undefined]);
     expect(
       _whereCacheGet(whereCache, schema, "X", {
-        where: {
+        field: "whoknows",
-          field: "whoknows",
+        column: "whatever",
-          column: "whatever",
+        value: "snork",
-          value: "snork",
-        },
       })
     ).toEqual([undefined]);
     expect(
       _whereCacheGet(whereCache, schema, "X", {
-        where: {
+        field: "var",
-          field: "var",
+        column: "whatever",
-          column: "whatever",
+        value: "snork",
-          value: "snork",
-        },
       })
     ).toEqual([undefined]);
     expect(
       _whereCacheGet(whereCache, schema, "X", {
-        where: {
+        field: "var",
-          field: "var",
+        column: "foo",
-          column: "foo",
+        value: "snork",
-          value: "snork",
-        },
       })
     ).toEqual([undefined]);
   });
 
-  test("whereCacheCreate, where query", () => {
+  test("whereCacheCreate", () => {
     const query = {
-      where: {
+      field: "queryField",
-        field: "queryField",
+      column: "queryColumn",
-        column: "queryColumn",
+      value: "queryValue",
-        value: "queryValue",
-      },
     };
     const wc = _whereCacheCreate(
       "field",
-      {
+      { field: "queryField", column: "queryColumn", value: "queryValue" },
-        where: {
-          field: "queryField",
-          column: "queryColumn",
-          value: "queryValue",
-        },
-      },
       [0, 1, 2]
     );
     expect(wc).toBeDefined();
     expect(wc).toEqual(
       expect.objectContaining({
-        where: {
+        field: {
-          field: {
+          queryField: expect.any(Map),
-            queryField: expect.any(Map),
-          },
         },
       })
     );
-    expect(wc.where.field.queryField.has("queryColumn")).toEqual(true);
+    expect(wc.field.queryField.has("queryColumn")).toEqual(true);
-    expect(wc.where.field.queryField.get("queryColumn")).toBeInstanceOf(Map);
+    expect(wc.field.queryField.get("queryColumn")).toBeInstanceOf(Map);
-    expect(
+    expect(wc.field.queryField.get("queryColumn").has("queryValue")).toEqual(
-      wc.where.field.queryField.get("queryColumn").has("queryValue")
+      true
-    ).toEqual(true);
+    );
     expect(_whereCacheGet(wc, schema, "field", query)).toEqual([0, 1, 2]);
   });
 
-  test("whereCacheCreate, summarize query", () => {
+  test("whereCacheMerge", () => {
-    const query = {
-      summarize: {
-        method: "method",
-        field: "queryField",
-        column: "queryColumn",
-        values: ["queryValue"],
-      },
-    };
-    const wc = _whereCacheCreate("field", query, [0, 1, 2]);
-    expect(_whereCacheGet(wc, schema, "field", query)).toEqual([0, 1, 2]);
-  });
-
-  test("whereCacheCreate, unknown query type", () => {
-    expect(_whereCacheCreate("field", { foobar: true }, [1])).toEqual({});
-  });
-
-  test("whereCacheMerge, where queries", () => {
     let wc;
 
     // remember, will mutate dst
     const src = _whereCacheCreate(
       "field",
-      { where: { field: "queryField", column: "queryColumn", value: "foo" } },
+      { field: "queryField", column: "queryColumn", value: "foo" },
       ["foo"]
     );
 
     const dst1 = _whereCacheCreate(
       "field",
-      { where: { field: "queryField", column: "queryColumn", value: "bar" } },
+      { field: "queryField", column: "queryColumn", value: "bar" },
       ["dst1"]
     );
     wc = _whereCacheMerge(dst1, src);
     expect(
       _whereCacheGet(wc, schema, "field", {
-        where: {
+        field: "queryField",
-          field: "queryField",
+        column: "queryColumn",
-          column: "queryColumn",
+        value: "foo",
-          value: "foo",
-        },
       })
     ).toEqual(["foo"]);
     expect(
       _whereCacheGet(wc, schema, "field", {
-        where: {
+        field: "queryField",
-          field: "queryField",
+        column: "queryColumn",
-          column: "queryColumn",
+        value: "bar",
-          value: "bar",
-        },
       })
     ).toEqual(["dst1"]);
 
     const dst2 = _whereCacheCreate(
       "field",
-      { where: { field: "queryField", column: "queryColumn", value: "bar" } },
+      { field: "queryField", column: "queryColumn", value: "bar" },
       ["dst2"]
     );
     wc = _whereCacheMerge(dst2, dst1, src);
     expect(
       _whereCacheGet(wc, schema, "field", {
-        where: {
+        field: "queryField",
-          field: "queryField",
+        column: "queryColumn",
-          column: "queryColumn",
+        value: "foo",
-          value: "foo",
-        },
       })
     ).toEqual(["foo"]);
     expect(
       _whereCacheGet(wc, schema, "field", {
-        where: {
+        field: "queryField",
-          field: "queryField",
+        column: "queryColumn",
-          column: "queryColumn",
+        value: "bar",
-          value: "bar",
-        },
       })
     ).toEqual(["dst1"]);
 
     wc = _whereCacheMerge({}, src);
     expect(wc).toEqual(src);
 
-    wc = _whereCacheMerge({ where: { field: { queryField: new Map() } } }, src);
+    wc = _whereCacheMerge({ field: { queryField: new Map() } }, src);
     expect(wc).toEqual(src);
   });
-
-  test("whereCacheMerge, mixed queries", () => {
-    const wc = _whereCacheMerge(
-      _whereCacheCreate(
-        "field",
-        {
-          where: {
-            field: "queryField",
-            column: "queryColumn",
-            value: "foo",
-          },
-        },
-        ["a"]
-      ),
-      _whereCacheCreate(
-        "field",
-        {
-          summarize: {
-            method: "mean",
-            field: "queryField",
-            column: "queryColumn",
-            values: ["foo", "bar", "baz"],
-          },
-        },
-        ["b"]
-      )
-    );
-
-    expect(
-      _whereCacheGet(wc, schema, "field", {
-        where: {
-          field: "queryField",
-          column: "queryColumn",
-          value: "foo",
-        },
-      })
-    ).toEqual(["a"]);
-
-    expect(
-      _whereCacheGet(wc, schema, "field", {
-        summarize: {
-          method: "mean",
-          field: "queryField",
-          column: "queryColumn",
-          values: ["foo", "bar", "baz"],
-        },
-      })
-    ).toEqual(["b"]);
-
-    expect(
-      _whereCacheGet(wc, schema, "field", {
-        where: {
-          field: "queryField",
-          column: "queryColumn",
-          value: "does-not-exist",
-        },
-      })
-    ).toEqual([undefined]);
-
-    expect(
-      _whereCacheGet(wc, schema, "field", {
-        summarize: {
-          method: "no-such-method",
-          field: "queryField",
-          column: "queryColumn",
-          values: ["does-not-exist"],
-        },
-      })
-    ).toEqual([undefined]);
-  });
 });

client/__tests__/util/centroid.test.js

@@ -1,11 +1,11 @@
-import cloneDeep from "lodash.clonedeep";
+import _ from "lodash";
 
 import calcCentroid from "../../src/util/centroid";
 import quantile from "../../src/util/quantile";
 import { matrixFBSToDataframe } from "../../src/util/stateManager/matrix";
 import * as REST from "./stateManager/sampleResponses";
 import { indexEntireSchema } from "../../src/util/stateManager/schemaHelpers";
-import { normalizeWritableCategoricalSchema } from "../../src/annoMatrix/normalize";
+import { _normalizeCategoricalSchema } from "../../src/annoMatrix/schema";
 
 describe("centroid", () => {
   let schema;
@@ -13,11 +13,11 @@ describe("centroid", () => {
   let obsLayout;
 
   beforeAll(() => {
-    schema = indexEntireSchema(cloneDeep(REST.schema.schema));
+    schema = indexEntireSchema(_.cloneDeep(REST.schema.schema));
     obsAnnotations = matrixFBSToDataframe(REST.annotationsObs);
     obsLayout = matrixFBSToDataframe(REST.layoutObs);
 
-    normalizeWritableCategoricalSchema(
+    _normalizeCategoricalSchema(
       schema.annotations.obsByName.field3,
       obsAnnotations.col("field3")
     );

client/__tests__/util/dataframe/dataframe.test.js

@@ -793,7 +793,9 @@ describe("dataframe factories", () => {
         [3, 3],
         [new Array(3).fill(0), new Array(3).fill(0), new Array(3).fill(0)]
       );
-      const dfB = dfA.mapColumns(() => new Array(3).fill(1));
+      const dfB = dfA.mapColumns(() => {
+        return new Array(3).fill(1);
+      });
       expect(dfA).not.toBe(dfB);
       expect(dfB.iat(0, 0)).toEqual(1);
       expect(dfB.iat(0, 1)).toEqual(1);

client/__tests__/util/dataframe/histogram.test.js

@@ -88,7 +88,16 @@ describe("Dataframe column histogram", () => {
     expect(df.col(1).histogram(5, [0, 100])).toEqual([5, 1, 0, 0, 2]);
     expect(df.col(0).histogram(2, [0, 10])).toEqual([2, 2]);
     expect(df.col(0).histogram(10, [0, 100])).toEqual([
-      3, 2, 1, 0, 0, 0, 0, 0, 0, 2,
+      3,
+      2,
+      1,
+      0,
+      0,
+      0,
+      0,
+      0,
+      0,
+      2,
     ]);
   });
 });

client/__tests__/util/quantile.test.js

@@ -19,7 +19,11 @@ describe("quantile", () => {
   test("multi q", () => {
     const arr = new Float32Array([9, 3, 5, 6, 0]);
     expect(quantile([0, 0.25, 0.5, 0.75, 1.0], arr)).toMatchObject([
-      0, 3, 5, 6, 9,
+      0,
+      3,
+      5,
+      6,
+      9,
     ]);
   });
 });

client/__tests__/util/stateManager/controlsHelpers.test.js

@@ -1,9 +1,38 @@
 /*
 test controls helpers
 */
-// TODO #2227 test: improve test coverage on control helper functions
+import { subsetAndResetGeneLists } from "../../../src/util/stateManager/controlsHelpers";
-// (`isSelectableCategoryName()`, `selectableCategoryNames()`, `createCategorySummaryFromDfCol()`, `createCategoricalSelection()`, )
+import * as globals from "../../../src/globals";
 
 describe("controls helpers", () => {
-  test("placeholder", () => {});
+  test("subsetAndResetGeneLists", () => {
+    const geneList = [];
+    const genRandGene = () => Math.random().toString(36).substring(2, 6);
+
+    // build a unique set of genes
+    for (let i = 0; i < 150; i += 1) {
+      let randGene = genRandGene();
+      while (geneList.includes(randGene)) randGene = genRandGene();
+      geneList.push(randGene);
+    }
+    // insert duplicates
+    geneList[0] = "dupl";
+    geneList[20] = "dupl";
+    const state = {
+      userDefinedGenes: geneList.slice(0, 20),
+      diffexpGenes: geneList.slice(20),
+    };
+    const [newUserDefinedGenes, newDiffExpGenes] = subsetAndResetGeneLists(
+      state
+    );
+    const expectedNewUserDefinedGenes = [
+      ...geneList.slice(0, 20),
+      ...geneList.slice(21),
+    ].slice(0, globals.maxGenes);
+    expect(globals.maxUserDefinedGenes).toBeLessThan(globals.maxGenes);
+    expect(geneList.length).toBeGreaterThan(globals.maxGenes);
+    expect(newUserDefinedGenes).toHaveLength(globals.maxGenes);
+    expect(newUserDefinedGenes).toStrictEqual(expectedNewUserDefinedGenes);
+    expect(newDiffExpGenes).toStrictEqual([]);
+  });
 });

client/__tests__/util/stateManager/sampleResponses.js

@@ -1,7 +1,3 @@
-import every from "lodash.every";
-import map from "lodash.map";
-import isNumber from "lodash.isnumber";
-import zip from "lodash.zip";
 import _ from "lodash";
 import { flatbuffers } from "flatbuffers";
 import { NetEncoding } from "../../../src/util/stateManager/matrix_generated";
@@ -125,10 +121,10 @@ function encodeMatrix(columns, colIndex = undefined) {
   */
   const utf8Encoder = new TextEncoder("utf-8");
   const builder = new flatbuffers.Builder(1024);
-  const cols = map(columns, (carr) => {
+  const cols = _.map(columns, (carr) => {
     let uType;
     let tarr;
-    if (every(carr, isNumber)) {
+    if (_.every(carr, _.isNumber)) {
       uType = NetEncoding.TypedArray.Float32Array;
       tarr = encodeTypedArray(builder, uType, new Float32Array(carr));
     } else {
@@ -171,12 +167,12 @@ function encodeMatrix(columns, colIndex = undefined) {
 }
 
 const anAnnotationsObsFBSResponse = (() => {
-  const columns = zip(...anAnnotationsObsJSONResponse.data).slice(1);
+  const columns = _.zip(...anAnnotationsObsJSONResponse.data).slice(1);
   return encodeMatrix(columns, anAnnotationsObsJSONResponse.names);
 })();
 
 const anAnnotationsVarFBSResponse = (() => {
-  const columns = zip(...anAnnotationsVarJSONResponse.data).slice(1);
+  const columns = _.zip(...anAnnotationsVarJSONResponse.data).slice(1);
   return encodeMatrix(columns, anAnnotationsVarJSONResponse.names);
 })();
 

client/__tests__/util/typedCrossfilter/bitArray.test.js

@@ -1,3 +1,5 @@
+// jshint esversion: 6
+
 import BitArray from "../../../src/util/typedCrossfilter/bitArray";
 
 const defaultTestLength = 8;

client/__tests__/util/typedCrossfilter/crossfilter.test.js

@@ -1,5 +1,4 @@
-import filter from "lodash.filter";
+import _ from "lodash";
-import zip from "lodash.zip";
 
 import Crossfilter from "../../../src/util/typedCrossfilter";
 
@@ -252,12 +251,12 @@ describe("ImmutableTypedCrossfilter", () => {
     test.each([[[]], [[2]], [[2, 1]], [[9, 82]], [[0, 1]]])("exact: %p", (v) =>
       expect(
         p.select("quantity", { mode: "exact", values: v }).countSelected()
-      ).toEqual(filter(someData, (d) => v.includes(d.quantity)).length)
+      ).toEqual(_.filter(someData, (d) => v.includes(d.quantity)).length)
     );
     test("single value exact", () => {
       expect(
         p.select("quantity", { mode: "exact", values: 2 }).countSelected()
-      ).toEqual(filter(someData, (d) => d.quantity === 2).length);
+      ).toEqual(_.filter(someData, (d) => d.quantity === 2).length);
     });
     test.each([
       [0, 1],
@@ -268,7 +267,7 @@ describe("ImmutableTypedCrossfilter", () => {
       expect(
         p.select("quantity", { mode: "range", lo, hi }).countSelected()
       ).toEqual(
-        filter(someData, (d) => d.quantity >= lo && d.quantity < hi).length
+        _.filter(someData, (d) => d.quantity >= lo && d.quantity < hi).length
       )
     );
     test("bad mode", () => {
@@ -299,12 +298,12 @@ describe("ImmutableTypedCrossfilter", () => {
     ])("exact: %p", (v) =>
       expect(
         p.select("type", { mode: "exact", values: v }).countSelected()
-      ).toEqual(filter(someData, (d) => v.includes(d.type)).length)
+      ).toEqual(_.filter(someData, (d) => v.includes(d.type)).length)
     );
     test("single value exact", () => {
       expect(
         p.select("type", { mode: "exact", values: "tab" }).countSelected()
-      ).toEqual(filter(someData, (d) => d.type === "tab").length);
+      ).toEqual(_.filter(someData, (d) => d.type === "tab").length);
     });
     test("range", () => {
       expect(() => p.select("type", { mode: "range", lo: 0, hi: 9 })).toThrow(
@@ -342,7 +341,7 @@ describe("ImmutableTypedCrossfilter", () => {
           .select("coords", { mode: "within-rect", minX, minY, maxX, maxY })
           .allSelected()
       ).toEqual(
-        filter(someData, (d) => {
+        _.filter(someData, (d) => {
           const [x, y] = d.coords;
           return minX <= x && x < maxX && minY <= y && y < maxY;
         })
@@ -398,7 +397,7 @@ describe("ImmutableTypedCrossfilter", () => {
       expect(
         p.select("coords", { mode: "within-polygon", polygon }).allSelected()
       ).toEqual(
-        zip(someData, expected)
+        _.zip(someData, expected)
           .filter((x) => x[1])
           .map((x) => x[0])
       );

client/__tests__/util/typedCrossfilter/positiveInterval.test.js

@@ -1,3 +1,5 @@
+// jshint esversion: 6
+
 // const PositiveIntervals = require("../../src/util/typedCrossfilter/positiveIntervals");
 import PositiveIntervals from "../../../src/util/typedCrossfilter/positiveIntervals";
 
@@ -151,9 +153,9 @@ describe("intersection", () => {
       [1, 2],
       [6, 9],
     ]);
-    expect(PositiveIntervals.intersection([[0, 2638]], [[1363, 2638]])).toEqual(
+    expect(
-      [[1363, 2638]]
+      PositiveIntervals.intersection([[0, 2638]], [[1363, 2638]])
-    );
+    ).toEqual([[1363, 2638]]);
     expect(PositiveIntervals.intersection([[1, 2]], [[1, 2]])).toEqual([
       [1, 2],
     ]);

client/configuration/babel/babel.dev.js

@@ -16,8 +16,6 @@ module.exports = {
     "@babel/plugin-proposal-function-bind",
     ["@babel/plugin-proposal-decorators", { legacy: true }],
     ["@babel/plugin-proposal-class-properties", { loose: true }],
-    ["@babel/plugin-proposal-private-methods", { loose: true }],
-    ["@babel/plugin-proposal-private-property-in-object", { loose: true }],
     "@babel/plugin-proposal-export-namespace-from",
     "@babel/plugin-proposal-optional-chaining",
     "@babel/plugin-proposal-nullish-coalescing-operator",

client/configuration/babel/babel.prod.js

@@ -15,8 +15,6 @@ module.exports = {
     "@babel/plugin-proposal-function-bind",
     ["@babel/plugin-proposal-decorators", { legacy: true }],
     ["@babel/plugin-proposal-class-properties", { loose: true }],
-    ["@babel/plugin-proposal-private-methods", { loose: true }],
-    ["@babel/plugin-proposal-private-property-in-object", { loose: true }],
     "@babel/plugin-proposal-export-namespace-from",
     "@babel/plugin-transform-react-constant-elements",
     "@babel/plugin-transform-runtime",

client/configuration/eslint/eslint.js

@@ -1,21 +1,24 @@
 module.exports = {
   root: true,
-
+  parser: "babel-eslint",
   extends: [
     "airbnb",
     "plugin:eslint-comments/recommended",
     "plugin:@blueprintjs/recommended",
     "plugin:compat/recommended",
-    "plugin:jsx-a11y/recommended",
+    "plugin:prettier/recommended",
-    // (thuang) disable eslint formatting rules, so prettier can do its job
+    "prettier/react",
-    // Do not use `plugin:prettier/recommended` per doc below:
-    // https://prettier.io/docs/en/integrating-with-linters.html
-    "prettier",
   ],
   settings: {
-    // AbortController is not supported in iOS Safari 10.3, Chrome 61
+    polyfills: [
-    // Headers is not supported in iOS Safari 10.3
+      "TextDecoder",
-    polyfills: ["Headers", "AbortController"],
+      "TextEncoder",
+      "fetch",
+      "Request",
+      "Response",
+      "Headers",
+      "AbortController",
+    ],
   },
   env: { browser: true, commonjs: true, es6: true },
   globals: {
@@ -28,7 +31,6 @@ module.exports = {
     context: true,
     beforeEach: true,
   },
-  parser: "@babel/eslint-parser",
   parserOptions: {
     ecmaVersion: 2017,
     sourceType: "module",
@@ -36,9 +38,6 @@ module.exports = {
       jsx: true,
       generators: true,
     },
-    babelOptions: {
-      configFile: "./configuration/babel/babel.prod.js",
-    },
   },
   rules: {
     "react/jsx-no-target-blank": "off",

client/configuration/lint-staged/lint-staged.config.js

@@ -1,4 +1,3 @@
 module.exports = {
   "*.js": "eslint --fix",
-  "**/*": "prettier --write --ignore-unknown",
 };

client/configuration/webpack/obsoleteHTMLTemplate.html

@@ -56,6 +56,13 @@
         />
         <div>Chrome &gt; 60</div>
       </a>
+      <a href="https://www.apple.com/safari/" aria-label="Download Safari">
+        <img
+          src="https://cellxgene.cziscience.com/s3/cellxgene/static/images/safari.png"
+          style="width: 80px; height: 80px;"
+        />
+        <div>Safari ≥ 10.1</div>
+      </a>
       <a href="https://www.mozilla.com/firefox/" aria-label="Download Firefox">
         <img
           src="https://cellxgene.cziscience.com/s3/cellxgene/static/images/firefox.png"

client/configuration/webpack/webpack.config.dev.js

@@ -1,6 +1,8 @@
 const path = require("path");
 const webpack = require("webpack");
 const HtmlWebpackPlugin = require("html-webpack-plugin");
+const FaviconsWebpackPlugin = require("favicons-webpack-plugin");
+const ScriptExtHtmlWebpackPlugin = require("script-ext-html-webpack-plugin");
 const MiniCssExtractPlugin = require("mini-css-extract-plugin");
 
 const { merge } = require("webpack-merge");
@@ -9,7 +11,6 @@ const sharedConfig = require("./webpack.config.shared");
 const babelOptions = require("../babel/babel.dev");
 
 const fonts = path.resolve("src/fonts");
-const images = path.resolve("src/images");
 const nodeModules = path.resolve("node_modules");
 
 const devConfig = {
@@ -29,11 +30,11 @@ const devConfig = {
       {
         test: /\.(jpg|png|gif|eot|svg|ttf|woff|woff2|otf)$/i,
         loader: "file-loader",
-        include: [nodeModules, fonts, images],
+        include: [nodeModules, fonts],
-        options: {
+        query: {
           name: "static/assets/[name].[ext]",
           // (thuang): This is needed to make sure @font url path is '/static/assets/'
-          publicPath: "..",
+          publicPath: "/",
         },
       },
     ],
@@ -43,6 +44,21 @@ const devConfig = {
       inject: true,
       template: path.resolve("index.html"),
     }),
+    new FaviconsWebpackPlugin({
+      logo: "./favicon.png",
+      prefix: "static/img/",
+      favicons: {
+        icons: {
+          android: false,
+          appleIcon: false,
+          appleStartup: false,
+          coast: false,
+          firefox: false,
+          windows: false,
+          yandex: false,
+        },
+      },
+    }),
     new MiniCssExtractPlugin({
       filename: "static/[name].css",
     }),
@@ -51,16 +67,14 @@ const devConfig = {
       __REACT_DEVTOOLS_GLOBAL_HOOK__: "({ isDisabled: true })",
     }),
     new webpack.DefinePlugin({
-      // webpack 5 no longer polyfills NodeJS modules, so fake the one we need
+      "process.env.CXG_SERVER_PORT": JSON.stringify(
-      "process.env": JSON.stringify({
+        process.env.CXG_SERVER_PORT
-        NODE_ENV: process.env.NODE_ENV || "development",
+      ),
-        CXG_SERVER_PORT: process.env.CXG_SERVER_PORT || "5005",
+    }),
-      }),
+    new ScriptExtHtmlWebpackPlugin({
+      async: "obsolete",
     }),
   ],
-  infrastructureLogging: {
-    level: "warn",
-  },
 };
 
 module.exports = merge(sharedConfig, devConfig);

client/configuration/webpack/webpack.config.prod.js

@@ -1,9 +1,10 @@
 const path = require("path");
-const webpack = require("webpack");
 const HtmlWebpackPlugin = require("html-webpack-plugin");
 const { CleanWebpackPlugin } = require("clean-webpack-plugin");
 const TerserJSPlugin = require("terser-webpack-plugin");
-const CssMinimizerPlugin = require("css-minimizer-webpack-plugin");
+const CleanCss = require("clean-css");
+const OptimizeCSSAssetsPlugin = require("optimize-css-assets-webpack-plugin");
+const FaviconsWebpackPlugin = require("favicons-webpack-plugin");
 const MiniCssExtractPlugin = require("mini-css-extract-plugin");
 
 const { merge } = require("webpack-merge");
@@ -14,7 +15,6 @@ const CspHashPlugin = require("./cspHashPlugin");
 const sharedConfig = require("./webpack.config.shared");
 
 const fonts = path.resolve("src/fonts");
-const images = path.resolve("src/images");
 const nodeModules = path.resolve("node_modules");
 
 const prodConfig = {
@@ -28,8 +28,8 @@ const prodConfig = {
     minimize: true,
     minimizer: [
       new TerserJSPlugin({}),
-      new CssMinimizerPlugin({
+      new OptimizeCSSAssetsPlugin({
-        minify: CssMinimizerPlugin.cleanCssMinify,
+        cssProcessor: CleanCss,
       }),
     ],
   },
@@ -44,11 +44,11 @@ const prodConfig = {
       {
         test: /\.(jpg|png|gif|eot|svg|ttf|woff|woff2|otf)$/i,
         loader: "file-loader",
-        include: [nodeModules, fonts, images],
+        include: [nodeModules, fonts],
-        options: {
+        query: {
           name: "static/assets/[name]-[contenthash].[ext]",
           // (thuang): This is needed to make sure @font url path is '../static/assets/'
-          publicPath: "..",
+          publicPath: "static/",
         },
       },
     ],
@@ -65,18 +65,27 @@ const prodConfig = {
       protectWebpackAssets: false,
       cleanAfterEveryBuildPatterns: ["main.js", "main.css"],
     }),
+    new FaviconsWebpackPlugin({
+      logo: "./favicon.png",
+      prefix: "static/assets/",
+      favicons: {
+        icons: {
+          android: false,
+          appleIcon: false,
+          appleStartup: false,
+          coast: false,
+          firefox: false,
+          windows: false,
+          yandex: false,
+        },
+      },
+    }),
     new MiniCssExtractPlugin({
       filename: "static/[name]-[contenthash].css",
     }),
     new CspHashPlugin({
       filename: "csp-hashes.json",
     }),
-    new webpack.DefinePlugin({
-      // webpack 5 no longer polyfills NodeJS modules, so fake the one we need
-      "process.env": JSON.stringify({
-        NODE_ENV: "production",
-      }),
-    }),
   ],
   performance: {
     maxEntrypointSize: 2000000,

client/configuration/webpack/webpack.config.shared.js

@@ -2,6 +2,7 @@ const path = require("path");
 const fs = require("fs");
 const MiniCssExtractPlugin = require("mini-css-extract-plugin");
 const ObsoleteWebpackPlugin = require("obsolete-webpack-plugin");
+const ScriptExtHtmlWebpackPlugin = require("script-ext-html-webpack-plugin");
 
 const src = path.resolve("src");
 const nodeModules = path.resolve("node_modules");
@@ -19,6 +20,7 @@ module.exports = {
   entry: [
     "core-js",
     "regenerator-runtime/runtime",
+    "fastestsmallesttextencoderdecoder",
     "whatwg-fetch",
     "abort-controller/polyfill",
     "./src/index",
@@ -39,7 +41,7 @@ module.exports = {
             loader: "css-loader",
             options: {
               modules: {
-                localIdentName: "[name]__[local]___[contenthash:base64:5]",
+                localIdentName: "[name]__[local]___[hash:base64:5]",
               },
               importLoaders: 1,
             },
@@ -65,5 +67,8 @@ module.exports = {
       template: obsoleteHTMLTemplate,
       promptOnNonTargetBrowser: false,
     }),
+    new ScriptExtHtmlWebpackPlugin({
+      async: "obsolete",
+    }),
   ],
 };

client/index.html

@@ -3,7 +3,7 @@
   <head>
     <meta charset="utf-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1" />
-    <title>CELL&times;GENE | Annotate</title>
+    <title>cell&times;gene</title>
     <style>
       html,
       body,

client/index_template.html

@@ -3,7 +3,7 @@
   <head>
     <meta charset="utf-8" />
     <meta name="viewport" content="width=device-width, initial-scale=1" />
-    <title>CELL&times;GENE | Annotate</title>
+    <title>cell&times;gene</title>
     <style>
       html,
       body,

client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cellxgene",
-  "version": "1.1.1",
+  "version": "0.16.8",
   "license": "MIT",
   "description": "cellxgene is a web application for the interactive exploration of single cell sequence data.",
   "repository": "https://github.com/chanzuckerberg/cellxgene",
@@ -10,11 +10,11 @@
     "dev": "npm run build -- configuration/webpack/webpack.config.dev.js",
     "e2e": "jest --config __tests__/e2e/e2eJestConfig.json e2e/e2e.test.js",
     "e2e-annotations": "jest --config __tests__/e2e/e2eJestConfig.json e2e/e2eAnnotations.test.js",
+    "e2e-prod": "CXG_URL_BASE='https://cellxgene.cziscience.com/d/pbmc3k.cxg/' jest --config __tests__/e2e/e2eJestConfig.json e2e/e2e.test.js",
     "fmt": "eslint --fix src __tests__",
     "lint": "eslint --fix src __tests__",
     "prod": "npm run build -- configuration/webpack/webpack.config.prod.js",
-    "test": "jest --testPathIgnorePatterns e2e",
+    "test": "jest --testPathIgnorePatterns e2e"
-    "prepare": "cd .. && husky install client/.husky"
   },
   "engineStrict": true,
   "engines": {
@@ -31,117 +31,113 @@
   },
   "browserslist": [
     "Chrome > 60",
+    "Safari >= 10.1",
     "iOS >= 10.3",
     "Firefox >= 60",
     "Edge >= 79",
-    "not Explorer > 0",
+    "not Explorer > 0"
-    "not Safari > 0"
   ],
   "dependencies": {
-    "@babel/eslint-parser": "^7.15.0",
+    "@blueprintjs/core": "^3.30.0",
-    "@blueprintjs/core": "^3.44.0",
     "@blueprintjs/icons": "^3.19.0",
-    "@blueprintjs/popover2": "^0.11.2",
+    "@blueprintjs/select": "^3.13.5",
-    "@blueprintjs/select": "^3.16.0",
     "abort-controller": "^3.0.0",
-    "core-js": "^3.16.3",
+    "core-js": "^3.6.5",
     "d3": "^4.10.0",
     "d3-scale-chromatic": "^1.5.0",
+    "fastestsmallesttextencoderdecoder": "^1.0.22",
     "flatbuffers": "^1.11.0",
     "fuzzysort": "^1.1.4",
     "gl-mat4": "^1.2.0",
     "gl-matrix": "^3.3.0",
     "gl-vec3": "^1.1.3",
     "is-number": "^7.0.0",
-    "lodash.clonedeep": "^4.5.0",
+    "lodash": "^4.17.20",
-    "lodash.difference": "^4.5.0",
-    "lodash.foreach": "^4.5.0",
-    "lodash.pull": "^4.1.0",
-    "lodash.sortby": "^4.7.0",
-    "lodash.uniq": "^4.5.0",
     "memoize-one": "^5.1.1",
-    "pako": "^2.0.3",
+    "pako": "^1.0.11",
-    "react": "^17.0.2",
+    "react": "^16.13.1",
     "react-async": "^10.0.1",
-    "react-dom": "^17.0.2",
+    "react-dom": "^16.13.1",
     "react-flip-toolkit": "^7.0.12",
-    "react-helmet": "^6.1.0",
+    "react-helmet": "^5.2.1",
-    "react-icons": "^4.2.0",
+    "react-icons": "^3.10.0",
     "react-redux": "^7.2.0",
     "redux": "^4.0.5",
     "redux-thunk": "^2.3.0",
     "regenerator-runtime": "^0.13.7",
-    "regl": "^2.1.0",
+    "regl": "^1.6.1",
-    "sha1": "^1.1.1",
+    "script-ext-html-webpack-plugin": "^2.1.4",
     "tinyqueue": "^2.0.3",
+    "webpack-merge": "^5.0.9",
     "whatwg-fetch": "^3.2.0"
   },
   "devDependencies": {
-    "@babel/core": "^7.13.16",
+    "@babel/core": "^7.10.5",
     "@babel/plugin-proposal-class-properties": "^7.10.4",
-    "@babel/plugin-proposal-decorators": "^7.13.15",
+    "@babel/plugin-proposal-decorators": "^7.10.5",
     "@babel/plugin-proposal-export-namespace-from": "^7.10.4",
     "@babel/plugin-proposal-function-bind": "^7.10.5",
     "@babel/plugin-proposal-nullish-coalescing-operator": "^7.10.4",
     "@babel/plugin-proposal-optional-chaining": "^7.10.4",
-    "@babel/plugin-transform-react-constant-elements": "^7.13.13",
+    "@babel/plugin-transform-react-constant-elements": "^7.10.4",
-    "@babel/plugin-transform-runtime": "^7.13.15",
+    "@babel/plugin-transform-runtime": "^7.10.5",
-    "@babel/preset-env": "^7.13.15",
+    "@babel/preset-env": "^7.10.4",
-    "@babel/preset-react": "^7.13.13",
+    "@babel/preset-react": "^7.10.4",
-    "@babel/register": "^7.13.16",
+    "@babel/register": "^7.10.5",
-    "@babel/runtime": "^7.13.16",
+    "@babel/runtime": "^7.10.5",
     "@blueprintjs/eslint-plugin": "^0.3.0",
-    "@sentry/webpack-plugin": "^1.15.0",
+    "@sentry/webpack-plugin": "^1.12.0",
+    "babel-eslint": "^10.1.0",
     "babel-jest": "^26.1.0",
     "babel-loader": "^8.1.0",
-    "babel-preset-modern-browsers": "^15.0.2",
+    "babel-preset-modern-browsers": "^14.2.1",
     "chalk": "^4.1.0",
-    "cheerio": "^1.0.0-rc.6",
+    "cheerio": "^1.0.0-rc.3",
-    "clean-css": "^5.1.2",
+    "clean-css": "^4.2.3",
-    "clean-webpack-plugin": "^4.0.0-alpha.0",
+    "clean-webpack-plugin": "^3.0.0",
     "codecov": "^3.7.1",
-    "css-loader": "^5.2.4",
+    "connect-history-api-fallback": "^1.6.0",
-    "css-minimizer-webpack-plugin": "^4.0.0",
+    "css-loader": "^3.6.0",
-    "eslint": "^7.24.0",
+    "eslint": "^7.4.0",
     "eslint-config-airbnb": "^18.2.0",
-    "eslint-config-prettier": "^8.2.0",
+    "eslint-config-prettier": "^6.11.0",
+    "eslint-loader": "^3.0.4",
     "eslint-plugin-compat": "^3.8.0",
     "eslint-plugin-eslint-comments": "^3.2.0",
     "eslint-plugin-filenames": "^1.3.2",
-    "eslint-plugin-import": "^2.24.2",
+    "eslint-plugin-import": "^2.22.0",
-    "eslint-plugin-jest": "^24.3.5",
+    "eslint-plugin-jest": "^23.18.0",
     "eslint-plugin-jsx-a11y": "^6.3.1",
-    "eslint-plugin-react": "^7.23.2",
+    "eslint-plugin-prettier": "^3.1.4",
+    "eslint-plugin-react": "^7.20.3",
     "eslint-plugin-react-hooks": "^4.0.8",
-    "expect-puppeteer": "^5.0.0",
+    "expect-puppeteer": "^4.4.0",
     "express": "^4.17.1",
+    "favicons-webpack-plugin": "^3.0.1",
     "file-loader": "^6.0.0",
-    "html-webpack-plugin": "^5.3.1",
+    "html-webpack-plugin": "^4.3.0",
-    "husky": "^7.0.2",
+    "husky": "^4.2.5",
-    "jest": "^27.0.6",
+    "jest": "^26.1.0",
-    "jest-circus": "^27.0.6",
+    "jest-circus": "^26.1.0",
-    "jest-environment-puppeteer": "^5.0.1",
+    "jest-environment-puppeteer": "^4.4.0",
     "jest-fetch-mock": "^3.0.3",
-    "jest-puppeteer": "^5.0.1",
+    "jest-puppeteer": "^4.4.0",
     "json-loader": "^0.5.7",
     "lint-staged": "^10.2.11",
-    "lodash": "^4.17.21",
+    "mini-css-extract-plugin": "^0.9.0",
-    "lodash.every": "^4.6.0",
-    "lodash.filter": "^4.6.0",
-    "lodash.isnumber": "^3.0.3",
-    "lodash.map": "^4.6.0",
-    "lodash.zip": "^4.2.0",
-    "mini-css-extract-plugin": "^1.5.0",
     "obsolete-webpack-plugin": "^0.5.6",
+    "optimize-css-assets-webpack-plugin": "^5.0.3",
     "prettier": "^2.0.5",
-    "puppeteer": "^8.0.0",
+    "puppeteer": "^3.3.0",
     "rimraf": "^3.0.2",
     "serve-favicon": "^2.5.0",
-    "terser-webpack-plugin": "^5.1.1",
+    "style-loader": "^1.2.1",
-    "webpack": "^5.34.0",
+    "sw-precache-webpack-plugin": "^1.0.0",
-    "webpack-cli": "^4.6.0",
+    "terser-webpack-plugin": "^3.0.7",
-    "webpack-dev-middleware": "^4.1.0",
+    "url-loader": "^4.1.0",
-    "webpack-merge": "^5.0.9"
+    "webpack": "^4.43.0",
+    "webpack-cli": "^3.3.12",
+    "webpack-dev-middleware": "^3.7.2"
   },
   "jest": {
     "testMatch": [
@@ -174,18 +170,6 @@
               "loose": true
             }
           ],
-          [
-            "@babel/plugin-proposal-private-methods",
-            {
-              "loose": true
-            }
-          ],
-          [
-            "@babel/plugin-proposal-private-property-in-object",
-            {
-              "loose": true
-            }
-          ],
           "@babel/plugin-proposal-export-namespace-from",
           "@babel/plugin-transform-react-constant-elements",
           "@babel/plugin-transform-runtime",
@@ -194,5 +178,10 @@
         ]
       }
     }
+  },
+  "husky": {
+    "hooks": {
+      "pre-commit": "lint-staged --config \"./configuration/lint-staged/lint-staged.config.js\""
+    }
   }
 }

client/server/development.js

@@ -32,6 +32,7 @@ const app = express();
 
 app.use(
   devMiddleware(compiler, {
+    logLevel: "warn",
     publicPath: config.output.publicPath,
     index: true,
   })
@@ -39,6 +40,14 @@ app.use(
 
 app.use(favicon("./favicon.png"));
 
+app.get("/login", async (req, res) => {
+  try {
+    res.redirect(`${API.prefix}login?dataset=http://localhost:${CLIENT_PORT}`);
+  } catch (err) {
+    console.error(err);
+  }
+});
+
 app.get("/logout", async (req, res) => {
   try {
     res.redirect(`${API.prefix}logout?dataset=http://localhost:${CLIENT_PORT}`);

client/server/utils.js

@@ -8,9 +8,7 @@ function isLikelyASyntaxError(message) {
   return message.indexOf(friendlySyntaxErrorLabel) !== -1;
 }
 
-function formatMessage(messageObject) {
+function formatMessage(message) {
-  let { message, details } = messageObject;
-  if (details) message = message + ": " + details;
   return message
     .replace("Module build failed: SyntaxError:", friendlySyntaxErrorLabel)
     .replace(

client/src/actions/annotation.js

@@ -1,260 +1,288 @@
 /*
 Action creators for user annotation
 */
-import difference from "lodash.difference";
+import _ from "lodash";
 import pako from "pako";
 import * as globals from "../globals";
 import { MatrixFBS, AnnotationsHelpers } from "../util/stateManager";
 
 const { isUserAnnotation } = AnnotationsHelpers;
 
-export const annotationCreateCategoryAction =
+export const annotationCreateCategoryAction = (
-  (newCategoryName, categoryToDuplicate) => async (dispatch, getState) => {
+  newCategoryName,
-    /*
+  categoryToDuplicate
+) => async (dispatch, getState) => {
+  /*
   Add a new user-created category to the obs annotations.
 
   Arguments:
     newCategoryName - string name for the category.
     categoryToDuplicate - obs category to use for initial values, or null.
   */
-    const { annoMatrix: prevAnnoMatrix, obsCrossfilter: prevObsCrossfilter } =
+  const {
-      getState();
+    annoMatrix: prevAnnoMatrix,
-    if (!prevAnnoMatrix || !prevObsCrossfilter) return;
+    obsCrossfilter: prevObsCrossfilter,
-    const { schema } = prevAnnoMatrix;
+  } = getState();
+  if (!prevAnnoMatrix || !prevObsCrossfilter) return;
+  const { schema } = prevAnnoMatrix;
 
-    /* name must be a string,  non-zero length */
+  /* name must be a string,  non-zero length */
-    if (typeof newCategoryName !== "string" || newCategoryName.length === 0)
+  if (typeof newCategoryName !== "string" || newCategoryName.length === 0)
-      throw new Error("user annotations require string name");
+    throw new Error("user annotations require string name");
 
-    /* ensure the name isn't already in use! */
+  /* ensure the name isn't already in use! */
-    if (schema.annotations.obsByName[newCategoryName])
+  if (schema.annotations.obsByName[newCategoryName])
-      throw new Error("name collision on annotation category create");
+    throw new Error("name collision on annotation category create");
 
-    let initialValue;
+  let initialValue;
-    let newSchema;
+  let newSchema;
-    let ctor;
+  let ctor;
-    if (categoryToDuplicate) {
+  if (categoryToDuplicate) {
-      /* if we are duplicating a category, retrieve it */
+    /* if we are duplicating a category, retrieve it */
-      const catDupSchema = schema.annotations.obsByName[categoryToDuplicate];
+    const catDupSchema = schema.annotations.obsByName[categoryToDuplicate];
-      const catDupType = catDupSchema?.type;
+    const catDupType = catDupSchema?.type;
-      if (catDupType !== "string" && catDupType !== "categorical")
+    if (catDupType !== "string" && catDupType !== "categorical")
-        throw new Error(
+      throw new Error("categoryToDuplicate does not exist or has invalid type");
-          "categoryToDuplicate does not exist or has invalid type"
-        );
 
-      const catToDupDf = await prevAnnoMatrix
+    const catToDupDf = await prevAnnoMatrix
-        .base()
+      .base()
-        .fetch("obs", categoryToDuplicate);
+      .fetch("obs", categoryToDuplicate);
-      const col = catToDupDf.col(categoryToDuplicate);
+    const col = catToDupDf.col(categoryToDuplicate);
-      initialValue = col.asArray();
+    initialValue = col.asArray();
-      const { categories } = col.summarizeCategorical();
+    const { categories } = col.summarizeCategorical();
-      // all user-created annotations must have the unassigned category
+    // all user-created annotations must have the unassigned category
-      if (!categories.includes(globals.unassignedCategoryLabel)) {
+    if (!categories.includes(globals.unassignedCategoryLabel)) {
-        categories.push(globals.unassignedCategoryLabel);
+      categories.push(globals.unassignedCategoryLabel);
-      }
-      ctor = initialValue.constructor;
-      newSchema = {
-        ...catDupSchema,
-        name: newCategoryName,
-        categories,
-        writable: true,
-      };
-    } else {
-      /* else assign to the standard default value */
-      initialValue = globals.unassignedCategoryLabel;
-      ctor = Array;
-      newSchema = {
-        name: newCategoryName,
-        type: "categorical",
-        categories: [globals.unassignedCategoryLabel],
-        writable: true,
-      };
     }
+    ctor = initialValue.constructor;
+    newSchema = {
+      ...catDupSchema,
+      name: newCategoryName,
+      categories,
+      writable: true,
+    };
+  } else {
+    /* else assign to the standard default value */
+    initialValue = globals.unassignedCategoryLabel;
+    ctor = Array;
+    newSchema = {
+      name: newCategoryName,
+      type: "categorical",
+      categories: [globals.unassignedCategoryLabel],
+      writable: true,
+    };
+  }
 
-    const obsCrossfilter = prevObsCrossfilter.addObsColumn(
+  const obsCrossfilter = prevObsCrossfilter.addObsColumn(
-      newSchema,
+    newSchema,
-      ctor,
+    ctor,
-      initialValue
+    initialValue
-    );
+  );
 
-    dispatch({
+  dispatch({
-      type: "annotation: create category",
+    type: "annotation: create category",
-      data: newCategoryName,
+    data: newCategoryName,
-      categoryToDuplicate,
+    categoryToDuplicate,
-      annoMatrix: obsCrossfilter.annoMatrix,
+    annoMatrix: obsCrossfilter.annoMatrix,
-      obsCrossfilter,
+    obsCrossfilter,
-    });
+  });
-  };
+};
 
-export const annotationRenameCategoryAction =
+export const annotationRenameCategoryAction = (
-  (oldCategoryName, newCategoryName) => (dispatch, getState) => {
+  oldCategoryName,
-    /*
+  newCategoryName
+) => (dispatch, getState) => {
+  /*
   Rename a user-created annotation category
   */
-    const { annoMatrix: prevAnnoMatrix, obsCrossfilter: prevObsCrossfilter } =
+  const {
-      getState();
+    annoMatrix: prevAnnoMatrix,
-    if (!prevAnnoMatrix || !prevObsCrossfilter) return;
+    obsCrossfilter: prevObsCrossfilter,
-    if (!isUserAnnotation(prevAnnoMatrix, oldCategoryName))
+  } = getState();
-      throw new Error("not a user annotation");
+  if (!prevAnnoMatrix || !prevObsCrossfilter) return;
+  if (!isUserAnnotation(prevAnnoMatrix, oldCategoryName))
+    throw new Error("not a user annotation");
 
-    /* name must be a string,  non-zero length */
+  /* name must be a string,  non-zero length */
-    if (typeof newCategoryName !== "string" || newCategoryName.length === 0)
+  if (typeof newCategoryName !== "string" || newCategoryName.length === 0)
-      throw new Error("user annotations require string name");
+    throw new Error("user annotations require string name");
 
-    if (oldCategoryName === newCategoryName) return;
+  if (oldCategoryName === newCategoryName) return;
 
-    const obsCrossfilter = prevObsCrossfilter.renameObsColumn(
+  const obsCrossfilter = prevObsCrossfilter.renameObsColumn(
-      oldCategoryName,
+    oldCategoryName,
-      newCategoryName
+    newCategoryName
-    );
+  );
 
-    dispatch({
+  dispatch({
-      type: "annotation: category edited",
+    type: "annotation: category edited",
-      annoMatrix: obsCrossfilter.annoMatrix,
+    annoMatrix: obsCrossfilter.annoMatrix,
-      obsCrossfilter,
+    obsCrossfilter,
-      metadataField: oldCategoryName,
+    metadataField: oldCategoryName,
-      newCategoryText: newCategoryName,
+    newCategoryText: newCategoryName,
-      data: newCategoryName,
+    data: newCategoryName,
-    });
+  });
-  };
+};
 
-export const annotationDeleteCategoryAction =
+export const annotationDeleteCategoryAction = (categoryName) => (
-  (categoryName) => (dispatch, getState) => {
+  dispatch,
-    /*
+  getState
+) => {
+  /*
   Delete a user-created category
   */
-    const { annoMatrix: prevAnnoMatrix, obsCrossfilter: prevObsCrossfilter } =
+  const {
-      getState();
+    annoMatrix: prevAnnoMatrix,
-    if (!prevAnnoMatrix || !prevObsCrossfilter) return;
+    obsCrossfilter: prevObsCrossfilter,
-    if (!isUserAnnotation(prevAnnoMatrix, categoryName))
+  } = getState();
-      throw new Error("not a user annotation");
+  if (!prevAnnoMatrix || !prevObsCrossfilter) return;
+  if (!isUserAnnotation(prevAnnoMatrix, categoryName))
+    throw new Error("not a user annotation");
 
-    const obsCrossfilter = prevObsCrossfilter.dropObsColumn(categoryName);
+  const obsCrossfilter = prevObsCrossfilter.dropObsColumn(categoryName);
-    dispatch({
+  dispatch({
-      type: "annotation: delete category",
+    type: "annotation: delete category",
-      annoMatrix: obsCrossfilter.annoMatrix,
+    annoMatrix: obsCrossfilter.annoMatrix,
-      obsCrossfilter,
+    obsCrossfilter,
-      metadataField: categoryName,
+    metadataField: categoryName,
-    });
+  });
-  };
+};
 
-export const annotationCreateLabelInCategory =
+export const annotationCreateLabelInCategory = (
-  (categoryName, labelName, assignSelected) => async (dispatch, getState) => {
+  categoryName,
-    /*
+  labelName,
+  assignSelected
+) => async (dispatch, getState) => {
+  /*
   Add a new label to a user-defined category.  If assignSelected is true, assign
   the label to all currently selected cells.
   */
-    const { annoMatrix: prevAnnoMatrix, obsCrossfilter: prevObsCrossfilter } =
+  const {
-      getState();
+    annoMatrix: prevAnnoMatrix,
-    if (!prevAnnoMatrix || !prevObsCrossfilter) return;
+    obsCrossfilter: prevObsCrossfilter,
-    if (!isUserAnnotation(prevAnnoMatrix, categoryName))
+  } = getState();
-      throw new Error("not a user annotation");
+  if (!prevAnnoMatrix || !prevObsCrossfilter) return;
+  if (!isUserAnnotation(prevAnnoMatrix, categoryName))
+    throw new Error("not a user annotation");
 
-    let obsCrossfilter = prevObsCrossfilter.addObsAnnoCategory(
+  let obsCrossfilter = prevObsCrossfilter.addObsAnnoCategory(
-      categoryName,
+    categoryName,
-      labelName
+    labelName
-    );
+  );
-    if (assignSelected) {
+  if (assignSelected) {
-      obsCrossfilter = await obsCrossfilter.setObsColumnValues(
+    obsCrossfilter = await obsCrossfilter.setObsColumnValues(
-        categoryName,
-        prevObsCrossfilter.allSelectedLabels(),
-        labelName
-      );
-    }
-
-    dispatch({
-      type: "annotation: add new label to category",
-      annoMatrix: obsCrossfilter.annoMatrix,
-      obsCrossfilter,
-      metadataField: categoryName,
-      newLabelText: labelName,
-      assignSelectedCells: assignSelected,
-    });
-  };
-
-export const annotationDeleteLabelFromCategory =
-  (categoryName, labelName) => async (dispatch, getState) => {
-    /*
-  delete a label from a user-defined category
-  */
-    const { annoMatrix: prevAnnoMatrix, obsCrossfilter: prevObsCrossfilter } =
-      getState();
-    if (!prevAnnoMatrix || !prevObsCrossfilter) return;
-    if (!isUserAnnotation(prevAnnoMatrix, categoryName))
-      throw new Error("not a user annotation");
-
-    const obsCrossfilter = await prevObsCrossfilter.removeObsAnnoCategory(
-      categoryName,
-      labelName,
-      globals.unassignedCategoryLabel
-    );
-
-    dispatch({
-      type: "annotation: delete label",
-      metadataField: categoryName,
-      label: labelName,
-      annoMatrix: obsCrossfilter.annoMatrix,
-      obsCrossfilter,
-    });
-  };
-
-export const annotationRenameLabelInCategory =
-  (categoryName, oldLabelName, newLabelName) => async (dispatch, getState) => {
-    /*
-  label name change
-  */
-    const { annoMatrix: prevAnnoMatrix, obsCrossfilter: prevObsCrossfilter } =
-      getState();
-    if (!prevAnnoMatrix || !prevObsCrossfilter) return;
-    if (!isUserAnnotation(prevAnnoMatrix, categoryName))
-      throw new Error("not a user annotation");
-
-    let obsCrossfilter = await prevObsCrossfilter.resetObsColumnValues(
-      categoryName,
-      oldLabelName,
-      newLabelName
-    );
-    obsCrossfilter = await obsCrossfilter.removeObsAnnoCategory(
-      categoryName,
-      oldLabelName,
-      globals.unassignedCategoryLabel
-    );
-
-    dispatch({
-      type: "annotation: label edited",
-      editedLabel: newLabelName,
-      metadataField: categoryName,
-      label: oldLabelName,
-      annoMatrix: obsCrossfilter.annoMatrix,
-      obsCrossfilter,
-    });
-  };
-
-export const annotationLabelCurrentSelection =
-  (categoryName, labelName) => async (dispatch, getState) => {
-    /*
-  set the label on all currently selected
-  */
-    const { annoMatrix: prevAnnoMatrix, obsCrossfilter: prevObsCrossfilter } =
-      getState();
-    if (!prevAnnoMatrix || !prevObsCrossfilter) return;
-    if (!isUserAnnotation(prevAnnoMatrix, categoryName))
-      throw new Error("not a user annotation");
-
-    const obsCrossfilter = await prevObsCrossfilter.setObsColumnValues(
       categoryName,
       prevObsCrossfilter.allSelectedLabels(),
       labelName
     );
+  }
 
-    dispatch({
+  dispatch({
-      type: "annotation: label current cell selection",
+    type: "annotation: add new label to category",
-      metadataField: categoryName,
+    annoMatrix: obsCrossfilter.annoMatrix,
-      label: labelName,
+    obsCrossfilter,
-      obsCrossfilter,
+    metadataField: categoryName,
-      annoMatrix: obsCrossfilter.annoMatrix,
+    newLabelText: labelName,
-    });
+    assignSelectedCells: assignSelected,
-  };
+  });
+};
+
+export const annotationDeleteLabelFromCategory = (
+  categoryName,
+  labelName
+) => async (dispatch, getState) => {
+  /*
+  delete a label from a user-defined category
+  */
+  const {
+    annoMatrix: prevAnnoMatrix,
+    obsCrossfilter: prevObsCrossfilter,
+  } = getState();
+  if (!prevAnnoMatrix || !prevObsCrossfilter) return;
+  if (!isUserAnnotation(prevAnnoMatrix, categoryName))
+    throw new Error("not a user annotation");
+
+  const obsCrossfilter = await prevObsCrossfilter.removeObsAnnoCategory(
+    categoryName,
+    labelName,
+    globals.unassignedCategoryLabel
+  );
+
+  dispatch({
+    type: "annotation: delete label",
+    metadataField: categoryName,
+    label: labelName,
+    annoMatrix: obsCrossfilter.annoMatrix,
+    obsCrossfilter,
+  });
+};
+
+export const annotationRenameLabelInCategory = (
+  categoryName,
+  oldLabelName,
+  newLabelName
+) => async (dispatch, getState) => {
+  /*
+  label name change
+  */
+  const {
+    annoMatrix: prevAnnoMatrix,
+    obsCrossfilter: prevObsCrossfilter,
+  } = getState();
+  if (!prevAnnoMatrix || !prevObsCrossfilter) return;
+  if (!isUserAnnotation(prevAnnoMatrix, categoryName))
+    throw new Error("not a user annotation");
+
+  let obsCrossfilter = await prevObsCrossfilter.resetObsColumnValues(
+    categoryName,
+    oldLabelName,
+    newLabelName
+  );
+  obsCrossfilter = await obsCrossfilter.removeObsAnnoCategory(
+    categoryName,
+    oldLabelName,
+    globals.unassignedCategoryLabel
+  );
+
+  dispatch({
+    type: "annotation: label edited",
+    editedLabel: newLabelName,
+    metadataField: categoryName,
+    label: oldLabelName,
+    annoMatrix: obsCrossfilter.annoMatrix,
+    obsCrossfilter,
+  });
+};
+
+export const annotationLabelCurrentSelection = (
+  categoryName,
+  labelName
+) => async (dispatch, getState) => {
+  /*
+  set the label on all currently selected
+  */
+  const {
+    annoMatrix: prevAnnoMatrix,
+    obsCrossfilter: prevObsCrossfilter,
+  } = getState();
+  if (!prevAnnoMatrix || !prevObsCrossfilter) return;
+  if (!isUserAnnotation(prevAnnoMatrix, categoryName))
+    throw new Error("not a user annotation");
+
+  const obsCrossfilter = await prevObsCrossfilter.setObsColumnValues(
+    categoryName,
+    prevObsCrossfilter.allSelectedLabels(),
+    labelName
+  );
+
+  dispatch({
+    type: "annotation: label current cell selection",
+    metadataField: categoryName,
+    label: labelName,
+    obsCrossfilter,
+    annoMatrix: obsCrossfilter.annoMatrix,
+  });
+};
 
 function writableAnnotations(annoMatrix) {
   return annoMatrix.schema.annotations.obs.columns
@@ -280,7 +308,7 @@ export const needToSaveObsAnnotations = (annoMatrix, lastSavedAnnoMatrix) => {
 
   // if the schema has changed, we need to save
   const currentWritable = writableAnnotations(annoMatrix);
-  if (difference(currentWritable, writableAnnotations(lastSavedAnnoMatrix))) {
+  if (_.difference(currentWritable, writableAnnotations(lastSavedAnnoMatrix))) {
     return true;
   }
 
@@ -359,97 +387,3 @@ export const saveObsAnnotationsAction = () => async (dispatch, getState) => {
     });
   }
 };
-
-export const saveGenesetsAction = () => async (dispatch, getState) => {
-  const state = getState();
-
-  // bail if gene sets not available, or in readonly mode.
-  const { config } = state;
-  const { lastTid, genesets } = state.genesets;
-
-  const genesetsAreAvailable =
-    config?.parameters?.annotations_genesets ?? false;
-  const genesetsReadonly =
-    config?.parameters?.annotations_genesets_readonly ?? true;
-  if (!genesetsAreAvailable || genesetsReadonly) {
-    // our non-save was completed!
-    return dispatch({
-      type: "autosave: genesets complete",
-      lastSavedGenesets: genesets,
-    });
-  }
-
-  dispatch({
-    type: "autosave: genesets started",
-  });
-
-  /* Create the JSON OTA data structure */
-  const tid = (lastTid ?? 0) + 1;
-  const genesetsOTA = [];
-  for (const [name, gs] of genesets) {
-    const genes = [];
-    for (const g of gs.genes.values()) {
-      genes.push({
-        gene_symbol: g.geneSymbol,
-        gene_description: g.geneDescription,
-      });
-    }
-    genesetsOTA.push({
-      geneset_name: name,
-      geneset_description: gs.genesetDescription,
-      genes,
-    });
-  }
-  const ota = {
-    tid,
-    genesets: genesetsOTA,
-  };
-
-  /* Save to server */
-  try {
-    const { dataCollectionNameIsReadOnly, dataCollectionName } =
-      state.annotations;
-    const queryString =
-      !dataCollectionNameIsReadOnly && !!dataCollectionName
-        ? `?annotation-collection-name=${encodeURIComponent(
-            dataCollectionName
-          )}`
-        : "";
-
-    const res = await fetch(
-      `${globals.API.prefix}${globals.API.version}genesets${queryString}`,
-      {
-        method: "PUT",
-        headers: new Headers({
-          Accept: "application/json",
-          "Content-Type": "application/json",
-        }),
-        body: JSON.stringify(ota),
-        credentials: "include",
-      }
-    );
-    if (!res.ok) {
-      return dispatch({
-        type: "autosave: genesets error",
-        message: `HTTP error ${res.status} - ${res.statusText}`,
-        res,
-      });
-    }
-    return Promise.all([
-      dispatch({
-        type: "autosave: genesets complete",
-        lastSavedGenesets: genesets,
-      }),
-      dispatch({
-        type: "geneset: set tid",
-        tid,
-      }),
-    ]);
-  } catch (error) {
-    return dispatch({
-      type: "autosave: genesets error",
-      message: error.toString(),
-      error,
-    });
-  }
-};

client/src/actions/embedding.js

@@ -11,7 +11,7 @@ export async function _switchEmbedding(
   newEmbeddingName
 ) {
   /*
-  DRY helper used by embedding action creators
+  DRY helper used by this and reembedding action creators
   */
   const base = prevAnnoMatrix.base();
   const embeddingDf = await base.fetch("emb", newEmbeddingName);
@@ -25,23 +25,27 @@ export async function _switchEmbedding(
   return [annoMatrix, obsCrossfilter];
 }
 
-export const layoutChoiceAction =
+export const layoutChoiceAction = (newLayoutChoice) => async (
-  (newLayoutChoice) => async (dispatch, getState) => {
+  dispatch,
-    /*
+  getState
+) => {
+  /*
   On layout choice, make sure we have selected all on the previous layout, AND the new
   layout.
   */
-    const { annoMatrix: prevAnnoMatrix, obsCrossfilter: prevCrossfilter } =
+  const {
-      getState();
+    annoMatrix: prevAnnoMatrix,
-    const [annoMatrix, obsCrossfilter] = await _switchEmbedding(
+    obsCrossfilter: prevCrossfilter,
-      prevAnnoMatrix,
+  } = getState();
-      prevCrossfilter,
+  const [annoMatrix, obsCrossfilter] = await _switchEmbedding(
-      newLayoutChoice
+    prevAnnoMatrix,
-    );
+    prevCrossfilter,
-    dispatch({
+    newLayoutChoice
-      type: "set layout choice",
+  );
-      layoutChoice: newLayoutChoice,
+  dispatch({
-      obsCrossfilter,
+    type: "set layout choice",
-      annoMatrix,
+    layoutChoice: newLayoutChoice,
-    });
+    obsCrossfilter,
-  };
+    annoMatrix,
+  });
+};

client/src/actions/geneset.js

@@ -1,149 +0,0 @@
-import { postUserErrorToast } from "../components/framework/toasters";
-/*
-Action creators for gene sets
-
-Primarily used to keep the crossfilter and underlying data in sync with the UI.
-
-The behavior manifest in these action creators:
-
-    Delete a gene set, will
-      * drop index & clear selection state on the gene set summary
-      * drop index & clear selection state of each gene in the geneset
-
-    Delete a gene from a gene set, will:
-      * drop index & clear selection state on the gene set summary
-      * drop index & clear selection state on the gene
-
-    Add a gene to a gene set, will:
-      * drop index & clear selection state on the gene set summary
-      * will NOT touch the selection state for the gene
-
-Note that crossfilter indices are lazy created, as needed.
-*/
-
-export const genesetDelete = (genesetName) => (dispatch, getState) => {
-  const state = getState();
-  const { genesets } = state;
-  const gs = genesets?.genesets?.get(genesetName) ?? {};
-  const geneSymbols = Array.from(gs.genes.keys());
-  const obsCrossfilter = dropGeneset(dispatch, state, genesetName, geneSymbols);
-  if (genesetName === state.colors.colorAccessor) {
-    dispatch({
-      type: "reset colorscale",
-    });
-  }
-  dispatch({
-    type: "geneset: delete",
-    genesetName,
-    obsCrossfilter,
-    annoMatrix: obsCrossfilter.annoMatrix,
-  });
-};
-
-export const genesetAddGenes =
-  (genesetName, genes) => async (dispatch, getState) => {
-    const state = getState();
-    const { obsCrossfilter: prevObsCrossfilter, annoMatrix } = state;
-    const { schema } = annoMatrix;
-    const varIndex = schema.annotations.var.index;
-    const df = await annoMatrix.fetch("var", varIndex);
-    const geneNames = df.col(varIndex).asArray();
-    genes = genes.reduce((acc, gene) => {
-      if (geneNames.indexOf(gene.geneSymbol) === -1) {
-        postUserErrorToast(
-          `${gene.geneSymbol} doesn't appear to be a valid gene name.`
-        );
-      } else acc.push(gene);
-      return acc;
-    }, []);
-
-    const obsCrossfilter = dropGenesetSummaryDimension(
-      prevObsCrossfilter,
-      state,
-      genesetName
-    );
-    dispatch({
-      type: "continuous metadata histogram cancel",
-      continuousNamespace: { isGeneSetSummary: true },
-      selection: genesetName,
-    });
-    return dispatch({
-      type: "geneset: add genes",
-      genesetName,
-      genes,
-      obsCrossfilter,
-      annoMatrix: obsCrossfilter.annoMatrix,
-    });
-  };
-
-export const genesetDeleteGenes =
-  (genesetName, geneSymbols) => (dispatch, getState) => {
-    const state = getState();
-    const obsCrossfilter = dropGeneset(
-      dispatch,
-      state,
-      genesetName,
-      geneSymbols
-    );
-    return dispatch({
-      type: "geneset: delete genes",
-      genesetName,
-      geneSymbols,
-      obsCrossfilter,
-      annoMatrix: obsCrossfilter.annoMatrix,
-    });
-  };
-
-/*
-Private
-*/
-
-function dropGenesetSummaryDimension(obsCrossfilter, state, genesetName) {
-  const { annoMatrix, genesets } = state;
-  const varIndex = annoMatrix.schema.annotations?.var?.index;
-  const gs = genesets?.genesets?.get(genesetName) ?? {};
-  const genes = Array.from(gs.genes.keys());
-  const query = {
-    summarize: {
-      method: "mean",
-      field: "var",
-      column: varIndex,
-      values: genes,
-    },
-  };
-  return obsCrossfilter.dropDimension("X", query);
-}
-
-function dropGeneDimension(obsCrossfilter, state, gene) {
-  const { annoMatrix } = state;
-  const varIndex = annoMatrix.schema.annotations?.var?.index;
-  const query = {
-    where: {
-      field: "var",
-      column: varIndex,
-      value: gene,
-    },
-  };
-  return obsCrossfilter.dropDimension("X", query);
-}
-
-function dropGeneset(dispatch, state, genesetName, geneSymbols) {
-  const { obsCrossfilter: prevObsCrossfilter } = state;
-  const obsCrossfilter = geneSymbols.reduce(
-    (crossfilter, gene) => dropGeneDimension(crossfilter, state, gene),
-    dropGenesetSummaryDimension(prevObsCrossfilter, state, genesetName)
-  );
-  dispatch({
-    type: "continuous metadata histogram cancel",
-    continuousNamespace: { isGeneSetSummary: true },
-    selection: genesetName,
-  });
-  geneSymbols.forEach((g) =>
-    dispatch({
-      type: "continuous metadata histogram cancel",
-      continuousNamespace: { isUserDefined: true },
-      selection: g,
-    })
-  );
-  return obsCrossfilter;
-}

client/src/actions/index.js

@@ -5,22 +5,14 @@ import {
   doJsonRequest,
   dispatchNetworkErrorMessageToUser,
 } from "../util/actionHelpers";
+import {
+  requestReembed /* , reembedResetWorldToUniverse -- disabled temporarily, TODO issue #1606 */,
+} from "./reembed";
 import { loadUserColorConfig } from "../util/stateManager/colorHelpers";
 import * as selnActions from "./selection";
 import * as annoActions from "./annotation";
 import * as viewActions from "./viewStack";
 import * as embActions from "./embedding";
-import * as genesetActions from "./geneset";
-
-function setGlobalConfig(config) {
-  /**
-   * Set any global run-time config not _exclusively_ managed by the config reducer.
-   * This should only set fields defined in globals.globalConfig.
-   */
-  globals.globalConfig.maxCategoricalOptionsToDisplay =
-    config?.parameters?.["max-category-items"] ??
-    globals.globalConfig.maxCategoricalOptionsToDisplay;
-}
 
 /*
 return promise fetching user-configured colors
@@ -41,9 +33,6 @@ async function schemaFetch() {
 async function configFetch(dispatch) {
   return fetchJson("config").then((response) => {
     const config = { ...globals.configDefaults, ...response.config };
-
-    setGlobalConfig(config);
-
     dispatch({
       type: "configuration load complete",
       config,
@@ -52,25 +41,15 @@ async function configFetch(dispatch) {
   });
 }
 
-async function genesetsFetch(dispatch, config) {
+async function userInfoFetch(dispatch) {
-  /* request genesets ONLY if the backend supports the feature */
+  return fetchJson("userinfo").then((response) => {
-  const defaultResponse = {
+    const { userinfo: userInfo } = response || {};
-    genesets: [],
-    tid: 0,
-  };
-  if (config?.parameters?.annotations_genesets ?? false) {
-    fetchJson("genesets").then((response) => {
-      dispatch({
-        type: "geneset: initial load",
-        data: response ?? defaultResponse,
-      });
-    });
-  } else {
     dispatch({
-      type: "geneset: initial load",
+      type: "userInfo load complete",
-      data: defaultResponse,
+      userInfo,
     });
-  }
+    return userInfo;
+  });
 }
 
 function prefetchEmbeddings(annoMatrix) {
@@ -94,10 +73,9 @@ const doInitialDataLoad = () =>
         configFetch(dispatch),
         schemaFetch(dispatch),
         userColorsFetchAndLoad(dispatch),
+        userInfoFetch(dispatch),
       ]);
 
-      genesetsFetch(dispatch, config);
-
       const baseDataUrl = `${globals.API.prefix}${globals.API.version}`;
       const annoMatrix = new AnnoMatrixLoader(baseDataUrl, schema.schema);
       const obsCrossfilter = new AnnoMatrixObsCrossfilter(annoMatrix);
@@ -110,7 +88,7 @@ const doInitialDataLoad = () =>
       });
       dispatch({ type: "initial data load complete" });
 
-      const defaultEmbedding = config?.parameters?.default_embedding;
+      const defaultEmbedding = config?.parameters?.["default_embedding"];
       const layoutSchema = schema?.schema?.layout?.obs ?? [];
       if (
         defaultEmbedding &&
@@ -160,74 +138,72 @@ const dispatchDiffExpErrors = (dispatch, response) => {
   }
 };
 
-const requestDifferentialExpression =
+const requestDifferentialExpression = (set1, set2, num_genes = 10) => async (
-  (set1, set2, num_genes = 50) =>
+  dispatch,
-  async (dispatch, getState) => {
+  getState
-    dispatch({ type: "request differential expression started" });
+) => {
-    try {
+  dispatch({ type: "request differential expression started" });
-      /*
+  try {
+    /*
     Steps:
     1. get the most differentially expressed genes
     2. get expression data for each
     */
-      const { annoMatrix } = getState();
+    const { annoMatrix } = getState();
-      const varIndexName = annoMatrix.schema.annotations.var.index;
+    const varIndexName = annoMatrix.schema.annotations.var.index;
 
-      // Legal values are null, Array or TypedArray.  Null is initial state.
+    // Legal values are null, Array or TypedArray.  Null is initial state.
-      if (!set1) set1 = [];
+    if (!set1) set1 = [];
-      if (!set2) set2 = [];
+    if (!set2) set2 = [];
 
-      // These lines ensure that we convert any TypedArray to an Array.
+    // These lines ensure that we convert any TypedArray to an Array.
-      // This is necessary because JSON.stringify() does some very strange
+    // This is necessary because JSON.stringify() does some very strange
-      // things with TypedArrays (they are marshalled to JSON objects, rather
+    // things with TypedArrays (they are marshalled to JSON objects, rather
-      // than being marshalled as a JSON array).
+    // than being marshalled as a JSON array).
-      set1 = Array.isArray(set1) ? set1 : Array.from(set1);
+    set1 = Array.isArray(set1) ? set1 : Array.from(set1);
-      set2 = Array.isArray(set2) ? set2 : Array.from(set2);
+    set2 = Array.isArray(set2) ? set2 : Array.from(set2);
 
-      const res = await fetch(
+    const res = await fetch(
-        `${globals.API.prefix}${globals.API.version}diffexp/obs`,
+      `${globals.API.prefix}${globals.API.version}diffexp/obs`,
-        {
+      {
-          method: "POST",
+        method: "POST",
-          headers: new Headers({
+        headers: new Headers({
-            Accept: "application/json",
+          Accept: "application/json",
-            "Content-Type": "application/json",
+          "Content-Type": "application/json",
-          }),
+        }),
-          body: JSON.stringify({
+        body: JSON.stringify({
-            mode: "topN",
+          mode: "topN",
-            count: num_genes,
+          count: num_genes,
-            set1: { filter: { obs: { index: set1 } } },
+          set1: { filter: { obs: { index: set1 } } },
-            set2: { filter: { obs: { index: set2 } } },
+          set2: { filter: { obs: { index: set2 } } },
-          }),
+        }),
-          credentials: "include",
+        credentials: "include",
-        }
-      );
-
-      if (!res.ok || res.headers.get("Content-Type") !== "application/json") {
-        return dispatchDiffExpErrors(dispatch, res);
       }
+    );
 
-      const response = await res.json();
+    if (!res.ok || res.headers.get("Content-Type") !== "application/json") {
-      const varIndex = await annoMatrix.fetch("var", varIndexName);
+      return dispatchDiffExpErrors(dispatch, res);
-      const diffexpLists = { negative: [], positive: [] };
-      for (const polarity of Object.keys(diffexpLists)) {
-        diffexpLists[polarity] = response[polarity].map((v) => [
-          varIndex.at(v[0], varIndexName),
-          ...v.slice(1),
-        ]);
-      }
-
-      /* then send the success case action through */
-      return dispatch({
-        type: "request differential expression success",
-        data: diffexpLists,
-      });
-    } catch (error) {
-      return dispatch({
-        type: "request differential expression error",
-        error,
-      });
     }
-  };
+
+    const response = await res.json();
+    const varIndex = await annoMatrix.fetch("var", varIndexName);
+    const data = response.map((v) => [
+      varIndex.at(v[0], varIndexName),
+      ...v.slice(1),
+    ]);
+
+    /* then send the success case action through */
+    return dispatch({
+      type: "request differential expression success",
+      data,
+    });
+  } catch (error) {
+    return dispatch({
+      type: "request differential expression error",
+      error,
+    });
+  }
+};
 
 function fetchJson(pathAndQuery) {
   return doJsonRequest(
@@ -240,6 +216,7 @@ export default {
   requestDifferentialExpression,
   requestSingleGeneExpressionCountsForColoringPOST,
   requestUserDefinedGene,
+  requestReembed,
   selectContinuousMetadataAction: selnActions.selectContinuousMetadataAction,
   selectCategoricalMetadataAction: selnActions.selectCategoricalMetadataAction,
   selectCategoricalAllMetadataAction:
@@ -265,11 +242,7 @@ export default {
   annotationRenameLabelInCategory: annoActions.annotationRenameLabelInCategory,
   annotationLabelCurrentSelection: annoActions.annotationLabelCurrentSelection,
   saveObsAnnotationsAction: annoActions.saveObsAnnotationsAction,
-  saveGenesetsAction: annoActions.saveGenesetsAction,
   needToSaveObsAnnotations: annoActions.needToSaveObsAnnotations,
   layoutChoiceAction: embActions.layoutChoiceAction,
   setCellSetFromSelection: selnActions.setCellSetFromSelection,
-  genesetDelete: genesetActions.genesetDelete,
-  genesetAddGenes: genesetActions.genesetAddGenes,
-  genesetDeleteGenes: genesetActions.genesetDeleteGenes,
 };

client/src/actions/reembed.js

@@ -0,0 +1,109 @@
+import { API } from "../globals";
+import {
+  postNetworkErrorToast,
+  postAsyncSuccessToast,
+  postAsyncFailureToast,
+} from "../components/framework/toasters";
+import { _switchEmbedding } from "./embedding";
+
+function abortableFetch(request, opts, timeout = 0) {
+  const controller = new AbortController();
+  const { signal } = controller;
+
+  return {
+    abort: () => controller.abort(),
+    isAborted: () => signal.aborted,
+    ready: () => {
+      if (timeout) {
+        setTimeout(() => controller.abort(), timeout);
+      }
+      return fetch(request, { ...opts, signal });
+    },
+  };
+}
+
+async function doReembedFetch(dispatch, getState) {
+  const state = getState();
+  let cells = state.annoMatrix.rowIndex.labels();
+
+  // These lines ensure that we convert any TypedArray to an Array.
+  // This is necessary because JSON.stringify() does some very strange
+  // things with TypedArrays (they are marshalled to JSON objects, rather
+  // than being marshalled as a JSON array).
+  cells = Array.isArray(cells) ? cells : Array.from(cells);
+
+  const af = abortableFetch(
+    `${API.prefix}${API.version}layout/obs`,
+    {
+      method: "PUT",
+      headers: new Headers({
+        Accept: "application/octet-stream",
+        "Content-Type": "application/json",
+      }),
+      body: JSON.stringify({
+        method: "umap",
+        filter: { obs: { index: cells } },
+      }),
+      credentials: "include",
+    },
+    60000 // 1 minute timeout
+  );
+  dispatch({
+    type: "reembed: request start",
+    abortableFetch: af,
+  });
+  const res = await af.ready();
+
+  if (res.ok && res.headers.get("Content-Type").includes("application/json")) {
+    return res;
+  }
+
+  // else an error
+  let msg = `Unexpected HTTP response ${res.status}, ${res.statusText}`;
+  const body = await res.text();
+  if (body && body.length > 0) {
+    msg = `${msg} -- ${body}`;
+  }
+  throw new Error(msg);
+}
+
+/*
+functions below are dispatch-able
+*/
+export function requestReembed() {
+  return async (dispatch, getState) => {
+    try {
+      const res = await doReembedFetch(dispatch, getState);
+      const schema = await res.json();
+      dispatch({
+        type: "reembed: request completed",
+      });
+
+      const { annoMatrix: prevAnnoMatrix, obsCrossfilter: prevCrossfilter } = getState();
+      const base = prevAnnoMatrix.base().addEmbedding(schema);
+      const [annoMatrix, obsCrossfilter] = await _switchEmbedding(
+        base,
+        prevCrossfilter,
+        schema.name
+      );
+      dispatch({
+        type: "reembed: add reembedding",
+        schema,
+        annoMatrix,
+        obsCrossfilter,
+      });
+
+      postAsyncSuccessToast("Re-embedding has completed.");
+    } catch (error) {
+      dispatch({
+        type: "reembed: request aborted",
+      });
+      if (error.name === "AbortError") {
+        postAsyncFailureToast("Re-embedding calculation was aborted.");
+      } else {
+        postNetworkErrorToast(`Re-embedding: ${error.message}`);
+      }
+      console.log("Reembed exception:", error, error.name, error.message);
+    }
+  };
+}

client/src/actions/selection.js

@@ -1,101 +1,104 @@
 /*
 Action creators for selection 
 */
-export const selectContinuousMetadataAction =
+export const selectContinuousMetadataAction = (
-  (type, query, range, oldProps = {}) =>
+  type,
-  async (dispatch, getState) => {
+  query,
-    const { obsCrossfilter: prevObsCrossfilter } = getState();
+  range,
+  oldProps = {}
+) => async (dispatch, getState) => {
+  const { obsCrossfilter: prevObsCrossfilter } = getState();
 
-    const selection = range
+  const selection = range
-      ? {
+    ? {
-          mode: "range",
+        mode: "range",
-          lo: range[0],
+        lo: range[0],
-          hi: range[1],
+        hi: range[1],
-          inclusive: true, // [lo, hi] incluisve selection
+        inclusive: true, // [lo, hi] incluisve selection
-        }
+      }
-      : { mode: "all" };
+    : { mode: "all" };
 
-    const obsCrossfilter = await prevObsCrossfilter.select(...query, selection);
+  const obsCrossfilter = await prevObsCrossfilter.select(...query, selection);
 
-    dispatch({
+  dispatch({
-      type,
+    type,
-      obsCrossfilter,
+    obsCrossfilter,
-      range,
+    range,
-      ...oldProps,
+    ...oldProps,
-    });
+  });
+};
+
+export const selectCategoricalMetadataAction = (
+  type, // action type
+  metadataField, // annotation category name
+  labels,
+  label, // the label being selected/deselected
+  isSelected, // bool
+  oldProps = {}
+) => async (dispatch, getState) => {
+  const {
+    obsCrossfilter: prevObsCrossfilter,
+    categoricalSelection,
+  } = getState();
+
+  const labelSelectionState = new Map(categoricalSelection[metadataField]);
+  labels.forEach(
+    (l) => labelSelectionState.has(l) || labelSelectionState.set(l, true)
+  );
+  labelSelectionState.set(label, isSelected);
+
+  const values = Array.from(labelSelectionState.keys()).filter((k) =>
+    labelSelectionState.get(k)
+  );
+  const selection = {
+    mode: "exact",
+    values,
   };
+  const obsCrossfilter = await prevObsCrossfilter.select(
+    "obs",
+    metadataField,
+    selection
+  );
 
-export const selectCategoricalMetadataAction =
+  dispatch({
-  (
+    type,
-    type, // action type
+    obsCrossfilter,
-    metadataField, // annotation category name
+    metadataField,
-    labels,
+    labelSelectionState,
-    label, // the label being selected/deselected
+    ...oldProps,
-    isSelected, // bool
+  });
-    oldProps = {}
+};
-  ) =>
-  async (dispatch, getState) => {
-    const { obsCrossfilter: prevObsCrossfilter, categoricalSelection } =
-      getState();
 
-    const labelSelectionState = new Map(categoricalSelection[metadataField]);
+export const selectCategoricalAllMetadataAction = (
-    labels.forEach(
+  type, // action type
-      (l) => labelSelectionState.has(l) || labelSelectionState.set(l, true)
+  metadataField, // annotation category name
-    );
+  labels,
-    labelSelectionState.set(label, isSelected);
+  isSelected, // bool, select all or none
+  oldProps = {}
+) => async (dispatch, getState) => {
+  const {
+    obsCrossfilter: prevObsCrossfilter,
+    categoricalSelection,
+  } = getState();
 
-    const values = Array.from(labelSelectionState.keys()).filter((k) =>
+  const labelSelectionState = new Map(categoricalSelection[metadataField]);
-      labelSelectionState.get(k)
+  labels.forEach((label) => labelSelectionState.set(label, isSelected));
-    );
-    const selection = {
-      mode: "exact",
-      values,
-    };
-    const obsCrossfilter = await prevObsCrossfilter.select(
-      "obs",
-      metadataField,
-      selection
-    );
 
-    dispatch({
+  const selection = { mode: isSelected ? "all" : "none" };
-      type,
+  const obsCrossfilter = await prevObsCrossfilter.select(
-      obsCrossfilter,
+    "obs",
-      metadataField,
+    metadataField,
-      labelSelectionState,
+    selection
-      ...oldProps,
+  );
-    });
-  };
 
-export const selectCategoricalAllMetadataAction =
+  dispatch({
-  (
+    type,
-    type, // action type
+    obsCrossfilter,
-    metadataField, // annotation category name
+    metadataField,
-    labels,
+    labelSelectionState,
-    isSelected, // bool, select all or none
+    ...oldProps,
-    oldProps = {}
+  });
-  ) =>
+};
-  async (dispatch, getState) => {
-    const { obsCrossfilter: prevObsCrossfilter, categoricalSelection } =
-      getState();
-
-    const labelSelectionState = new Map(categoricalSelection[metadataField]);
-    labels.forEach((label) => labelSelectionState.set(label, isSelected));
-
-    const selection = { mode: isSelected ? "all" : "none" };
-    const obsCrossfilter = await prevObsCrossfilter.select(
-      "obs",
-      metadataField,
-      selection
-    );
-
-    dispatch({
-      type,
-      obsCrossfilter,
-      metadataField,
-      labelSelectionState,
-      ...oldProps,
-    });
-  };
 
 /**
  ** Graph selection-related actions
@@ -105,23 +108,25 @@ export const graphBrushStartAction = () =>
   /* no change to crossfilter until a change fires */
   ({ type: "graph brush start" });
 
-const _graphBrushWithinRectAction =
+const _graphBrushWithinRectAction = (type, embName, brushCoords) => async (
-  (type, embName, brushCoords) => async (dispatch, getState) => {
+  dispatch,
-    const { obsCrossfilter: prevObsCrossfilter } = getState();
+  getState
+) => {
+  const { obsCrossfilter: prevObsCrossfilter } = getState();
 
-    const selection = { mode: "within-rect", ...brushCoords };
+  const selection = { mode: "within-rect", ...brushCoords };
-    const obsCrossfilter = await prevObsCrossfilter.select(
+  const obsCrossfilter = await prevObsCrossfilter.select(
-      "emb",
+    "emb",
-      embName,
+    embName,
-      selection
+    selection
-    );
+  );
 
-    dispatch({
+  dispatch({
-      type,
+    type,
-      obsCrossfilter,
+    obsCrossfilter,
-      brushCoords,
+    brushCoords,
-    });
+  });
-  };
+};
 
 const _graphAllAction = (type, embName) => async (dispatch, getState) => {
   const { obsCrossfilter: prevObsCrossfilter } = getState();
@@ -157,26 +162,28 @@ export const graphLassoCancelAction = (embName) =>
 export const graphLassoDeselectAction = (embName) =>
   _graphAllAction("graph lasso cancel", embName);
 
-export const graphLassoEndAction =
+export const graphLassoEndAction = (embName, polygon) => async (
-  (embName, polygon) => async (dispatch, getState) => {
+  dispatch,
-    const { obsCrossfilter: prevObsCrossfilter } = getState();
+  getState
+) => {
+  const { obsCrossfilter: prevObsCrossfilter } = getState();
 
-    const selection = {
+  const selection = {
-      mode: "within-polygon",
+    mode: "within-polygon",
-      polygon,
+    polygon,
-    };
-    const obsCrossfilter = await prevObsCrossfilter.select(
-      "emb",
-      embName,
-      selection
-    );
-
-    dispatch({
-      type: "graph lasso end",
-      obsCrossfilter,
-      polygon,
-    });
   };
+  const obsCrossfilter = await prevObsCrossfilter.select(
+    "emb",
+    embName,
+    selection
+  );
+
+  dispatch({
+    type: "graph lasso end",
+    obsCrossfilter,
+    polygon,
+  });
+};
 
 /*
 Differential expression set selection

client/src/actions/viewStack.js

@@ -42,8 +42,10 @@ export const subsetAction = () => (dispatch, getState) => {
   By convention, a clip view is ALWAYS the top view, so if present, pop
   off and re-apply
   */
-  const { annoMatrix: prevAnnoMatrix, obsCrossfilter: prevObsCrossfilter } =
+  const {
-    getState();
+    annoMatrix: prevAnnoMatrix,
+    obsCrossfilter: prevObsCrossfilter,
+  } = getState();
   const annoMatrix = _userSubsetAnnoMatrix(
     prevAnnoMatrix,
     prevObsCrossfilter.allSelectedMask()

client/src/annoMatrix/annoMatrix.js

@@ -12,7 +12,6 @@ import {
 import { indexEntireSchema } from "../util/stateManager/schemaHelpers";
 import { _whereCacheGet, _whereCacheMerge } from "./whereCache";
 import _shallowClone from "./clone";
-import { _queryValidate, _queryCacheKey } from "./query";
 
 const _dataframeCache = dataframeMemo(128);
 
@@ -182,7 +181,7 @@ export default class AnnoMatrix {
 		Returns a Promise for the query result, which will resolve to a dataframe.
 
 		Field must be one of the matrix fields: 'obs', 'var', 'X', 'emb'.  Value
-		represents the underlying object upon which the query is occurring.
+		represents the underlying object upon which the query is occuring.
 
 		Query is one of:
 			* a string, representing a single column name from the field, eg,
@@ -198,6 +197,12 @@ export default class AnnoMatrix {
 		field/column, similar to a join.  Currently only supported on the var
 		dimension, allowing query of X columns by var value (eg, gene name)
 
+    The query filter	is a single value filter:
+			{ "field name": [
+				{name: "column name", values: [ list of values ]}
+			]}
+		One and only one value filter is allowed in a value query.
+
 		Examples:
 
     1. Fetch the "n_genes" column the "obs":
@@ -215,9 +220,7 @@ export default class AnnoMatrix {
        value "TYMP" in the var index.
 
 			fetch("X", { 
-				where: {
+				where: {field: "var", column: this.schema.annotations.var.index, value: "TYMP"}
-          field: "var", column: this.schema.annotations.var.index, value: "TYMP"
-        }
 			})
 
       In AnnData & Pandas DataFrame API, this is equivalent to:
@@ -407,14 +410,6 @@ export default class AnnoMatrix {
     _subclassResponsibility();
   }
 
-  getCacheKeys(field, query) {
-    /*
-    Return cache keys for columns associated with this query.  May return
-    [unknown] if no keys are known (ie, nothing is or was cached).
-    */
-    return _whereCacheGet(this._whereCache, this.schema, field, query);
-  }
-
   /**
    ** Private interfaces below.
    **/
@@ -432,7 +427,6 @@ export default class AnnoMatrix {
   async _fetch(field, q) {
     if (!AnnoMatrix.fields().includes(field)) return undefined;
     const queries = Array.isArray(q) ? q : [q];
-    queries.forEach(_queryValidate);
 
     /* find cached columns we need, and GC the rest */
     const cachedColumns = this._resolveCachedQueries(field, queries);
@@ -513,7 +507,7 @@ export default class AnnoMatrix {
     * obs, var and emb do not grow without bounds, and are needed constantly 
       for rendering.  
       a) There is no upside to GC'ing these in the base (loader)
-      b) The undo/redo cache can hold a large number in views, which is worth GC'ing
+      b) The undo/redo cache can hold a large number in views, which is worht GC'ing
     * X is often much larger than memory, and the UI allows add/del from
       this.  Most of the GC potential is here in both the base and views.
 
@@ -528,7 +522,7 @@ export default class AnnoMatrix {
   as much of the cache is pinned by that data structure.
   */
   _gcField(field, isHot, pinnedColumns) {
-    const maxColumns = isHot ? 256 : 10; // maybe to aggressive?
+    const maxColumns = isHot ? 256 : 10; // maybe to aggessive?
 
     const cache = this._cache[field];
     if (cache.colIndex.size() < maxColumns) return; // trivial rejection
@@ -596,18 +590,23 @@ export default class AnnoMatrix {
     called each time a query is performed, allowing the gc to update any bookkeeping
     information.  Currently, this is just a simple last-fetched timestamp, stored
     in a Map.
+
+    Map objects preserve order of insertion. This is leveraged as a cheap way to
+    do LRU, by removing and re-inserting keys.  IMPORTANT: the cleanup code assumes
+    the map insertion order is least-recently-used first.
     */
     const cols = dataframe.colIndex.labels();
     const { _gcInfo } = this;
     const now = Date.now();
     cols.forEach((c) => {
+      // gcInfo.delete(c);
       _gcInfo.set(_columnCacheKey(field, c), now);
     });
   }
 
   /**
-  Cloning subclass protocol - we rely in cloning to preserve immutable
+  Cloning sublcass protocol - we rely in cloning to preserve immutable
-  semantics while not causing races or other side effects in internal
+  symantics while not causing races or other side effects in internal
   cache management.
 
   Subclasses must override _cloneDeeper() if they have state which requires 
@@ -640,6 +639,15 @@ export default class AnnoMatrix {
 /*
 private utility functions below
 */
+
+function _queryCacheKey(field, query) {
+  if (typeof query === "object") {
+    const { field: queryField, column: queryColumn, value: queryValue } = query;
+    return `${field}/${queryField}/${queryColumn}/${queryValue}`;
+  }
+  return `${field}/${query}`;
+}
+
 function _columnCacheKey(field, column) {
   return `${field}/${column}`;
 }

client/src/annoMatrix/crossfilter.js

@@ -123,24 +123,6 @@ export default class AnnoMatrixObsCrossfilter {
     return new AnnoMatrixObsCrossfilter(annoMatrix, this.obsCrossfilter);
   }
 
-  /**
-   * Drop the crossfilter dimension. Do not change the annoMatrix. Useful when we
-   * want to stop trackin the selection state, but aren't sure we want to blow the
-   * annomatrix cache.
-   */
-  dropDimension(field, query) {
-    const { annoMatrix } = this;
-    let { obsCrossfilter } = this;
-    const keys = annoMatrix
-      .getCacheKeys(field, query)
-      .filter((k) => k !== undefined);
-    const dimName = _dimensionName(field, keys);
-    if (obsCrossfilter.hasDimension(dimName)) {
-      obsCrossfilter = obsCrossfilter.delDimension(dimName);
-    }
-    return new AnnoMatrixObsCrossfilter(annoMatrix, obsCrossfilter);
-  }
-
   /**
   Selection state - API is identical to ImmutableTypedCrossfilter, as these
   are just wrappers to lazy create indices.
@@ -182,7 +164,9 @@ export default class AnnoMatrixObsCrossfilter {
 		*/
     const { annoMatrix } = this;
     const currentDims = this.obsCrossfilter.dimensionNames();
-    const obsCrossfilter = currentDims.reduce((xfltr, dim) => xfltr.select(dim, { mode: "all" }), this.obsCrossfilter);
+    const obsCrossfilter = currentDims.reduce((xfltr, dim) => {
+      return xfltr.select(dim, { mode: "all" });
+    }, this.obsCrossfilter);
     return new AnnoMatrixObsCrossfilter(annoMatrix, obsCrossfilter);
   }
 

client/src/annoMatrix/fetchHelpers.js

@@ -1,4 +1,4 @@
-export { doBinaryRequest, doFetch } from "../util/actionHelpers";
+export { doBinaryRequest } from "../util/actionHelpers";
 
 /* double URI encode - needed for query-param filters */
 export function _dubEncURIComp(s) {
@@ -19,7 +19,9 @@ export function _fetchResult(promise) {
     }
   );
 
-  res.status = () => _status;
+  res.status = () => {
+    return _status;
+  };
 
   return res;
 }

client/src/annoMatrix/loader.js

@@ -1,6 +1,6 @@
-import { doBinaryRequest, doFetch } from "./fetchHelpers";
+import { doBinaryRequest, _dubEncURIComp } from "./fetchHelpers";
 import { matrixFBSToDataframe } from "../util/stateManager/matrix";
-import { _getColumnSchema } from "./schema";
+import { _getColumnSchema, _normalizeCategoricalSchema } from "./schema";
 import {
   addObsAnnoColumn,
   removeObsAnnoColumn,
@@ -12,17 +12,6 @@ import { isArrayOrTypedArray } from "../util/typeHelpers";
 import { _whereCacheCreate } from "./whereCache";
 import AnnoMatrix from "./annoMatrix";
 import PromiseLimit from "../util/promiseLimit";
-import {
-  _expectSimpleQuery,
-  _expectComplexQuery,
-  _urlEncodeLabelQuery,
-  _urlEncodeComplexQuery,
-  _hashStringValues,
-} from "./query";
-import {
-  normalizeResponse,
-  normalizeWritableCategoricalSchema,
-} from "./normalize";
 
 const promiseThrottle = new PromiseLimit(5);
 
@@ -126,7 +115,7 @@ export default class AnnoMatrixLoader extends AnnoMatrix {
       data = new Ctor(this.nObs).fill(value);
     }
     newAnnoMatrix._cache.obs = this._cache.obs.withCol(colName, data);
-    normalizeWritableCategoricalSchema(
+    _normalizeCategoricalSchema(
       colSchema,
       newAnnoMatrix._cache.obs.col(colName)
     );
@@ -234,23 +223,27 @@ export default class AnnoMatrixLoader extends AnnoMatrix {
     /*
     _doLoad - evaluates the query against the field. Returns:
       * whereCache update: column query map mapping the query to the column labels
-      * Dataframe containing the new columns (one per dimension)
+      * Dataframe containing the new colums (one per dimension)
     */
-    let doRequest;
+    let urlQuery;
+    let urlBase;
     let priority = 10; // default fetch priority
 
     switch (field) {
       case "obs":
       case "var": {
-        doRequest = _obsOrVarLoader(this.baseURL, field, query);
+        urlBase = `${this.baseURL}annotations/${field}`;
+        urlQuery = _encodeQuery("annotation-name", query);
         break;
       }
       case "X": {
-        doRequest = _XLoader(this.baseURL, field, query);
+        urlBase = `${this.baseURL}data/var`;
+        urlQuery = _encodeQuery(undefined, query);
         break;
       }
       case "emb": {
-        doRequest = _embLoader(this.baseURL, field, query);
+        urlBase = `${this.baseURL}layout/obs`;
+        urlQuery = _encodeQuery("layout-name", query);
         priority = 0; // high prio load for embeddings
         break;
       }
@@ -258,8 +251,13 @@ export default class AnnoMatrixLoader extends AnnoMatrix {
         throw new Error("Unknown field name");
     }
 
-    const buffer = await promiseThrottle.priorityAdd(priority, doRequest);
+    const url = `${urlBase}?${urlQuery}`;
-    let result = matrixFBSToDataframe(buffer);
+    const buffer = await promiseThrottle.priorityAdd(
+      priority,
+      doBinaryRequest,
+      url
+    );
+    const result = matrixFBSToDataframe(buffer);
     if (!result || result.isEmpty()) throw Error("Unknown field/col");
 
     const whereCacheUpdate = _whereCacheCreate(
@@ -268,7 +266,13 @@ export default class AnnoMatrixLoader extends AnnoMatrix {
       result.colIndex.labels()
     );
 
-    result = normalizeResponse(field, query, this.schema, result);
+    if (field === "obs") {
+      /* cough, cough - see comment on method */
+      _normalizeCategoricalSchema(
+        this.schema.annotations.obsByName[query],
+        result.col(query)
+      );
+    }
 
     return [whereCacheUpdate, result];
   }
@@ -278,6 +282,17 @@ export default class AnnoMatrixLoader extends AnnoMatrix {
 Utility functions below
 */
 
+function _encodeQuery(colKey, q) {
+  if (typeof q === "object") {
+    const { field: queryField, column: queryColumn, value: queryValue } = q;
+    return `${_dubEncURIComp(queryField)}:${_dubEncURIComp(
+      queryColumn
+    )}=${_dubEncURIComp(queryValue)}`;
+  }
+  if (!colKey) throw new Error("Unsupported query by name");
+  return `${colKey}=${encodeURIComponent(q)}`;
+}
+
 function _writableCheck(colSchema) {
   if (!colSchema?.writable) {
     throw new Error("Unknown or readonly obs column");
@@ -290,57 +305,3 @@ function _writableCategoryTypeCheck(colSchema) {
     throw new Error("column must be categorical");
   }
 }
-
-function _embLoader(baseURL, _field, query) {
-  _expectSimpleQuery(query);
-
-  const urlBase = `${baseURL}layout/obs`;
-  const urlQuery = _urlEncodeLabelQuery("layout-name", query);
-  const url = `${urlBase}?${urlQuery}`;
-  return () => doBinaryRequest(url);
-}
-
-function _obsOrVarLoader(baseURL, field, query) {
-  _expectSimpleQuery(query);
-
-  const urlBase = `${baseURL}annotations/${field}`;
-  const urlQuery = _urlEncodeLabelQuery("annotation-name", query);
-  const url = `${urlBase}?${urlQuery}`;
-  return () => doBinaryRequest(url);
-}
-
-function _XLoader(baseURL, field, query) {
-  _expectComplexQuery(query);
-
-  if (query.where) {
-    const urlBase = `${baseURL}data/var`;
-    const urlQuery = _urlEncodeComplexQuery(query);
-    const url = `${urlBase}?${urlQuery}`;
-    return () => doBinaryRequest(url);
-  }
-
-  if (query.summarize) {
-    const urlBase = `${baseURL}summarize/var`;
-    const urlQuery = _urlEncodeComplexQuery(query);
-
-    if (urlBase.length + urlQuery.length < 2000) {
-      const url = `${urlBase}?${urlQuery}`;
-      return () => doBinaryRequest(url);
-    }
-
-    const url = `${urlBase}?key=${_hashStringValues([urlQuery])}`;
-    return async () => {
-      const res = await doFetch(url, {
-        method: "POST",
-        body: urlQuery,
-        headers: new Headers({
-          Accept: "application/octet-stream",
-          "Content-Type": "application/x-www-form-urlencoded",
-        }),
-      });
-      return res.arrayBuffer();
-    };
-  }
-
-  throw new Error("Unknown query structure");
-}

client/src/annoMatrix/normalize.js

@@ -1,150 +0,0 @@
-import { _getColumnSchema, _isIndex } from "./schema";
-import catLabelSort from "../util/catLabelSort";
-import {
-  unassignedCategoryLabel,
-  overflowCategoryLabel,
-  globalConfig,
-} from "../globals";
-
-export function normalizeResponse(field, query, schema, response) {
-  /**
-   * There are a number of assumptions in the front-end about data typing and data
-   * characteristics. This routine will normalize a server response dataframe
-   * to match front-end expectations and UI conventions. This includes cast/transform
-   * of the data and schema updates.
-   *
-   * This consolidates all assumptions into one location, for ease of update.
-   *
-   * Currently, this includes normalization for obs/var columns only:
-   *
-   * - Dataframe columns in var/obs that are declared type: boolean may be sent by
-   *   the server in a variety of formats (eg uint8, etc).  Cast to JS Array[boolean]
-   *
-   * - "Categorical" columns may not have all categories represented in the server-provided
-   *   schema (for valid reasons, eg, floating point rounding differences).  For all
-   *   types we treat as categorical in the UI (string, boolean, categorical), update
-   *   the schema to contain all categories as a convenience.
-   *
-   * - "Categorical" columns (ie, string, boolean, categorical) may contain an excess
-   *   of category values (aka labels).  Consolidate any excess into an "all other"
-   *   category.
-   */
-
-  // currently no data or schema normalization necessary for X or emb
-  if (field !== "obs" && field !== "var") return response;
-
-  const colLabels = response.colIndex.labels();
-  for (const colLabel of colLabels) {
-    const colSchema = _getColumnSchema(schema, field, colLabel);
-    const isIndex = _isIndex(schema, field, colLabel);
-    const { type, writable } = colSchema;
-
-    // Boolean data -- cast entire array to Array[bool]
-    if (type === "boolean") {
-      response = castColumnToBoolean(response, colLabel);
-    }
-
-    // Types that are categorical in UI (string, boolean, categorical) OR are writable
-    // are introspected to ensure the schema `categories` field and data values match,
-    // and that we do not have an excess of category values (for non-writable columns)
-    const isEnumType =
-      type === "boolean" ||
-      type === "string" ||
-      type === "categorical" ||
-      writable;
-    if (!isIndex && isEnumType) {
-      response = normalizeCategorical(response, colLabel, colSchema);
-    }
-  }
-  return response;
-}
-
-function castColumnToBoolean(df, label) {
-  const colData = df.col(label).asArray();
-  const newColData = new Array(colData.length);
-  for (let i = 0; i < colData.length; i += 1) newColData[i] = !!colData[i];
-  df = df.replaceColData(label, newColData);
-  return df;
-}
-
-export function normalizeWritableCategoricalSchema(colSchema, col) {
-  /*
-  Ensure all enum writable / categorical schema have a categories array, that
-  the categories array contains all unique values in the data array, AND that
-  the array is UI sorted.
-  */
-  const categorySet = new Set(
-    col.summarizeCategorical().categories.concat(colSchema.categories ?? [])
-  );
-  if (!categorySet.has(unassignedCategoryLabel)) {
-    categorySet.add(unassignedCategoryLabel);
-  }
-  colSchema.categories = catLabelSort(true, Array.from(categorySet));
-  return colSchema;
-}
-
-export function normalizeCategorical(df, colLabel, colSchema) {
-  /*
-  If writable, ensure schema matches data and we have an unassigned label
-  If not writable, ensure schema matches data and that we consolidate labels in excess
-  of "top N" into an overflow labels.
-  */
-  const { writable } = colSchema;
-  const col = df.col(colLabel);
-
-  if (writable) {
-    // writable (aka user) annotations
-    normalizeWritableCategoricalSchema(colSchema, col);
-    return df;
-  }
-
-  // else read-only, categorical columns
-  const TopN = globalConfig.maxCategoricalOptionsToDisplay;
-
-  // consolidate all categories from data and schema into a single list
-  const colDataSummary = col.summarizeCategorical();
-  const allCategories = new Set(
-    colDataSummary.categories.concat(colSchema.categories ?? [])
-  );
-
-  // if no overflow, just UI sort schema categories and return
-  if (allCategories.size <= TopN) {
-    colSchema.categories = catLabelSort(writable, [...allCategories.keys()]);
-    return df;
-  }
-
-  // Otherwise, pick top N categories by count and rewrite data
-
-  // choose unique overflow category label
-  let overflowCatName = `${colLabel}${overflowCategoryLabel}`;
-  while (allCategories.has(overflowCatName)) {
-    overflowCatName += "_";
-  }
-
-  // pick top N category labels and add overflow label
-  const topNCategories = new Set(
-    [...colDataSummary.categoryCounts.keys()].slice(0, TopN)
-  );
-  topNCategories.add(overflowCatName);
-
-  // rewrite data - consolidate all excess labels into overflow label
-  const newColData = Array.from(col.asArray());
-  for (let i = 0; i < newColData.length; i += 1) {
-    if (!topNCategories.has(newColData[i])) {
-      newColData[i] = overflowCatName;
-    }
-  }
-
-  // replace data in dataframe
-  df = df.replaceColData(colLabel, newColData);
-
-  // Update schema with categories, in UI sort order. Ensure overflow label is at end
-  // of list for display purposes.
-  const revisedCategories = df.col(colLabel).summarizeCategorical().categories;
-  revisedCategories.push(
-    revisedCategories.splice(revisedCategories.indexOf(overflowCatName), 1)[0]
-  );
-  colSchema.categories = catLabelSort(writable, revisedCategories);
-
-  return df;
-}

client/src/annoMatrix/query.js

@@ -1,126 +0,0 @@
-import sha1 from "sha1";
-import { _dubEncURIComp } from "./fetchHelpers";
-
-/**
- * Query utilities, mostly for debugging support and validation.
- */
-
-/**
- * Normalize & error check the query.
- * @param {object | string} query - the query
- * @returns {object | string} - the normalized query
- */
-export function _queryValidate(query) {
-  if (typeof query !== "object") return query;
-
-  if (query.where && query.summarize)
-    throw new Error("query may not specify both where and summarize");
-  if (query.where) {
-    const {
-      field: queryField,
-      column: queryColumn,
-      value: queryValue,
-    } = query.where;
-    if (!queryField || !queryColumn || !queryValue)
-      throw new Error("Incomplete where query");
-    return query;
-  }
-  if (query.summarize) {
-    const {
-      field: queryField,
-      column: queryColumn,
-      values: queryValues,
-    } = query.summarize;
-    if (!queryField || !queryColumn || !queryValues)
-      throw new Error("Incomplete where query");
-    if (!Array.isArray(queryValues))
-      throw new Error("Summarize query values must be an array");
-    return query;
-  }
-  throw new Error("query must specify one of where or summarize");
-}
-
-export function _expectSimpleQuery(query) {
-  if (typeof query === "object") throw new Error("expected simple query");
-}
-
-export function _expectComplexQuery(query) {
-  if (typeof query !== "object") throw new Error("expected complex query");
-}
-
-/**
- * Generate a unique key which can be used to reference this query.
- *
- * @param {string} field
- * @param {string|object} query
- * @returns the key
- */
-export function _queryCacheKey(field, query) {
-  if (typeof query === "object") {
-    // complex query
-    if (query.where) {
-      const {
-        field: queryField,
-        column: queryColumn,
-        value: queryValue,
-      } = query.where;
-      return `${field}/${queryField}/${queryColumn}/${queryValue}`;
-    }
-    if (query.summarize) {
-      const {
-        method,
-        field: queryField,
-        column: queryColumn,
-        values: queryValues,
-      } = query.summarize;
-      return `${field}/${method}/${queryField}/${queryColumn}/${queryValues.join(
-        ","
-      )}`;
-    }
-    throw new Error("Unrecognized complex query type");
-  }
-
-  // simple query
-  return `${field}/${query}`;
-}
-
-function _urlEncodeWhereQuery(q) {
-  const { field: queryField, column: queryColumn, value: queryValue } = q;
-  return `${_dubEncURIComp(queryField)}:${_dubEncURIComp(
-    queryColumn
-  )}=${_dubEncURIComp(queryValue)}`;
-}
-
-function _urlEncodeSummarizeQuery(q) {
-  const { method, field, column, values } = q;
-  const filter = values
-    .map((value) => _urlEncodeWhereQuery({ field, column, value }))
-    .join("&");
-  return `method=${method}&${filter}`;
-}
-
-export function _urlEncodeComplexQuery(q) {
-  if (typeof q === "object") {
-    if (q.where) {
-      return _urlEncodeWhereQuery(q.where);
-    }
-    if (q.summarize) {
-      return _urlEncodeSummarizeQuery(q.summarize);
-    }
-  }
-  throw new Error("Unrecognized complex query type");
-}
-
-export function _urlEncodeLabelQuery(colKey, q) {
-  if (!colKey) throw new Error("Unsupported query by name");
-  if (typeof q !== "string") throw new Error("Query must be a simple label.");
-  return `${colKey}=${encodeURIComponent(q)}`;
-}
-
-/**
- * Generate the column key the server will send us for this query.
- */
-export function _hashStringValues(arrayOfString) {
-  const hash = sha1(arrayOfString.join(""));
-  return hash;
-}

client/src/annoMatrix/schema.js

@@ -1,6 +1,9 @@
 /*
 Private helper functions related to schema
 */
+import catLabelSort from "../util/catLabelSort";
+import { unassignedCategoryLabel } from "../globals";
+
 export function _getColumnSchema(schema, field, col) {
   /* look up the column definition */
   switch (field) {
@@ -23,15 +26,10 @@ export function _getColumnSchema(schema, field, col) {
   }
 }
 
-export function _isIndex(schema, field, col) {
-  const index = schema.annotations?.[field].index;
-  return index && index === col;
-}
-
 export function _getColumnDimensionNames(schema, field, col) {
   /*
 		field/col may be an alias for multiple columns. Currently used to map ND 
-		values to 1D dataframe columns for embeddings/layout. Signified by the presence
+		values to 1D dataframe columns for embeddings/layout. Signfied by the presence
 		of the "dims" value in the schema.
 		*/
   const colSchema = _getColumnSchema(schema, field, col);
@@ -65,3 +63,26 @@ export function _isContinuousType(schema) {
   const { type } = schema;
   return !(type === "string" || type === "boolean" || type === "categorical");
 }
+
+export function _normalizeCategoricalSchema(colSchema, col) {
+  const { type, writable } = colSchema;
+  if (
+    type === "string" ||
+    type === "boolean" ||
+    type === "categorical" ||
+    writable
+  ) {
+    const categorySet = new Set(
+      col.summarizeCategorical().categories.concat(colSchema.categories ?? [])
+    );
+    if (writable && !categorySet.has(unassignedCategoryLabel)) {
+      categorySet.add(unassignedCategoryLabel);
+    }
+    colSchema.categories = Array.from(categorySet);
+  }
+
+  if (colSchema.categories) {
+    colSchema.categories = catLabelSort(writable, colSchema.categories);
+  }
+  return colSchema;
+}

client/src/annoMatrix/viewCreators.js

@@ -58,11 +58,10 @@ function _maskToList(mask) {
   if (!mask) {
     return null;
   }
-  const [...m] = mask;
+  const list = new Int32Array(mask.length);
-  const list = new Int32Array(m.length);
   let elems = 0;
-  for (let i = 0, l = m.length; i < l; i += 1) {
+  for (let i = 0, l = mask.length; i < l; i += 1) {
-    if (m[i]) {
+    if (mask[i]) {
       list[elems] = i;
       elems += 1;
     }

client/src/annoMatrix/whereCache.js

@@ -1,55 +1,27 @@
 /*
 Private support functions.
 
-This implements a query resolver cache, mapping a query onto the column labels
+Support for a "where" query, eg,
-resolved by that query. These labels are then used to manage the acutal data cache,
-which stores data by the resolved label.
 
-There are three query forms:
+	{ where: { field: "var", column: "gene", value: "FOXP2" }}
-  * primitive (string, number) - which is just reference the column label of same value
-  * where query (object) - eg, { where: { field: "var", column: "gene", value: "FOXP2" }}
-  * summary query (object) - eg, { summarize: { method: "mean", field: "var", column: "gene", values: ["FOXP2", "GNE", "F5"]}}
 
-These queries all resolve to one or more column labels on a field.  This
+These evaluate to a given column label.
-cache maintains a record of this, allowing direct access to the data caches
-without a server round-trip.
 
-The data structure for where queries, the following query against X as an example:
+The "where cache" is a map that saves evaluated queries and points 
-  { where: { field: "var", column: "column_label_in_var", value: "value_in_var_column" } }
+to the column label they resolve to.
-results in the following cached entry:
-{
-  where: {
-    X: {
-      var: Map(
-        column_label_in_var => Map(
-          value_in_var_column => [column_label_in_X, ...]
-        )
-      )
-    }
-  },
-  summarize: {},
-}
 
-And for summarize queries, for the following summary on X:
+Data structure, using X as the example field being queried, and var as
-  { summarize: { method: "mean", field: "var", column: "gene", values: ["G1", "G2"]}}
+the index.
-creates a cache entry of:
+
-{
+{ 
-  where: {},
+	X: {
-  summarize: {
+		var: Map(
-    X: {
+			column_label_in_var => Map(value_in_var_column => [column_label_in_X, ...])
-      mean: {
+		)
-        var: Map(
+	}
-          "gene" => Map(
-            "G1,G2" => [summary_column_label, ...]
-          )
-        )
-      }
-    },
-  },
 }
 */
 import { _getColumnDimensionNames } from "./schema";
-import { _hashStringValues } from "./query";
 
 export function _whereCacheGet(whereCache, schema, field, query) {
   /* 
@@ -59,30 +31,20 @@ export function _whereCacheGet(whereCache, schema, field, query) {
 	*/
 
   if (typeof query === "object") {
-    if (query.where) {
+    const { field: queryField, column: queryColumn, value: queryValue } = query;
-      const {
+
-        field: queryField,
+    const columnMap = whereCache?.[field]?.[queryField];
-        column: queryColumn,
+    if (columnMap === undefined) return [undefined];
-        value: queryValue,
+
-      } = query.where;
+    const valueMap = columnMap.get(queryColumn);
-      const columnMap = whereCache?.where?.[field]?.[queryField];
+    if (valueMap === undefined) return [undefined];
-      return columnMap?.get(queryColumn)?.get(queryValue) ?? [undefined];
+
-    }
+    const columnLabels = valueMap.get(queryValue);
-    if (query.summarize) {
+    return columnLabels === undefined ? [undefined] : columnLabels;
-      const {
-        method,
-        field: queryField,
-        column: queryColumn,
-        values: queryValues,
-      } = query.summarize;
-      const columnMap = whereCache?.summarize?.[field]?.[method]?.[queryField];
-      const queryValueHash = _hashStringValues(queryValues);
-      return columnMap?.get(queryColumn)?.get(queryValueHash) ?? [undefined];
-    }
-    return [undefined];
   }
 
-  return _getColumnDimensionNames(schema, field, query) ?? [undefined];
+  const colDims = _getColumnDimensionNames(schema, field, query);
+  return colDims === undefined ? [undefined] : colDims;
 }
 
 export function _whereCacheCreate(field, query, columnLabels) {
@@ -91,83 +53,40 @@ export function _whereCacheCreate(field, query, columnLabels) {
 	*/
   if (typeof query !== "object") return null;
 
-  const { where, summarize } = query;
+  const { field: queryField, column: queryColumn, value: queryValue } = query;
-  if (where) {
+  const whereCache = {
-    const { field: queryField, column: queryColumn, value: queryValue } = where;
+    [field]: {
-    return {
+      [queryField]: new Map([
-      where: {
+        [queryColumn, new Map([[queryValue, columnLabels]])],
-        [field]: {
+      ]),
-          [queryField]: new Map([
+    },
-            [queryColumn, new Map([[queryValue, columnLabels]])],
+  };
-          ]),
+  return whereCache;
-        },
-      },
-    };
-  }
-  if (summarize) {
-    const {
-      method,
-      field: queryField,
-      column: queryColumn,
-      values: queryValues,
-    } = summarize;
-    const queryValueHash = _hashStringValues(queryValues);
-    return {
-      summarize: {
-        [field]: {
-          [method]: {
-            [queryField]: new Map([
-              [queryColumn, new Map([[queryValueHash, columnLabels]])],
-            ]),
-          },
-        },
-      },
-    };
-  }
-  // oops, not sure what that query is!
-  return {};
-}
-
-function __mergeQueries(dst, src) {
-  for (const [queryField, columnMap] of Object.entries(src)) {
-    dst[queryField] = dst[queryField] || new Map();
-    for (const [queryColumn, valueMap] of columnMap) {
-      if (!dst[queryField].has(queryColumn))
-        dst[queryField].set(queryColumn, new Map());
-      for (const [queryValue, columnLabels] of valueMap) {
-        dst[queryField].get(queryColumn).set(queryValue, columnLabels);
-      }
-    }
-  }
 }
 
 function __whereCacheMerge(dst, src) {
   /*
 	merge src into dst (modifies dst)
 	*/
+  if (!dst) dst = {};
   if (!src || typeof src !== "object") return dst;
-
+  Object.entries(src).forEach(([field, query]) => {
-  if (src.where) {
+    if (!Object.prototype.hasOwnProperty.call(dst, field)) dst[field] = {};
-    dst.where = dst.where || {};
+    Object.entries(query).forEach(([queryField, columnMap]) => {
-    for (const [field, query] of Object.entries(src.where)) {
+      if (!Object.prototype.hasOwnProperty.call(dst[field], queryField))
-      dst.where[field] = dst.where[field] || {};
+        dst[field][queryField] = new Map();
-      __mergeQueries(dst.where[field], query);
+      columnMap.forEach((valueMap, queryColumn) => {
-    }
+        if (!dst[field][queryField].has(queryColumn))
-  }
+          dst[field][queryField].set(queryColumn, new Map());
-  if (src.summarize) {
+        valueMap.forEach((columnLabels, queryValue) => {
-    dst.summarize = dst.summarize || {};
+          dst[field][queryField].get(queryColumn).set(queryValue, columnLabels);
-    for (const [field, method] of Object.entries(src.summarize)) {
+        });
-      dst.summarize[field] = dst.summarize[field] || {};
+      });
-      for (const [methodName, query] of Object.entries(method)) {
+    });
-        dst.summarize[field][methodName] =
+  });
-          dst.summarize[field][methodName] || {};
-        __mergeQueries(dst.summarize[field][methodName], query);
-      }
-    }
-  }
   return dst;
 }
 
 export function _whereCacheMerge(...caches) {
-  return caches.reduce(__whereCacheMerge, {});
+  return caches.reduce((dst, src) => __whereCacheMerge(dst, src), {});
 }

client/src/components/app.js

@@ -1,3 +1,4 @@
+// jshint esversion: 6
 import React from "react";
 import Helmet from "react-helmet";
 import { connect } from "react-redux";
@@ -11,6 +12,7 @@ import Graph from "./graph/graph";
 import MenuBar from "./menubar";
 import Autosave from "./autosave";
 import Embedding from "./embedding";
+import TermsOfServicePrompt from "./termsPrompt";
 
 import actions from "../actions";
 
@@ -41,7 +43,7 @@ class App extends React.Component {
     const { loading, error, graphRenderCounter } = this.props;
     return (
       <Container>
-        <Helmet title="CELL&times;GENE | Annotate" />
+        <Helmet title="cellxgene" />
         {loading ? (
           <div
             style={{
@@ -74,6 +76,7 @@ class App extends React.Component {
                 <MenuBar />
                 <Embedding />
                 <Autosave />
+                <TermsOfServicePrompt />
                 <Legend viewportRef={viewportRef} />
                 <Graph key={graphRenderCounter} viewportRef={viewportRef} />
               </>

client/src/components/autosave/filenameDialog.js

@@ -14,10 +14,9 @@ import {
 @connect((state) => ({
   idhash: state.config?.parameters?.["annotations-user-data-idhash"] ?? null,
   annotations: state.annotations,
+  auth: state.config?.authentication,
+  userInfo: state.userInfo,
   writableCategoriesEnabled: state.config?.parameters?.annotations ?? false,
-  writableGenesetsEnabled: !(
-    state.config?.parameters?.annotations_genesets_readonly ?? true
-  ),
 }))
 class FilenameDialog extends React.Component {
   constructor(props) {
@@ -96,19 +95,20 @@ class FilenameDialog extends React.Component {
   render() {
     const {
       writableCategoriesEnabled,
-      writableGenesetsEnabled,
       annotations,
       idhash,
+      userInfo,
     } = this.props;
     const { filenameText } = this.state;
 
-    return (writableCategoriesEnabled || writableGenesetsEnabled) &&
+    return writableCategoriesEnabled &&
       annotations.promptForFilename &&
       !annotations.dataCollectionNameIsReadOnly &&
-      !annotations.dataCollectionName ? (
+      !annotations.dataCollectionName &&
+      userInfo.is_authenticated ? (
       <Dialog
         icon="tag"
-        title="User Generated Data Directory"
+        title="Annotations Collection"
         isOpen={!annotations.dataCollectionName}
         onClose={this.dismissFilenameDialog}
       >
@@ -120,7 +120,7 @@ class FilenameDialog extends React.Component {
         >
           <div className={Classes.DIALOG_BODY} data-testid="annotation-dialog">
             <div style={{ marginBottom: 20 }}>
-              <p>Name your user generated data directory:</p>
+              <p>Name your annotations collection:</p>
               <InputGroup
                 autoFocus
                 value={filenameText}
@@ -145,15 +145,9 @@ class FilenameDialog extends React.Component {
             </div>
             <div>
               <p>
-                {"Your annotations are stored in this file: "}
+                Your annotations are stored in this file:
                 <Code>
-                  {filenameText}-cell-labels-{idhash}.csv
+                  {filenameText}-{idhash}.csv
-                </Code>
-              </p>
-              <p>
-                {"Your gene sets are stored in this file: "}
-                <Code>
-                  {filenameText}-gene-sets-{idhash}.csv
                 </Code>
               </p>
               <p style={{ fontStyle: "italic" }}>
@@ -173,7 +167,7 @@ class FilenameDialog extends React.Component {
                 type="submit"
                 data-testid="submit-annotation"
               >
-                Create user generated data directory
+                Create annotations collection
               </Button>
             </div>
           </div>

client/src/components/autosave/index.js

@@ -5,18 +5,11 @@ import FilenameDialog from "./filenameDialog";
 
 @connect((state) => ({
   annotations: state.annotations,
-  obsAnnotationSaveInProgress:
+  saveInProgress: state.autosave?.saveInProgress ?? false,
-    state.autosave?.obsAnnotationSaveInProgress ?? false,
-  genesetSaveInProgress: state.autosave?.genesetSaveInProgress ?? false,
   error: state.autosave?.error,
   writableCategoriesEnabled: state.config?.parameters?.annotations ?? false,
-  writableGenesetsEnabled: !(
-    state.config?.parameters?.annotations_genesets_readonly ?? true
-  ),
   annoMatrix: state.annoMatrix,
-  genesets: state.genesets,
   lastSavedAnnoMatrix: state.autosave?.lastSavedAnnoMatrix,
-  lastSavedGenesets: state.autosave?.lastSavedGenesets,
 }))
 class Autosave extends React.Component {
   constructor(props) {
@@ -27,11 +20,11 @@ class Autosave extends React.Component {
   }
 
   componentDidMount() {
-    const { writableCategoriesEnabled, writableGenesetsEnabled } = this.props;
+    const { writableCategoriesEnabled } = this.props;
 
     let { timer } = this.state;
     if (timer) clearInterval(timer);
-    if (writableCategoriesEnabled || writableGenesetsEnabled) {
+    if (writableCategoriesEnabled) {
       timer = setInterval(this.tick, 2500);
     } else {
       timer = null;
@@ -45,37 +38,18 @@ class Autosave extends React.Component {
   }
 
   tick = () => {
-    const { dispatch, obsAnnotationSaveInProgress, genesetSaveInProgress } =
+    const { dispatch, saveInProgress } = this.props;
-      this.props;
+    if (this.needToSave() && !saveInProgress) {
-    if (!obsAnnotationSaveInProgress && this.needToSaveObsAnnotations()) {
       dispatch(actions.saveObsAnnotationsAction());
     }
-    if (!genesetSaveInProgress && this.needToSaveGenesets()) {
-      dispatch(actions.saveGenesetsAction());
-    }
   };
 
-  needToSaveObsAnnotations = () => {
+  needToSave = () => {
-    /* return true if we need to save obs cell labels, false if we don't */
+    /* return true if we need to save, false if we don't */
     const { annoMatrix, lastSavedAnnoMatrix } = this.props;
     return actions.needToSaveObsAnnotations(annoMatrix, lastSavedAnnoMatrix);
   };
 
-  needToSaveGenesets = () => {
-    /* return true if we need to save gene ses, false if we do not */
-    const { genesets, lastSavedGenesets } = this.props;
-    return genesets.initialized && genesets.genesets !== lastSavedGenesets;
-  };
-
-  needToSave() {
-    return this.needToSaveGenesets() || this.needToSaveObsAnnotations();
-  }
-
-  saveInProgress() {
-    const { obsAnnotationSaveInProgress, genesetSaveInProgress } = this.props;
-    return obsAnnotationSaveInProgress || genesetSaveInProgress;
-  }
-
   statusMessage() {
     const { error } = this.props;
     if (error) {
@@ -87,12 +61,12 @@ class Autosave extends React.Component {
   render() {
     const {
       writableCategoriesEnabled,
-      writableGenesetsEnabled,
+      saveInProgress,
       lastSavedAnnoMatrix,
     } = this.props;
     const initialDataLoadComplete = lastSavedAnnoMatrix;
 
-    if (!writableCategoriesEnabled && !writableGenesetsEnabled) return null;
+    if (!writableCategoriesEnabled) return null;
 
     return (
       <div
@@ -100,7 +74,7 @@ class Autosave extends React.Component {
         data-testclass={
           !initialDataLoadComplete
             ? "autosave-init"
-            : this.saveInProgress() || this.needToSave()
+            : this.needToSave() || saveInProgress
             ? "autosave-incomplete"
             : "autosave-complete"
         }

client/src/components/brushableHistogram/error.js

@@ -1,15 +0,0 @@
-import React from "react";
-import * as globals from "../../globals";
-
-const ErrorLoading = ({ displayName, zebra }) => (
-    <div
-      style={{
-        backgroundColor: zebra ? globals.lightestGrey : "white",
-        fontStyle: "italic",
-      }}
-    >
-      <span>{`Failure loading ${displayName}`}</span>
-    </div>
-  );
-
-export default ErrorLoading;

client/src/components/brushableHistogram/footer.js

@@ -1,63 +0,0 @@
-import React from "react";
-
-const HistogramFooter = React.memo(
-  ({
-    displayName,
-    hideRanges,
-    rangeMin,
-    rangeMax,
-    rangeColorMin,
-    rangeColorMax,
-    isObs,
-    isGeneSetSummary,
-  }) => 
-    /*
-    Footer of each histogram.  Will render range and title.
-
-    Required props:
-      * displayName - the displayName, aka "n_genes", "FOXP2", etc.
-      * hideRanges - true/false, enables/disable rendering of ranges
-      * range - length two array, [min, max], containing the range values to display
-      * rangeColor - length two array, [mincolor, maxcolor], each a CSS color
-    */
-     (
-      <div>
-        <div
-          style={{
-            display: "flex",
-            justifyContent: hideRanges ? "center" : "space-between",
-          }}
-        >
-          <span
-            style={{
-              color: rangeColorMin,
-              display: hideRanges ? "none" : "block",
-            }}
-          >
-            min {rangeMin.toPrecision(4)}
-          </span>
-          <span
-            data-testclass="brushable-histogram-field-name"
-            style={{ fontStyle: "italic" }}
-          >
-            {isObs && displayName}
-            {isGeneSetSummary && "gene set mean expression"}
-          </span>
-          <div style={{ display: hideRanges ? "block" : "none" }}>
-            : {rangeMin}
-          </div>
-          <span
-            style={{
-              color: rangeColorMax,
-              display: hideRanges ? "none" : "block",
-            }}
-          >
-            max {rangeMax.toPrecision(4)}
-          </span>
-        </div>
-      </div>
-    )
-  
-);
-
-export default HistogramFooter;

client/src/components/brushableHistogram/header.js

@@ -1,100 +0,0 @@
-import React, { useCallback } from "react";
-import { Button, ButtonGroup, Tooltip, Icon } from "@blueprintjs/core";
-import { IconNames } from "@blueprintjs/icons";
-import * as globals from "../../globals";
-
-const HistogramHeader = React.memo(
-  ({
-    fieldId,
-    isColorBy,
-    onColorByClick,
-    onRemoveClick,
-    isScatterPlotX,
-    isScatterPlotY,
-    onScatterPlotXClick,
-    onScatterPlotYClick,
-    isObs,
-  }) => {
-    /*
-        Render the toolbar for the histogram.  Props:
-          * fieldId - field identifier, used for various IDs
-          * isColorBy - true/false, is this the current color-by
-          * onColorByClick - color-by click handler
-          * onRemoveClick - optional handler for remove.  Button will not render if not defined.
-          * isScatterPlotX - optional, true/false if currently the X scatterplot field
-          * isScatterPlotY - optional, true/false if currently the Y scatterplot field
-          * onScatterPlotXClick - optional, handler for scatterPlot X button.
-          * onScatterPlotYClick - optional, handler for scatterPlot X button.
-  
-        Scatterplot controls will not render if either handler unspecified.
-      */
-
-    const memoizedColorByCallback = useCallback(
-      () => onColorByClick(fieldId, isObs),
-      [fieldId, isObs]
-    );
-
-    return (
-      <div
-        style={{
-          display: "flex",
-          justifyContent: "flex-end",
-          paddingBottom: "8px",
-        }}
-      >
-        {onScatterPlotXClick && onScatterPlotYClick ? (
-          <span>
-            <Icon icon={IconNames.SCATTER_PLOT} style={{ marginRight: 7 }} />
-            <ButtonGroup style={{ marginRight: 7 }}>
-              <Button
-                data-testid={`plot-x-${fieldId}`}
-                onClick={onScatterPlotXClick}
-                active={isScatterPlotX}
-                intent={isScatterPlotX ? "primary" : "none"}
-              >
-                plot x
-              </Button>
-              <Button
-                data-testid={`plot-y-${fieldId}`}
-                onClick={onScatterPlotYClick}
-                active={isScatterPlotY}
-                intent={isScatterPlotY ? "primary" : "none"}
-              >
-                plot y
-              </Button>
-            </ButtonGroup>
-          </span>
-        ) : null}
-        {onRemoveClick ? (
-          <Button
-            minimal
-            onClick={onRemoveClick}
-            style={{
-              color: globals.blue,
-              cursor: "pointer",
-              marginLeft: 7,
-            }}
-          >
-            remove
-          </Button>
-        ) : null}
-        <Tooltip
-          content="Use as color scale"
-          position="bottom"
-          hoverOpenDelay={globals.tooltipHoverOpenDelay}
-        >
-          <Button
-            onClick={memoizedColorByCallback}
-            active={isColorBy}
-            intent={isColorBy ? "primary" : "none"}
-            data-testclass="colorby"
-            data-testid={`colorby-${fieldId}`}
-            icon="tint"
-          />
-        </Tooltip>
-      </div>
-    );
-  }
-);
-
-export default HistogramHeader;

client/src/components/brushableHistogram/histogram.js

@@ -1,191 +0,0 @@
-import React, { useEffect, useRef, useState } from "react";
-import { interpolateCool } from "d3-scale-chromatic";
-import * as d3 from "d3";
-
-import maybeScientific from "../../util/maybeScientific";
-import clamp from "../../util/clamp";
-
-const Histogram = ({
-  field,
-  fieldForId,
-  display,
-  histogram,
-  width,
-  height,
-  onBrush,
-  onBrushEnd,
-  margin,
-  isColorBy,
-  selectionRange,
-  mini,
-}) => {
-  const svgRef = useRef(null);
-  const [brush, setBrush] = useState(null);
-
-  useEffect(() => {
-    /*
-      Create the d3 histogram
-      */
-    //  This is just a constant that's flipped by parent's `mini` boolean
-    const {
-      LEFT: marginLeft,
-      RIGHT: marginRight,
-      BOTTOM: marginBottom,
-      TOP: marginTop,
-    } = margin;
-    const { x, y, bins, binStart, binEnd, binWidth } = histogram;
-    const svg = d3.select(svgRef.current);
-    const binPadding = mini ? 0 : -1;
-    const defaultBarColor = mini ? "black" : "#bbb";
-
-    /* Remove everything */
-    svg.selectAll("*").remove();
-
-    /* Set margins within the SVG */
-    const container = svg
-      .attr("width", width + marginLeft + marginRight)
-      .attr("height", height + marginTop + marginBottom)
-      .append("g")
-      .attr("class", "histogram-container")
-      .attr("transform", `translate(${marginLeft},${marginTop})`);
-
-    const colorScale = d3
-      .scaleSequential(interpolateCool)
-      .domain([0, bins.length]);
-
-    const histogramScale = d3
-      .scaleLinear()
-      .domain(x.domain())
-      .range([
-        colorScale.domain()[1],
-        colorScale.domain()[0],
-      ]); /* we flip this to make colors dark if high in the color scale */
-
-    if (binWidth > 0) {
-      /* BINS */
-      container
-        .insert("g", "*")
-        .selectAll("rect")
-        .data(bins)
-        .enter()
-        .append("rect")
-        .attr("x", (d, i) => x(binStart(i)) + 1)
-        .attr("y", (d) => y(d))
-        .attr("width", (d, i) => x(binEnd(i)) - x(binStart(i)) - binPadding)
-        .attr("height", (d) => y(0) - y(d))
-        .style(
-          "fill",
-          isColorBy
-            ? (d, i) => colorScale(histogramScale(binStart(i)))
-            : defaultBarColor
-        );
-    }
-
-    if (!mini) {
-      // BRUSH
-      // Note the brushable area is bounded by the data on three sides, but goes down to cover the x-axis
-      const brushX = d3
-        .brushX()
-        .extent([
-          [x.range()[0], y.range()[1]],
-          [x.range()[1], marginTop + height + marginBottom],
-        ])
-        /*
-        emit start so that the Undoable history can save an undo point
-        upon drag start, and ignore the subsequent intermediate drag events.
-        */
-        .on("start", onBrush(field, x.invert, "start"))
-        .on("brush", onBrush(field, x.invert, "brush"))
-        .on("end", onBrushEnd(field, x.invert));
-
-      const brushXselection = container
-        .insert("g")
-        .attr("class", "brush")
-        .attr("data-testid", `${svgRef.current.dataset.testid}-brushable-area`)
-        .call(brushX);
-
-      /* X AXIS */
-      container
-        .insert("g")
-        .attr("class", "axis axis--x")
-        .attr("transform", `translate(0,${marginTop + height})`)
-        .call(
-          d3
-            .axisBottom(x)
-            .ticks(4)
-            .tickFormat(d3.format(maybeScientific(x)))
-        );
-
-      /* Y AXIS */
-      container
-        .insert("g")
-        .attr("class", "axis axis--y")
-        .attr("transform", `translate(${marginLeft + width},0)`)
-        .call(
-          d3
-            .axisRight(y)
-            .ticks(3)
-            .tickFormat(
-              d3.format(
-                y.domain().some((n) => Math.abs(n) >= 10000) ? ".0e" : ","
-              )
-            )
-        );
-
-      /* axis style */
-      svg.selectAll(".axis text").style("fill", "rgb(80,80,80)");
-      svg.selectAll(".axis path").style("stroke", "rgb(230,230,230)");
-      svg.selectAll(".axis line").style("stroke", "rgb(230,230,230)");
-
-      setBrush({ brushX, brushXselection });
-    }
-  }, [histogram, isColorBy]);
-
-  useEffect(() => {
-    /*
-      paint/update selection brush
-      */
-    if (!brush) return;
-    const { brushX, brushXselection } = brush;
-    const selection = d3.brushSelection(brushXselection.node());
-    if (!selectionRange && selection) {
-      /* no active selection - clear brush */
-      brushXselection.call(brushX.move, null);
-    } else if (selectionRange) {
-      const { x, domain } = histogram;
-      const [min, max] = domain;
-      const x0 = x(clamp(selectionRange[0], [min, max]));
-      const x1 = x(clamp(selectionRange[1], [min, max]));
-      if (!selection) {
-        /* there is an active selection, but no brush - set the brush */
-        brushXselection.call(brushX.move, [x0, x1]);
-      } else {
-        /* there is an active selection and a brush - make sure they match */
-        const moveDeltaThreshold = 1;
-        const dX0 = Math.abs(x0 - selection[0]);
-        const dX1 = Math.abs(x1 - selection[1]);
-        /*
-          only update the brush if it is grossly incorrect,
-          as defined by the moveDeltaThreshold
-          */
-        if (dX0 > moveDeltaThreshold || dX1 > moveDeltaThreshold) {
-          brushXselection.call(brushX.move, [x0, x1]);
-        }
-      }
-    }
-  }, [brush, selectionRange]);
-
-  return (
-    <svg
-      style={{ display }}
-      width={width}
-      height={height}
-      id={`histogram_${fieldForId}_svg`}
-      data-testclass="histogram-plot"
-      data-testid={`histogram-${field}-plot`}
-      ref={svgRef}
-    />
-  );
-};
-
-export default Histogram;

client/src/components/brushableHistogram/index.js

@@ -1,38 +1,447 @@
-import React from "react";
+/*
-import { connect, shallowEqual } from "react-redux";
+https://bl.ocks.org/mbostock/4341954
+https://bl.ocks.org/mbostock/34f08d5e11952a80609169b7917d4172
+https://bl.ocks.org/SpaceActuary/2f004899ea1b2bd78d6f1dbb2febf771
+https://bl.ocks.org/mbostock/3019563
+*/
+import React, { useEffect, useRef, useState, useCallback } from "react";
+import { Button, ButtonGroup, Icon, Tooltip } from "@blueprintjs/core";
+import { connect } from "react-redux";
 import * as d3 from "d3";
+import { interpolateCool } from "d3-scale-chromatic";
 import Async from "react-async";
 import memoize from "memoize-one";
+import { IconNames } from "@blueprintjs/icons";
 import * as globals from "../../globals";
 import actions from "../../actions";
+import { histogramContinuous } from "../../util/dataframe/histogram";
 import { makeContinuousDimensionName } from "../../util/nameCreators";
-import HistogramHeader from "./header";
+import significantDigits from "../../util/significantDigits";
-import Histogram from "./histogram";
-import HistogramFooter from "./footer";
-import StillLoading from "./loading";
-import ErrorLoading from "./error";
 
-const MARGIN = {
+function clamp(val, rng) {
-  LEFT: 10, // Space for 0 tick label on X axis
+  return Math.max(Math.min(val, rng[1]), rng[0]);
-  RIGHT: 54, // space for Y axis & labels
+}
-  BOTTOM: 25, // space for X axis & labels
+
-  TOP: 3,
+function maybeScientific(x) {
+  let format = ",";
+  const _ticks = x.ticks(4);
+
+  if (x.domain().some((n) => Math.abs(n) >= 10000)) {
+    /*
+      heuristic: if the last tick d3 wants to render has one significant
+      digit ie., 2000, render 2e+3, but if it's anything else ie., 42000000 render
+      4.20e+n
+    */
+    format = significantDigits(_ticks[_ticks.length - 1]) === 1 ? ".0e" : ".2e";
+  }
+
+  return format;
+}
+
+const StillLoading = ({ zebra, displayName }) => {
+  /*
+  Render a loading indicator for the field.
+  */
+  return (
+    <div
+      style={{
+        padding: globals.leftSidebarSectionPadding,
+        backgroundColor: zebra ? globals.lightestGrey : "white",
+      }}
+    >
+      <div
+        style={{
+          display: "flex",
+          justifyContent: "space-between",
+          justifyItems: "center",
+          alignItems: "center",
+        }}
+      >
+        <div style={{ minWidth: 30 }} />
+        <div style={{ display: "flex", alignSelf: "center" }}>
+          <span style={{ fontStyle: "italic" }}>{displayName}</span>
+        </div>
+        <div
+          style={{
+            display: "flex",
+            justifyContent: "flex-end",
+          }}
+        >
+          <Button minimal loading intent="primary" />
+        </div>
+      </div>
+    </div>
+  );
 };
-const WIDTH = 340 - MARGIN.LEFT - MARGIN.RIGHT;
+
-const HEIGHT = 135 - MARGIN.TOP - MARGIN.BOTTOM;
+const ErrorLoading = ({ displayName, error, zebra }) => {
-const MARGIN_MINI = {
+  console.log(error); // log to console as this is unexpected
-  LEFT: 0, // Space for 0 tick label on X axis
+  return (
-  RIGHT: 0, // space for Y axis & labels
+    <div
-  BOTTOM: 0, // space for X axis & labels
+      style={{
-  TOP: 0,
+        padding: globals.leftSidebarSectionPadding,
+        backgroundColor: zebra ? globals.lightestGrey : "white",
+      }}
+    >
+      <span>{`Failure loading ${displayName}`}</span>
+    </div>
+  );
+};
+
+const HistogramFooter = React.memo(
+  ({
+    displayName,
+    hideRanges,
+    rangeMin,
+    rangeMax,
+    rangeColorMin,
+    rangeColorMax,
+    logFoldChange,
+    pvalAdj,
+  }) => {
+    /*
+  Footer of each histogram.  Will render range, title, and optionally
+  differential expression info.
+
+  Required props:
+    * displayName - the displayName, aka "n_genes", "FOXP2", etc.
+    * hideRanges - true/false, enables/disable rendering of ranges
+    * range - length two array, [min, max], containing the range values to display
+    * rangeColor - length two array, [mincolor, maxcolor], each a CSS color
+    * logFoldChange - lfc to display, optional.
+    * pValue - pValue to display, optional.
+  */
+    return (
+      <div>
+        <div
+          style={{
+            display: "flex",
+            justifyContent: hideRanges ? "center" : "space-between",
+          }}
+        >
+          <span
+            style={{
+              color: rangeColorMin,
+              display: hideRanges ? "none" : "block",
+            }}
+          >
+            min {rangeMin.toPrecision(4)}
+          </span>
+          <span
+            data-testclass="brushable-histogram-field-name"
+            style={{ fontStyle: "italic" }}
+          >
+            {displayName}
+          </span>
+          <div style={{ display: hideRanges ? "block" : "none" }}>
+            : {rangeMin}
+          </div>
+          <span
+            style={{
+              color: rangeColorMax,
+              display: hideRanges ? "none" : "block",
+            }}
+          >
+            max {rangeMax.toPrecision(4)}
+          </span>
+        </div>
+
+        {logFoldChange !== undefined && pvalAdj !== undefined ? (
+          <div
+            style={{
+              display: "flex",
+              justifyContent: "center",
+              alignItems: "baseline",
+            }}
+          >
+            <span>
+              <strong>log fold change:</strong>
+              {` ${logFoldChange.toPrecision(4)}`}
+            </span>
+            <span
+              style={{
+                marginLeft: 7,
+                padding: 2,
+              }}
+            >
+              <strong>p-value (adj):</strong>
+              {pvalAdj < 0.0001 ? " < 0.0001" : ` ${pvalAdj.toFixed(4)}`}
+            </span>
+          </div>
+        ) : null}
+      </div>
+    );
+  }
+);
+
+const HistogramHeader = React.memo(
+  ({
+    fieldId,
+    isColorBy,
+    onColorByClick,
+    onRemoveClick,
+    isScatterPlotX,
+    isScatterPlotY,
+    onScatterPlotXClick,
+    onScatterPlotYClick,
+    isObs,
+  }) => {
+    /*
+      Render the toolbar for the histogram.  Props:
+        * fieldId - field identifier, used for various IDs
+        * isColorBy - true/false, is this the current color-by
+        * onColorByClick - color-by click handler
+        * onRemoveClick - optional handler for remove.  Button will not render if not defined.
+        * isScatterPlotX - optional, true/false if currently the X scatterplot field
+        * isScatterPlotY - optional, true/false if currently the Y scatterplot field
+        * onScatterPlotXClick - optional, handler for scatterPlot X button.
+        * onScatterPlotYClick - optional, handler for scatterPlot X button.
+
+      Scatterplot controls will not render if either handler unspecified.
+    */
+
+    const memoizedColorByCallback = useCallback(
+      () => onColorByClick(fieldId, isObs),
+      [fieldId, isObs]
+    );
+
+    return (
+      <div
+        style={{
+          display: "flex",
+          justifyContent: "flex-end",
+          paddingBottom: "8px",
+        }}
+      >
+        {onScatterPlotXClick && onScatterPlotYClick ? (
+          <span>
+            <Icon icon={IconNames.SCATTER_PLOT} style={{ marginRight: 7 }} />
+            <ButtonGroup style={{ marginRight: 7 }}>
+              <Button
+                data-testid={`plot-x-${fieldId}`}
+                onClick={onScatterPlotXClick}
+                active={isScatterPlotX}
+                intent={isScatterPlotX ? "primary" : "none"}
+              >
+                plot x
+              </Button>
+              <Button
+                data-testid={`plot-y-${fieldId}`}
+                onClick={onScatterPlotYClick}
+                active={isScatterPlotY}
+                intent={isScatterPlotY ? "primary" : "none"}
+              >
+                plot y
+              </Button>
+            </ButtonGroup>
+          </span>
+        ) : null}
+        {onRemoveClick ? (
+          <Button
+            minimal
+            onClick={onRemoveClick}
+            style={{
+              color: globals.blue,
+              cursor: "pointer",
+              marginLeft: 7,
+            }}
+          >
+            remove
+          </Button>
+        ) : null}
+        <Tooltip
+          content="Use as color scale"
+          position="bottom"
+          hoverOpenDelay={globals.tooltipHoverOpenDelay}
+        >
+          <Button
+            onClick={memoizedColorByCallback}
+            active={isColorBy}
+            intent={isColorBy ? "primary" : "none"}
+            data-testclass="colorby"
+            data-testid={`colorby-${fieldId}`}
+            icon="tint"
+          />
+        </Tooltip>
+      </div>
+    );
+  }
+);
+
+const Histogram = ({
+  field,
+  fieldForId,
+  display,
+  histogram,
+  width,
+  height,
+  onBrush,
+  onBrushEnd,
+  margin,
+  isColorBy,
+  selectionRange,
+}) => {
+  const svgRef = useRef(null);
+  const [brush, setBrush] = useState(null);
+
+  useEffect(() => {
+    /*
+    Create the d3 histogram
+    */
+    const { marginLeft, marginRight, marginBottom, marginTop } = margin;
+    const { x, y, bins, binStart, binEnd, binWidth } = histogram;
+    const svg = d3.select(svgRef.current);
+
+    /* Remove everything */
+    svg.selectAll("*").remove();
+
+    /* Set margins within the SVG */
+    const container = svg
+      .attr("width", width + marginLeft + marginRight)
+      .attr("height", height + marginTop + marginBottom)
+      .append("g")
+      .attr("class", "histogram-container")
+      .attr("transform", `translate(${marginLeft},${marginTop})`);
+
+    const colorScale = d3
+      .scaleSequential(interpolateCool)
+      .domain([0, bins.length]);
+
+    const histogramScale = d3
+      .scaleLinear()
+      .domain(x.domain())
+      .range([
+        colorScale.domain()[1],
+        colorScale.domain()[0],
+      ]); /* we flip this to make colors dark if high in the color scale */
+
+    if (binWidth > 0) {
+      /* BINS */
+      container
+        .insert("g", "*")
+        .selectAll("rect")
+        .data(bins)
+        .enter()
+        .append("rect")
+        .attr("x", (d, i) => x(binStart(i)) + 1)
+        .attr("y", (d) => y(d))
+        .attr("width", (d, i) => x(binEnd(i)) - x(binStart(i)) - 1)
+        .attr("height", (d) => y(0) - y(d))
+        .style(
+          "fill",
+          isColorBy ? (d, i) => colorScale(histogramScale(binStart(i))) : "#bbb"
+        );
+    }
+
+    // BRUSH
+    // Note the brushable area is bounded by the data on three sides, but goes down to cover the x-axis
+    const brushX = d3
+      .brushX()
+      .extent([
+        [x.range()[0], y.range()[1]],
+        [x.range()[1], marginTop + height + marginBottom],
+      ])
+      /*
+      emit start so that the Undoable history can save an undo point
+      upon drag start, and ignore the subsequent intermediate drag events.
+      */
+      .on("start", onBrush(field, x.invert, "start"))
+      .on("brush", onBrush(field, x.invert, "brush"))
+      .on("end", onBrushEnd(field, x.invert));
+
+    const brushXselection = container
+      .insert("g")
+      .attr("class", "brush")
+      .attr("data-testid", `${svgRef.current.dataset.testid}-brushable-area`)
+      .call(brushX);
+
+    /* X AXIS */
+    container
+      .insert("g")
+      .attr("class", "axis axis--x")
+      .attr("transform", `translate(0,${marginTop + height})`)
+      .call(
+        d3
+          .axisBottom(x)
+          .ticks(4)
+          .tickFormat(d3.format(maybeScientific(x)))
+      );
+
+    /* Y AXIS */
+    container
+      .insert("g")
+      .attr("class", "axis axis--y")
+      .attr("transform", `translate(${marginLeft + width},0)`)
+      .call(
+        d3
+          .axisRight(y)
+          .ticks(3)
+          .tickFormat(
+            d3.format(
+              y.domain().some((n) => Math.abs(n) >= 10000) ? ".0e" : ","
+            )
+          )
+      );
+
+    /* axis style */
+    svg.selectAll(".axis text").style("fill", "rgb(80,80,80)");
+    svg.selectAll(".axis path").style("stroke", "rgb(230,230,230)");
+    svg.selectAll(".axis line").style("stroke", "rgb(230,230,230)");
+
+    setBrush({ brushX, brushXselection });
+  }, [histogram, isColorBy]);
+
+  useEffect(() => {
+    /*
+    paint/update selection brush
+    */
+    if (!brush) return;
+    const { brushX, brushXselection } = brush;
+    const selection = d3.brushSelection(brushXselection.node());
+    if (!selectionRange && selection) {
+      /* no active selection - clear brush */
+      brushXselection.call(brushX.move, null);
+    } else if (selectionRange) {
+      const { x, domain } = histogram;
+      const [min, max] = domain;
+      const x0 = x(clamp(selectionRange[0], [min, max]));
+      const x1 = x(clamp(selectionRange[1], [min, max]));
+      if (!selection) {
+        /* there is an active selection, but no brush - set the brush */
+        brushXselection.call(brushX.move, [x0, x1]);
+      } else {
+        /* there is an active selection and a brush - make sure they match */
+        const moveDeltaThreshold = 1;
+        const dX0 = Math.abs(x0 - selection[0]);
+        const dX1 = Math.abs(x1 - selection[1]);
+        /*
+        only update the brush if it is grossly incorrect,
+        as defined by the moveDeltaThreshold
+        */
+        if (dX0 > moveDeltaThreshold || dX1 > moveDeltaThreshold) {
+          brushXselection.call(brushX.move, [x0, x1]);
+        }
+      }
+    }
+  }, [brush, selectionRange]);
+
+  return (
+    <svg
+      style={{ display }}
+      width={width}
+      height={height}
+      id={`histogram_${fieldForId}_svg`}
+      data-testclass="histogram-plot"
+      data-testid={`histogram-${field}-plot`}
+      ref={svgRef}
+    />
+  );
 };
-const WIDTH_MINI = 120 - MARGIN_MINI.LEFT - MARGIN_MINI.RIGHT;
-const HEIGHT_MINI = 15 - MARGIN_MINI.TOP - MARGIN_MINI.BOTTOM;
 
 @connect((state, ownProps) => {
-  const { isObs, isUserDefined, isGeneSetSummary, field } = ownProps;
+  const { isObs, isUserDefined, isDiffExp, field } = ownProps;
   const myName = makeContinuousDimensionName(
-    { isObs, isUserDefined, isGeneSetSummary },
+    { isObs, isUserDefined, isDiffExp },
     field
   );
   return {
@@ -44,10 +453,6 @@ const HEIGHT_MINI = 15 - MARGIN_MINI.TOP - MARGIN_MINI.BOTTOM;
   };
 })
 class HistogramBrush extends React.PureComponent {
-  static watchAsync(props, prevProps) {
-    return !shallowEqual(props.watchProps, prevProps.watchProps);
-  }
-
   /* memoized closure to prevent HistogramHeader unecessary repaint */
   handleColorAction = memoize((dispatch) => (field, isObs) => {
     if (isObs) {
@@ -60,11 +465,27 @@ class HistogramBrush extends React.PureComponent {
     }
   });
 
+  constructor(props) {
+    super(props);
+
+    const marginLeft = 10; // Space for 0 tick label on X axis
+    const marginRight = 54; // space for Y axis & labels
+    const marginBottom = 25; // space for X axis & labels
+    const marginTop = 3;
+    this.margin = {
+      marginLeft,
+      marginRight,
+      marginBottom,
+      marginTop,
+    };
+    this.width = 340 - marginLeft - marginRight;
+    this.height = 135 - marginTop - marginBottom;
+  }
+
   onBrush = (selection, x, eventType) => {
     const type = `continuous metadata histogram ${eventType}`;
     return () => {
-      const { dispatch, field, isObs, isUserDefined, isGeneSetSummary } =
+      const { dispatch, field, isObs, isUserDefined, isDiffExp } = this.props;
-        this.props;
 
       // ignore programmatically generated events
       if (!d3.event.sourceEvent) return;
@@ -80,7 +501,7 @@ class HistogramBrush extends React.PureComponent {
         continuousNamespace: {
           isObs,
           isUserDefined,
-          isGeneSetSummary,
+          isDiffExp,
         },
       };
       dispatch(
@@ -89,9 +510,9 @@ class HistogramBrush extends React.PureComponent {
     };
   };
 
-  onBrushEnd = (selection, x) => () => {
+  onBrushEnd = (selection, x) => {
-      const { dispatch, field, isObs, isUserDefined, isGeneSetSummary } =
+    return () => {
-        this.props;
+      const { dispatch, field, isObs, isUserDefined, isDiffExp } = this.props;
       const minAllowedBrushSize = 10;
       const smallAmountToAvoidInfiniteLoop = 0.1;
 
@@ -130,13 +551,14 @@ class HistogramBrush extends React.PureComponent {
         continuousNamespace: {
           isObs,
           isUserDefined,
-          isGeneSetSummary,
+          isDiffExp,
         },
       };
       dispatch(
         actions.selectContinuousMetadataAction(type, query, range, otherProps)
       );
     };
+  };
 
   handleSetGeneAsScatterplotX = () => {
     const { dispatch, field } = this.props;
@@ -186,8 +608,7 @@ class HistogramBrush extends React.PureComponent {
   };
 
   fetchAsyncProps = async () => {
-    const { annoMatrix, width } = this.props;
+    const { annoMatrix } = this.props;
-
     const { isClipped } = annoMatrix;
 
     const query = this.createQuery();
@@ -217,15 +638,9 @@ class HistogramBrush extends React.PureComponent {
 
     const histogram = this.calcHistogramCache(
       column,
-      MARGIN,
+      this.margin,
-      width || WIDTH,
+      this.width,
-      HEIGHT
+      this.height
-    );
-    const miniHistogram = this.calcHistogramCache(
-      column,
-      MARGIN_MINI,
-      width || WIDTH_MINI,
-      HEIGHT_MINI
     );
 
     const isSingleValue = summary.min === summary.max;
@@ -239,7 +654,6 @@ class HistogramBrush extends React.PureComponent {
 
     return {
       histogram,
-      miniHistogram,
       range,
       unclippedRange,
       unclippedRangeColor,
@@ -249,33 +663,27 @@ class HistogramBrush extends React.PureComponent {
   };
 
   // eslint-disable-next-line class-methods-use-this -- instance method allows for memoization per annotation
-  calcHistogramCache(col, newMargin, newWidth, newHeight) {
+  calcHistogramCache(col, margin, width, height) {
     /*
      recalculate expensive stuff, notably bins, summaries, etc.
     */
-    const histogramCache = {}; /* maybe change this so that it computes ... */
+    const histogramCache = {};
-    const summary =
+    const summary = col.summarize();
-      col.summarize(); /* this is memoized, so it's free the second time you call it */
     const { min: domainMin, max: domainMax } = summary;
     const numBins = 40;
-    const { TOP: topMargin, LEFT: leftMargin } =
+    const { marginTop, marginLeft } = margin;
-      newMargin; /* changes with mini */
 
-    histogramCache.domain = [
+    histogramCache.domain = [domainMin, domainMax];
-      domainMin,
-      domainMax,
-    ]; /* doesn't change with mini */
 
     histogramCache.x = d3
       .scaleLinear()
       .domain([domainMin, domainMax])
-      .range([leftMargin, leftMargin + newWidth]);
+      .range([marginLeft, marginLeft + width]);
 
-    histogramCache.bins = col.histogram(numBins, [
+    histogramCache.bins = histogramContinuous(col, numBins, [
       domainMin,
       domainMax,
-    ]); /* memoized */
+    ]);
-
     histogramCache.binWidth = (domainMax - domainMin) / numBins;
 
     histogramCache.binStart = (i) => domainMin + i * histogramCache.binWidth;
@@ -287,43 +695,25 @@ class HistogramBrush extends React.PureComponent {
     histogramCache.y = d3
       .scaleLinear()
       .domain([0, yMax])
-      .range([topMargin + newHeight, topMargin]);
+      .range([marginTop + height, marginTop]);
 
     return histogramCache;
   }
 
   createQuery() {
-    const { isObs, isGeneSetSummary, field, setGenes, annoMatrix } = this.props;
+    const { isObs, field, annoMatrix } = this.props;
     const { schema } = annoMatrix;
     if (isObs) {
       return ["obs", field];
     }
     const varIndex = schema?.annotations?.var?.index;
     if (!varIndex) return null;
-
-    if (isGeneSetSummary) {
-      return [
-        "X",
-        {
-          summarize: {
-            method: "mean",
-            field: "var",
-            column: varIndex,
-            values: [...setGenes.keys()],
-          },
-        },
-      ];
-    }
-
-    // else, we assume it is a gene expression
     return [
       "X",
       {
-        where: {
+        field: "var",
-          field: "var",
+        column: varIndex,
-          column: varIndex,
+        value: field,
-          value: field,
-        },
       },
     ];
   }
@@ -335,34 +725,20 @@ class HistogramBrush extends React.PureComponent {
       field,
       isColorAccessor,
       isUserDefined,
-      isGeneSetSummary,
+      isDiffExp,
+      logFoldChange,
+      pvalAdj,
       isScatterplotXXaccessor,
       isScatterplotYYaccessor,
       zebra,
       continuousSelectionRange,
       isObs,
-      mini,
-      setGenes,
     } = this.props;
-
-    let { width } = this.props;
-    if (!width) {
-      width = mini ? WIDTH_MINI : WIDTH;
-    }
-
     const fieldForId = field.replace(/\s/g, "_");
-    const showScatterPlot = isUserDefined;
+    const showScatterPlot = isDiffExp || isUserDefined;
-
-    let testClass = "histogram-continuous-metadata";
-    if (isUserDefined) testClass = "histogram-user-gene";
-    else if (isGeneSetSummary) testClass = "histogram-gene-set-summary";
 
     return (
-      <Async
+      <Async watch={annoMatrix} promiseFn={this.fetchAsyncProps}>
-        watchFn={HistogramBrush.watchAsync}
-        promiseFn={this.fetchAsyncProps}
-        watchProps={{ annoMatrix, setGenes }}
-      >
         <Async.Pending initial>
           <StillLoading displayName={field} zebra={zebra} />
         </Async.Pending>
@@ -377,57 +753,56 @@ class HistogramBrush extends React.PureComponent {
               <div
                 id={`histogram_${fieldForId}`}
                 data-testid={`histogram-${field}`}
-                data-testclass={testClass}
+                data-testclass={
+                  isDiffExp
+                    ? "histogram-diffexp"
+                    : isUserDefined
+                    ? "histogram-user-gene"
+                    : "histogram-continuous-metadata"
+                }
                 style={{
-                  padding: mini ? 0 : globals.leftSidebarSectionPadding,
+                  padding: globals.leftSidebarSectionPadding,
                   backgroundColor: zebra ? globals.lightestGrey : "white",
                 }}
               >
-                {!mini && isObs ? (
+                <HistogramHeader
-                  <HistogramHeader
+                  fieldId={field}
-                    fieldId={field}
+                  isColorBy={isColorAccessor}
-                    isColorBy={isColorAccessor}
+                  isObs={isObs}
-                    isObs={isObs}
+                  onColorByClick={this.handleColorAction(dispatch)}
-                    onColorByClick={this.handleColorAction(dispatch)}
+                  onRemoveClick={isUserDefined ? this.removeHistogram : null}
-                    onRemoveClick={isUserDefined ? this.removeHistogram : null}
+                  isScatterPlotX={isScatterplotXXaccessor}
-                    isScatterPlotX={isScatterplotXXaccessor}
+                  isScatterPlotY={isScatterplotYYaccessor}
-                    isScatterPlotY={isScatterplotYYaccessor}
+                  onScatterPlotXClick={
-                    onScatterPlotXClick={
+                    showScatterPlot ? this.handleSetGeneAsScatterplotX : null
-                      showScatterPlot ? this.handleSetGeneAsScatterplotX : null
+                  }
-                    }
+                  onScatterPlotYClick={
-                    onScatterPlotYClick={
+                    showScatterPlot ? this.handleSetGeneAsScatterplotY : null
-                      showScatterPlot ? this.handleSetGeneAsScatterplotY : null
+                  }
-                    }
+                />
-                  />
-                ) : null}
                 <Histogram
                   field={field}
                   fieldForId={fieldForId}
                   display={asyncProps.isSingleValue ? "none" : "block"}
-                  histogram={
+                  histogram={asyncProps.histogram}
-                    mini ? asyncProps.miniHistogram : asyncProps.histogram
+                  width={this.width}
-                  }
+                  height={this.height}
-                  width={width}
-                  height={mini ? HEIGHT_MINI : HEIGHT}
                   onBrush={this.onBrush}
                   onBrushEnd={this.onBrushEnd}
-                  margin={mini ? MARGIN_MINI : MARGIN}
+                  margin={this.margin}
                   isColorBy={isColorAccessor}
                   selectionRange={continuousSelectionRange}
-                  mini={mini}
                 />
-                {!mini && (
+                <HistogramFooter
-                  <HistogramFooter
+                  displayName={field}
-                    isGeneSetSummary={isGeneSetSummary}
+                  hideRanges={asyncProps.isSingleValue}
-                    isObs={isObs}
+                  rangeMin={asyncProps.unclippedRange[0]}
-                    displayName={field}
+                  rangeMax={asyncProps.unclippedRange[1]}
-                    hideRanges={asyncProps.isSingleValue}
+                  rangeColorMin={asyncProps.unclippedRangeColor[0]}
-                    rangeMin={asyncProps.unclippedRange[0]}
+                  rangeColorMax={asyncProps.unclippedRangeColor[1]}
-                    rangeMax={asyncProps.unclippedRange[1]}
+                  logFoldChange={logFoldChange}
-                    rangeColorMin={asyncProps.unclippedRangeColor[0]}
+                  pvalAdj={pvalAdj}
-                    rangeColorMax={asyncProps.unclippedRangeColor[1]}
+                />
-                  />
-                )}
               </div>
             ) : null
           }

client/src/components/brushableHistogram/loading.js

@@ -1,43 +0,0 @@
-import React from "react";
-import { Button } from "@blueprintjs/core";
-
-import * as globals from "../../globals";
-
-const StillLoading = ({ zebra, displayName }) => 
-  /*
-    Render a loading indicator for the field.
-    */
-   (
-    <div
-      data-testclass="gene-loading-spinner"
-      style={{
-        padding: globals.leftSidebarSectionPadding,
-        backgroundColor: zebra ? globals.lightestGrey : "white",
-      }}
-    >
-      <div
-        style={{
-          display: "flex",
-          justifyContent: "space-between",
-          justifyItems: "center",
-          alignItems: "center",
-        }}
-      >
-        <div style={{ minWidth: 30 }} />
-        <div style={{ display: "flex", alignSelf: "center" }}>
-          <span style={{ fontStyle: "italic" }}>{displayName}</span>
-        </div>
-        <div
-          style={{
-            display: "flex",
-            justifyContent: "flex-end",
-          }}
-        >
-          <Button minimal loading intent="primary" />
-        </div>
-      </div>
-    </div>
-  )
-;
-
-export default StillLoading;

client/src/components/categorical/annoDialog.js

@@ -18,8 +18,6 @@ class AnnoDialog extends React.PureComponent {
       validationError,
       annoSelect,
       annoInput,
-      secondaryInstructions,
-      secondaryInput,
       handleCancel,
       handleSubmit,
       primaryButtonText,
@@ -42,19 +40,12 @@ class AnnoDialog extends React.PureComponent {
               <p
                 style={{
                   marginTop: 7,
-                  visibility: errorMessage !== "" ? "visible" : "hidden",
+                  visibility: validationError ? "visible" : "hidden",
                   color: Colors.ORANGE3,
                 }}
               >
                 {errorMessage}
               </p>
-              {/* we might rename, secondary button and secondary input are not related */}
-              {secondaryInstructions && (
-                <p style={{ marginTop: secondaryInstructions ? 20 : 0 }}>
-                  {secondaryInstructions}
-                </p>
-              )}
-              {secondaryInput || null}
             </div>
             {annoSelect || null}
           </div>
@@ -63,7 +54,6 @@ class AnnoDialog extends React.PureComponent {
               <Tooltip content={cancelTooltipContent}>
                 <Button onClick={handleCancel}>Cancel</Button>
               </Tooltip>
-              {/* we might rename, secondary button and secondary input are not related */}
               {handleSecondaryButtonSubmit && secondaryButtonText ? (
                 <Button
                   onClick={handleSecondaryButtonSubmit}

client/src/components/categorical/annoSelect.js

@@ -26,14 +26,16 @@ class DuplicateCategorySelect extends React.PureComponent {
             [] /* this is a placeholder, could be  a subcomponent to avoid this */
           }
           filterable={false}
-          itemRenderer={(d, { handleClick }) => (
+          itemRenderer={(d, { handleClick }) => {
+            return (
               <MenuItem
                 data-testclass="duplicate-category-dropdown-option"
                 onClick={handleClick}
                 key={d}
                 text={d}
               />
-            )}
+            );
+          }}
           noResults={<MenuItem disabled text="No results." />}
           onItemSelect={(d) => {
             handleModalDuplicateCategorySelection(d);

client/src/components/categorical/category/annoDialogAddLabel.js

@@ -1,13 +1,14 @@
 import React from "react";
 import { connect } from "react-redux";
-import AnnoDialog from "../../annoDialog";
+import AnnoDialog from "../annoDialog";
-import LabelInput from "../../labelInput";
+import LabelInput from "../labelInput";
 import { labelPrompt, isLabelErroneous } from "../labelUtil";
 import actions from "../../../actions";
 
 @connect((state) => ({
   annotations: state.annotations,
   schema: state.annoMatrix?.schema,
+  ontology: state.ontology,
   obsCrossfilter: state.obsCrossfilter,
 }))
 class Category extends React.PureComponent {
@@ -56,11 +57,13 @@ class Category extends React.PureComponent {
   };
 
   labelNameError = (name) => {
-    const { metadataField, schema } = this.props;
+    const { metadataField, ontology, schema } = this.props;
-    return isLabelErroneous(name, metadataField, schema);
+    return isLabelErroneous(name, metadataField, ontology, schema);
   };
 
-  instruction = (label) => labelPrompt(this.labelNameError(label), "New, unique label", ":");
+  instruction = (label) => {
+    return labelPrompt(this.labelNameError(label), "New, unique label", ":");
+  };
 
   handleChangeOrSelect = (label) => {
     this.setState({ newLabelText: label });
@@ -68,7 +71,8 @@ class Category extends React.PureComponent {
 
   render() {
     const { newLabelText } = this.state;
-    const { metadataField, annotations, obsCrossfilter } = this.props;
+    const { metadataField, annotations, ontology, obsCrossfilter } = this.props;
+    const ontologyEnabled = ontology?.enabled ?? false;
 
     return (
       <>
@@ -93,7 +97,7 @@ class Category extends React.PureComponent {
           handleCancel={this.disableAddNewLabelMode}
           annoInput={
             <LabelInput
-              labelSuggestions={null}
+              labelSuggestions={ontologyEnabled ? ontology.terms : null}
               onChange={this.handleChangeOrSelect}
               onSelect={this.handleChangeOrSelect}
               inputProps={{

client/src/components/categorical/category/annoDialogEditCategoryName.js

@@ -1,7 +1,7 @@
 import React from "react";
 import { connect } from "react-redux";
-import AnnoDialog from "../../annoDialog";
+import AnnoDialog from "../annoDialog";
-import LabelInput from "../../labelInput";
+import LabelInput from "../labelInput";
 import { labelPrompt } from "../labelUtil";
 
 import { AnnotationsHelpers } from "../../../util/stateManager";
@@ -10,6 +10,7 @@ import actions from "../../../actions";
 @connect((state) => ({
   annotations: state.annotations,
   schema: state.annoMatrix?.schema,
+  ontology: state.ontology,
 }))
 class AnnoDialogEditCategoryName extends React.PureComponent {
   constructor(props) {
@@ -89,11 +90,13 @@ class AnnoDialogEditCategoryName extends React.PureComponent {
     return false;
   };
 
-  instruction = (name) => labelPrompt(
+  instruction = (name) => {
+    return labelPrompt(
       this.editedCategoryNameError(name),
       "New, unique category name",
       ":"
     );
+  };
 
   allCategoryNames() {
     const { schema } = this.props;
@@ -102,7 +105,8 @@ class AnnoDialogEditCategoryName extends React.PureComponent {
 
   render() {
     const { newCategoryText } = this.state;
-    const { metadataField, annotations } = this.props;
+    const { metadataField, annotations, ontology } = this.props;
+    const ontologyEnabled = ontology?.enabled ?? false;
 
     return (
       <>
@@ -128,7 +132,7 @@ class AnnoDialogEditCategoryName extends React.PureComponent {
           annoInput={
             <LabelInput
               label={newCategoryText}
-              labelSuggestions={null}
+              labelSuggestions={ontologyEnabled ? ontology.terms : null}
               onChange={this.handleChangeOrSelect}
               onSelect={this.handleChangeOrSelect}
               inputProps={{

client/src/components/categorical/category/annoMenuCategory.js

@@ -9,9 +9,7 @@ import {
   Tooltip,
   Icon,
   PopoverInteractionKind,
-  Intent,
 } from "@blueprintjs/core";
-import { IconNames } from "@blueprintjs/icons";
 
 import * as globals from "../../../globals";
 import actions from "../../../actions";
@@ -91,8 +89,8 @@ class AnnoMenuCategory extends React.PureComponent {
                     text={editText}
                   />
                   <MenuItem
-                    icon={IconNames.TRASH}
+                    icon="delete"
-                    intent={Intent.DANGER}
+                    intent="danger"
                     data-testclass="handleDeleteCategory"
                     data-testid={`${metadataField}:delete-category`}
                     onClick={this.handleDeleteCategory}

client/src/components/categorical/category/index.js

@@ -44,7 +44,6 @@ const LABEL_WIDTH_ANNO = LABEL_WIDTH - ANNO_BUTTON_WIDTH;
     schema,
     crossfilter: state.obsCrossfilter,
     isUserAnno,
-    genesets: state.genesets.genesets,
   };
 })
 class Category extends React.PureComponent {
@@ -143,15 +142,9 @@ class Category extends React.PureComponent {
     */
     const { schema } = annoMatrix;
     const { colorAccessor, colorMode } = colors;
-    const { genesets } = this.props;
     let colorDataPromise = Promise.resolve(null);
     if (colorAccessor) {
-      const query = createColorQuery(
+      const query = createColorQuery(colorMode, colorAccessor, schema);
-        colorMode,
-        colorAccessor,
-        schema,
-        genesets
-      );
       if (query) colorDataPromise = annoMatrix.fetch(...query);
     }
     const [categoryData, colorData] = await Promise.all([
@@ -258,12 +251,14 @@ class Category extends React.PureComponent {
                 isColorAccessor,
                 handleCategoryToggleAllClick,
               } = asyncProps;
+              const isTruncated = !!categorySummary?.isTruncated;
               const selectionState = this.getSelectionState(categorySummary);
               return (
                 <CategoryRender
                   metadataField={metadataField}
                   checkboxID={checkboxID}
                   isUserAnno={isUserAnno}
+                  isTruncated={isTruncated}
                   isExpanded={isExpanded}
                   isColorAccessor={isColorAccessor}
                   selectionState={selectionState}
@@ -288,54 +283,57 @@ class Category extends React.PureComponent {
 
 export default Category;
 
-const StillLoading = ({ metadataField, checkboxID }) => (
+const StillLoading = ({ metadataField, checkboxID }) => {
   /*
   We are still loading this category, so render a "busy" signal.
   */
-  <div
+  return (
-    style={{
-      maxWidth: globals.maxControlsWidth,
-    }}
-  >
     <div
       style={{
-        display: "flex",
+        maxWidth: globals.maxControlsWidth,
-        justifyContent: "space-between",
-        alignItems: "baseline",
       }}
     >
       <div
         style={{
           display: "flex",
-          justifyContent: "flex-start",
+          justifyContent: "space-between",
-          alignItems: "flex-start",
+          alignItems: "baseline",
         }}
       >
-        <label
+        <div
-          htmlFor={checkboxID}
+          style={{
-          className={`${Classes.CONTROL} ${Classes.CHECKBOX}`}
+            display: "flex",
+            justifyContent: "flex-start",
+            alignItems: "flex-start",
+          }}
         >
-          <input disabled id={checkboxID} checked type="checkbox" />
+          <label
-          <span className={Classes.CONTROL_INDICATOR} />
+            htmlFor={checkboxID}
-        </label>
+            className={`${Classes.CONTROL} ${Classes.CHECKBOX}`}
-        <Truncate>
-          <span
-            style={{
-              cursor: "pointer",
-              display: "inline-block",
-              width: LABEL_WIDTH,
-            }}
           >
-            {metadataField}
+            <input disabled id={checkboxID} checked type="checkbox" />
-          </span>
+            <span className={Classes.CONTROL_INDICATOR} />
-        </Truncate>
+          </label>
-      </div>
+          <Truncate>
-      <div>
+            <span
-        <Button minimal loading intent="primary" />
+              style={{
+                cursor: "pointer",
+                display: "inline-block",
+                width: LABEL_WIDTH,
+              }}
+            >
+              {metadataField}
+            </span>
+          </Truncate>
+        </div>
+        <div>
+          <Button minimal loading intent="primary" />
+        </div>
       </div>
     </div>
-  </div>
+  );
-);
+};
+
 const ErrorLoading = ({ metadataField, error }) => {
   console.error(error); // log error to console as it is unexpected.
   return (
@@ -359,6 +357,7 @@ const CategoryHeader = React.memo(
     metadataField,
     checkboxID,
     isUserAnno,
+    isTruncated,
     isColorAccessor,
     isExpanded,
     selectionState,
@@ -417,7 +416,6 @@ const CategoryHeader = React.memo(
                   maxWidth: isUserAnno ? LABEL_WIDTH_ANNO : LABEL_WIDTH,
                 }}
                 data-testid={`${metadataField}:category-label`}
-                tabIndex="-1"
               >
                 {metadataField}
               </span>
@@ -435,8 +433,8 @@ const CategoryHeader = React.memo(
             )}
           </span>
         </div>
-        <AnnoDialogEditCategoryName metadataField={metadataField} />
+        {<AnnoDialogEditCategoryName metadataField={metadataField} />}
-        <AnnoDialogAddLabel metadataField={metadataField} />
+        {<AnnoDialogAddLabel metadataField={metadataField} />}
         <div>
           <AnnoMenu
             metadataField={metadataField}
@@ -447,7 +445,11 @@ const CategoryHeader = React.memo(
           />
 
           <Tooltip
-            content="Use as color scale"
+            content={
+              isTruncated
+                ? `Coloring by ${metadataField} is disabled, as it exceeds the limit of ${globals.maxCategoricalOptionsToDisplay} labels`
+                : "Use as color scale"
+            }
             position={Position.LEFT}
             usePortal
             hoverOpenDelay={globals.tooltipHoverOpenDelay}
@@ -462,6 +464,7 @@ const CategoryHeader = React.memo(
               onClick={onColorChangeClick}
               active={isColorAccessor}
               intent={isColorAccessor ? "primary" : "none"}
+              disabled={isTruncated}
               icon="tint"
             />
           </Tooltip>
@@ -476,6 +479,7 @@ const CategoryRender = React.memo(
     metadataField,
     checkboxID,
     isUserAnno,
+    isTruncated,
     isColorAccessor,
     isExpanded,
     selectionState,
@@ -494,23 +498,12 @@ const CategoryRender = React.memo(
     */
     const { numCategoryValues } = categorySummary;
     const isSingularValue = !isUserAnno && numCategoryValues === 1;
+
     if (isSingularValue) {
       /*
       Entire category has a single value, special case.
       */
-      const theOneValue = categorySummary.categoryValues[0];
+      return null;
-      return (
-        <div style={{ marginBottom: 10, marginTop: 4 }}>
-          <Truncate>
-            <span style={{ maxWidth: 150, fontWeight: 700 }}>
-              {metadataField}
-            </span>
-          </Truncate>
-          <Truncate>
-            <span style={{ maxWidth: 150 }}>{`: ${theOneValue}`}</span>
-          </Truncate>
-        </div>
-      );
     }
 
     /*
@@ -535,6 +528,7 @@ const CategoryRender = React.memo(
             metadataField={metadataField}
             checkboxID={checkboxID}
             isUserAnno={isUserAnno}
+            isTruncated={isTruncated}
             isExpanded={isExpanded}
             isColorAccessor={isColorAccessor}
             selectionState={selectionState}
@@ -560,6 +554,11 @@ const CategoryRender = React.memo(
             ) : null
           }
         </div>
+        <div>
+          {isExpanded && isTruncated ? (
+            <p style={{ paddingLeft: 15 }}>... truncated list ...</p>
+          ) : null}
+        </div>
       </div>
     );
   }

client/src/components/categorical/index.js

@@ -1,18 +1,21 @@
+// jshint esversion: 6
 import React from "react";
 import { AnchorButton, Tooltip, Position } from "@blueprintjs/core";
 import { connect } from "react-redux";
 import * as globals from "../../globals";
 import Category from "./category";
 import { AnnotationsHelpers, ControlsHelpers } from "../../util/stateManager";
-import AnnoDialog from "../annoDialog";
+import AnnoDialog from "./annoDialog";
 import AnnoSelect from "./annoSelect";
-import LabelInput from "../labelInput";
+import LabelInput from "./labelInput";
 import { labelPrompt } from "./labelUtil";
 import actions from "../../actions";
 
 @connect((state) => ({
   writableCategoriesEnabled: state.config?.parameters?.annotations ?? false,
   schema: state.annoMatrix?.schema,
+  ontology: state.ontology,
+  userInfo: state.userInfo,
 }))
 class Categories extends React.Component {
   constructor(props) {
@@ -97,8 +100,13 @@ class Categories extends React.Component {
     this.setState({ newCategoryText: name });
   };
 
-  instruction = (name) =>
+  instruction = (name) => {
-    labelPrompt(this.categoryNameError(name), "New, unique category name", ":");
+    return labelPrompt(
+      this.categoryNameError(name),
+      "New, unique category name",
+      ":"
+    );
+  };
 
   onExpansionChange = (catName) => {
     const { expandedCats } = this.state;
@@ -120,10 +128,17 @@ class Categories extends React.Component {
       newCategoryText,
       expandedCats,
     } = this.state;
-    const { writableCategoriesEnabled, schema } = this.props;
+    const {
+      writableCategoriesEnabled,
+      schema,
+      ontology,
+      userInfo,
+    } = this.props;
+    const ontologyEnabled = ontology?.enabled ?? false;
     /* all names, sorted in display order.  Will be rendered in this order */
-    const allCategoryNames =
+    const allCategoryNames = ControlsHelpers.selectableCategoryNames(
-      ControlsHelpers.selectableCategoryNames(schema).sort();
+      schema
+    ).sort();
 
     return (
       <div
@@ -144,7 +159,7 @@ class Categories extends React.Component {
           handleCancel={this.handleDisableAnnoMode}
           annoInput={
             <LabelInput
-              labelSuggestions={null}
+              labelSuggestions={ontologyEnabled ? ontology.terms : null}
               onChange={this.handleChange}
               onSelect={this.handleSelect}
               inputProps={{
@@ -167,43 +182,20 @@ class Categories extends React.Component {
           }
         />
 
-        {writableCategoriesEnabled ? (
-          <div style={{ marginBottom: 10 }}>
-            <Tooltip
-              content="Create a new category"
-              position={Position.RIGHT}
-              boundary="viewport"
-              hoverOpenDelay={globals.tooltipHoverOpenDelay}
-              modifiers={{
-                preventOverflow: { enabled: false },
-                hide: { enabled: false },
-              }}
-            >
-              <AnchorButton
-                type="button"
-                data-testid="open-annotation-dialog"
-                onClick={this.handleEnableAnnoMode}
-                intent="primary"
-              >
-                Create new <strong>category</strong>
-              </AnchorButton>
-            </Tooltip>
-          </div>
-        ) : null}
-
         {/* READ ONLY CATEGORICAL FIELDS */}
         {/* this is duplicative but flat, could be abstracted */}
-        {allCategoryNames.map(
+        {allCategoryNames.map((catName) =>
-          (catName) =>
+          !schema.annotations.obsByName[catName].writable &&
-            !schema.annotations.obsByName[catName].writable && (
+          (schema.annotations.obsByName[catName].categories?.length > 1 ||
-              <Category
+            !schema.annotations.obsByName[catName].categories) ? (
-                key={catName}
+            <Category
-                metadataField={catName}
+              key={catName}
-                onExpansionChange={this.onExpansionChange}
+              metadataField={catName}
-                isExpanded={expandedCats.has(catName)}
+              onExpansionChange={this.onExpansionChange}
-                createAnnoModeActive={createAnnoModeActive}
+              isExpanded={expandedCats.has(catName)}
-              />
+              createAnnoModeActive={createAnnoModeActive}
-            )
+            />
+          ) : null
         )}
         {/* WRITEABLE FIELDS */}
         {allCategoryNames.map((catName) =>
@@ -217,6 +209,33 @@ class Categories extends React.Component {
             />
           ) : null
         )}
+
+        {writableCategoriesEnabled ? (
+          <Tooltip
+            content={
+              userInfo.is_authenticated
+                ? "Create a new category"
+                : "You must be logged in to create new categorical fields"
+            }
+            position={Position.RIGHT}
+            boundary="viewport"
+            hoverOpenDelay={globals.tooltipHoverOpenDelay}
+            modifiers={{
+              preventOverflow: { enabled: false },
+              hide: { enabled: false },
+            }}
+          >
+            <AnchorButton
+              type="button"
+              data-testid="open-annotation-dialog"
+              onClick={this.handleEnableAnnoMode}
+              intent="primary"
+              disabled={!userInfo.is_authenticated}
+            >
+              Create new category
+            </AnchorButton>
+          </Tooltip>
+        ) : null}
       </div>
     );
   }

client/src/components/categorical/labelUtil.js

@@ -3,7 +3,7 @@ import { Colors } from "@blueprintjs/core";
 
 import { AnnotationsHelpers } from "../../util/stateManager";
 
-export function isLabelErroneous(label, metadataField, schema) {
+export function isLabelErroneous(label, metadataField, ontology, schema) {
   /*
     return false if this is a LEGAL/acceptable category name or NULL/empty string,
     or return an error type.
@@ -12,9 +12,10 @@ export function isLabelErroneous(label, metadataField, schema) {
   /* allow empty string */
   if (label === "") return false;
 
-  /* check for label syntax errors */
+  /* check for label syntax errors, but allow terms in ontology */
+  const termInOntology = ontology?.termSet.has(label) ?? false;
   const error = AnnotationsHelpers.annotationNameIsErroneous(label);
-  if (error) return error;
+  if (error && !termInOntology) return error;
 
   /* disallow duplicates */
   const { obsByName } = schema.annotations;

client/src/components/categorical/value/index.js

@@ -14,8 +14,8 @@ import {
 } from "@blueprintjs/core";
 import * as globals from "../../../globals";
 import styles from "../categorical.css";
-import AnnoDialog from "../../annoDialog";
+import AnnoDialog from "../annoDialog";
-import LabelInput from "../../labelInput";
+import LabelInput from "../labelInput";
 import Truncate from "../../util/truncate";
 
 import { AnnotationsHelpers } from "../../../util/stateManager";
@@ -25,8 +25,8 @@ import MiniHistogram from "../../miniHistogram";
 import MiniStackedBar from "../../miniStackedBar";
 import { CategoryCrossfilterContext } from "../categoryContext";
 
-const STACKED_BAR_HEIGHT = 11;
+const VALUE_HEIGHT = 11;
-const STACKED_BAR_WIDTH = 100;
+const CHART_WIDTH = 100;
 
 /* this is defined outside of the class so we can use it in connect() */
 function _currentLabelAsString(ownProps) {
@@ -50,6 +50,7 @@ function _currentLabelAsString(ownProps) {
   return {
     annotations: state.annotations,
     schema: state.annoMatrix?.schema,
+    ontology: state.ontology,
     isDilated,
     isSelected,
     label,
@@ -117,12 +118,14 @@ class CategoryValue extends React.Component {
   };
 
   labelNameError = (name) => {
-    const { metadataField, schema } = this.props;
+    const { metadataField, ontology, schema } = this.props;
     if (name === this.currentLabelAsString()) return false;
-    return isLabelErroneous(name, metadataField, schema);
+    return isLabelErroneous(name, metadataField, ontology, schema);
   };
 
-  instruction = (label) => labelPrompt(this.labelNameError(label), "New, unique label", ":");
+  instruction = (label) => {
+    return labelPrompt(this.labelNameError(label), "New, unique label", ":");
+  };
 
   activateEditLabelMode = () => {
     const { dispatch, metadataField, categoryIndex, label } = this.props;
@@ -148,8 +151,12 @@ class CategoryValue extends React.Component {
   };
 
   toggleOff = () => {
-    const { dispatch, metadataField, categoryIndex, categorySummary } =
+    const {
-      this.props;
+      dispatch,
+      metadataField,
+      categoryIndex,
+      categorySummary,
+    } = this.props;
     const label = categorySummary.categoryValues[categoryIndex];
     dispatch(
       actions.selectCategoricalMetadataAction(
@@ -214,8 +221,12 @@ class CategoryValue extends React.Component {
   };
 
   toggleOn = () => {
-    const { dispatch, metadataField, categoryIndex, categorySummary } =
+    const {
-      this.props;
+      dispatch,
+      metadataField,
+      categoryIndex,
+      categorySummary,
+    } = this.props;
     const label = categorySummary.categoryValues[categoryIndex];
     dispatch(
       actions.selectCategoricalMetadataAction(
@@ -399,7 +410,7 @@ class CategoryValue extends React.Component {
         label,
         colorTable,
         schema,
-        STACKED_BAR_WIDTH
+        CHART_WIDTH
       ) ?? {};
 
     if (!domainValues || !scale || !domain || !occupancy) {
@@ -415,8 +426,8 @@ class CategoryValue extends React.Component {
           domain,
           occupancy,
         }}
-        height={STACKED_BAR_HEIGHT}
+        height={VALUE_HEIGHT}
-        width={STACKED_BAR_WIDTH}
+        width={CHART_WIDTH}
       />
     );
   };
@@ -449,8 +460,8 @@ class CategoryValue extends React.Component {
         colorAccessor,
         colorData,
         label,
-        STACKED_BAR_WIDTH,
+        CHART_WIDTH,
-        STACKED_BAR_HEIGHT
+        VALUE_HEIGHT
       ) ?? {}; // if createHistogramBins returns empty object assign null to deconstructed
 
     if (!xScale || !yScale || !bins) return null;
@@ -465,8 +476,8 @@ class CategoryValue extends React.Component {
         }}
         obsOrVarContinuousFieldDisplayName={colorAccessor}
         domainLabel={label}
-        height={STACKED_BAR_HEIGHT}
+        height={VALUE_HEIGHT}
-        width={STACKED_BAR_WIDTH}
+        width={CHART_WIDTH}
       />
     );
   };
@@ -479,12 +490,14 @@ class CategoryValue extends React.Component {
       colorTable,
       isUserAnno,
       annotations,
+      ontology,
       isDilated,
       isSelected,
       categorySummary,
       label,
     } = this.props;
     const colorScale = colorTable?.scale;
+    const ontologyEnabled = ontology?.enabled ?? false;
 
     const { editedLabelText } = this.state;
 
@@ -521,7 +534,7 @@ class CategoryValue extends React.Component {
       colorAccessor && !isColorBy
         ? globals.leftSidebarWidth -
           otherElementsWidth -
-          STACKED_BAR_WIDTH -
+          CHART_WIDTH -
           CHART_MARGIN
         : globals.leftSidebarWidth - otherElementsWidth;
 
@@ -577,7 +590,6 @@ class CategoryValue extends React.Component {
               <span
                 data-testid={`categorical-value-${metadataField}-${displayString}`}
                 data-testclass="categorical-value"
-                tabIndex="-1"
                 style={{
                   width: labelWidth,
                   color:
@@ -621,7 +633,7 @@ class CategoryValue extends React.Component {
                   annoInput={
                     <LabelInput
                       label={editedLabelText}
-                      labelSuggestions={null}
+                      labelSuggestions={ontologyEnabled ? ontology.terms : null}
                       onChange={this.handleTextChange}
                       onSelect={this.handleTextChange}
                       inputProps={{
@@ -665,8 +677,8 @@ class CategoryValue extends React.Component {
               display={isColorBy && categoryValueIndices ? "auto" : "none"}
               style={{
                 marginLeft: 5,
-                width: 15,
+                width: VALUE_HEIGHT,
-                height: 15,
+                height: VALUE_HEIGHT,
                 backgroundColor:
                   isColorBy && categoryValueIndices
                     ? colorScale(categoryValueIndices.get(label))
@@ -727,7 +739,7 @@ class CategoryValue extends React.Component {
                       ) : null}
                       {displayString !== globals.unassignedCategoryLabel ? (
                         <MenuItem
-                          icon="trash"
+                          icon="delete"
                           intent="danger"
                           data-testclass="handleDeleteValue"
                           data-testid={`${metadataField}:${displayString}:delete-label`}

client/src/components/categorical/value/occupancy.js

@@ -1,3 +1,4 @@
+// jshint esversion: 6
 import React from "react";
 import { connect } from "react-redux";
 import * as d3 from "d3";
@@ -22,8 +23,12 @@ class Occupancy extends React.PureComponent {
       createHistogram fetches the continous data in relation to the cells releveant to the catagory value.
       It then seperates that data into 50 bins for drawing the mini-histogram
     */
-    const { metadataField, categoryData, colorData, categoryValue } =
+    const {
-      this.props;
+      metadataField,
+      categoryData,
+      colorData,
+      categoryValue,
+    } = this.props;
 
     if (!this.canvas) return;
 

client/src/components/continuous/setupParallelCoordinates.js

@@ -3,6 +3,7 @@
       Setup SVG & Canvas elements
 ******************************************
 ******************************************/
+// jshint esversion: 6
 import * as d3 from "d3";
 
 const setupParallelCoordinates = (width, height, margin) => {

client/src/components/continuous/util.js

@@ -1,4 +1,5 @@
-import each from "lodash.foreach";
+// jshint esversion: 6
+import _ from "lodash";
 import * as d3 from "d3";
 
 const paddingRight = 120;
@@ -14,7 +15,7 @@ export const devicePixelRatio = window.devicePixelRatio || 1;
 
 export const createDimensions = (data) => {
   const newArr = [];
-  each(data, (value, key) => {
+  _.each(data, (value, key) => {
     if (value.range) {
       newArr.push({
         key /* room for confusion: lodash calls this key, it's also the name of the property parallel coords code is looking for */,

client/src/components/continuousLegend/index.js

@@ -1,3 +1,4 @@
+// jshint esversion: 6
 import React from "react";
 import { connect } from "react-redux";
 import * as d3 from "d3";
@@ -9,6 +10,7 @@ import {
 } from "../../util/stateManager/colorHelpers";
 
 // create continuous color legend
+// http://bl.ocks.org/syntagmatic/e8ccca52559796be775553b467593a9f
 const continuous = (selectorId, colorScale, colorAccessor) => {
   const legendHeight = 200;
   const legendWidth = 80;
@@ -98,8 +100,6 @@ const continuous = (selectorId, colorScale, colorAccessor) => {
     .attr("y", 2)
     .attr("x", 0 - legendHeight / 2)
     .attr("dy", "1em")
-    .attr("data-testid", "continuous_legend_color_by_label")
-    .attr("aria-label", colorAccessor)
     .style("text-anchor", "middle")
     .style("fill", "white")
     .text(colorAccessor);
@@ -108,24 +108,16 @@ const continuous = (selectorId, colorScale, colorAccessor) => {
 @connect((state) => ({
   annoMatrix: state.annoMatrix,
   colors: state.colors,
-  genesets: state.genesets.genesets,
 }))
 class ContinuousLegend extends React.Component {
   async componentDidUpdate(prevProps) {
-    const { annoMatrix, colors, genesets } = this.props;
+    const { annoMatrix, colors } = this.props;
     if (!colors || !annoMatrix) return;
 
     if (colors !== prevProps?.colors || annoMatrix !== prevProps?.annoMatrix) {
       const { schema } = annoMatrix;
       const { colorMode, colorAccessor, userColors } = colors;
-
+      const colorQuery = createColorQuery(colorMode, colorAccessor, schema);
-      const colorQuery = createColorQuery(
-        colorMode,
-        colorAccessor,
-        schema,
-        genesets
-      );
-
       const colorDf = colorQuery ? await annoMatrix.fetch(...colorQuery) : null;
       const colorTable = createColorTable(
         colorMode,

client/src/components/embedding/index.js

@@ -15,11 +15,13 @@ import * as globals from "../../globals";
 import actions from "../../actions";
 import { getDiscreteCellEmbeddingRowIndex } from "../../util/stateManager/viewStackHelpers";
 
-@connect((state) => ({
+@connect((state) => {
+  return {
     layoutChoice: state.layoutChoice, // TODO: really should clean up naming, s/layout/embedding/g
     schema: state.annoMatrix?.schema,
     crossfilter: state.obsCrossfilter,
-  }))
+  };
+})
 class Embedding extends React.PureComponent {
   constructor(props) {
     super(props);

client/src/components/framework/container.js

@@ -1,3 +1,4 @@
+// jshint esversion: 6
 import React from "react";
 
 function Container(props) {