Bump version: 0.16.6-rc.0 → 0.16.6

* Updates due dependency version changes.

h5py recently changes and now values once returned as str are now returned as bytes.
This would have caused a much larger change, so instead the version is restricted to <3.0.0.

This caused the bulk of the testing failues.
A few other changes were needed to make a few other tests pass.

 #1959

.bumpversion.cfg

@@ -1,5 +1,16 @@
 [bumpversion]
-current_version = 0.16.2
+current_version = 0.16.6
+commit = True
+parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(?:-(?P<prerel>rc)\.(?P<prerelversion>\d+))?
+serialize = 
+	{major}.{minor}.{patch}-{prerel}.{prerelversion}
+	{major}.{minor}.{patch}
+
+[bumpversion:part:prerel]
+optional_value = release
+values = 
+	rc
+	release
 
 [bumpversion:file:setup.py]
 search = version="{current_version}"

.github/workflows/compatibility_tests.yml

@@ -25,6 +25,7 @@ jobs:
   cellxgene-main-with-python-and-anndata-versions:
     name: python versions x anndata versions
     runs-on: ubuntu-latest
+    continue-on-error: true
     strategy:
       matrix:
         python-version: [3.6, 3.7, 3.8]
@@ -67,11 +68,6 @@ jobs:
         uses: actions/checkout@v2
         with:
           path: cellxgene
-      - name: Checkout tools repo
-        uses: actions/checkout@v2
-        with:
-          repository: theislab/anndata
-          path: anndata
       - name: Install dependencies
         run: |
           cd cellxgene
@@ -82,7 +78,7 @@ jobs:
           # 2. install cellxgene
           pip install --upgrade cellxgene
           # 3. install anndata
-          cd ../anndata && pip install -e .
+          pip install git+https://github.com/theislab/anndata
       - name: Tests
         run: cd cellxgene && make unit-test ${{ matrix.test-suite }}
 
@@ -102,17 +98,11 @@ jobs:
         uses: actions/checkout@v2
         with:
           path: cellxgene
-      - name: Checkout tools repo
-        uses: actions/checkout@v2
-        with:
-          repository: theislab/anndata
-          path: anndata
       - name: Install dependencies
         run: |
           cd cellxgene
           sed -i -E 's/^anndata[>=]=[0-9]+.[0-9]+.[0-9]+$/anndata/g' server/requirements.txt
           make pydist install-dist dev-env
-          cd ../anndata
-          pip install -e .
+          pip install git+https://github.com/theislab/anndata
       - name: Tests
         run: cd cellxgene && make unit-test ${{ matrix.test-suite }}

.github/workflows/push_tests.yml

@@ -31,9 +31,10 @@ jobs:
       - name: Install dependencies
         run: |
           pip install flake8
+          pip install black
           cd client
           npm install
-      - name: Lint with flake8
+      - name: Format with black and lint with flake8
         run: |
           make lint-server
       - name: Lint src with eslint

Dockerfile

@@ -5,6 +5,7 @@ ENV LANG=C.UTF-8
 
 RUN apt-get update && \
     apt-get install -y build-essential libxml2-dev python3-dev python3-pip zlib1g-dev python3-requests python3-aiohttp && \
+    python3 -m pip install --upgrade pip && \
     pip3 install cellxgene
 
 ENTRYPOINT ["cellxgene"]

LICENSE.txt

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2013
+Copyright (c) 2017-2020 Chan Zuckerberg Initiative
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of
 this software and associated documentation files (the "Software"), to deal in
@@ -17,4 +17,4 @@ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
 FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
 COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
 IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
-CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

Makefile

@@ -82,8 +82,9 @@ fmt-py:
 lint: lint-server lint-client
 
 .PHONY: lint-server
-lint-server:
-	flake8 server
+lint-server: fmt-py
+	flake8 server --per-file-ignores='server/test/fixtures/dataset_config_outline.py:F821 server/test/fixtures/server_config_outline.py:F821 server/test/performance/scale_test_annotations.py:E501'
+
 
 .PHONY: lint-client
 lint-client:
@@ -99,22 +100,32 @@ pydist: build
 
 # RELEASE HELPERS
 
-# create new version to commit to main
-.PHONY: release-stage-1
-release-stage-1: dev-env bump clean-lite gen-package-lock
+# Create new version to commit to main
+.PHONY: create-release-candidate
+create-release-candidate: dev-env bump-version clean-lite gen-package-lock
 	@echo "Version bumped part:$(PART) and client built. Ready to commit and push"
 
-# build dist and release to dev pypi
-.PHONY: release-stage-2
-release-stage-2: dev-env pydist twine
+# Bump the release candidate version if needed (i.e. the previous release candidate had errors).
+.PHONY: recreate-release-candidate
+recreate-release-candidate: dev-env bump-release-candidate clean-lite gen-package-lock
+	@echo "Version bumped part:$(PART) and client built. Ready to commit and push"
+
+# Build dist and release to Test PyPI
+.PHONY: release-candidate-to-test-pypi
+release-candidate-to-test-pypi: dev-env pydist twine
 	@echo "Dist built and uploaded to test.pypi.org"
 	@echo "Test the install:"
 	@echo "    make install-release-test"
-	@echo "Then upload to Pypi prod:"
-	@echo "    make twine-prod"
 
-.PHONY: release-stage-final
-release-stage-final: twine-prod
+# Build final dist (gets rid of the rc tag) and release final candidate to TestPyPI
+.PHONY: release-final-to-test-pypi
+release-final-to-test-pypi: dev-env bump-release clean-lite gen-package-lock pydist twine
+	@echo "Final release dist built and uploaded to test.pypi.org"
+	@echo "Test the install:"
+	@echo "    make install-release-test"
+
+.PHONY: release-final
+release-final: twine-prod
 	@echo "Release uploaded to pypi.org"
 
 # DANGER: releases directly to prod
@@ -136,11 +147,22 @@ dev-env-client:
 dev-env-server:
 	pip install -r server/requirements-dev.txt
 
-# give PART=[major, minor, part] as param to make bump
-.PHONY: bump
-bump:
+# Set PART=[major, minor, patch] as param to make bump.
+# This will create a release candidate. (i.e. 0.16.1 -> 0.16.2-rc.0 for a patch bump)
+.PHONY: bump-version
+bump-version:
 	bumpversion --config-file .bumpversion.cfg $(PART)
 
+# Increments the release candidate version (i.e. 0.16.2-rc.1 -> 0.16.2-rc.2)
+.PHONY: bump-release-candidate
+bump-release-candidate:
+	bumpversion --config-file .bumpversion.cfg prerelversion --allow-dirty
+
+# Finalizes the release candidate by removing the release candidate tag (i.e. 0.16.2-rc.2 -> 0.16.2).
+.PHONY: bump-release
+bump-release:
+	bumpversion --config-file .bumpversion.cfg prerel --allow-dirty
+
 .PHONY: twine
 twine:
 	twine upload --repository-url https://test.pypi.org/legacy/ dist/*

README.md

@@ -79,25 +79,9 @@ This project was started with the sole goal of empowering the scientific communi
 
 If you believe you have found a security issue, we would appreciate notification. Please send email to <security@chanzuckerberg.com>.
 
-# About
+# Inspiration
 
-### Core team
-
-The current core team:
-
-- Colin Megill, frontend & product design
-- Bruce Martin, software engineer
-- Sidney Bell, computational biologist
-- Lia Prins, designer
-- Severiano Badajoz, software engineer
-
-We would also like to gratefully acknowledge contributions from past core team members:
-
-- Charlotte Weaver, software engineer
-
-### Inspiration
-
-We've been heavily inspired by several other related single-cell visualization projects, including the [UCSC Cell Browswer](http://cells.ucsc.edu/), [Cytoscape](http://www.cytoscape.org/), [Xena](https://xena.ucsc.edu/), [ASAP](https://asap.epfl.ch/), [Gene Pattern](http://genepattern-notebook.org/), and many others. We hope to explore collaborations where useful as this community works together on improving interactive visualization for single-cell data.
+We've been heavily inspired by several other related single-cell visualization projects, including the [UCSC Cell Browser](http://cells.ucsc.edu/), [Cytoscape](http://www.cytoscape.org/), [Xena](https://xena.ucsc.edu/), [ASAP](https://asap.epfl.ch/), [GenePattern](http://genepattern-notebook.org/), and many others. We hope to explore collaborations where useful as this community works together on improving interactive visualization for single-cell data.
 
 We were inspired by Mike Bostock and the [crossfilter](https://github.com/crossfilter) team for the design of our filtering implementation.
 

client/Makefile

@@ -3,6 +3,8 @@ include ../common.mk
 ANNOTATIONS := $(if $(ANNOTATIONS),$(ANNOTATIONS),../server/test/fixtures/pbmc3k-annotations.csv)
 ANNOTATIONS_FILENAME := $(shell basename $(ANNOTATIONS))
 
+CXG_CONFIG := $(if $(CXG_CONFIG), $(CXG_CONFIG), ./__tests__/e2e/test_config.yaml)
+
 # Packaging
 .PHONY: clean
 clean:
@@ -31,9 +33,9 @@ start-frontend:
 .PHONY: smoke-test
 smoke-test:
 	start_server_and_test \
-		'CXG_OPTIONS="--disable-annotations" $(MAKE) start-server' \
+		'CXG_OPTIONS="--config-file $(CXG_CONFIG)" $(MAKE) start-server' \
 		$(CXG_SERVER_PORT) \
-		'CXG_URL_BASE="http://localhost:$(CXG_SERVER_PORT)" npm run e2e -- --verbose false'
+		'CXG_URL_BASE="http://localhost:$(CXG_SERVER_PORT)" CXG_AUTH_TYPE="test" npm run e2e -- --verbose false'
 
 # start an instance of cellxgene and run the end-to-end annotations tests
 .PHONY: smoke-test-annotations

client/__tests__/e2e/__snapshots__/e2e.test.js.snap

@@ -1,5 +1,5 @@
 // Jest Snapshot v1, https://goo.gl/fbAQLP
 
-exports[`did launch page launched 1`] = `"<span style=\\"width: 185px; display: flex; overflow: hidden; justify-content: flex-start;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">pbm</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">c3k</span><span style=\\"position: absolute; right: 0px; color: inherit;\\">c3k</span></span></span>"`;
+exports[`did launch page launched 1`] = `"<span style=\\"max-width: 155px; display: flex; overflow: hidden; justify-content: flex-start; width: 100%; padding: 0px;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">pbm</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">c3k</span><span style=\\"position: absolute; right: 0px; color: inherit;\\">c3k</span></span></span>"`;
 
-exports[`metadata loads categories and values from dataset appear 1`] = `"<div style=\\"display: flex; justify-content: space-between; align-items: baseline;\\"><div style=\\"display: flex; justify-content: flex-start; align-items: flex-start;\\"><label class=\\"bp3-control bp3-checkbox\\" for=\\"category-select-louvain\\"><input id=\\"category-select-louvain\\" data-testclass=\\"category-select\\" data-testid=\\"louvain:category-select\\" type=\\"checkbox\\" checked=\\"\\"><span class=\\"bp3-control-indicator\\"></span></label><span role=\\"menuitem\\" tabindex=\\"0\\" data-testclass=\\"category-expand\\" data-testid=\\"louvain:category-expand\\" style=\\"cursor: pointer;\\"><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\"><span data-testid=\\"louvain:category-label\\" aria-label=\\"louvain\\" class=\\"\\" tabindex=\\"0\\" style=\\"max-width: 265px;\\"><span style=\\"max-width: 265px; display: flex; overflow: hidden; justify-content: flex-start;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">lou</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">vain</span><span style=\\"position: absolute; right: 0px; color: inherit;\\">vain</span></span></span></span></span></span><svg stroke=\\"currentColor\\" fill=\\"currentColor\\" stroke-width=\\"0\\" viewBox=\\"0 0 320 512\\" data-testclass=\\"category-expand-is-not-expanded\\" height=\\"1em\\" width=\\"1em\\" xmlns=\\"http://www.w3.org/2000/svg\\" style=\\"font-size: 10px; margin-left: 5px;\\"><path d=\\"M285.476 272.971L91.132 467.314c-9.373 9.373-24.569 9.373-33.941 0l-22.667-22.667c-9.357-9.357-9.375-24.522-.04-33.901L188.505 256 34.484 101.255c-9.335-9.379-9.317-24.544.04-33.901l22.667-22.667c9.373-9.373 24.569-9.373 33.941 0L285.475 239.03c9.373 9.372 9.373 24.568.001 33.941z\\"></path></svg></span></div><div><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\"><a role=\\"button\\" data-testclass=\\"colorby\\" data-testid=\\"colorby-louvain\\" class=\\"bp3-button\\" tabindex=\\"0\\"><span icon=\\"tint\\" class=\\"bp3-icon bp3-icon-tint\\"><svg data-icon=\\"tint\\" width=\\"16\\" height=\\"16\\" viewBox=\\"0 0 16 16\\"><desc>tint</desc><path d=\\"M7.88 1s-4.9 6.28-4.9 8.9c.01 2.82 2.34 5.1 4.99 5.1 2.65-.01 5.03-2.3 5.03-5.13C12.99 7.17 7.88 1 7.88 1z\\" fill-rule=\\"evenodd\\"></path></svg></span></a></span></span></div></div><div style=\\"margin-left: 26px;\\"></div><div></div>"`;
+exports[`metadata loads categories and values from dataset appear 1`] = `"<div style=\\"display: flex; justify-content: space-between; align-items: baseline;\\"><div style=\\"display: flex; justify-content: flex-start; align-items: flex-start;\\"><label class=\\"bp3-control bp3-checkbox\\" for=\\"category-select-louvain\\"><input id=\\"category-select-louvain\\" data-testclass=\\"category-select\\" data-testid=\\"louvain:category-select\\" type=\\"checkbox\\" checked=\\"\\"><span class=\\"bp3-control-indicator\\"></span></label><span role=\\"menuitem\\" tabindex=\\"0\\" data-testclass=\\"category-expand\\" data-testid=\\"louvain:category-expand\\" style=\\"cursor: pointer;\\"><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\" style=\\"max-width: 265px;\\"><span data-testid=\\"louvain:category-label\\" aria-label=\\"louvain\\" class=\\"\\" tabindex=\\"0\\" style=\\"max-width: 265px;\\"><span style=\\"max-width: 265px; display: flex; overflow: hidden; justify-content: flex-start; width: 100%; padding: 0px;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">lou</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">vain</span><span style=\\"position: absolute; right: 0px; color: inherit;\\">vain</span></span></span></span></span></span><svg stroke=\\"currentColor\\" fill=\\"currentColor\\" stroke-width=\\"0\\" viewBox=\\"0 0 320 512\\" data-testclass=\\"category-expand-is-not-expanded\\" height=\\"1em\\" width=\\"1em\\" xmlns=\\"http://www.w3.org/2000/svg\\" style=\\"font-size: 10px; margin-left: 5px;\\"><path d=\\"M285.476 272.971L91.132 467.314c-9.373 9.373-24.569 9.373-33.941 0l-22.667-22.667c-9.357-9.357-9.375-24.522-.04-33.901L188.505 256 34.484 101.255c-9.335-9.379-9.317-24.544.04-33.901l22.667-22.667c9.373-9.373 24.569-9.373 33.941 0L285.475 239.03c9.373 9.372 9.373 24.568.001 33.941z\\"></path></svg></span></div><div><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\"><a role=\\"button\\" data-testclass=\\"colorby\\" data-testid=\\"colorby-louvain\\" class=\\"bp3-button\\" tabindex=\\"0\\"><span icon=\\"tint\\" class=\\"bp3-icon bp3-icon-tint\\"><svg data-icon=\\"tint\\" width=\\"16\\" height=\\"16\\" viewBox=\\"0 0 16 16\\"><desc>tint</desc><path d=\\"M7.88 1s-4.9 6.28-4.9 8.9c.01 2.82 2.34 5.1 4.99 5.1 2.65-.01 5.03-2.3 5.03-5.13C12.99 7.17 7.88 1 7.88 1z\\" fill-rule=\\"evenodd\\"></path></svg></span></a></span></span></div></div><div style=\\"margin-left: 26px;\\"></div><div></div>"`;

client/__tests__/e2e/__snapshots__/e2eAnnotations.test.js.snap

@@ -2,22 +2,22 @@
 
 exports[`annotations stacked bar graph renders 1`] = `
 Array [
-  "<div class=\\"categorical__value___2RKaC\\" data-testclass=\\"categorical-row\\" style=\\"padding: 4px 0px 4px 7px; display: flex; align-items: baseline; justify-content: space-between; margin-bottom: 2px; border-radius: 2px;\\"><div style=\\"margin: 0px; padding: 0px; user-select: none; width: 220px; display: flex; justify-content: space-between;\\"><div style=\\"display: flex; align-items: baseline;\\"><label for=\\"value-toggle-checkbox-TEST-CATEGORY-TEST-LABEL\\" class=\\"bp3-control bp3-checkbox\\" style=\\"margin: 0px;\\"><input id=\\"value-toggle-checkbox-TEST-CATEGORY-TEST-LABEL\\" data-testclass=\\"categorical-value-select\\" data-testid=\\"categorical-value-select-TEST-CATEGORY-TEST-LABEL\\" type=\\"checkbox\\" checked=\\"\\"><span class=\\"bp3-control-indicator\\"></span></label><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\"><span data-testid=\\"categorical-value-TEST-CATEGORY-TEST-LABEL\\" data-testclass=\\"categorical-value\\" aria-label=\\"TEST-LABEL\\" class=\\"\\" tabindex=\\"0\\" style=\\"width: 63px; color: black; font-style: normal; display: inline-block; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px;\\"><span style=\\"width: 63px; color: black; font-style: normal; display: flex; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px; justify-content: flex-start;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">TEST-</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">LABEL</span><span style=\\"position: absolute; right: 0px; color: black;\\">LABEL</span></span></span></span></span></span></div><span style=\\"flex-shrink: 0;\\"></span></div><div><span><span data-testclass=\\"categorical-value-count\\" data-testid=\\"categorical-value-count-TEST-CATEGORY-TEST-LABEL\\" style=\\"color: black;\\">0</span><svg display=\\"none\\" style=\\"margin-left: 5px; width: 11px; height: 11px; background-color: inherit;\\"></svg><span><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\"><button type=\\"button\\" data-testclass=\\"seeActions\\" data-testid=\\"TEST-CATEGORY:TEST-LABEL:see-actions\\" class=\\"bp3-button bp3-minimal bp3-small\\" tabindex=\\"0\\" style=\\"margin-left: 2px; position: relative; top: -1px; min-height: 16px;\\"><span icon=\\"more\\" class=\\"bp3-icon bp3-icon-more\\"><svg data-icon=\\"more\\" width=\\"10\\" height=\\"10\\" viewBox=\\"0 0 16 16\\"><desc>more</desc><path d=\\"M2 6.03a2 2 0 100 4 2 2 0 100-4zM14 6.03a2 2 0 100 4 2 2 0 100-4zM8 6.03a2 2 0 100 4 2 2 0 100-4z\\" fill-rule=\\"evenodd\\"></path></svg></span></button></span></span></span></span></div></div>",
-  "<div class=\\"categorical__value___2RKaC\\" data-testclass=\\"categorical-row\\" style=\\"padding: 4px 0px 4px 7px; display: flex; align-items: baseline; justify-content: space-between; margin-bottom: 2px; border-radius: 2px;\\"><div style=\\"margin: 0px; padding: 0px; user-select: none; width: 220px; display: flex; justify-content: space-between;\\"><div style=\\"display: flex; align-items: baseline;\\"><label for=\\"value-toggle-checkbox-TEST-CATEGORY-unassigned\\" class=\\"bp3-control bp3-checkbox\\" style=\\"margin: 0px;\\"><input id=\\"value-toggle-checkbox-TEST-CATEGORY-unassigned\\" data-testclass=\\"categorical-value-select\\" data-testid=\\"categorical-value-select-TEST-CATEGORY-unassigned\\" type=\\"checkbox\\" checked=\\"\\"><span class=\\"bp3-control-indicator\\"></span></label><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\"><span data-testid=\\"categorical-value-TEST-CATEGORY-unassigned\\" data-testclass=\\"categorical-value\\" aria-label=\\"unassigned\\" class=\\"\\" tabindex=\\"0\\" style=\\"width: 63px; color: rgb(171, 171, 171); font-style: italic; display: inline-block; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px;\\"><span style=\\"width: 63px; color: rgb(171, 171, 171); font-style: italic; display: flex; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px; justify-content: flex-start;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">unass</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">igned</span><span style=\\"position: absolute; right: 0px; color: rgb(171, 171, 171);\\">igned</span></span></span></span></span></span></div><span style=\\"flex-shrink: 0;\\"><canvas class=\\"bp3-popover-targer\\" width=\\"100\\" height=\\"11\\" style=\\"margin-right: 5px; width: 100px; height: 11px;\\"></canvas></span></div><div><span><span data-testclass=\\"categorical-value-count\\" data-testid=\\"categorical-value-count-TEST-CATEGORY-unassigned\\" style=\\"color: rgb(171, 171, 171); font-style: italic;\\">2133</span><svg display=\\"none\\" style=\\"margin-left: 5px; width: 11px; height: 11px; background-color: inherit;\\"></svg><span><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\"><button type=\\"button\\" data-testclass=\\"seeActions\\" data-testid=\\"TEST-CATEGORY:unassigned:see-actions\\" class=\\"bp3-button bp3-minimal bp3-small\\" tabindex=\\"0\\" style=\\"margin-left: 2px; position: relative; top: -1px; min-height: 16px;\\"><span icon=\\"more\\" class=\\"bp3-icon bp3-icon-more\\"><svg data-icon=\\"more\\" width=\\"10\\" height=\\"10\\" viewBox=\\"0 0 16 16\\"><desc>more</desc><path d=\\"M2 6.03a2 2 0 100 4 2 2 0 100-4zM14 6.03a2 2 0 100 4 2 2 0 100-4zM8 6.03a2 2 0 100 4 2 2 0 100-4z\\" fill-rule=\\"evenodd\\"></path></svg></span></button></span></span></span></span></div></div>",
+  "<div class=\\"categorical__value___2RKaC\\" data-testclass=\\"categorical-row\\" style=\\"padding: 4px 0px 4px 7px; display: flex; align-items: baseline; justify-content: space-between; margin-bottom: 2px; border-radius: 2px;\\"><div style=\\"margin: 0px; padding: 0px; user-select: none; width: 220px; display: flex; justify-content: space-between;\\"><div style=\\"display: flex; align-items: baseline;\\"><label for=\\"value-toggle-checkbox-TEST-CATEGORY-TEST-LABEL\\" class=\\"bp3-control bp3-checkbox\\" style=\\"margin: 0px;\\"><input id=\\"value-toggle-checkbox-TEST-CATEGORY-TEST-LABEL\\" data-testclass=\\"categorical-value-select\\" data-testid=\\"categorical-value-select-TEST-CATEGORY-TEST-LABEL\\" type=\\"checkbox\\" checked=\\"\\"><span class=\\"bp3-control-indicator\\"></span></label><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\" style=\\"width: 63px; color: black; font-style: normal; display: inline-block; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px;\\"><span data-testid=\\"categorical-value-TEST-CATEGORY-TEST-LABEL\\" data-testclass=\\"categorical-value\\" aria-label=\\"TEST-LABEL\\" class=\\"\\" tabindex=\\"0\\" style=\\"width: 63px; color: black; font-style: normal; display: inline-block; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px;\\"><span style=\\"width: 100%; color: black; font-style: normal; display: flex; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px; justify-content: flex-start; padding: 0px;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">TEST-</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">LABEL</span><span style=\\"position: absolute; right: 0px; color: black;\\">LABEL</span></span></span></span></span></span></div><span style=\\"flex-shrink: 0;\\"></span></div><div><span><span data-testclass=\\"categorical-value-count\\" data-testid=\\"categorical-value-count-TEST-CATEGORY-TEST-LABEL\\" style=\\"color: black;\\">0</span><svg display=\\"none\\" style=\\"margin-left: 5px; width: 11px; height: 11px; background-color: inherit;\\"></svg><span><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\"><button type=\\"button\\" data-testclass=\\"seeActions\\" data-testid=\\"TEST-CATEGORY:TEST-LABEL:see-actions\\" class=\\"bp3-button bp3-minimal bp3-small\\" tabindex=\\"0\\" style=\\"margin-left: 2px; position: relative; top: -1px; min-height: 16px;\\"><span icon=\\"more\\" class=\\"bp3-icon bp3-icon-more\\"><svg data-icon=\\"more\\" width=\\"10\\" height=\\"10\\" viewBox=\\"0 0 16 16\\"><desc>more</desc><path d=\\"M2 6.03a2 2 0 100 4 2 2 0 100-4zM14 6.03a2 2 0 100 4 2 2 0 100-4zM8 6.03a2 2 0 100 4 2 2 0 100-4z\\" fill-rule=\\"evenodd\\"></path></svg></span></button></span></span></span></span></div></div>",
+  "<div class=\\"categorical__value___2RKaC\\" data-testclass=\\"categorical-row\\" style=\\"padding: 4px 0px 4px 7px; display: flex; align-items: baseline; justify-content: space-between; margin-bottom: 2px; border-radius: 2px;\\"><div style=\\"margin: 0px; padding: 0px; user-select: none; width: 220px; display: flex; justify-content: space-between;\\"><div style=\\"display: flex; align-items: baseline;\\"><label for=\\"value-toggle-checkbox-TEST-CATEGORY-unassigned\\" class=\\"bp3-control bp3-checkbox\\" style=\\"margin: 0px;\\"><input id=\\"value-toggle-checkbox-TEST-CATEGORY-unassigned\\" data-testclass=\\"categorical-value-select\\" data-testid=\\"categorical-value-select-TEST-CATEGORY-unassigned\\" type=\\"checkbox\\" checked=\\"\\"><span class=\\"bp3-control-indicator\\"></span></label><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\" style=\\"width: 63px; color: rgb(171, 171, 171); font-style: italic; display: inline-block; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px;\\"><span data-testid=\\"categorical-value-TEST-CATEGORY-unassigned\\" data-testclass=\\"categorical-value\\" aria-label=\\"unassigned\\" class=\\"\\" tabindex=\\"0\\" style=\\"width: 63px; color: rgb(171, 171, 171); font-style: italic; display: inline-block; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px;\\"><span style=\\"width: 100%; color: rgb(171, 171, 171); font-style: italic; display: flex; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px; justify-content: flex-start; padding: 0px;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">unass</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">igned</span><span style=\\"position: absolute; right: 0px; color: rgb(171, 171, 171);\\">igned</span></span></span></span></span></span></div><span style=\\"flex-shrink: 0;\\"><canvas width=\\"100\\" height=\\"11\\" style=\\"margin-right: 5px; width: 100px; height: 11px;\\"></canvas></span></div><div><span><span data-testclass=\\"categorical-value-count\\" data-testid=\\"categorical-value-count-TEST-CATEGORY-unassigned\\" style=\\"color: rgb(171, 171, 171); font-style: italic;\\">2133</span><svg display=\\"none\\" style=\\"margin-left: 5px; width: 11px; height: 11px; background-color: inherit;\\"></svg><span><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\"><button type=\\"button\\" data-testclass=\\"seeActions\\" data-testid=\\"TEST-CATEGORY:unassigned:see-actions\\" class=\\"bp3-button bp3-minimal bp3-small\\" tabindex=\\"0\\" style=\\"margin-left: 2px; position: relative; top: -1px; min-height: 16px;\\"><span icon=\\"more\\" class=\\"bp3-icon bp3-icon-more\\"><svg data-icon=\\"more\\" width=\\"10\\" height=\\"10\\" viewBox=\\"0 0 16 16\\"><desc>more</desc><path d=\\"M2 6.03a2 2 0 100 4 2 2 0 100-4zM14 6.03a2 2 0 100 4 2 2 0 100-4zM8 6.03a2 2 0 100 4 2 2 0 100-4z\\" fill-rule=\\"evenodd\\"></path></svg></span></button></span></span></span></span></div></div>",
 ]
 `;
 
 exports[`annotations stacked bar graph renders 2`] = `
 Array [
-  "<div class=\\"categorical__value___2RKaC\\" data-testclass=\\"categorical-row\\" style=\\"padding: 4px 0px 4px 7px; display: flex; align-items: baseline; justify-content: space-between; margin-bottom: 2px; border-radius: 2px;\\"><div style=\\"margin: 0px; padding: 0px; user-select: none; width: 220px; display: flex; justify-content: space-between;\\"><div style=\\"display: flex; align-items: baseline;\\"><label for=\\"value-toggle-checkbox-TEST-CATEGORY-TEST-LABEL\\" class=\\"bp3-control bp3-checkbox\\" style=\\"margin: 0px;\\"><input id=\\"value-toggle-checkbox-TEST-CATEGORY-TEST-LABEL\\" data-testclass=\\"categorical-value-select\\" data-testid=\\"categorical-value-select-TEST-CATEGORY-TEST-LABEL\\" type=\\"checkbox\\" checked=\\"\\"><span class=\\"bp3-control-indicator\\"></span></label><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\"><span data-testid=\\"categorical-value-TEST-CATEGORY-TEST-LABEL\\" data-testclass=\\"categorical-value\\" aria-label=\\"TEST-LABEL\\" class=\\"\\" tabindex=\\"0\\" style=\\"width: 63px; color: black; font-style: normal; display: inline-block; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px;\\"><span style=\\"width: 63px; color: black; font-style: normal; display: flex; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px; justify-content: flex-start;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">TEST-</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">LABEL</span><span style=\\"position: absolute; right: 0px; color: black;\\">LABEL</span></span></span></span></span></span></div><span style=\\"flex-shrink: 0;\\"></span></div><div><span><span data-testclass=\\"categorical-value-count\\" data-testid=\\"categorical-value-count-TEST-CATEGORY-TEST-LABEL\\" style=\\"color: black;\\">0</span><svg display=\\"none\\" style=\\"margin-left: 5px; width: 11px; height: 11px; background-color: inherit;\\"></svg><span><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\"><button type=\\"button\\" data-testclass=\\"seeActions\\" data-testid=\\"TEST-CATEGORY:TEST-LABEL:see-actions\\" class=\\"bp3-button bp3-minimal bp3-small\\" tabindex=\\"0\\" style=\\"margin-left: 2px; position: relative; top: -1px; min-height: 16px;\\"><span icon=\\"more\\" class=\\"bp3-icon bp3-icon-more\\"><svg data-icon=\\"more\\" width=\\"10\\" height=\\"10\\" viewBox=\\"0 0 16 16\\"><desc>more</desc><path d=\\"M2 6.03a2 2 0 100 4 2 2 0 100-4zM14 6.03a2 2 0 100 4 2 2 0 100-4zM8 6.03a2 2 0 100 4 2 2 0 100-4z\\" fill-rule=\\"evenodd\\"></path></svg></span></button></span></span></span></span></div></div>",
-  "<div class=\\"categorical__value___2RKaC\\" data-testclass=\\"categorical-row\\" style=\\"padding: 4px 0px 4px 7px; display: flex; align-items: baseline; justify-content: space-between; margin-bottom: 2px; border-radius: 2px;\\"><div style=\\"margin: 0px; padding: 0px; user-select: none; width: 220px; display: flex; justify-content: space-between;\\"><div style=\\"display: flex; align-items: baseline;\\"><label for=\\"value-toggle-checkbox-TEST-CATEGORY-unassigned\\" class=\\"bp3-control bp3-checkbox\\" style=\\"margin: 0px;\\"><input id=\\"value-toggle-checkbox-TEST-CATEGORY-unassigned\\" data-testclass=\\"categorical-value-select\\" data-testid=\\"categorical-value-select-TEST-CATEGORY-unassigned\\" type=\\"checkbox\\" checked=\\"\\"><span class=\\"bp3-control-indicator\\"></span></label><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\"><span data-testid=\\"categorical-value-TEST-CATEGORY-unassigned\\" data-testclass=\\"categorical-value\\" aria-label=\\"unassigned\\" class=\\"\\" tabindex=\\"0\\" style=\\"width: 63px; color: rgb(171, 171, 171); font-style: italic; display: inline-block; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px;\\"><span style=\\"width: 63px; color: rgb(171, 171, 171); font-style: italic; display: flex; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px; justify-content: flex-start;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">unass</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">igned</span><span style=\\"position: absolute; right: 0px; color: rgb(171, 171, 171);\\">igned</span></span></span></span></span></span></div><span style=\\"flex-shrink: 0;\\"><canvas class=\\"bp3-popover-targer\\" width=\\"100\\" height=\\"11\\" style=\\"margin-right: 5px; width: 100px; height: 11px;\\"></canvas></span></div><div><span><span data-testclass=\\"categorical-value-count\\" data-testid=\\"categorical-value-count-TEST-CATEGORY-unassigned\\" style=\\"color: rgb(171, 171, 171); font-style: italic;\\">2638</span><svg display=\\"none\\" style=\\"margin-left: 5px; width: 11px; height: 11px; background-color: inherit;\\"></svg><span><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\"><button type=\\"button\\" data-testclass=\\"seeActions\\" data-testid=\\"TEST-CATEGORY:unassigned:see-actions\\" class=\\"bp3-button bp3-minimal bp3-small\\" tabindex=\\"0\\" style=\\"margin-left: 2px; position: relative; top: -1px; min-height: 16px;\\"><span icon=\\"more\\" class=\\"bp3-icon bp3-icon-more\\"><svg data-icon=\\"more\\" width=\\"10\\" height=\\"10\\" viewBox=\\"0 0 16 16\\"><desc>more</desc><path d=\\"M2 6.03a2 2 0 100 4 2 2 0 100-4zM14 6.03a2 2 0 100 4 2 2 0 100-4zM8 6.03a2 2 0 100 4 2 2 0 100-4z\\" fill-rule=\\"evenodd\\"></path></svg></span></button></span></span></span></span></div></div>",
+  "<div class=\\"categorical__value___2RKaC\\" data-testclass=\\"categorical-row\\" style=\\"padding: 4px 0px 4px 7px; display: flex; align-items: baseline; justify-content: space-between; margin-bottom: 2px; border-radius: 2px;\\"><div style=\\"margin: 0px; padding: 0px; user-select: none; width: 220px; display: flex; justify-content: space-between;\\"><div style=\\"display: flex; align-items: baseline;\\"><label for=\\"value-toggle-checkbox-TEST-CATEGORY-TEST-LABEL\\" class=\\"bp3-control bp3-checkbox\\" style=\\"margin: 0px;\\"><input id=\\"value-toggle-checkbox-TEST-CATEGORY-TEST-LABEL\\" data-testclass=\\"categorical-value-select\\" data-testid=\\"categorical-value-select-TEST-CATEGORY-TEST-LABEL\\" type=\\"checkbox\\" checked=\\"\\"><span class=\\"bp3-control-indicator\\"></span></label><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\" style=\\"width: 63px; color: black; font-style: normal; display: inline-block; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px;\\"><span data-testid=\\"categorical-value-TEST-CATEGORY-TEST-LABEL\\" data-testclass=\\"categorical-value\\" aria-label=\\"TEST-LABEL\\" class=\\"\\" tabindex=\\"0\\" style=\\"width: 63px; color: black; font-style: normal; display: inline-block; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px;\\"><span style=\\"width: 100%; color: black; font-style: normal; display: flex; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px; justify-content: flex-start; padding: 0px;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">TEST-</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">LABEL</span><span style=\\"position: absolute; right: 0px; color: black;\\">LABEL</span></span></span></span></span></span></div><span style=\\"flex-shrink: 0;\\"></span></div><div><span><span data-testclass=\\"categorical-value-count\\" data-testid=\\"categorical-value-count-TEST-CATEGORY-TEST-LABEL\\" style=\\"color: black;\\">0</span><svg display=\\"none\\" style=\\"margin-left: 5px; width: 11px; height: 11px; background-color: inherit;\\"></svg><span><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\"><button type=\\"button\\" data-testclass=\\"seeActions\\" data-testid=\\"TEST-CATEGORY:TEST-LABEL:see-actions\\" class=\\"bp3-button bp3-minimal bp3-small\\" tabindex=\\"0\\" style=\\"margin-left: 2px; position: relative; top: -1px; min-height: 16px;\\"><span icon=\\"more\\" class=\\"bp3-icon bp3-icon-more\\"><svg data-icon=\\"more\\" width=\\"10\\" height=\\"10\\" viewBox=\\"0 0 16 16\\"><desc>more</desc><path d=\\"M2 6.03a2 2 0 100 4 2 2 0 100-4zM14 6.03a2 2 0 100 4 2 2 0 100-4zM8 6.03a2 2 0 100 4 2 2 0 100-4z\\" fill-rule=\\"evenodd\\"></path></svg></span></button></span></span></span></span></div></div>",
+  "<div class=\\"categorical__value___2RKaC\\" data-testclass=\\"categorical-row\\" style=\\"padding: 4px 0px 4px 7px; display: flex; align-items: baseline; justify-content: space-between; margin-bottom: 2px; border-radius: 2px;\\"><div style=\\"margin: 0px; padding: 0px; user-select: none; width: 220px; display: flex; justify-content: space-between;\\"><div style=\\"display: flex; align-items: baseline;\\"><label for=\\"value-toggle-checkbox-TEST-CATEGORY-unassigned\\" class=\\"bp3-control bp3-checkbox\\" style=\\"margin: 0px;\\"><input id=\\"value-toggle-checkbox-TEST-CATEGORY-unassigned\\" data-testclass=\\"categorical-value-select\\" data-testid=\\"categorical-value-select-TEST-CATEGORY-unassigned\\" type=\\"checkbox\\" checked=\\"\\"><span class=\\"bp3-control-indicator\\"></span></label><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\" style=\\"width: 63px; color: rgb(171, 171, 171); font-style: italic; display: inline-block; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px;\\"><span data-testid=\\"categorical-value-TEST-CATEGORY-unassigned\\" data-testclass=\\"categorical-value\\" aria-label=\\"unassigned\\" class=\\"\\" tabindex=\\"0\\" style=\\"width: 63px; color: rgb(171, 171, 171); font-style: italic; display: inline-block; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px;\\"><span style=\\"width: 100%; color: rgb(171, 171, 171); font-style: italic; display: flex; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px; justify-content: flex-start; padding: 0px;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">unass</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">igned</span><span style=\\"position: absolute; right: 0px; color: rgb(171, 171, 171);\\">igned</span></span></span></span></span></span></div><span style=\\"flex-shrink: 0;\\"><canvas width=\\"100\\" height=\\"11\\" style=\\"margin-right: 5px; width: 100px; height: 11px;\\"></canvas></span></div><div><span><span data-testclass=\\"categorical-value-count\\" data-testid=\\"categorical-value-count-TEST-CATEGORY-unassigned\\" style=\\"color: rgb(171, 171, 171); font-style: italic;\\">2638</span><svg display=\\"none\\" style=\\"margin-left: 5px; width: 11px; height: 11px; background-color: inherit;\\"></svg><span><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\"><button type=\\"button\\" data-testclass=\\"seeActions\\" data-testid=\\"TEST-CATEGORY:unassigned:see-actions\\" class=\\"bp3-button bp3-minimal bp3-small\\" tabindex=\\"0\\" style=\\"margin-left: 2px; position: relative; top: -1px; min-height: 16px;\\"><span icon=\\"more\\" class=\\"bp3-icon bp3-icon-more\\"><svg data-icon=\\"more\\" width=\\"10\\" height=\\"10\\" viewBox=\\"0 0 16 16\\"><desc>more</desc><path d=\\"M2 6.03a2 2 0 100 4 2 2 0 100-4zM14 6.03a2 2 0 100 4 2 2 0 100-4zM8 6.03a2 2 0 100 4 2 2 0 100-4z\\" fill-rule=\\"evenodd\\"></path></svg></span></button></span></span></span></span></div></div>",
 ]
 `;
 
-exports[`annotations truncate midpoint whitespace 1`] = `"<span data-testid=\\"categorical-value-TEST-CATEGORY-123 456\\" data-testclass=\\"categorical-value\\" aria-label=\\"123 456\\" class=\\"\\" tabindex=\\"0\\" style=\\"width: 187px; color: black; font-style: normal; display: inline-block; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px;\\"><span style=\\"width: 187px; color: black; font-style: normal; display: flex; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px; justify-content: flex-start;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">123</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">&nbsp;456</span><span style=\\"position: absolute; right: 0px; color: black;\\">&nbsp;456</span></span></span></span>"`;
+exports[`annotations truncate midpoint whitespace 1`] = `"<span data-testid=\\"categorical-value-TEST-CATEGORY-123 456\\" data-testclass=\\"categorical-value\\" aria-label=\\"123 456\\" class=\\"\\" tabindex=\\"0\\" style=\\"width: 187px; color: black; font-style: normal; display: inline-block; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px;\\"><span style=\\"width: 100%; color: black; font-style: normal; display: flex; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px; justify-content: flex-start; padding: 0px;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">123</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">&nbsp;456</span><span style=\\"position: absolute; right: 0px; color: black;\\">&nbsp;456</span></span></span></span>"`;
 
-exports[`annotations truncate midpoint whitespace 2`] = `"<span data-testid=\\"categorical-value-TEST-CATEGORY-123 456\\" data-testclass=\\"categorical-value\\" aria-label=\\"123 456\\" class=\\"\\" tabindex=\\"0\\" style=\\"width: 187px; color: black; font-style: normal; display: inline-block; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px;\\"><span style=\\"width: 187px; color: black; font-style: normal; display: flex; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px; justify-content: flex-start;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">123</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">&nbsp;456</span><span style=\\"position: absolute; right: 0px; color: black;\\">&nbsp;456</span></span></span></span>"`;
+exports[`annotations truncate midpoint whitespace 2`] = `"<span data-testid=\\"categorical-value-TEST-CATEGORY-123 456\\" data-testclass=\\"categorical-value\\" aria-label=\\"123 456\\" class=\\"\\" tabindex=\\"0\\" style=\\"width: 187px; color: black; font-style: normal; display: inline-block; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px;\\"><span style=\\"width: 100%; color: black; font-style: normal; display: flex; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px; justify-content: flex-start; padding: 0px;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">123</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">&nbsp;456</span><span style=\\"position: absolute; right: 0px; color: black;\\">&nbsp;456</span></span></span></span>"`;
 
-exports[`annotations truncate single character 1`] = `"<span data-testid=\\"categorical-value-TEST-CATEGORY-T\\" data-testclass=\\"categorical-value\\" aria-label=\\"T\\" class=\\"\\" tabindex=\\"0\\" style=\\"width: 187px; color: black; font-style: normal; display: inline-block; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px;\\"><span style=\\"width: 187px; color: black; font-style: normal; display: flex; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px; justify-content: flex-start;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">T</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\"></span><span style=\\"position: absolute; right: 0px; color: black;\\"></span></span></span></span>"`;
+exports[`annotations truncate single character 1`] = `"<span data-testid=\\"categorical-value-TEST-CATEGORY-T\\" data-testclass=\\"categorical-value\\" aria-label=\\"T\\" class=\\"\\" tabindex=\\"0\\" style=\\"width: 187px; color: black; font-style: normal; display: inline-block; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px;\\"><span style=\\"width: 100%; color: black; font-style: normal; display: flex; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px; justify-content: flex-start; padding: 0px;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">T</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\"></span><span style=\\"position: absolute; right: 0px; color: black;\\"></span></span></span></span>"`;
 
-exports[`annotations truncate single character 2`] = `"<span data-testid=\\"categorical-value-TEST-CATEGORY-T\\" data-testclass=\\"categorical-value\\" aria-label=\\"T\\" class=\\"\\" tabindex=\\"0\\" style=\\"width: 187px; color: black; font-style: normal; display: inline-block; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px;\\"><span style=\\"width: 187px; color: black; font-style: normal; display: flex; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px; justify-content: flex-start;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">T</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\"></span><span style=\\"position: absolute; right: 0px; color: black;\\"></span></span></span></span>"`;
+exports[`annotations truncate single character 2`] = `"<span data-testid=\\"categorical-value-TEST-CATEGORY-T\\" data-testclass=\\"categorical-value\\" aria-label=\\"T\\" class=\\"\\" tabindex=\\"0\\" style=\\"width: 187px; color: black; font-style: normal; display: inline-block; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px;\\"><span style=\\"width: 100%; color: black; font-style: normal; display: flex; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px; justify-content: flex-start; padding: 0px;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">T</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\"></span><span style=\\"position: absolute; right: 0px; color: black;\\"></span></span></span></span>"`;

client/__tests__/e2e/cellxgeneActions.js

@@ -13,8 +13,11 @@ import {
   getTestClass,
   getTestId,
   isElementPresent,
+  goToPage,
 } from "./puppeteerUtils";
 
+import { appUrlBase } from "./config";
+
 export async function drag(testId, start, end, lasso = false) {
   const layout = await waitByID(testId);
   const elBox = await layout.boxModel();
@@ -312,4 +315,74 @@ export async function assertCategoryDoesNotExist(categoryName) {
 
   await expect(result).toBe(false);
 }
+
+export async function login() {
+  const email = `cellxgene-smoke-test+${process.env.DEPLOYMENT_STAGE}@chanzuckerberg.com`;
+  const password = "Test1111";
+
+  await goToPage(appUrlBase);
+
+  await clickOn("log-in");
+
+  // (thuang): Auth0 form is unstable and unsafe for input until verified
+  await waitUntilFormFieldStable('[name="email"]');
+
+  await expect(page).toFillForm("form", {
+    email,
+    password,
+  });
+
+  await Promise.all([
+    page.waitForNavigation({ waitUntil: "networkidle0" }),
+    expect(page).toClick('[name="submit"]'),
+  ]);
+
+  expect(page.url()).toContain(appUrlBase);
+}
+
+export async function logout() {
+  await clickOnUntil("user-info", async () => {
+    await waitByID("log-out");
+    await Promise.all([
+      page.waitForNavigation({ waitUntil: "networkidle0" }),
+      clickOn("log-out"),
+    ]);
+  });
+
+  await waitByID("log-in");
+}
+
+async function waitUntilFormFieldStable(selector) {
+  const MAX_RETRY = 10;
+  const WAIT_FOR_MS = 200;
+
+  const EXPECTED_VALUE = "aaa";
+
+  let retry = 0;
+
+  while (retry < MAX_RETRY) {
+    try {
+      await expect(page).toFill(selector, EXPECTED_VALUE);
+
+      const fieldHandle = await expect(page).toMatchElement(selector);
+
+      const fieldValue = await page.evaluate(
+        (input) => input.value,
+        fieldHandle
+      );
+
+      expect(fieldValue).toBe(EXPECTED_VALUE);
+
+      break;
+    } catch (error) {
+      retry += 1;
+
+      await page.waitFor(WAIT_FOR_MS);
+    }
+  }
+
+  if (retry === MAX_RETRY) {
+    throw Error("clickOnUntil() assertion failed!");
+  }
+}
 /* eslint-enable no-await-in-loop -- await in loop is needed to emulate sequential user actions */

client/__tests__/e2e/e2e.test.js

@@ -17,6 +17,7 @@ import {
   goToPage,
   typeInto,
   waitByID,
+  clickOnUntil,
 } from "./puppeteerUtils";
 
 import {
@@ -31,6 +32,8 @@ import {
   runDiffExp,
   selectCategory,
   subset,
+  login,
+  logout,
 } from "./cellxgeneActions";
 
 const data = datasets[DATASET];
@@ -518,4 +521,34 @@ test("lasso moves after pan", async () => {
 
   expect(panCount).toBe(initialCount);
 });
+
+const describeIfCalledByMakeFileTarget =
+  process.env.CXG_AUTH_TYPE?.toLowerCase() === "test"
+    ? describe
+    : describe.skip;
+
+describeIfCalledByMakeFileTarget("auth buttons", () => {
+  test("login then logout", async () => {
+    await goToPage(appUrlBase);
+    await clickOnUntil("log-in", async () => {
+      await page.waitForNavigation({ waitUntil: "networkidle0" });
+      await waitByID("user-info");
+    });
+    await logout();
+  });
+});
+
+const conditionalDescribe =
+  process.env.TEST_AUTH_INTEGRATION === "true" ? describe : describe.skip;
+
+conditionalDescribe("AuthN Integration", () => {
+  it("logs in", async () => {
+    await login();
+  });
+
+  it("logs out", async () => {
+    await login();
+    await logout();
+  });
+});
 /* eslint-enable no-await-in-loop -- await in loop is needed to emulate sequential user actions */

client/__tests__/e2e/puppeteer.setup.js

@@ -17,7 +17,9 @@ setDefaultOptions({ timeout: 20 * 1000 });
 
 jest.retryTimes(ENV_DEFAULT.RETRY_ATTEMPTS);
 
-(async () => {
+beforeEach(async () => {
+  await jestPuppeteer.resetBrowser();
+
   const userAgent = await browser.userAgent();
   await page.setUserAgent(`${userAgent}bot`);
 
@@ -53,6 +55,4 @@ jest.retryTimes(ENV_DEFAULT.RETRY_ATTEMPTS);
       }
     }
   });
-})().catch((error) => {
-  console.error("puppeteer.setup.js error", error);
 });

client/__tests__/e2e/test_config.yaml

@@ -0,0 +1,47 @@
+server:
+  app:
+    force_https: true
+
+    # By default, cellxgene will serve api requests from the same base url as the webpage.
+    # In general api_base_url and web_base_url will not need to be set.
+    # There are two reasons to set these parameters:
+    #  1. Oauth authentication is used; the oauth server will redirect back to the api_base_url after login,
+    #     which then redirects back to the web_base_url.  If the web_base_url is not set, it will default to
+    #     the api_base_url.  If oauth authentication is used, the api_base_url must be set.
+    #     For a local test (where the server runs on "http://localhost:<port>"), then the api_base_url may be
+    #     set to the string "local".
+    #  2. The cellxgene deploymnent is in an environment where the webpage and api have
+    #     different base urls.  In this case both api_base_url and web_base_url must be set.
+    #     It is up to the server admin to ensure that the networking is setup correctly for this environment.
+    api_base_url: http://localhost:5005
+    web_base_url: http://localhost:3000
+
+  authentication:
+    # The authentication types may be "none", "session", "oauth"
+    # none:  No authentication support, features like user_annotations must not be enabled.
+    # session:  A session based userid is automatically generated. (no params needed)
+    # oauth: oauth2 is used for authentication;  parameters are defined in params_oauth.
+    type: test
+
+dataset:
+  app:
+    about_legal_tos: null
+    about_legal_privacy: null
+
+  presentation:
+    max_categories: 1000
+    custom_colors: true
+
+  user_annotations:
+    enable: false
+    type: local_file_csv
+    local_file_csv:
+      directory: null
+      file: null
+    ontology:
+      enable: false
+      obo_location: null
+
+  embeddings:
+    names: []
+    enable_reembedding: false

client/configuration/eslint/eslint.js

@@ -4,6 +4,7 @@ module.exports = {
   extends: [
     "airbnb",
     "plugin:eslint-comments/recommended",
+    "plugin:@blueprintjs/recommended",
     "plugin:compat/recommended",
     "plugin:prettier/recommended",
     "prettier/react",
@@ -39,6 +40,7 @@ module.exports = {
     },
   },
   rules: {
+    "react/jsx-no-target-blank": "off",
     "eslint-comments/require-description": ["error"],
     "no-magic-numbers": "off",
     "no-nested-ternary": "off",
@@ -64,6 +66,12 @@ module.exports = {
       "LabeledStatement",
       "WithStatement",
     ],
+    "import/no-extraneous-dependencies": [
+      "error",
+      {
+        devDependencies: true,
+      },
+    ],
   },
   overrides: [
     {

client/configuration/webpack/obsoleteHTMLTemplate.html

@@ -75,7 +75,7 @@
           src="https://cellxgene.cziscience.com/s3/cellxgene/static/images/edge.png"
           style="width: 80px; height: 80px;"
         />
-        <div>Edge ≥ 15</div>
+        <div>Edge ≥ 79</div>
       </a>
     </div>
   </div>

client/configuration/webpack/webpack.config.dev.js

@@ -31,7 +31,11 @@ const devConfig = {
         test: /\.(jpg|png|gif|eot|svg|ttf|woff|woff2|otf)$/i,
         loader: "file-loader",
         include: [nodeModules, fonts],
-        query: { name: "static/assets/[name].[ext]" },
+        query: {
+          name: "static/assets/[name].[ext]",
+          // (thuang): This is needed to make sure @font url path is '/static/assets/'
+          publicPath: "/",
+        },
       },
     ],
   },

client/configuration/webpack/webpack.config.prod.js

@@ -45,7 +45,11 @@ const prodConfig = {
         test: /\.(jpg|png|gif|eot|svg|ttf|woff|woff2|otf)$/i,
         loader: "file-loader",
         include: [nodeModules, fonts],
-        query: { name: "static/assets/[name]-[contenthash].[ext]" },
+        query: {
+          name: "static/assets/[name]-[contenthash].[ext]",
+          // (thuang): This is needed to make sure @font url path is '../static/assets/'
+          publicPath: "static/",
+        },
       },
     ],
   },

client/configuration/webpack/webpack.config.shared.js

@@ -7,7 +7,7 @@ const ScriptExtHtmlWebpackPlugin = require("script-ext-html-webpack-plugin");
 const src = path.resolve("src");
 const nodeModules = path.resolve("node_modules");
 
-const publicPath = "/";
+const publicPath = "";
 
 const rawObsoleteHTMLTemplate = fs.readFileSync(
   `${__dirname}/obsoleteHTMLTemplate.html`,

client/package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "cellxgene",
-  "version": "0.16.2",
+  "version": "0.16.6",
   "lockfileVersion": 1,
   "requires": true,
   "dependencies": {
@@ -4156,6 +4156,216 @@
         "tslib": "~1.10.0"
       }
     },
+    "@blueprintjs/eslint-plugin": {
+      "version": "0.3.0",
+      "resolved": "https://registry.npmjs.org/@blueprintjs/eslint-plugin/-/eslint-plugin-0.3.0.tgz",
+      "integrity": "sha512-bQEdE4ApEHxCDV8hT9uIxeRbDFKOtRLBT3/Zy3Ku+nowDAYl/8jwZKp6lJuR/nqvsfuIXTnVef6ivwdBEieQfA==",
+      "dev": true,
+      "requires": {
+        "@typescript-eslint/experimental-utils": "^4.2.0",
+        "eslint": "^7.9.0"
+      },
+      "dependencies": {
+        "@typescript-eslint/experimental-utils": {
+          "version": "4.3.0",
+          "resolved": "https://registry.npmjs.org/@typescript-eslint/experimental-utils/-/experimental-utils-4.3.0.tgz",
+          "integrity": "sha512-cmmIK8shn3mxmhpKfzMMywqiEheyfXLV/+yPDnOTvQX/ztngx7Lg/OD26J8gTZfkLKUmaEBxO2jYP3keV7h2OQ==",
+          "dev": true,
+          "requires": {
+            "@types/json-schema": "^7.0.3",
+            "@typescript-eslint/scope-manager": "4.3.0",
+            "@typescript-eslint/types": "4.3.0",
+            "@typescript-eslint/typescript-estree": "4.3.0",
+            "eslint-scope": "^5.0.0",
+            "eslint-utils": "^2.0.0"
+          }
+        },
+        "@typescript-eslint/typescript-estree": {
+          "version": "4.3.0",
+          "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-4.3.0.tgz",
+          "integrity": "sha512-ZAI7xjkl+oFdLV/COEz2tAbQbR3XfgqHEGy0rlUXzfGQic6EBCR4s2+WS3cmTPG69aaZckEucBoTxW9PhzHxxw==",
+          "dev": true,
+          "requires": {
+            "@typescript-eslint/types": "4.3.0",
+            "@typescript-eslint/visitor-keys": "4.3.0",
+            "debug": "^4.1.1",
+            "globby": "^11.0.1",
+            "is-glob": "^4.0.1",
+            "lodash": "^4.17.15",
+            "semver": "^7.3.2",
+            "tsutils": "^3.17.1"
+          }
+        },
+        "acorn": {
+          "version": "7.4.0",
+          "resolved": "https://registry.npmjs.org/acorn/-/acorn-7.4.0.tgz",
+          "integrity": "sha512-+G7P8jJmCHr+S+cLfQxygbWhXy+8YTVGzAkpEbcLo2mLoL7tij/VG41QSHACSf5QgYRhMZYHuNc6drJaO0Da+w==",
+          "dev": true
+        },
+        "cross-spawn": {
+          "version": "7.0.3",
+          "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz",
+          "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==",
+          "dev": true,
+          "requires": {
+            "path-key": "^3.1.0",
+            "shebang-command": "^2.0.0",
+            "which": "^2.0.1"
+          }
+        },
+        "eslint": {
+          "version": "7.10.0",
+          "resolved": "https://registry.npmjs.org/eslint/-/eslint-7.10.0.tgz",
+          "integrity": "sha512-BDVffmqWl7JJXqCjAK6lWtcQThZB/aP1HXSH1JKwGwv0LQEdvpR7qzNrUT487RM39B5goWuboFad5ovMBmD8yA==",
+          "dev": true,
+          "requires": {
+            "@babel/code-frame": "^7.0.0",
+            "@eslint/eslintrc": "^0.1.3",
+            "ajv": "^6.10.0",
+            "chalk": "^4.0.0",
+            "cross-spawn": "^7.0.2",
+            "debug": "^4.0.1",
+            "doctrine": "^3.0.0",
+            "enquirer": "^2.3.5",
+            "eslint-scope": "^5.1.1",
+            "eslint-utils": "^2.1.0",
+            "eslint-visitor-keys": "^1.3.0",
+            "espree": "^7.3.0",
+            "esquery": "^1.2.0",
+            "esutils": "^2.0.2",
+            "file-entry-cache": "^5.0.1",
+            "functional-red-black-tree": "^1.0.1",
+            "glob-parent": "^5.0.0",
+            "globals": "^12.1.0",
+            "ignore": "^4.0.6",
+            "import-fresh": "^3.0.0",
+            "imurmurhash": "^0.1.4",
+            "is-glob": "^4.0.0",
+            "js-yaml": "^3.13.1",
+            "json-stable-stringify-without-jsonify": "^1.0.1",
+            "levn": "^0.4.1",
+            "lodash": "^4.17.19",
+            "minimatch": "^3.0.4",
+            "natural-compare": "^1.4.0",
+            "optionator": "^0.9.1",
+            "progress": "^2.0.0",
+            "regexpp": "^3.1.0",
+            "semver": "^7.2.1",
+            "strip-ansi": "^6.0.0",
+            "strip-json-comments": "^3.1.0",
+            "table": "^5.2.3",
+            "text-table": "^0.2.0",
+            "v8-compile-cache": "^2.0.3"
+          },
+          "dependencies": {
+            "eslint-scope": {
+              "version": "5.1.1",
+              "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-5.1.1.tgz",
+              "integrity": "sha512-2NxwbF/hZ0KpepYN0cNbo+FN6XoK7GaHlQhgx/hIZl6Va0bF45RQOOwhLIy8lQDbuCiadSLCBnH2CFYquit5bw==",
+              "dev": true,
+              "requires": {
+                "esrecurse": "^4.3.0",
+                "estraverse": "^4.1.1"
+              }
+            }
+          }
+        },
+        "eslint-visitor-keys": {
+          "version": "1.3.0",
+          "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-1.3.0.tgz",
+          "integrity": "sha512-6J72N8UNa462wa/KFODt/PJ3IU60SDpC3QXC1Hjc1BXXpfL2C9R5+AU7jhe0F6GREqVMh4Juu+NY7xn+6dipUQ==",
+          "dev": true
+        },
+        "espree": {
+          "version": "7.3.0",
+          "resolved": "https://registry.npmjs.org/espree/-/espree-7.3.0.tgz",
+          "integrity": "sha512-dksIWsvKCixn1yrEXO8UosNSxaDoSYpq9reEjZSbHLpT5hpaCAKTLBwq0RHtLrIr+c0ByiYzWT8KTMRzoRCNlw==",
+          "dev": true,
+          "requires": {
+            "acorn": "^7.4.0",
+            "acorn-jsx": "^5.2.0",
+            "eslint-visitor-keys": "^1.3.0"
+          }
+        },
+        "esrecurse": {
+          "version": "4.3.0",
+          "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz",
+          "integrity": "sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==",
+          "dev": true,
+          "requires": {
+            "estraverse": "^5.2.0"
+          },
+          "dependencies": {
+            "estraverse": {
+              "version": "5.2.0",
+              "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.2.0.tgz",
+              "integrity": "sha512-BxbNGGNm0RyRYvUdHpIwv9IWzeM9XClbOxwoATuFdOE7ZE6wHL+HQ5T8hoPM+zHvmKzzsEqhgy0GrQ5X13afiQ==",
+              "dev": true
+            }
+          }
+        },
+        "glob-parent": {
+          "version": "5.1.1",
+          "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.1.tgz",
+          "integrity": "sha512-FnI+VGOpnlGHWZxthPGR+QhR78fuiK0sNLkHQv+bL9fQi57lNNdquIbna/WrfROrolq8GK5Ek6BiMwqL/voRYQ==",
+          "dev": true,
+          "requires": {
+            "is-glob": "^4.0.1"
+          }
+        },
+        "globals": {
+          "version": "12.4.0",
+          "resolved": "https://registry.npmjs.org/globals/-/globals-12.4.0.tgz",
+          "integrity": "sha512-BWICuzzDvDoH54NHKCseDanAhE3CeDorgDL5MT6LMXXj2WCnd9UC2szdk4AWLfjdgNBCXLUanXYcpBBKOSWGwg==",
+          "dev": true,
+          "requires": {
+            "type-fest": "^0.8.1"
+          }
+        },
+        "ignore": {
+          "version": "4.0.6",
+          "resolved": "https://registry.npmjs.org/ignore/-/ignore-4.0.6.tgz",
+          "integrity": "sha512-cyFDKrqc/YdcWFniJhzI42+AzS+gNwmUzOSFcRCQYwySuBBBy/KjuxWLZ/FHEH6Moq1NizMOBWyTcv8O4OZIMg==",
+          "dev": true
+        },
+        "path-key": {
+          "version": "3.1.1",
+          "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz",
+          "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==",
+          "dev": true
+        },
+        "semver": {
+          "version": "7.3.2",
+          "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.2.tgz",
+          "integrity": "sha512-OrOb32TeeambH6UrhtShmF7CRDqhL6/5XpPNp2DuRH6+9QLw/orhp72j87v8Qa1ScDkvrrBNpZcDejAirJmfXQ==",
+          "dev": true
+        },
+        "shebang-command": {
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz",
+          "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==",
+          "dev": true,
+          "requires": {
+            "shebang-regex": "^3.0.0"
+          }
+        },
+        "shebang-regex": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz",
+          "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==",
+          "dev": true
+        },
+        "which": {
+          "version": "2.0.2",
+          "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz",
+          "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==",
+          "dev": true,
+          "requires": {
+            "isexe": "^2.0.0"
+          }
+        }
+      }
+    },
     "@blueprintjs/icons": {
       "version": "3.19.0",
       "resolved": "https://registry.npmjs.org/@blueprintjs/icons/-/icons-3.19.0.tgz",
@@ -4184,6 +4394,76 @@
         "minimist": "^1.2.0"
       }
     },
+    "@eslint/eslintrc": {
+      "version": "0.1.3",
+      "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-0.1.3.tgz",
+      "integrity": "sha512-4YVwPkANLeNtRjMekzux1ci8hIaH5eGKktGqR0d3LWsKNn5B2X/1Z6Trxy7jQXl9EBGE6Yj02O+t09FMeRllaA==",
+      "dev": true,
+      "requires": {
+        "ajv": "^6.12.4",
+        "debug": "^4.1.1",
+        "espree": "^7.3.0",
+        "globals": "^12.1.0",
+        "ignore": "^4.0.6",
+        "import-fresh": "^3.2.1",
+        "js-yaml": "^3.13.1",
+        "lodash": "^4.17.19",
+        "minimatch": "^3.0.4",
+        "strip-json-comments": "^3.1.1"
+      },
+      "dependencies": {
+        "acorn": {
+          "version": "7.4.0",
+          "resolved": "https://registry.npmjs.org/acorn/-/acorn-7.4.0.tgz",
+          "integrity": "sha512-+G7P8jJmCHr+S+cLfQxygbWhXy+8YTVGzAkpEbcLo2mLoL7tij/VG41QSHACSf5QgYRhMZYHuNc6drJaO0Da+w==",
+          "dev": true
+        },
+        "ajv": {
+          "version": "6.12.5",
+          "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.5.tgz",
+          "integrity": "sha512-lRF8RORchjpKG50/WFf8xmg7sgCLFiYNNnqdKflk63whMQcWR5ngGjiSXkL9bjxy6B2npOK2HSMN49jEBMSkag==",
+          "dev": true,
+          "requires": {
+            "fast-deep-equal": "^3.1.1",
+            "fast-json-stable-stringify": "^2.0.0",
+            "json-schema-traverse": "^0.4.1",
+            "uri-js": "^4.2.2"
+          }
+        },
+        "eslint-visitor-keys": {
+          "version": "1.3.0",
+          "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-1.3.0.tgz",
+          "integrity": "sha512-6J72N8UNa462wa/KFODt/PJ3IU60SDpC3QXC1Hjc1BXXpfL2C9R5+AU7jhe0F6GREqVMh4Juu+NY7xn+6dipUQ==",
+          "dev": true
+        },
+        "espree": {
+          "version": "7.3.0",
+          "resolved": "https://registry.npmjs.org/espree/-/espree-7.3.0.tgz",
+          "integrity": "sha512-dksIWsvKCixn1yrEXO8UosNSxaDoSYpq9reEjZSbHLpT5hpaCAKTLBwq0RHtLrIr+c0ByiYzWT8KTMRzoRCNlw==",
+          "dev": true,
+          "requires": {
+            "acorn": "^7.4.0",
+            "acorn-jsx": "^5.2.0",
+            "eslint-visitor-keys": "^1.3.0"
+          }
+        },
+        "globals": {
+          "version": "12.4.0",
+          "resolved": "https://registry.npmjs.org/globals/-/globals-12.4.0.tgz",
+          "integrity": "sha512-BWICuzzDvDoH54NHKCseDanAhE3CeDorgDL5MT6LMXXj2WCnd9UC2szdk4AWLfjdgNBCXLUanXYcpBBKOSWGwg==",
+          "dev": true,
+          "requires": {
+            "type-fest": "^0.8.1"
+          }
+        },
+        "ignore": {
+          "version": "4.0.6",
+          "resolved": "https://registry.npmjs.org/ignore/-/ignore-4.0.6.tgz",
+          "integrity": "sha512-cyFDKrqc/YdcWFniJhzI42+AzS+gNwmUzOSFcRCQYwySuBBBy/KjuxWLZ/FHEH6Moq1NizMOBWyTcv8O4OZIMg==",
+          "dev": true
+        }
+      }
+    },
     "@hapi/address": {
       "version": "2.1.4",
       "resolved": "https://registry.npmjs.org/@hapi/address/-/address-2.1.4.tgz",
@@ -5137,6 +5417,32 @@
         }
       }
     },
+    "@nodelib/fs.scandir": {
+      "version": "2.1.3",
+      "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.3.tgz",
+      "integrity": "sha512-eGmwYQn3gxo4r7jdQnkrrN6bY478C3P+a/y72IJukF8LjB6ZHeB3c+Ehacj3sYeSmUXGlnA67/PmbM9CVwL7Dw==",
+      "dev": true,
+      "requires": {
+        "@nodelib/fs.stat": "2.0.3",
+        "run-parallel": "^1.1.9"
+      }
+    },
+    "@nodelib/fs.stat": {
+      "version": "2.0.3",
+      "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.3.tgz",
+      "integrity": "sha512-bQBFruR2TAwoevBEd/NWMoAAtNGzTRgdrqnYCc7dhzfoNvqPzLyqlEQnzZ3kVnNrSp25iyxE00/3h2fqGAGArA==",
+      "dev": true
+    },
+    "@nodelib/fs.walk": {
+      "version": "1.2.4",
+      "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.4.tgz",
+      "integrity": "sha512-1V9XOY4rDW0rehzbrcqAmHnz8e7SKvX27gh8Gt2WgB0+pdzdiLV83p72kZPU+jvMbS1qU5mauP2iOvO8rhmurQ==",
+      "dev": true,
+      "requires": {
+        "@nodelib/fs.scandir": "2.1.3",
+        "fastq": "^1.6.0"
+      }
+    },
     "@npmcli/move-file": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/@npmcli/move-file/-/move-file-1.0.1.tgz",
@@ -5489,6 +5795,22 @@
         "eslint-utils": "^2.0.0"
       }
     },
+    "@typescript-eslint/scope-manager": {
+      "version": "4.3.0",
+      "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-4.3.0.tgz",
+      "integrity": "sha512-cTeyP5SCNE8QBRfc+Lgh4Xpzje46kNUhXYfc3pQWmJif92sjrFuHT9hH4rtOkDTo/si9Klw53yIr+djqGZS1ig==",
+      "dev": true,
+      "requires": {
+        "@typescript-eslint/types": "4.3.0",
+        "@typescript-eslint/visitor-keys": "4.3.0"
+      }
+    },
+    "@typescript-eslint/types": {
+      "version": "4.3.0",
+      "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-4.3.0.tgz",
+      "integrity": "sha512-Cx9TpRvlRjOppGsU6Y6KcJnUDOelja2NNCX6AZwtVHRzaJkdytJWMuYiqi8mS35MRNA3cJSwDzXePfmhU6TANw==",
+      "dev": true
+    },
     "@typescript-eslint/typescript-estree": {
       "version": "2.34.0",
       "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-2.34.0.tgz",
@@ -5512,6 +5834,24 @@
         }
       }
     },
+    "@typescript-eslint/visitor-keys": {
+      "version": "4.3.0",
+      "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-4.3.0.tgz",
+      "integrity": "sha512-xZxkuR7XLM6RhvLkgv9yYlTcBHnTULzfnw4i6+z2TGBLy9yljAypQaZl9c3zFvy7PNI7fYWyvKYtohyF8au3cw==",
+      "dev": true,
+      "requires": {
+        "@typescript-eslint/types": "4.3.0",
+        "eslint-visitor-keys": "^2.0.0"
+      },
+      "dependencies": {
+        "eslint-visitor-keys": {
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-2.0.0.tgz",
+          "integrity": "sha512-QudtT6av5WXels9WjIM7qz1XD1cWGvX4gGXvp/zBn9nXG02D0utdU3Em2m/QjTnrsk6bBjmCygl3rmj118msQQ==",
+          "dev": true
+        }
+      }
+    },
     "@webassemblyjs/ast": {
       "version": "1.9.0",
       "resolved": "https://registry.npmjs.org/@webassemblyjs/ast/-/ast-1.9.0.tgz",
@@ -6387,9 +6727,9 @@
       }
     },
     "bl": {
-      "version": "4.0.2",
-      "resolved": "https://registry.npmjs.org/bl/-/bl-4.0.2.tgz",
-      "integrity": "sha512-j4OH8f6Qg2bGuWfRiltT2HYGx0e1QcBTrK9KAHNMwMZdQnDZFk0ZSYIpADjYCB3U12nicC5tVJwSIhwOWjb4RQ==",
+      "version": "4.0.3",
+      "resolved": "https://registry.npmjs.org/bl/-/bl-4.0.3.tgz",
+      "integrity": "sha512-fs4G6/Hu4/EE+F75J8DuN/0IpQqNjAdC7aEQv7Qt8MHGUH7Ckv2MwTEEeN9QehD0pfIDkMI1bkHYkKy7xHyKIg==",
       "dev": true,
       "requires": {
         "buffer": "^5.5.0",
@@ -6397,16 +6737,6 @@
         "readable-stream": "^3.4.0"
       },
       "dependencies": {
-        "buffer": {
-          "version": "5.5.0",
-          "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.5.0.tgz",
-          "integrity": "sha512-9FTEDjLjwoAkEwyMGDjYJQN2gfRgOKBKRfiglhvibGbpeeU/pQn1bJxQqm32OD/AIeEuHxU9roxXxg34Byp/Ww==",
-          "dev": true,
-          "requires": {
-            "base64-js": "^1.0.2",
-            "ieee754": "^1.1.4"
-          }
-        },
         "readable-stream": {
           "version": "3.6.0",
           "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz",
@@ -8796,6 +9126,15 @@
         }
       }
     },
+    "dir-glob": {
+      "version": "3.0.1",
+      "resolved": "https://registry.npmjs.org/dir-glob/-/dir-glob-3.0.1.tgz",
+      "integrity": "sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA==",
+      "dev": true,
+      "requires": {
+        "path-type": "^4.0.0"
+      }
+    },
     "doctrine": {
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-3.0.0.tgz",
@@ -10187,6 +10526,31 @@
       "integrity": "sha512-xJuoT5+L99XlZ8twedaRf6Ax2TgQVxvgZOYoPKqZufmJib0tL2tegPBOZb1pVNgIhlqDlA0eO0c3wBvQcmzx4w==",
       "dev": true
     },
+    "fast-glob": {
+      "version": "3.2.4",
+      "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.2.4.tgz",
+      "integrity": "sha512-kr/Oo6PX51265qeuCYsyGypiO5uJFgBS0jksyG7FUeCyQzNwYnzrNIMR1NXfkZXsMYXYLRAHgISHBz8gQcxKHQ==",
+      "dev": true,
+      "requires": {
+        "@nodelib/fs.stat": "^2.0.2",
+        "@nodelib/fs.walk": "^1.2.3",
+        "glob-parent": "^5.1.0",
+        "merge2": "^1.3.0",
+        "micromatch": "^4.0.2",
+        "picomatch": "^2.2.1"
+      },
+      "dependencies": {
+        "glob-parent": {
+          "version": "5.1.1",
+          "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.1.tgz",
+          "integrity": "sha512-FnI+VGOpnlGHWZxthPGR+QhR78fuiK0sNLkHQv+bL9fQi57lNNdquIbna/WrfROrolq8GK5Ek6BiMwqL/voRYQ==",
+          "dev": true,
+          "requires": {
+            "is-glob": "^4.0.1"
+          }
+        }
+      }
+    },
     "fast-json-stable-stringify": {
       "version": "2.1.0",
       "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz",
@@ -10202,6 +10566,15 @@
       "resolved": "https://registry.npmjs.org/fastestsmallesttextencoderdecoder/-/fastestsmallesttextencoderdecoder-1.0.22.tgz",
       "integrity": "sha512-Pb8d48e+oIuY4MaM64Cd7OW1gt4nxCHs7/ddPPZ/Ic3sg8yVGM7O9wDvZ7us6ScaUupzM+pfBolwtYhN1IxBIw=="
     },
+    "fastq": {
+      "version": "1.8.0",
+      "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.8.0.tgz",
+      "integrity": "sha512-SMIZoZdLh/fgofivvIkmknUXyPnvxRE3DhtZ5Me3Mrsk5gyPL42F0xr51TdRXskBxHfMp+07bcYzfsYEsSQA9Q==",
+      "dev": true,
+      "requires": {
+        "reusify": "^1.0.4"
+      }
+    },
     "favicons": {
       "version": "5.5.0",
       "resolved": "https://registry.npmjs.org/favicons/-/favicons-5.5.0.tgz",
@@ -11159,6 +11532,28 @@
       "resolved": "https://registry.npmjs.org/globals/-/globals-11.12.0.tgz",
       "integrity": "sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA=="
     },
+    "globby": {
+      "version": "11.0.1",
+      "resolved": "https://registry.npmjs.org/globby/-/globby-11.0.1.tgz",
+      "integrity": "sha512-iH9RmgwCmUJHi2z5o2l3eTtGBtXek1OYlHrbcxOYugyHLmAsZrPj43OtHThd62Buh/Vv6VyCBD2bdyWcGNQqoQ==",
+      "dev": true,
+      "requires": {
+        "array-union": "^2.1.0",
+        "dir-glob": "^3.0.1",
+        "fast-glob": "^3.1.1",
+        "ignore": "^5.1.4",
+        "merge2": "^1.3.0",
+        "slash": "^3.0.0"
+      },
+      "dependencies": {
+        "array-union": {
+          "version": "2.1.0",
+          "resolved": "https://registry.npmjs.org/array-union/-/array-union-2.1.0.tgz",
+          "integrity": "sha512-HGyxoOTYUyCM6stUe6EJgnd4EoewAI7zMdfqO+kGjnlZmBDz/cR5pf8r/cR4Wq60sL/p0IkcjUEEPwS3GFrIyw==",
+          "dev": true
+        }
+      }
+    },
     "got": {
       "version": "6.7.1",
       "resolved": "https://registry.npmjs.org/got/-/got-6.7.1.tgz",
@@ -11691,6 +12086,12 @@
       "integrity": "sha1-xg7taebY/bazEEofy8ocGS3FtQE=",
       "dev": true
     },
+    "ignore": {
+      "version": "5.1.8",
+      "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.1.8.tgz",
+      "integrity": "sha512-BMpfD7PpiETpBl/A6S498BaIJ6Y/ABT93ETbby2fP00v4EbvPBXWEoaR1UBPKs3iR53pJY7EtZk5KACI57i1Uw==",
+      "dev": true
+    },
     "ignore-walk": {
       "version": "3.0.3",
       "resolved": "https://registry.npmjs.org/ignore-walk/-/ignore-walk-3.0.3.tgz",
@@ -13787,9 +14188,9 @@
       }
     },
     "lodash": {
-      "version": "4.17.19",
-      "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.19.tgz",
-      "integrity": "sha512-JNvd8XER9GQX0v2qJgsaN/mzFCNA5BRe/j8JN9d+tWyGLSodKQHKFicdwNYzWwI3wjRnaKPsGj1XkBjx/F96DQ=="
+      "version": "4.17.20",
+      "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.20.tgz",
+      "integrity": "sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA=="
     },
     "lodash._reinterpolate": {
       "version": "3.0.0",
@@ -14189,6 +14590,12 @@
       "resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz",
       "integrity": "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w=="
     },
+    "merge2": {
+      "version": "1.4.1",
+      "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz",
+      "integrity": "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==",
+      "dev": true
+    },
     "methods": {
       "version": "1.1.2",
       "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz",
@@ -15251,6 +15658,12 @@
       "integrity": "sha1-32BBeABfUi8V60SQ5yR6G/qmf4w=",
       "dev": true
     },
+    "path-type": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/path-type/-/path-type-4.0.0.tgz",
+      "integrity": "sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==",
+      "dev": true
+    },
     "pbkdf2": {
       "version": "3.1.1",
       "resolved": "https://registry.npmjs.org/pbkdf2/-/pbkdf2-3.1.1.tgz",
@@ -17065,6 +17478,12 @@
       "resolved": "https://registry.npmjs.org/ret/-/ret-0.1.15.tgz",
       "integrity": "sha512-TTlYpa+OL+vMMNG24xSlQGEJ3B/RzEfUlLct7b5G/ytav+wPrplCpVMFuwzXbkecJrb6IYo1iFb0S9v37754mg=="
     },
+    "reusify": {
+      "version": "1.0.4",
+      "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.0.4.tgz",
+      "integrity": "sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==",
+      "dev": true
+    },
     "rgb-regex": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/rgb-regex/-/rgb-regex-1.0.1.tgz",
@@ -17100,6 +17519,12 @@
       "resolved": "https://registry.npmjs.org/rsvp/-/rsvp-4.8.5.tgz",
       "integrity": "sha512-nfMOlASu9OnRJo1mbEk2cz0D56a1MBNrJ7orjRZQG10XDyuvwksKbuXNp6qa+kbn839HwjwhBzhFmdsaEAfauA=="
     },
+    "run-parallel": {
+      "version": "1.1.9",
+      "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.1.9.tgz",
+      "integrity": "sha512-DEqnSRTDw/Tc3FXf49zedI638Z9onwUotBMiUFKmrO2sdFKIbXamXGQ3Axd4qgphxKB4kw/qP1w5kTxnfU1B9Q==",
+      "dev": true
+    },
     "run-queue": {
       "version": "1.0.3",
       "resolved": "https://registry.npmjs.org/run-queue/-/run-queue-1.0.3.tgz",

client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cellxgene",
-  "version": "0.16.2",
+  "version": "0.16.6",
   "license": "MIT",
   "description": "cellxgene is a web application for the interactive exploration of single cell sequence data.",
   "repository": "https://github.com/chanzuckerberg/cellxgene",
@@ -34,7 +34,7 @@
     "Safari >= 10.1",
     "iOS >= 10.3",
     "Firefox >= 60",
-    "Edge >= 15",
+    "Edge >= 79",
     "not Explorer > 0"
   ],
   "dependencies": {
@@ -52,7 +52,7 @@
     "gl-matrix": "^3.3.0",
     "gl-vec3": "^1.1.3",
     "is-number": "^7.0.0",
-    "lodash": "^4.17.19",
+    "lodash": "^4.17.20",
     "memoize-one": "^5.1.1",
     "react": "^16.13.1",
     "react-async": "^10.0.1",
@@ -84,6 +84,7 @@
     "@babel/preset-react": "^7.10.4",
     "@babel/register": "^7.10.5",
     "@babel/runtime": "^7.10.5",
+    "@blueprintjs/eslint-plugin": "^0.3.0",
     "@sentry/webpack-plugin": "^1.12.0",
     "babel-eslint": "^10.1.0",
     "babel-jest": "^26.1.0",

client/server/development.js

@@ -1,46 +1,59 @@
-/* eslint-disable */
-// jshint esversion: 6
-var path = require("path");
-var historyApiFallback = require("connect-history-api-fallback");
-var chalk = require("chalk");
-var express = require("express");
-var favicon = require("serve-favicon");
-var webpack = require("webpack");
-var config = require("../configuration/webpack/webpack.config.dev");
-var utils = require("./utils");
+const chalk = require("chalk");
+const express = require("express");
+const favicon = require("serve-favicon");
+const webpack = require("webpack");
+const devMiddleware = require("webpack-dev-middleware");
+const config = require("../configuration/webpack/webpack.config.dev");
+const utils = require("./utils");
 
 process.env.NODE_ENV = "development";
 
 const CLIENT_PORT = process.env.CXG_CLIENT_PORT;
+const { CXG_SERVER_PORT } = process.env;
+
+const API = {
+  prefix: `http://localhost:${CXG_SERVER_PORT}/`,
+};
 
 // Set up compiler
-var compiler = webpack(config);
+const compiler = webpack(config);
 
-compiler.plugin("invalid", () => {
+compiler.hooks.invalid.tap("invalid", () => {
   utils.clearConsole();
   console.log("Compiling...");
 });
 
-compiler.plugin("done", (stats) => {
+compiler.hooks.done.tap("done", (stats) => {
   utils.formatStats(stats, CLIENT_PORT);
 });
 
 // Launch server
-var app = express();
-
-app.use(historyApiFallback({ verbose: false }));
+const app = express();
 
 app.use(
-  require("webpack-dev-middleware")(compiler, {
+  devMiddleware(compiler, {
     logLevel: "warn",
     publicPath: config.output.publicPath,
+    index: true,
   })
 );
 
 app.use(favicon("./favicon.png"));
 
-app.get("*", (req, res) => {
-  res.sendFile(path.resolve("index.html"));
+app.get("/login", async (req, res) => {
+  try {
+    res.redirect(`${API.prefix}login?dataset=http://localhost:${CLIENT_PORT}`);
+  } catch (err) {
+    console.error(err);
+  }
+});
+
+app.get("/logout", async (req, res) => {
+  try {
+    res.redirect(`${API.prefix}logout?dataset=http://localhost:${CLIENT_PORT}`);
+  } catch (err) {
+    console.error(err);
+  }
 });
 
 app.listen(CLIENT_PORT, (err) => {

client/src/actions/embedding.js

@@ -5,20 +5,23 @@ action creators related to embeddings choice
 import { AnnoMatrixObsCrossfilter } from "../annoMatrix";
 import { _setEmbeddingSubset } from "../util/stateManager/viewStackHelpers";
 
-export async function _switchEmbedding(prevAnnoMatrix, newEmbeddingName) {
+export async function _switchEmbedding(
+  prevAnnoMatrix,
+  prevCrossfilter,
+  newEmbeddingName
+) {
   /*
   DRY helper used by this and reembedding action creators
   */
   const base = prevAnnoMatrix.base();
   const embeddingDf = await base.fetch("emb", newEmbeddingName);
   const annoMatrix = _setEmbeddingSubset(prevAnnoMatrix, embeddingDf);
-  const obsCrossfilter = await new AnnoMatrixObsCrossfilter(annoMatrix).select(
-    "emb",
-    newEmbeddingName,
-    {
-      mode: "all",
-    }
-  );
+  const obsCrossfilter = await new AnnoMatrixObsCrossfilter(
+    annoMatrix,
+    prevCrossfilter.obsCrossfilter
+  ).select("emb", newEmbeddingName, {
+    mode: "all",
+  });
   return [annoMatrix, obsCrossfilter];
 }
 
@@ -30,9 +33,13 @@ export const layoutChoiceAction = (newLayoutChoice) => async (
   On layout choice, make sure we have selected all on the previous layout, AND the new
   layout.
   */
-  const { annoMatrix: prevAnnoMatrix } = getState();
+  const {
+    annoMatrix: prevAnnoMatrix,
+    obsCrossfilter: prevCrossfilter,
+  } = getState();
   const [annoMatrix, obsCrossfilter] = await _switchEmbedding(
     prevAnnoMatrix,
+    prevCrossfilter,
     newLayoutChoice
   );
   dispatch({

client/src/actions/index.js

@@ -41,6 +41,17 @@ async function configFetch(dispatch) {
   });
 }
 
+async function userInfoFetch(dispatch) {
+  return fetchJson("userinfo").then((response) => {
+    const { userinfo: userInfo } = response || {};
+    dispatch({
+      type: "userInfo load complete",
+      userInfo,
+    });
+    return userInfo;
+  });
+}
+
 function prefetchEmbeddings(annoMatrix) {
   /*
   prefetch requests for all embeddings
@@ -62,6 +73,7 @@ const doInitialDataLoad = () =>
         configFetch(dispatch),
         schemaFetch(dispatch),
         userColorsFetchAndLoad(dispatch),
+        userInfoFetch(dispatch),
       ]);
 
       const baseDataUrl = `${globals.API.prefix}${globals.API.version}`;

client/src/actions/reembed.js

@@ -79,10 +79,11 @@ export function requestReembed() {
         type: "reembed: request completed",
       });
 
-      const { annoMatrix: prevAnnoMatrix } = getState();
+      const { annoMatrix: prevAnnoMatrix, obsCrossfilter: prevCrossfilter } = getState();
       const base = prevAnnoMatrix.base().addEmbedding(schema);
       const [annoMatrix, obsCrossfilter] = await _switchEmbedding(
         base,
+        prevCrossfilter,
         schema.name
       );
       dispatch({

client/src/components/autosave/filenameDialog.js

@@ -3,17 +3,19 @@ import { connect } from "react-redux";
 
 import {
   Button,
-  Tooltip,
-  InputGroup,
-  Dialog,
   Classes,
+  Code,
   Colors,
+  Dialog,
+  InputGroup,
+  Tooltip,
 } from "@blueprintjs/core";
 
 @connect((state) => ({
   idhash: state.config?.parameters?.["annotations-user-data-idhash"] ?? null,
   annotations: state.annotations,
   auth: state.config?.authentication,
+  userInfo: state.userInfo,
   writableCategoriesEnabled: state.config?.parameters?.annotations ?? false,
 }))
 class FilenameDialog extends React.Component {
@@ -91,13 +93,19 @@ class FilenameDialog extends React.Component {
   };
 
   render() {
-    const { writableCategoriesEnabled, annotations, idhash, auth } = this.props;
+    const {
+      writableCategoriesEnabled,
+      annotations,
+      idhash,
+      userInfo,
+    } = this.props;
     const { filenameText } = this.state;
 
     return writableCategoriesEnabled &&
+      annotations.promptForFilename &&
       !annotations.dataCollectionNameIsReadOnly &&
       !annotations.dataCollectionName &&
-      auth.is_authenticated ? (
+      userInfo.is_authenticated ? (
       <Dialog
         icon="tag"
         title="Annotations Collection"
@@ -138,9 +146,9 @@ class FilenameDialog extends React.Component {
             <div>
               <p>
                 Your annotations are stored in this file:
-                <code className="bp3-code">
+                <Code>
                   {filenameText}-{idhash}.csv
-                </code>
+                </Code>
               </p>
               <p style={{ fontStyle: "italic" }}>
                 (We added a unique ID to your filename)

client/src/components/brushableHistogram/index.js

@@ -5,12 +5,13 @@ https://bl.ocks.org/SpaceActuary/2f004899ea1b2bd78d6f1dbb2febf771
 https://bl.ocks.org/mbostock/3019563
 */
 import React, { useEffect, useRef, useState, useCallback } from "react";
-import { Button, ButtonGroup, Tooltip } from "@blueprintjs/core";
+import { Button, ButtonGroup, Icon, Tooltip } from "@blueprintjs/core";
 import { connect } from "react-redux";
 import * as d3 from "d3";
 import { interpolateCool } from "d3-scale-chromatic";
 import Async from "react-async";
 import memoize from "memoize-one";
+import { IconNames } from "@blueprintjs/icons";
 import * as globals from "../../globals";
 import actions from "../../actions";
 import { histogramContinuous } from "../../util/dataframe/histogram";
@@ -26,7 +27,7 @@ function maybeScientific(x) {
   const _ticks = x.ticks(4);
 
   if (x.domain().some((n) => Math.abs(n) >= 10000)) {
-    /* 
+    /*
       heuristic: if the last tick d3 wants to render has one significant
       digit ie., 2000, render 2e+3, but if it's anything else ie., 42000000 render
       4.20e+n
@@ -99,7 +100,7 @@ const HistogramFooter = React.memo(
     pvalAdj,
   }) => {
     /*
-  Footer of each histogram.  Will render range, title, and optionally 
+  Footer of each histogram.  Will render range, title, and optionally
   differential expression info.
 
   Required props:
@@ -214,10 +215,7 @@ const HistogramHeader = React.memo(
       >
         {onScatterPlotXClick && onScatterPlotYClick ? (
           <span>
-            <span
-              style={{ marginRight: 7 }}
-              className="bp3-icon-standard bp3-icon-scatter-plot"
-            />
+            <Icon icon={IconNames.SCATTER_PLOT} style={{ marginRight: 7 }} />
             <ButtonGroup style={{ marginRight: 7 }}>
               <Button
                 data-testid={`plot-x-${fieldId}`}
@@ -452,6 +450,7 @@ const Histogram = ({
     isScatterplotYYaccessor: state.controls.scatterplotYYaccessor === field,
     continuousSelectionRange: state.continuousSelection[myName],
     isColorAccessor: state.colors.colorAccessor === field,
+    singleContinuousValues: state.singleContinuousValue.singleContinuousValues,
   };
 })
 class HistogramBrush extends React.PureComponent {
@@ -610,18 +609,44 @@ class HistogramBrush extends React.PureComponent {
   };
 
   fetchAsyncProps = async () => {
-    const { annoMatrix } = this.props;
+    const { annoMatrix, field, dispatch, singleContinuousValues } = this.props;
     const { isClipped } = annoMatrix;
-
+    if (singleContinuousValues.has(field)) {
+      return {
+        histogram: undefined,
+        range: undefined,
+        unclippedRange: undefined,
+        unclippedRangeColor: globals.blue,
+        isSingleValue: true,
+        OK2Render: false,
+      };
+    }
     const query = this.createQuery();
     const df = await annoMatrix.fetch(...query);
     const column = df.icol(0);
 
-    // if we are clipped, fetch both our value and our unclipped value,
-    // as we need the absolute min/max range, not just the clipped min/max.
     const summary = column.summarize();
     const range = [summary.min, summary.max];
 
+    if (summary.min === summary.max && !isClipped) {
+      dispatch({
+        type: "add single continuous value",
+        field,
+        value: summary.min,
+      });
+      return {
+        histogram: undefined,
+        range,
+        unclippedRange: range,
+        unclippedRangeColor: globals.blue,
+        isSingleValue: true,
+        OK2Render: false,
+      };
+    }
+
+    const isSingleValue = summary.min === summary.max;
+    // if we are clipped, fetch both our value and our unclipped value,
+    // as we need the absolute min/max range, not just the clipped min/max.
     let unclippedRange = [...range];
     if (isClipped) {
       const parent = await annoMatrix.viewOf.fetch(...query);
@@ -645,7 +670,6 @@ class HistogramBrush extends React.PureComponent {
       this.height
     );
 
-    const isSingleValue = summary.min === summary.max;
     const nonFiniteExtent =
       summary.min === undefined ||
       summary.max === undefined ||

client/src/components/categorical/category/index.js

@@ -1,7 +1,13 @@
 import React, { useRef, useEffect } from "react";
 import { connect, shallowEqual } from "react-redux";
 import { FaChevronRight, FaChevronDown } from "react-icons/fa";
-import { AnchorButton, Button, Tooltip, Position } from "@blueprintjs/core";
+import {
+  AnchorButton,
+  Button,
+  Classes,
+  Position,
+  Tooltip,
+} from "@blueprintjs/core";
 import { Flipper, Flipped } from "react-flip-toolkit";
 import Async from "react-async";
 import memoize from "memoize-one";
@@ -301,9 +307,12 @@ const StillLoading = ({ metadataField, checkboxID }) => {
             alignItems: "flex-start",
           }}
         >
-          <label htmlFor={checkboxID} className="bp3-control bp3-checkbox">
+          <label
+            htmlFor={checkboxID}
+            className={`${Classes.CONTROL} ${Classes.CHECKBOX}`}
+          >
             <input disabled id={checkboxID} checked type="checkbox" />
-            <span className="bp3-control-indicator" />
+            <span className={Classes.CONTROL_INDICATOR} />
           </label>
           <Truncate>
             <span
@@ -375,7 +384,10 @@ const CategoryHeader = React.memo(
             alignItems: "flex-start",
           }}
         >
-          <label className="bp3-control bp3-checkbox" htmlFor={checkboxID}>
+          <label
+            className={`${Classes.CONTROL} ${Classes.CHECKBOX}`}
+            htmlFor={checkboxID}
+          >
             <input
               id={checkboxID}
               data-testclass="category-select"
@@ -385,7 +397,7 @@ const CategoryHeader = React.memo(
               checked={selectionState === "all"}
               type="checkbox"
             />
-            <span className="bp3-control-indicator" />
+            <span className={Classes.CONTROL_INDICATOR} />
           </label>
           <span
             role="menuitem"
@@ -491,19 +503,7 @@ const CategoryRender = React.memo(
       /*
       Entire category has a single value, special case.
       */
-      const theOneValue = categorySummary.categoryValues[0];
-      return (
-        <div style={{ marginBottom: 10, marginTop: 4 }}>
-          <Truncate>
-            <span style={{ maxWidth: 150, fontWeight: 700 }}>
-              {metadataField}
-            </span>
-          </Truncate>
-          <Truncate>
-            <span style={{ maxWidth: 150 }}>{`: ${theOneValue}`}</span>
-          </Truncate>
-        </div>
-      );
+      return null;
     }
 
     /*

client/src/components/categorical/index.js

@@ -1,6 +1,6 @@
 // jshint esversion: 6
 import React from "react";
-import { Button } from "@blueprintjs/core";
+import { AnchorButton, Tooltip, Position } from "@blueprintjs/core";
 import { connect } from "react-redux";
 import * as globals from "../../globals";
 import Category from "./category";
@@ -15,6 +15,7 @@ import actions from "../../actions";
   writableCategoriesEnabled: state.config?.parameters?.annotations ?? false,
   schema: state.annoMatrix?.schema,
   ontology: state.ontology,
+  userInfo: state.userInfo,
 }))
 class Categories extends React.Component {
   constructor(props) {
@@ -127,7 +128,12 @@ class Categories extends React.Component {
       newCategoryText,
       expandedCats,
     } = this.state;
-    const { writableCategoriesEnabled, schema, ontology } = this.props;
+    const {
+      writableCategoriesEnabled,
+      schema,
+      ontology,
+      userInfo,
+    } = this.props;
     const ontologyEnabled = ontology?.enabled ?? false;
     /* all names, sorted in display order.  Will be rendered in this order */
     const allCategoryNames = ControlsHelpers.selectableCategoryNames(
@@ -179,7 +185,9 @@ class Categories extends React.Component {
         {/* READ ONLY CATEGORICAL FIELDS */}
         {/* this is duplicative but flat, could be abstracted */}
         {allCategoryNames.map((catName) =>
-          !schema.annotations.obsByName[catName].writable ? (
+          !schema.annotations.obsByName[catName].writable &&
+          (schema.annotations.obsByName[catName].categories?.length > 1 ||
+            !schema.annotations.obsByName[catName].categories) ? (
             <Category
               key={catName}
               metadataField={catName}
@@ -203,15 +211,30 @@ class Categories extends React.Component {
         )}
 
         {writableCategoriesEnabled ? (
-          <div>
-            <Button
+          <Tooltip
+            content={
+              userInfo.is_authenticated
+                ? "Create a new category"
+                : "You must be logged in to create new categorical fields"
+            }
+            position={Position.RIGHT}
+            boundary="viewport"
+            hoverOpenDelay={globals.tooltipHoverOpenDelay}
+            modifiers={{
+              preventOverflow: { enabled: false },
+              hide: { enabled: false },
+            }}
+          >
+            <AnchorButton
+              type="button"
               data-testid="open-annotation-dialog"
               onClick={this.handleEnableAnnoMode}
               intent="primary"
+              disabled={!userInfo.is_authenticated}
             >
               Create new category
-            </Button>
-          </div>
+            </AnchorButton>
+          </Tooltip>
         ) : null}
       </div>
     );

client/src/components/categorical/value/index.js

@@ -4,12 +4,13 @@ import * as d3 from "d3";
 
 import {
   Button,
+  Classes,
+  Icon,
   Menu,
   MenuItem,
   Popover,
-  Position,
-  Icon,
   PopoverInteractionKind,
+  Position,
 } from "@blueprintjs/core";
 import * as globals from "../../../globals";
 import styles from "../categorical.css";
@@ -81,7 +82,7 @@ class CategoryValue extends React.Component {
   get shouldRenderStackedBarOrHistogram() {
     const { colorAccessor, isColorBy, annotations } = this.props;
 
-    return colorAccessor && !isColorBy && !annotations.isEditingLabelName;
+    return !!colorAccessor && !isColorBy && !annotations.isEditingLabelName;
   }
 
   handleDeleteValue = () => {
@@ -410,7 +411,6 @@ class CategoryValue extends React.Component {
 
     return (
       <MiniStackedBar
-        /* eslint-disable react/jsx-props-no-spreading -- Disable unneeded on next release of eslint-config-airbnb */
         {...{
           colorTable,
           domainValues,
@@ -418,7 +418,6 @@ class CategoryValue extends React.Component {
           domain,
           occupancy,
         }}
-        /* eslint-enable react/jsx-props-no-spreading -- enable */
         height={VALUE_HEIGHT}
         width={CHART_WIDTH}
       />
@@ -439,7 +438,9 @@ class CategoryValue extends React.Component {
 
     if (
       !this.shouldRenderStackedBarOrHistogram ||
-      !AnnotationsHelpers.isContinuousAnnotation(schema, colorAccessor)
+      // This function returns true on categorical annotations(when stacked bar should not render),
+      //  in cases where the colorAccessor is a gene this function will return undefined since genes do not live on the schema
+      AnnotationsHelpers.isCategoricalAnnotation(schema, colorAccessor) === true
     ) {
       return null;
     }
@@ -459,14 +460,12 @@ class CategoryValue extends React.Component {
 
     return (
       <MiniHistogram
-        /* eslint-disable react/jsx-props-no-spreading -- Disable unneeded on next release of eslint-config-airbnb */
         {...{
           colorScale,
           xScale,
           yScale,
           bins,
         }}
-        /* eslint-enable react/jsx-props-no-spreading -- enable */
         obsOrVarContinuousFieldDisplayName={colorAccessor}
         domainLabel={label}
         height={VALUE_HEIGHT}
@@ -562,7 +561,7 @@ class CategoryValue extends React.Component {
           <div style={{ display: "flex", alignItems: "baseline" }}>
             <label
               htmlFor={valueToggleLabel}
-              className="bp3-control bp3-checkbox"
+              className={`${Classes.CONTROL} ${Classes.CHECKBOX}`}
               style={{ margin: 0 }}
             >
               <input
@@ -574,7 +573,7 @@ class CategoryValue extends React.Component {
                 type="checkbox"
               />
               <span
-                className="bp3-control-indicator"
+                className={Classes.CONTROL_INDICATOR}
                 onMouseEnter={this.handleMouseExit}
                 onMouseLeave={this.handleMouseEnter}
               />

client/src/components/categorical/value/occupancy.js

@@ -3,10 +3,10 @@ import React from "react";
 import { connect } from "react-redux";
 import * as d3 from "d3";
 import {
+  Classes,
   Popover,
   PopoverInteractionKind,
   Position,
-  Classes,
 } from "@blueprintjs/core";
 
 @connect((state) => ({
@@ -18,8 +18,8 @@ class Occupancy extends React.PureComponent {
   _HEIGHT = 11;
 
   createHistogram = () => {
-    /* 
-      Knowing that colorScale is based off continous data, 
+    /*
+      Knowing that colorScale is based off continous data,
       createHistogram fetches the continous data in relation to the cells releveant to the catagory value.
       It then seperates that data into 50 bins for drawing the mini-histogram
     */
@@ -75,8 +75,8 @@ class Occupancy extends React.PureComponent {
   };
 
   createOccupancyStack = () => {
-    /* 
-      Knowing that the color scale is based off of catagorical data, 
+    /*
+      Knowing that the color scale is based off of catagorical data,
       createOccupancyStack obtains a map showing the number if cells per colored value
       Using the colorScale a stack of colored bars is drawn representing the map
      */
@@ -155,7 +155,7 @@ class Occupancy extends React.PureComponent {
         popoverClassName={Classes.POPOVER_CONTENT_SIZING}
       >
         <canvas
-          className="bp3-popover-targer"
+          className={Classes.POPOVER_TARGET}
           style={{
             marginRight: 5,
             width: this._WIDTH,

client/src/components/embedding/index.js

@@ -2,13 +2,14 @@ import React from "react";
 import { connect } from "react-redux";
 import { useAsync } from "react-async";
 import {
-  ButtonGroup,
-  Popover,
   Button,
+  ButtonGroup,
+  H4,
+  Popover,
+  Position,
   Radio,
   RadioGroup,
   Tooltip,
-  Position,
 } from "@blueprintjs/core";
 import * as globals from "../../globals";
 import actions from "../../actions";
@@ -80,7 +81,7 @@ class Embedding extends React.PureComponent {
                 width: 400,
               }}
             >
-              <h1>Embedding Choice</h1>
+              <H4>Embedding Choice</H4>
               <p style={{ fontStyle: "italic" }}>
                 There are {schema?.dataframe?.nObs} cells in the entire dataset.
               </p>

client/src/components/geneExpression/addGenes.js

@@ -6,11 +6,12 @@ import fuzzysort from "fuzzysort";
 import { connect } from "react-redux";
 import { Suggest } from "@blueprintjs/select";
 import {
-  MenuItem,
   Button,
+  ControlGroup,
   FormGroup,
   InputGroup,
-  ControlGroup,
+  Intent,
+  MenuItem,
 } from "@blueprintjs/core";
 import * as globals from "../../globals";
 import actions from "../../actions";
@@ -278,7 +279,7 @@ class AddGenes extends React.Component {
               popoverProps={{ minimal: true }}
             />
             <Button
-              className="bp3-button bp3-intent-primary"
+              intent={Intent.PRIMARY}
               data-testid="add-gene"
               loading={userDefinedGenesLoading}
               onClick={() => this.handleClick(activeItem)}

client/src/components/graph/setupLasso.js

@@ -10,6 +10,7 @@ const Lasso = () => {
     let lassoPolygon;
     let lassoPath;
     let closePath;
+    let lassoInProgress;
 
     const polygonToPath = (polygon) =>
       `M${polygon.map((d) => d.join(",")).join("L")}`;
@@ -25,8 +26,18 @@ const Lasso = () => {
       lassoPolygon = [d3.mouse(svg.node())]; // current x y of mouse within element
 
       if (lassoPath) {
+        // If the existing path is in progress
+        if (lassoInProgress) {
+          // cancel the existing lasso
+          handleCancel();
+          // Don't continue with current drag start
+          return;
+        }
+
         lassoPath.remove();
       }
+      // We're starting a new drag
+      lassoInProgress = true;
 
       lassoPath = g
         .append("path")
@@ -67,25 +78,33 @@ const Lasso = () => {
       }
     };
 
+    const handleCancel = () => {
+      lassoPath.remove();
+      closePath = closePath?.remove();
+      lassoPath = null;
+      lassoPolygon = null;
+      closePath = null;
+      dispatch.call("cancel");
+    };
+
     const handleDragEnd = () => {
       // remove the close path
       closePath.remove();
       closePath = null;
 
-      // succesfully closed
+      // successfully closed
       if (
         distance(lassoPolygon[0], lassoPolygon[lassoPolygon.length - 1]) <
         closeDistance
       ) {
+        lassoInProgress = false;
+
         lassoPath.attr("d", `${polygonToPath(lassoPolygon)}Z`);
         dispatch.call("end", lasso, lassoPolygon);
 
         // otherwise cancel
       } else {
-        lassoPath.remove();
-        lassoPath = null;
-        lassoPolygon = null;
-        dispatch.call("cancel");
+        handleCancel();
       }
     };
 

client/src/components/infoDrawer/infoDrawer.js

@@ -0,0 +1,72 @@
+import React, { PureComponent } from "react";
+import { connect, shallowEqual } from "react-redux";
+import { Drawer } from "@blueprintjs/core";
+import InfoFormat from "./infoFormat";
+import { selectableCategoryNames } from "../../util/stateManager/controlsHelpers";
+
+@connect((state) => {
+  return {
+    annoMatrix: state.annoMatrix,
+    schema: state.annoMatrix.schema,
+    datasetTitle: state.config?.displayNames?.dataset ?? "",
+    aboutURL: state.config?.links?.["about-dataset"],
+    isOpen: state.controls.datasetDrawer,
+    dataPortalProps: state.config?.["corpora_props"] ?? {},
+    singleContinuousValues: state.singleContinuousValue.singleContinuousValues,
+  };
+})
+class InfoDrawer extends PureComponent {
+  static watchAsync(props, prevProps) {
+    return !shallowEqual(props.watchProps, prevProps.watchProps);
+  }
+
+  handleClose = () => {
+    const { dispatch } = this.props;
+
+    dispatch({ type: "toggle dataset drawer" });
+  };
+
+  render() {
+    const {
+      position,
+      aboutURL,
+      datasetTitle,
+      schema,
+      isOpen,
+      dataPortalProps,
+      singleContinuousValues,
+    } = this.props;
+
+    const allCategoryNames = selectableCategoryNames(schema).sort();
+    const allSingleValues = new Map();
+
+    allCategoryNames.forEach((catName) => {
+      const isUserAnno = schema?.annotations?.obsByName[catName]?.writable;
+      const colSchema = schema.annotations.obsByName[catName];
+      if (!isUserAnno && colSchema.categories?.length === 1) {
+        allSingleValues.set(catName, colSchema.categories[0]);
+      }
+    });
+    singleContinuousValues.forEach((value, catName) => {
+      allSingleValues.set(catName, value);
+    });
+
+    return (
+      <Drawer
+        title="Dataset Overview"
+        onClose={this.handleClose}
+        {...{ isOpen, position }}
+      >
+        <InfoFormat
+          {...{
+            datasetTitle,
+            aboutURL,
+            allSingleValues,
+            dataPortalProps,
+          }}
+        />
+      </Drawer>
+    );
+  }
+}
+export default InfoDrawer;

client/src/components/infoDrawer/infoFormat.js

@@ -0,0 +1,200 @@
+import { H3, H1, UL } from "@blueprintjs/core";
+import React from "react";
+
+import Truncate from "../util/truncate";
+
+const renderContributors = (contributors, affiliations) => {
+  // eslint-disable-next-line no-constant-condition --  Temp removed contributor section to avoid publishing PII
+  if (!contributors || contributors.length === 0 || true) return null;
+  return (
+    <>
+      <H3>Contributors</H3>
+      <p>
+        {contributors.map((contributor) => {
+          const { email, name, institution } = contributor;
+
+          return (
+            <span key={name}>
+              {name}
+              {email && `(${email})`}
+              <sup>{affiliations.indexOf(institution) + 1}</sup>
+            </span>
+          );
+        })}
+      </p>
+      {renderAffiliations(affiliations)}
+    </>
+  );
+};
+
+// generates a list of unique institutions by order of appearance in contributors
+const buildAffiliations = (contributors = []) => {
+  const affiliations = [];
+  contributors.forEach((contributor) => {
+    const { institution } = contributor;
+    if (affiliations.indexOf(institution) === -1) {
+      affiliations.push(institution);
+    }
+  });
+  return affiliations;
+};
+
+const renderAffiliations = (affiliations) => {
+  if (affiliations.length === 0) return null;
+  return (
+    <>
+      <H3>Affiliations</H3>
+      <UL>
+        {affiliations.map((item, index) => (
+          <div key={item}>
+            <sup>{index + 1}</sup>
+            {"  "}
+            {item}
+          </div>
+        ))}
+      </UL>
+    </>
+  );
+};
+
+const renderDOILink = (type, doi) => {
+  if (!doi) return null;
+  return (
+    <>
+      <H3>{type}</H3>
+      <p>
+        <a href={doi} target="_blank" rel="noopener">
+          {doi}
+        </a>
+      </p>
+    </>
+  );
+};
+
+const renderOrganism = (organism) => {
+  if (!organism) return null;
+  return (
+    <>
+      <H3>Organism</H3>
+      <p>{organism}</p>
+    </>
+  );
+};
+
+const ONTOLOGY_KEY = "ontology_term_id";
+const CAT_WIDTH = "30%";
+const VAL_WIDTH = "35%";
+// Render list of metadata attributes found in categorical field
+const renderSingleValues = (singleValues) => {
+  if (singleValues.size === 0) return null;
+  return (
+    <>
+      <H3>Dataset Metadata</H3>
+      <UL>
+        {Array.from(singleValues).reduce((elems, pair) => {
+          const [category, value] = pair;
+          // If the value is empty skip it
+          if (!value) return elems;
+
+          // If this category is a ontology term, let's add its value to the previous node
+          if (String(category).includes(ONTOLOGY_KEY)) {
+            const prevElem = elems.pop();
+            // Props aren't extensible so we must clone and alter the component to append the new child
+            elems.push(
+              React.cloneElement(
+                prevElem,
+                prevElem.props,
+                // Concat returns a new array
+                prevElem.props.children.concat([
+                  <Truncate key="ontology">
+                    <span style={{ width: VAL_WIDTH }}>{value}</span>
+                  </Truncate>,
+                ])
+              )
+            );
+          } else {
+            // Create the list item
+            elems.push(
+              <li key={category} style={{ width: "100%" }}>
+                <Truncate>
+                  <span style={{ width: CAT_WIDTH }}>{`${category}:`}</span>
+                </Truncate>
+                <Truncate>
+                  <span style={{ width: VAL_WIDTH }}>{value}</span>
+                </Truncate>
+              </li>
+            );
+          }
+          return elems;
+        }, [])}
+      </UL>
+    </>
+  );
+};
+
+// Renders any links found in the config where link_type is not "SUMMARY"
+// If there are no links in the config, render the aboutURL
+const renderLinks = (projectLinks, aboutURL) => {
+  if (!projectLinks && !aboutURL) return null;
+  if (projectLinks)
+    return (
+      <>
+        <H3>Project Links</H3>
+        <UL>
+          {projectLinks.map((link) => {
+            if (link.link_type === "SUMMARY") return null;
+            return (
+              <li key={link.link_name}>
+                <a href={link.link_url} target="_blank" rel="noopener">
+                  {link.link_name}
+                </a>
+              </li>
+            );
+          })}
+        </UL>
+      </>
+    );
+
+  return (
+    <>
+      <H3>More Info</H3>
+      <p>
+        <a href={aboutURL} target="_blank" rel="noopener">
+          {aboutURL}
+        </a>
+      </p>
+    </>
+  );
+};
+
+const InfoFormat = React.memo(
+  ({ datasetTitle, allSingleValues, aboutURL, dataPortalProps = {} }) => {
+    if (dataPortalProps.version?.["corpora_schema_version"] !== "1.0.0") {
+      dataPortalProps = {};
+    }
+    const {
+      title,
+      publication_doi: doi,
+      preprint_doi: preprintDOI,
+      organism,
+      contributors,
+      project_links: projectLinks,
+    } = dataPortalProps;
+
+    const affiliations = buildAffiliations(contributors);
+
+    return (
+      <div style={{ margin: 24, overflow: "auto" }}>
+        <H1>{title ?? datasetTitle}</H1>
+        {renderContributors(contributors, affiliations)}
+        {renderDOILink("DOI", doi)}
+        {renderDOILink("Preprint DOI", preprintDOI)}
+        {renderOrganism(organism)}
+        {renderSingleValues(allSingleValues)}
+        {renderLinks(projectLinks, aboutURL)}
+      </div>
+    );
+  }
+);
+
+export default InfoFormat;

client/src/components/leftSidebar/infoMenu.js

@@ -0,0 +1,72 @@
+// jshint esversion: 6
+import React from "react";
+import { Button, Menu, MenuItem, Popover, Position } from "@blueprintjs/core";
+import { IconNames } from "@blueprintjs/icons";
+
+const InformationMenu = React.memo((props) => {
+  const { libraryVersions, tosURL, privacyURL } = props;
+  return (
+    <Popover
+      content={
+        <Menu>
+          <MenuItem
+            href="https://chanzuckerberg.github.io/cellxgene/"
+            target="_blank"
+            icon="book"
+            text="Documentation"
+            rel="noopener"
+          />
+          <MenuItem
+            href="https://join-cellxgene-users.herokuapp.com/"
+            target="_blank"
+            icon="chat"
+            text="Chat"
+            rel="noopener"
+          />
+          <MenuItem
+            href="https://github.com/chanzuckerberg/cellxgene"
+            target="_blank"
+            icon="git-branch"
+            text="Github"
+            rel="noopener"
+          />
+          <MenuItem target="_blank" text={libraryVersions?.cellxgene || null} />
+          <MenuItem text="MIT License" />
+          {tosURL && (
+            <MenuItem
+              href={tosURL}
+              target="_blank"
+              text="Terms of Service"
+              rel="noopener"
+            />
+          )}
+          {privacyURL && (
+            <MenuItem
+              href={privacyURL}
+              target="_blank"
+              text="Privacy Policy"
+              rel="noopener"
+            />
+          )}
+        </Menu>
+      }
+      position={Position.BOTTOM_RIGHT}
+      modifiers={{
+        preventOverflow: { enabled: false },
+        hide: { enabled: false },
+      }}
+    >
+      <Button
+        data-testid="menu"
+        type="button"
+        icon={IconNames.INFO_SIGN}
+        style={{
+          cursor: "pointer",
+          verticalAlign: "middle",
+        }}
+      />
+    </Popover>
+  );
+});
+
+export default InformationMenu;

client/src/components/leftSidebar/topLeftLogoAndTitle.js

@@ -1,22 +1,38 @@
 // jshint esversion: 6
 import React from "react";
 import { connect } from "react-redux";
+import { Button } from "@blueprintjs/core";
+
 import * as globals from "../../globals";
 import Logo from "../framework/logo";
 import Truncate from "../util/truncate";
+import InfoDrawer from "../infoDrawer/infoDrawer";
+import InformationMenu from "./infoMenu";
 
-const DATASET_TITLE_WIDTH = 190;
 const DATASET_TITLE_FONT_SIZE = 14;
 
 @connect((state) => ({
   datasetTitle: state.config?.displayNames?.dataset ?? "",
-  aboutURL: state.config?.links?.["about-dataset"],
-  scatterplotXXaccessor: state.controls.scatterplotXXaccessor,
-  scatterplotYYaccessor: state.controls.scatterplotYYaccessor,
+  libraryVersions: state.config?.["library_versions"],
+  aboutLink: state.config?.links?.["about-dataset"],
+  tosURL: state.config?.parameters?.["about_legal_tos"],
+  privacyURL: state.config?.parameters?.["about_legal_privacy"],
 }))
 class LeftSideBar extends React.Component {
+  handleClick = () => {
+    const { dispatch } = this.props;
+    dispatch({ type: "toggle dataset drawer" });
+  };
+
   render() {
-    const { datasetTitle, aboutURL } = this.props;
+    const {
+      datasetTitle,
+      libraryVersions,
+      aboutLink,
+      privacyURL,
+      tosURL,
+      dispatch,
+    } = this.props;
 
     return (
       <div
@@ -26,65 +42,64 @@ class LeftSideBar extends React.Component {
           width: globals.leftSidebarWidth,
           zIndex: 1,
           borderBottom: `1px solid ${globals.lighterGrey}`,
+          display: "flex",
+          justifyContent: "space-between",
+          alignItems: "center",
         }}
       >
-        <Logo size={30} />
-        <span
-          style={{
-            fontSize: 28,
-            position: "relative",
-            top: -6,
-            fontWeight: "bold",
-            marginLeft: 5,
-            color: globals.logoColor,
-            userSelect: "none",
-          }}
-        >
-          cell
+        <div>
+          <Logo size={28} />
           <span
             style={{
-              position: "relative",
-              top: 1,
-              fontWeight: 300,
               fontSize: 24,
+              position: "relative",
+              top: -6,
+              fontWeight: "bold",
+              marginLeft: 5,
+              color: globals.logoColor,
+              userSelect: "none",
             }}
           >
-            ×
+            cell
+            <span
+              style={{
+                position: "relative",
+                top: 1,
+                fontWeight: 300,
+                fontSize: 24,
+              }}
+            >
+              ×
+            </span>
+            gene
           </span>
-          gene
-        </span>
-        <div
-          style={{
-            fontSize: DATASET_TITLE_FONT_SIZE,
-            position: "relative",
-            top: -6,
-            display: "inline-block",
-            width: DATASET_TITLE_WIDTH,
-            marginLeft: "7px",
-            height: "1.2em",
-            overflow: "hidden",
-            wordBreak: "break-all",
-          }}
-        >
-          {aboutURL ? (
+        </div>
+        <div style={{ marginRight: 5, height: "100%" }}>
+          <Button
+            minimal
+            style={{
+              fontSize: DATASET_TITLE_FONT_SIZE,
+              position: "relative",
+              top: -1,
+            }}
+            onClick={this.handleClick}
+          >
             <Truncate>
-              <a
-                style={{ width: 185 }}
-                href={aboutURL}
-                data-testid="header"
-                target="_blank"
-                rel="noopener noreferrer"
-              >
-                {datasetTitle}
-              </a>
-            </Truncate>
-          ) : (
-            <Truncate>
-              <span style={{ width: 185 }} data-testid="header">
+              <span style={{ maxWidth: 155 }} data-testid="header">
                 {datasetTitle}
               </span>
             </Truncate>
-          )}
+          </Button>
+          <InfoDrawer />
+          <InformationMenu
+            {...{
+              libraryVersions,
+              aboutLink,
+              tosURL,
+              privacyURL,
+              dispatch,
+            }}
+          />
         </div>
       </div>
     );

client/src/components/menubar/authButtons.js

@@ -1,32 +1,175 @@
-import React from "react";
-import { AnchorButton, Tooltip } from "@blueprintjs/core";
+import React, { useState } from "react";
+
+import {
+  AnchorButton,
+  Button,
+  MenuItem,
+  Tooltip,
+  Popover,
+  Menu,
+  Elevation,
+  PopoverPosition,
+  Checkbox,
+  Card,
+} from "@blueprintjs/core";
+
+import { IconNames } from "@blueprintjs/icons";
+
 import * as globals from "../../globals";
+
 import styles from "./menubar.css";
 
+import { storageGet, storageSet, KEYS } from "../util/localStorage";
+
+const BASE_EMOJI = [0x1f9d1, 0x1f468, 0x1f469];
+const SKIN_TONES = [0x1f3fb, 0x1f3fc, 0x1f3fd, 0x1f3fe, 0x1f3ff];
+const MICROSCOPE = 0x1f52c;
+const ZERO_WIDTH_JOINER = 0x0200d;
+
+const LOGIN_PROMPT_OFF = "off";
+
 const Auth = React.memo((props) => {
-  const { auth } = props;
+  const [isPromptOpen, setIsPromptOpen] = useState(shouldShowPrompt());
 
-  if (!auth || (auth && !auth.requires_client_login)) return null;
+  const { auth, userInfo } = props;
 
-  return (
-    <div className={`bp3-button-group ${styles.menubarButton}`}>
-      <Tooltip
-        content="Log in or log out of cellxgene"
-        position="bottom"
-        hoverOpenDelay={globals.tooltipHoverOpenDelay}
-      >
-        <AnchorButton
-          type="button"
-          data-testid="auth-button"
-          disabled={false}
-          icon={!auth.is_authenticated ? "log-in" : "log-out"}
-          href={!auth.is_authenticated ? auth.login : auth.logout}
-        >
-          {!auth.is_authenticated ? "Log In" : "Log Out"}
-        </AnchorButton>
-      </Tooltip>
-    </div>
+  const isAuthenticated = userInfo && userInfo.is_authenticated;
+
+  window.userInfo = userInfo;
+
+  const randomInt = Math.random() * 15;
+  const sexIndex = Math.floor(randomInt / 5);
+  const skinToneIndex = Math.floor(randomInt % 5);
+
+  const scientist = String.fromCodePoint(
+    BASE_EMOJI[sexIndex],
+    SKIN_TONES[skinToneIndex],
+    ZERO_WIDTH_JOINER,
+    MICROSCOPE
   );
+
+  if (!shouldShowAuth()) return null;
+
+  if (isAuthenticated) {
+    const PopoverContent = (
+      <Menu>
+        <MenuItem
+          data-testid="user-email"
+          text={`Logged in as: ${userInfo.email}`}
+        />
+        <MenuItem
+          data-testid="log-out"
+          text="Log Out"
+          href={auth.logout}
+          icon={IconNames.LOG_OUT}
+        />
+      </Menu>
+    );
+
+    return (
+      <Popover content={PopoverContent}>
+        <Button
+          data-testid="user-info"
+          className={styles.menubarButton}
+          style={{ padding: 0 }}
+        >
+          {/*  eslint-disable-next-line no-constant-condition -- disable profile picture until CSP is tweaked */}
+          {userInfo?.picture && false ? (
+            <img alt="profile" size="21px" src={userInfo?.picture} />
+          ) : (
+            <span style={{ fontSize: "18px" }}>{scientist}</span>
+          )}
+        </Button>
+      </Popover>
+    );
+  }
+
+  const LoginButton = (
+    <Tooltip
+      content="Log in to cellxgene"
+      position="bottom"
+      hoverOpenDelay={globals.tooltipHoverOpenDelay}
+    >
+      <AnchorButton
+        type="button"
+        data-testid="log-in"
+        href={auth.login}
+        className={styles.menubarButton}
+      >
+        Log In
+      </AnchorButton>
+    </Tooltip>
+  );
+
+  if (isPromptOpen) {
+    return (
+      <Popover
+        position={PopoverPosition.AUTO_END}
+        isOpen
+        content={<PromptContent setIsPromptOpen={setIsPromptOpen} />}
+        onInteraction={setIsPromptOpen}
+      >
+        {LoginButton}
+      </Popover>
+    );
+  }
+
+  return LoginButton;
+
+  function shouldShowAuth() {
+    return auth && auth.requires_client_login;
+  }
+
+  function shouldShowPrompt() {
+    if (storageGet(KEYS.LOGIN_PROMPT) === LOGIN_PROMPT_OFF) return false;
+
+    return shouldShowAuth && !isAuthenticated;
+  }
 });
 
+function PromptContent({ setIsPromptOpen }) {
+  const [isChecked, setIsChecked] = useState(false);
+
+  function handleOKClick() {
+    if (isChecked) {
+      storageSet(KEYS.LOGIN_PROMPT, LOGIN_PROMPT_OFF);
+    }
+
+    setIsPromptOpen(false);
+  }
+
+  function handleCheckboxChange() {
+    setIsChecked(!isChecked);
+  }
+
+  return (
+    <Card style={{ width: "500px" }} elevation={Elevation.TWO}>
+      <p>
+        Logging in will enable you to create your own categories and labels.
+        Logging in later will reset cellxgene to the default view and cause you
+        to lose progress.
+      </p>
+      <Checkbox
+        style={{ width: "230px" }}
+        checked={isChecked}
+        onChange={handleCheckboxChange}
+        data-testid="login-hint-do-not-show-again"
+      >
+        Do not show me this message again
+      </Checkbox>
+      <div
+        style={{ display: "flex", justifyContent: "flex-end", marginTop: 15 }}
+      >
+        <Button
+          onClick={handleOKClick}
+          intent="primary"
+          data-testid="login-hint-yes"
+        >
+          Acknowledge
+        </Button>
+      </div>
+    </Card>
+  );
+}
+
 export default Auth;

client/src/components/menubar/clip.js

@@ -1,12 +1,16 @@
 import React from "react";
 import {
-  Position,
   Button,
-  Popover,
-  NumericInput,
+  ButtonGroup,
   Icon,
+  Intent,
+  NumericInput,
+  Popover,
+  Position,
   Tooltip,
 } from "@blueprintjs/core";
+import { IconNames } from "@blueprintjs/icons";
+
 import { tooltipHoverOpenDelay } from "../../globals";
 import styles from "./menubar.css";
 
@@ -28,13 +32,13 @@ const Clip = React.memo((props) => {
     pendingClipPercentiles?.clipPercentileMin ?? clipPercentileMin;
   const clipMax =
     pendingClipPercentiles?.clipPercentileMax ?? clipPercentileMax;
-  const activeClipClass =
+  const intent =
     clipPercentileMin > 0 || clipPercentileMax < 100
-      ? " bp3-intent-warning"
-      : "";
+      ? Intent.INTENT_WARNING
+      : Intent.NONE;
 
   return (
-    <div className={`bp3-button-group ${styles.menubarButton}`}>
+    <ButtonGroup className={`${styles.menubarButton}`}>
       <Popover
         target={
           <Tooltip
@@ -45,7 +49,8 @@ const Clip = React.memo((props) => {
             <Button
               type="button"
               data-testid="visualization-settings"
-              className={`bp3-button bp3-icon-timeline-bar-chart ${activeClipClass}`}
+              intent={intent}
+              icon={IconNames.TIMELINE_BAR_CHART}
               style={{
                 cursor: "pointer",
               }}
@@ -126,7 +131,7 @@ const Clip = React.memo((props) => {
           </div>
         }
       />
-    </div>
+    </ButtonGroup>
   );
 });
 

client/src/components/menubar/index.js

@@ -6,8 +6,8 @@ import * as globals from "../../globals";
 import styles from "./menubar.css";
 import actions from "../../actions";
 import Clip from "./clip";
+
 import AuthButtons from "./authButtons";
-import InformationMenu from "./infoMenu";
 import Subset from "./subset";
 import UndoRedoReset from "./undoRedo";
 import DiffexpButtons from "./diffexpButtons";
@@ -42,6 +42,7 @@ import { getEmbSubsetView } from "../../util/stateManager/viewStackHelpers";
     celllist2: state.differential.celllist2,
     libraryVersions: state.config?.["library_versions"],
     auth: state.config?.authentication,
+    userInfo: state.userInfo,
     undoDisabled: state["@@undoable/past"].length === 0,
     redoDisabled: state["@@undoable/future"].length === 0,
     aboutLink: state.config?.links?.["about-dataset"],
@@ -203,7 +204,6 @@ class MenuBar extends React.PureComponent {
   render() {
     const {
       dispatch,
-      libraryVersions,
       disableDiffexp,
       undoDisabled,
       redoDisabled,
@@ -211,15 +211,13 @@ class MenuBar extends React.PureComponent {
       clipPercentileMin,
       clipPercentileMax,
       graphInteractionMode,
-      aboutLink,
       showCentroidLabels,
-      privacyURL,
-      tosURL,
       categoricalSelection,
       colorAccessor,
       subsetPossible,
       subsetResetPossible,
       enableReembedding,
+      userInfo,
       auth,
     } = this.props;
     const { pendingClipPercentiles } = this.state;
@@ -246,13 +244,7 @@ class MenuBar extends React.PureComponent {
           zIndex: 3,
         }}
       >
-        <AuthButtons auth={auth} />
-        <InformationMenu
-          libraryVersions={libraryVersions}
-          aboutLink={aboutLink}
-          tosURL={tosURL}
-          privacyURL={privacyURL}
-        />
+        <AuthButtons {...{ auth, userInfo }} />
         <UndoRedoReset
           dispatch={dispatch}
           undoDisabled={undoDisabled}

client/src/components/menubar/infoMenu.js

@@ -1,77 +0,0 @@
-// jshint esversion: 6
-import React from "react";
-import { Button, Popover, Menu, MenuItem, Position } from "@blueprintjs/core";
-import styles from "./menubar.css";
-
-const InformationMenu = React.memo((props) => {
-  const { libraryVersions, aboutLink, tosURL, privacyURL } = props;
-  return (
-    <div className={`bp3-button-group ${styles.menubarButton}`}>
-      <Popover
-        content={
-          <Menu>
-            {aboutLink ? (
-              <MenuItem
-                href={aboutLink}
-                target="_blank"
-                icon="document-open"
-                text="About this dataset"
-              />
-            ) : (
-              ""
-            )}
-
-            <MenuItem
-              href="https://chanzuckerberg.github.io/cellxgene/"
-              target="_blank"
-              icon="help"
-              text="Help"
-            />
-            <MenuItem
-              href="https://join-cellxgene-users.herokuapp.com/"
-              target="_blank"
-              icon="chat"
-              text="Chat"
-            />
-            <MenuItem
-              href="https://github.com/chanzuckerberg/cellxgene"
-              target="_blank"
-              icon="git-branch"
-              text="Github"
-            />
-            <MenuItem
-              target="_blank"
-              text={
-                libraryVersions && libraryVersions.cellxgene
-                  ? libraryVersions.cellxgene
-                  : null
-              }
-            />
-            <MenuItem text="MIT License" />
-            {tosURL ? (
-              <MenuItem href={tosURL} target="_blank" text="Terms of Service" />
-            ) : null}
-            {privacyURL ? (
-              <MenuItem
-                href={privacyURL}
-                target="_blank"
-                text="Privacy Policy"
-              />
-            ) : null}
-          </Menu>
-        }
-        position={Position.BOTTOM_RIGHT}
-      >
-        <Button
-          type="button"
-          className="bp3-button bp3-icon-info-sign"
-          style={{
-            cursor: "pointer",
-          }}
-        />
-      </Popover>
-    </div>
-  );
-});
-
-export default InformationMenu;

client/src/components/menubar/undoRedo.js

@@ -1,12 +1,13 @@
 import React from "react";
-import { AnchorButton, Tooltip } from "@blueprintjs/core";
+import { AnchorButton, ButtonGroup, Tooltip } from "@blueprintjs/core";
+import { IconNames } from "@blueprintjs/icons";
 import { tooltipHoverOpenDelay } from "../../globals";
 import styles from "./menubar.css";
 
 const UndoRedo = React.memo((props) => {
   const { undoDisabled, redoDisabled, dispatch } = props;
   return (
-    <div className={`bp3-button-group ${styles.menubarButton}`}>
+    <ButtonGroup className={`${styles.menubarButton}`}>
       <Tooltip
         content="Undo"
         position="bottom"
@@ -14,7 +15,7 @@ const UndoRedo = React.memo((props) => {
       >
         <AnchorButton
           type="button"
-          className="bp3-button bp3-icon-undo"
+          icon={IconNames.UNDO}
           disabled={undoDisabled}
           onClick={() => {
             dispatch({ type: "@@undoable/undo" });
@@ -32,7 +33,7 @@ const UndoRedo = React.memo((props) => {
       >
         <AnchorButton
           type="button"
-          className="bp3-button bp3-icon-redo"
+          icon={IconNames.REDO}
           disabled={redoDisabled}
           onClick={() => {
             dispatch({ type: "@@undoable/redo" });
@@ -43,7 +44,7 @@ const UndoRedo = React.memo((props) => {
           data-testid="redo"
         />
       </Tooltip>
-    </div>
+    </ButtonGroup>
   );
 });
 

client/src/components/miniHistogram/index.js

@@ -72,7 +72,6 @@ export default class MiniHistogram extends React.PureComponent {
         popoverClassName={Classes.POPOVER_CONTENT_SIZING}
       >
         <canvas
-          className="bp3-popover-targer"
           style={{
             marginRight: 5,
             width,

client/src/components/miniStackedBar/index.js

@@ -1,4 +1,3 @@
-// jshint esversion: 6
 import React from "react";
 
 export default class MiniStackedBar extends React.PureComponent {
@@ -59,7 +58,6 @@ export default class MiniStackedBar extends React.PureComponent {
 
     return (
       <canvas
-        className="bp3-popover-targer"
         style={{
           marginRight: 5,
           width,

client/src/components/termsPrompt/index.js

@@ -8,26 +8,7 @@ import {
   Colors,
   Icon,
 } from "@blueprintjs/core";
-
-const CookieDecision = "cxg.cookieDecision";
-
-function storageGet(key, defaultValue = null) {
-  try {
-    const val = window.localStorage.getItem(key);
-    if (val === null) return defaultValue;
-    return val;
-  } catch (e) {
-    return defaultValue;
-  }
-}
-
-function storageSet(key, value) {
-  try {
-    window.localStorage.setItem(key, value);
-  } catch {
-    // continue
-  }
-}
+import { storageGet, storageSet, KEYS } from "../util/localStorage";
 
 @connect((state) => ({
   tosURL: state.config?.parameters?.["about_legal_tos"],
@@ -37,7 +18,7 @@ class TermsPrompt extends React.PureComponent {
   constructor(props) {
     super(props);
     const { tosURL, privacyURL } = this.props;
-    const cookieDecision = storageGet(CookieDecision, null);
+    const cookieDecision = storageGet(KEYS.COOKIE_DECISION, null);
     const hasDecided = cookieDecision !== null;
     this.state = {
       hasDecided,
@@ -55,7 +36,7 @@ class TermsPrompt extends React.PureComponent {
 
   handleOK = () => {
     this.setState({ isOpen: false });
-    storageSet(CookieDecision, "yes");
+    storageSet(KEYS.COOKIE_DECISION, "yes");
     if (window.cookieDecisionCallback instanceof Function) {
       try {
         window.cookieDecisionCallback();
@@ -67,7 +48,7 @@ class TermsPrompt extends React.PureComponent {
 
   handleNo = () => {
     this.setState({ isOpen: false });
-    storageSet(CookieDecision, "no");
+    storageSet(KEYS.COOKIE_DECISION, "no");
   };
 
   renderTos() {
@@ -84,7 +65,7 @@ class TermsPrompt extends React.PureComponent {
           }}
           href={tosURL}
           target="_blank"
-          rel="noopener noreferrer"
+          rel="noopener"
         >
           terms of service
         </a>
@@ -106,7 +87,7 @@ class TermsPrompt extends React.PureComponent {
           }}
           href={privacyURL}
           target="_blank"
-          rel="noopener noreferrer"
+          rel="noopener"
         >
           privacy policy
         </a>

client/src/components/util/localStorage.js

@@ -0,0 +1,22 @@
+export const KEYS = {
+  COOKIE_DECISION: "cxg.cookieDecision",
+  LOGIN_PROMPT: "cxg.LOGIN_PROMPT",
+};
+
+export function storageGet(key, defaultValue = null) {
+  try {
+    const val = window.localStorage.getItem(key);
+    if (val === null) return defaultValue;
+    return val;
+  } catch (e) {
+    return defaultValue;
+  }
+}
+
+export function storageSet(key, value) {
+  try {
+    window.localStorage.setItem(key, value);
+  } catch {
+    // continue
+  }
+}

client/src/components/util/truncate.js

@@ -7,6 +7,8 @@ const SPLIT_STYLE = {
   display: "flex",
   overflow: "hidden",
   justifyContent: "flex-start",
+  width: "100%", // There are probably additional styles that we don't want to stack
+  padding: 0,
 };
 
 const FIRST_HALF_STYLE = {
@@ -40,7 +42,7 @@ export default (props) => {
   ) {
     throw Error("Only pass a single child with text to Truncate");
   }
-  const originalString = children.props.children;
+  const originalString = String(children.props.children);
 
   let firstString;
   let secondString;
@@ -58,7 +60,7 @@ export default (props) => {
     }
   }
 
-  const inheritedColor = children.props.style.color;
+  const inheritedColor = children.props.style?.color;
 
   const splitStyle = { ...children.props.style, ...SPLIT_STYLE };
   const secondHalfContentStyle = {
@@ -93,6 +95,7 @@ export default (props) => {
         preventOverflow: { enabled: false },
         hide: { enabled: false },
       }}
+      targetProps={{ style: children.props.style }}
     >
       {newChildren}
     </Tooltip>

client/src/reducers/annotations.js

@@ -26,6 +26,7 @@ const Annotations = (
     categoryBeingEdited: null,
     categoryAddingNewLabel: null,
     labelEditable: { category: null, label: null },
+    promptForFilename: true,
   },
   action
 ) => {
@@ -37,10 +38,13 @@ const Annotations = (
         action.config.parameters?.[
           "annotations-data-collection-name-is-read-only"
         ] ?? false;
+      const promptForFilename =
+        action.config.parameters?.["user_annotation_collection_name_enabled"];
       return {
         ...state,
         dataCollectionNameIsReadOnly,
         dataCollectionName,
+        promptForFilename,
       };
     }
 

client/src/reducers/controls.js

@@ -20,6 +20,8 @@ const Controls = (
     scatterplotXXaccessor: null, // just easier to read
     scatterplotYYaccessor: null,
     graphRenderCounter: 0 /* integer as <Component key={graphRenderCounter} - a change in key forces a remount */,
+
+    datasetDrawer: false,
   },
   action
 ) => {
@@ -162,6 +164,12 @@ const Controls = (
         scatterplotYYaccessor: null,
       };
 
+    /**************************
+          Dataset Drawer
+     **************************/
+    case "toggle dataset drawer":
+      return { ...state, datasetDrawer: !state.datasetDrawer };
+
     default:
       return state;
   }

client/src/reducers/index.js

@@ -4,6 +4,7 @@ import thunk from "redux-thunk";
 import cascadeReducers from "./cascade";
 import undoable from "./undoable";
 import config from "./config";
+import userInfo from "./userInfo";
 import annoMatrix from "./annoMatrix";
 import obsCrossfilter from "./obsCrossfilter";
 import categoricalSelection from "./categoricalSelection";
@@ -20,7 +21,7 @@ import centroidLabels from "./centroidLabels";
 import pointDialation from "./pointDilation";
 import { reembedController } from "./reembed";
 import { gcMiddleware as annoMatrixGC } from "../annoMatrix";
-
+import singleContinuousValue from "./singleContinuousValue";
 import undoableConfig from "./undoableConfig";
 
 const Reducer = undoable(
@@ -31,6 +32,7 @@ const Reducer = undoable(
     ["ontology", ontology],
     ["annotations", annotations],
     ["layoutChoice", layoutChoice],
+    ["singleContinuousValue", singleContinuousValue],
     ["categoricalSelection", categoricalSelection],
     ["continuousSelection", continuousSelection],
     ["graphSelection", graphSelection],
@@ -41,6 +43,7 @@ const Reducer = undoable(
     ["pointDilation", pointDialation],
     ["reembedController", reembedController],
     ["autosave", autosave],
+    ["userInfo", userInfo],
   ]),
   [
     "annoMatrix",

client/src/reducers/singleContinuousValue.js

@@ -0,0 +1,14 @@
+const initialState = {
+  singleContinuousValues: new Map(),
+};
+const singleContinuousValue = (state = initialState, action) => {
+  switch (action.type) {
+    case "add single continuous value":
+      state.singleContinuousValues.set(action.field, action.value);
+      return state;
+    default:
+      return state;
+  }
+};
+
+export default singleContinuousValue;

client/src/reducers/userInfo.js

@@ -0,0 +1,26 @@
+const UserInfo = (state = {}, action) => {
+  switch (action.type) {
+    case "initial data load start":
+      return {
+        ...state,
+        loading: true,
+        error: null,
+      };
+    case "userInfo load complete":
+      return {
+        ...state,
+        loading: false,
+        error: null,
+        ...action.userInfo,
+      };
+    case "initial data load error":
+      return {
+        ...state,
+        error: action.error,
+      };
+    default:
+      return state;
+  }
+};
+
+export default UserInfo;

client/src/util/stateManager/colorHelpers.js

@@ -94,16 +94,31 @@ export const createColorTable = memoize(_createColorTable);
 export function loadUserColorConfig(userColors) {
   const convertedUserColors = {};
   Object.keys(userColors).forEach((category) => {
-    const [colors, scaleMap] = Object.keys(userColors[category]).reduce(
-      (acc, label, i) => {
-        const color = parseRGB(userColors[category][label]);
-        acc[0][label] = color;
-        acc[1][i] = d3.rgb(255 * color[0], 255 * color[1], 255 * color[2]);
-        return acc;
-      },
-      [{}, {}]
-    );
-    const scale = (i) => scaleMap[i];
+    // We cannot iterate over keys without sorting
+    // because we handle categorical values in alphabetical order __ignoring case__
+    //  while Object.keys() _usually_ is ordered alphabetically where all upper characters are less than lowercase (A, B, C, a, b, c)
+    const [colors, scaleMap] = Object.keys(userColors[category])
+      .sort((a, b) => {
+        a = a.toLowerCase();
+        b = b.toLowerCase();
+        if (a === b) return 0;
+        if (a > b) return 1;
+        return -1;
+      })
+      .reduce(
+        (acc, label) => {
+          const color = parseRGB(userColors[category][label]);
+          acc[0][label] = color;
+          acc[1][label] = d3.rgb(
+            255 * color[0],
+            255 * color[1],
+            255 * color[2]
+          );
+          return acc;
+        },
+        [{}, {}]
+      );
+    const scale = (label) => scaleMap[label];
     convertedUserColors[category] = { colors, scale };
   });
   return convertedUserColors;

dev_docs/cxg.md

@@ -1,3 +1,5 @@
+## UPDATE (9/30/2020): Starting today, the name Corpora will only be used as the internal project name, with cellxgene Data Portal being the official product name
+
 # CXG Data Format Specification
 
 Document Status: _draft_

dev_docs/release_process.md

@@ -1,4 +1,4 @@
-# cellxgene release process
+# cellxgene Release Process
 
 _This document defines the release process for cellxgene_
 
@@ -16,71 +16,65 @@ The release process should result in the following side-effects:
 
 Note all release tags pushed to GitHub MUST follow semantic versioning.
 
-## Recipe
+## Releasing a Major or Minor Version of cellxgene
 
-Follow these steps to create a release.
+Please scroll down the section below for how to release a patch version. Follow these steps to create a major or minor release.
 
 1.  Preparation:
     -   python3.6 environment, and a cellxgene clone
-    -   Define the release version number, using [semantic versioning](https://semver.org/),
-        and specifying all three digits (eg, 0.3.0)
-    -   Write the release title and release notes and add to
-        [release notes document](https://docs.google.com/document/d/1KnHwkYfhyWO5H8BDcMu7y3ogjvq5Yi4OwpmZ8DB6w0Y/edit)
-2.  Create a release branch, eg, `release-version`
-3.  In the release branch:
-    -   Run `make release-stage-1 PART=[major | minor | patch]` where you choose major/minor/patch depending on which part of the version
-        is being bumped (eg, 0.2.9->0.3 is minor).
-4.  Commit and push the new branch
-5.  Create a PR for the release.
-    -   [optional] As needed, conduct PR review.
-6.  Merge to the `main` branch
-7.  Publish to pypi by performing the following steps (assumes you that you have registered for pypi,
-    and that you have write access to the cellxgene pypi package):
-    -   Build the distribution and upload to test pypi `make release-stage-2`
-    -   Test the test installation in a fresh virtual environment using `make install-release-test`
-    -   Upload the package to real pypi using `make release-stage-final`
-    -   Test the installation in a fresh virtual environment using `pip install cellxgene`
-8.  Create Github release using the version number and release notes
-    ([instructions](https://help.github.com/articles/creating-releases/)).
-    -   Draft new release
-    -   Type version name matching release version number from (1)
-    -   Select `main` as release branch (ensure you merged the release PR)
-    -   Type title `Release {version num}`
-    -   [optional] Check pre-release if this release is not ready for production
-    -   Publish Release
+    -   Define the release version number, using [semantic versioning](https://semver.org/), and specifying all three digits (e.g., 0.3.0)
+    -   Write the release title and release notes and add to [release notes document](https://docs.google.com/document/d/1KnHwkYfhyWO5H8BDcMu7y3ogjvq5Yi4OwpmZ8DB6w0Y/edit)
+2.  Create a release branch, eg, `release-version-0.16.0`
+3.  In the release branch, run `make create-release-candidate PART=[major | minor | patch]` where you choose major/minor/patch depending on which part of the version is being bumped (e.g., `0.2.9` -> `0.3.0` is minor version bump). This will bump the version and create a release *candidate* version (i.e. `0.3.0-rc.0`).
+4.  Commit and push the new branch. This will trigger tests to ensure that your branch isn't broken.
+5. Upload the release candidate to Test PyPI by running the command `make release-candidate-to-test-pypi`. (Make sure you are registered for PyPI and Test PyPI and you have write access to the cellxgene PyPI package for both).
+6. Verify the release candidate in a fresh virtual environment by running `make install-release-test` which installs the cellxgene build you just uploaded the Test PyPI.
+7. If you find errors with the release candidate, run `make recreate-release-candidate` to increment the release candidate version (i.e. `0.3.0-rc.0` -> `0.3.0-rc.1`). Then go back to Steps 5 and 6 to re-upload and re-test the new release candidate.
+8. If everything looks good, push the release to Test PyPI without the release candidate tag by running the command `make release-final-to-test-pypi` (i.e. `0.3.0-rc.1` -> `0.3.0`).
+    - **NOTE:** Once you push the final release version to Test PyPI, you cannot ever re-upload the build again. If you need to make changes to the build, you will have to "burn" the version number and bump the part again and go back to step 1 with a brand new version number. For example, if you upload `0.3.0` to Test PyPI and realize there's a bug, you will have to create a new version `0.4.0` and there will be no `0.3.0` version of cellxgene. This is why testing the release candidate is very important.
+9.  Create a PR for the release and conduct a PR review.
+10.  Merge to the `main` branch.
+11.  Publish to PyPI (prod) (assuming you that you have registered for PyPI, and that you have write access to the cellxgene pypi package) by running `make release-final`.
+12. Test the installation in a fresh virtual environment by running `pip install --no-cache-dir cellxgene`.
+13.  Create Github release using the version number and release notes ([instructions](https://help.github.com/articles/creating-releases/)):
+     - Draft new release
+     - Type version name matching release version number from (1)
+     - Select `main` as release branch (ensure you merged the release PR)
+     - Type title `Release {version num}`
+     - [optional] Check pre-release if this release is not ready for production
+     - Publish Release
 
-The optional steps are for testing purposes, and are recommended
-for publishing any major releases, and any releases that significantly
-change the packaging (e.g. new bundled files, new dependencies, etc.)
+The optional steps are for testing purposes, and are recommended for publishing any major releases, and any releases that significantly change the packaging (e.g. new bundled files, new dependencies, etc.)
 
-### Point release (special case)
+### Releasing a Patch Version of cellxgene (special case)
 
-To make a bugfix release (a point release) when there are already other changes in `main` we need to do a modified version of our release process. The difference is that instead of using `main` we are going make our release branch off of the tag for the release we want to patch. We cherrypick the commits that we want to include in the patch. Then instead of merging to `main`, we create the release directly off of the branch.
+To make a bugfix release (a point release/patch release) when there are already other changes in `main` we need to do a modified version of our release process. The difference is that instead of using `main` we are going make our release branch off of the tag for the release we want to patch. We cherrypick the commits that we want to include in the patch. Then instead of merging to `main`, we create the release directly off of the branch.
 
 1.  (same as above) Preparation:
     -   python3.6 environment, and a cellxgene clone
-    -   Define the release version number, using [semantic versioning](https://semver.org/),
-        and specifying all three digits (eg, 0.3.0) (for this you will update the last digit to represent a bugfix change)
-    -   Write the release title and release notes and add to
-        [release notes document](https://docs.google.com/document/d/1KnHwkYfhyWO5H8BDcMu7y3ogjvq5Yi4OwpmZ8DB6w0Y/edit)
+    -   Define the release version number, using [semantic versioning](https://semver.org/), and specifying all three digits (e.g., 0.3.2) (for this you will update the last digit to represent a bugfix change).
+    -   Write the release title and release notes and add to [release notes document](https://docs.google.com/document/d/1KnHwkYfhyWO5H8BDcMu7y3ogjvq5Yi4OwpmZ8DB6w0Y/edit)
 2.  Create a release branch off of the tag for the release you want to update.
-    -   Checkout the tag for the release you want to fix. ex. if we are fixing 0.9.0: `git checkout 0.9.0`
-    -   Create a branch from that tag. `git branch release-0.9.1`
+    -   Checkout the tag for the release you want to fix. For example, if we are fixing 0.9.0: `git checkout 0.9.0`.
+    -   Create a branch from that tag. `git branch release-version-0.9.1`
 3.  Cherrypick the commits that you want included in this patch.
-    -   Test that the cherrypicked commits landed and fixed the issue
-    -   We WILL NOT merge this branch back into `main`, these commits should already exist in `main`.
-4.  In the release branch:
-    -   Run `make release-stage-1 PART=patch`.
-5.  Commit and push the new branch. DO NOT MAKE A PR OR MERGE TO `main`.
-    -   wait for release to pass the tests
-6.  Publish to pypi by performing the following steps (assumes you that you have registered for pypi,
-    and that you have write access to the cellxgene pypi package): - Build the distribution and upload to test pypi `make release-stage-2` - Test the test installation in a fresh virtual environment using `make install-release-test` - Upload the package to real pypi using `make release-stage-final` - Test the installation in a fresh virtual environment using
-    `pip install --no-cache-dir cellxgene`
-7.  Create Github release using the version number and release notes
+    -   Test that the cherrypicked commits landed and fixed the issue locally.
+    -   We **WILL NOT** merge this branch back into `main` as these commits should already exist in `main`.
+4.  In the release branch (i.e. `release-version-0.9.1`), run `make create-release-candidate PART=patch` to bump the patch version and create the first release candidate (i.e. `0.9.1-rc.0`).
+5. Run `make release-candidate-to-test-pypi` to upload the release candidate to Test PyPI.
+6. Verify the release candidate in a fresh virtual environment by running `make install-release-test` which installs the cellxgene build you just uploaded the Test PyPI.
+7. If you find errors with the release candidate, run `make recreate-release-candidate` to increment the release candidate version (i.e. `0.9.1-rc.0` -> `0.9.1-rc.1`). Then go back to Steps 5 and 6 to re-upload and re-test the new release candidate.
+8. If everything looks good, push the final version of the release to Test PyPI without the release candidate tag by running the command `make release-final-to-test-pypi` (i.e. `0.9.1-rc.1` -> `0.9.1`).
+    - **NOTE:** Once you push the final release version to Test PyPI, you cannot ever re-upload the build again. If you need to make changes to the build, you will have to "burn" the version number and bump the part again and go back to step 1 with a brand new version number. For example, if you upload `0.9.1` to Test PyPI and realize there's a bug, you will have to create a new version `0.9.2` and there will be no `0.9.1` version of cellxgene. This is why testing the release candidate is very important.
+9.  Commit and push the new branch. DO NOT MAKE A PR OR MERGE TO `main`.
+    -   Wait for release to pass the tests.
+10.  Publish to PyPI (prod) (assuming you that you have registered for PyPI, and that you have write access to the cellxgene pypi package) by running `make release-final`.
+11. Test the installation in a fresh virtual environment by running `pip install --no-cache-dir cellxgene`.
+12.  Create Github release using the version number and release notes
     ([instructions](https://help.github.com/articles/creating-releases/)).
     -   Draft new release
     -   Type version name matching release version number from (1)
-    -   _Different than above_ Select the release-branch you pushed at step 5 as release branch
+    -   [**_Different than above_**] Select the release-branch you pushed at step 5 as release branch
     -   Type title `Release {version num}`
     -   [optional] Check pre-release if this release is not ready for production
     -   Publish Release
@@ -93,36 +87,52 @@ _PyPi doesn't allow you to reupload a release with the same version number_
 If you accidentally burned a release number you want to use on prod, you have a few options:
 
 1. OPTION 1: Create distribution `make pydist`; test release locally `pip install dist/<release tarball>`;
-   then upload to prod `make release-stage-final`.
+   then upload to prod `make release-final`.
 2. OPTION 2: (DANGER) release directly to prod: `make release-directly-to-prod`.
-3. OPTION 3: If the release was burned on prod as well run from Step 3 again with option
-   PART=patch until you get to an unburned version.
+3. OPTION 3: If the release was burned on prod as well run from Step 3 again with option PART=patch until you get to an unburned version.
 
 ### The release doesn't install or fails your tests when you install it
 
 Delete it from pypi - Go to pypi.org -> sign in -> go to the cellxgene package -> click manage -> then in the options drop down click delete -> follow the instructions. You will not be able to use that release number again. If it is a minor bug and not a major regression, you can just release a patch.
 
-### If you need to run stage final on a different computer than stage 2
+### If you need to run the final upload to PyPI (prod) on a different computer than where you ran the command to upload to Test PyPI.
 
-If you run stage final without running stage 2 first, the dist will not have been build on the computer running stage final. The solution is to run `make release-directly-to-prod`. This both builds the distribution files and then releases directly to prod pypi.org.
+If you run `make release-final` without running `make release-final-to-test-pypi` first, the dist will not have been build on the computer running the final PyPI push. The solution is to run `make release-directly-to-prod`. This both builds the distribution files and then releases directly to prod pypi.org.
 
-## Stage Details
+## Command Details
 
-### Stage 1 - `make release-stage-1`
+### Initial creation stage - `make create-release-candidate PART=[major | minor | patch]`
 
     1. Pip installs requirements-dev
-    2. Bumps version by [PART]
+    2. Bumps version by [PART] and creates the first release candidate.
     3. Deletes build directory, client/build, dist and cellxgene.egg-info
     4. Creates the package-lock.json
 
-### Stage 2 - `make release-stage-2`
+### Test PyPI upload stage - `make release-candidate-to-test-pypi`
 
     1. Pip installs requirements-dev
     2. Builds client and server
     3. Creates distribution release (sdist)
     4. Uploads to test.pypi.org
+    
+### Recreating release candidate stage(s) - `make recreate-release-candidate`
 
-### Stage final - `make release-stage-final`
+    1. Pip installs requirements-dev
+    2. Bumps release candidate version number.
+    3. Deletes build directory, client/build, dist and cellxgene.egg-info
+    4. Creates the package-lock.json
+
+### Penultimate stage, final release to Test PyPI - `make release-final-to-test-pypi`
+    1. Pip installs requirements-dev
+    2. Removes release candidate tag from the version number.
+    3. Deletes build directory, client/build, dist and cellxgene.egg-info
+    4. Creates the package-lock.json
+    5. Pip installs requirements-dev
+    6. Builds client and server
+    7. Creates distribution release (sdist)
+    8. Uploads to test.pypi.org
+
+### Final stage - `make release-final`
 
     ** Does not build distribution **
     1. Uploads to pypi.org

docs/_config.yml

@@ -13,6 +13,8 @@ nav:
     url: posts/install
   - title: Gallery
     url: posts/gallery
+  - title: Cellxgene data portal
+    url: https://cellxgene.cziscience.com/
   - title: Demo datasets
     url: posts/demo-data
   - title: Preparing your data
@@ -33,5 +35,3 @@ nav:
     url: posts/contribute
   - title: Contact & finding help
     url: posts/contact
-  - title: cellxgene.cziscience.com
-    url: posts/cellxgene_cziscience_com

docs/_site/index.html

@@ -7,7 +7,7 @@
 
 <!-- Begin Jekyll SEO tag v2.6.1 -->
 <title>Index | cellxgene</title>
-<meta name="generator" content="Jekyll v3.8.7" />
+<meta name="generator" content="Jekyll v3.9.0" />
 <meta property="og:title" content="Index" />
 <meta property="og:locale" content="en_US" />
 <meta name="description" content="An interactive explorer for single-cell transcriptomics data" />
@@ -16,10 +16,10 @@
 <meta property="og:url" content="https://chanzuckerberg.github.io/cellxgene/" />
 <meta property="og:site_name" content="cellxgene" />
 <script type="application/ld+json">
-{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"An interactive explorer for single-cell transcriptomics data","@type":"WebSite","headline":"Index","url":"https://chanzuckerberg.github.io/cellxgene/","name":"cellxgene","@context":"https://schema.org"}</script>
+{"url":"https://chanzuckerberg.github.io/cellxgene/","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"headline":"Index","name":"cellxgene","description":"An interactive explorer for single-cell transcriptomics data","@type":"WebSite","@context":"https://schema.org"}</script>
 <!-- End Jekyll SEO tag -->
 
-    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=6bda27f5542fb7f469425e1cd99f2f37268b095f">
+    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=3718e894edc8a8f6e7776946695ab37c5c96ec9f">
     <!--[if lt IE 9]>
     <script src="https://cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv.min.js"></script>
     <![endif]-->
@@ -46,6 +46,10 @@
         
         
         
+        <a href="https://cellxgene.cziscience.com/" class="btn">Cellxgene data portal</a><br>
+        
+        
+        
         <a href="/cellxgene/posts/demo-data" class="btn">Demo datasets</a><br>
         
         
@@ -85,10 +89,6 @@
         <a href="/cellxgene/posts/contact" class="btn">Contact & finding help</a><br>
         
         
-        
-        <a href="/cellxgene/posts/cellxgene_cziscience_com" class="btn">cellxgene.cziscience.com</a><br>
-        
-        
 
         <a href="https://github.com/chanzuckerberg/cellxgene" class="btn" target="_blank">Code</a>
     </p>

docs/_site/posts/annotations.html

@@ -7,7 +7,7 @@
 
 <!-- Begin Jekyll SEO tag v2.6.1 -->
 <title>annotations | cellxgene</title>
-<meta name="generator" content="Jekyll v3.8.7" />
+<meta name="generator" content="Jekyll v3.9.0" />
 <meta property="og:title" content="annotations" />
 <meta property="og:locale" content="en_US" />
 <meta name="description" content="Creating annotations" />
@@ -16,10 +16,10 @@
 <meta property="og:url" content="https://chanzuckerberg.github.io/cellxgene/posts/annotations.html" />
 <meta property="og:site_name" content="cellxgene" />
 <script type="application/ld+json">
-{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"Creating annotations","@type":"WebPage","headline":"annotations","url":"https://chanzuckerberg.github.io/cellxgene/posts/annotations.html","@context":"https://schema.org"}</script>
+{"url":"https://chanzuckerberg.github.io/cellxgene/posts/annotations.html","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"headline":"annotations","description":"Creating annotations","@type":"WebPage","@context":"https://schema.org"}</script>
 <!-- End Jekyll SEO tag -->
 
-    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=6bda27f5542fb7f469425e1cd99f2f37268b095f">
+    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=3718e894edc8a8f6e7776946695ab37c5c96ec9f">
     <!--[if lt IE 9]>
     <script src="https://cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv.min.js"></script>
     <![endif]-->
@@ -46,6 +46,10 @@
         
         
         
+        <a href="https://cellxgene.cziscience.com/" class="btn">Cellxgene data portal</a><br>
+        
+        
+        
         <a href="/cellxgene/posts/demo-data" class="btn">Demo datasets</a><br>
         
         
@@ -85,10 +89,6 @@
         <a href="/cellxgene/posts/contact" class="btn">Contact & finding help</a><br>
         
         
-        
-        <a href="/cellxgene/posts/cellxgene_cziscience_com" class="btn">cellxgene.cziscience.com</a><br>
-        
-        
 
         <a href="https://github.com/chanzuckerberg/cellxgene" class="btn" target="_blank">Code</a>
     </p>

docs/_site/posts/contact.html

@@ -7,7 +7,7 @@
 
 <!-- Begin Jekyll SEO tag v2.6.1 -->
 <title>Contact | cellxgene</title>
-<meta name="generator" content="Jekyll v3.8.7" />
+<meta name="generator" content="Jekyll v3.9.0" />
 <meta property="og:title" content="Contact" />
 <meta property="og:locale" content="en_US" />
 <meta name="description" content="Contact" />
@@ -16,10 +16,10 @@
 <meta property="og:url" content="https://chanzuckerberg.github.io/cellxgene/posts/contact.html" />
 <meta property="og:site_name" content="cellxgene" />
 <script type="application/ld+json">
-{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"Contact","@type":"WebPage","headline":"Contact","url":"https://chanzuckerberg.github.io/cellxgene/posts/contact.html","@context":"https://schema.org"}</script>
+{"url":"https://chanzuckerberg.github.io/cellxgene/posts/contact.html","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"headline":"Contact","description":"Contact","@type":"WebPage","@context":"https://schema.org"}</script>
 <!-- End Jekyll SEO tag -->
 
-    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=6bda27f5542fb7f469425e1cd99f2f37268b095f">
+    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=3718e894edc8a8f6e7776946695ab37c5c96ec9f">
     <!--[if lt IE 9]>
     <script src="https://cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv.min.js"></script>
     <![endif]-->
@@ -46,6 +46,10 @@
         
         
         
+        <a href="https://cellxgene.cziscience.com/" class="btn">Cellxgene data portal</a><br>
+        
+        
+        
         <a href="/cellxgene/posts/demo-data" class="btn">Demo datasets</a><br>
         
         
@@ -85,10 +89,6 @@
         <a href="/cellxgene/posts/contact" class="btn"><b>Contact & finding help</b></a><br>
         
         
-        
-        <a href="/cellxgene/posts/cellxgene_cziscience_com" class="btn">cellxgene.cziscience.com</a><br>
-        
-        
 
         <a href="https://github.com/chanzuckerberg/cellxgene" class="btn" target="_blank">Code</a>
     </p>

docs/_site/posts/contribute.html

@@ -7,7 +7,7 @@
 
 <!-- Begin Jekyll SEO tag v2.6.1 -->
 <title>Code of conduct | cellxgene</title>
-<meta name="generator" content="Jekyll v3.8.7" />
+<meta name="generator" content="Jekyll v3.9.0" />
 <meta property="og:title" content="Code of conduct" />
 <meta property="og:locale" content="en_US" />
 <meta name="description" content="An interactive explorer for single-cell transcriptomics data" />
@@ -16,10 +16,10 @@
 <meta property="og:url" content="https://chanzuckerberg.github.io/cellxgene/posts/contribute.html" />
 <meta property="og:site_name" content="cellxgene" />
 <script type="application/ld+json">
-{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"An interactive explorer for single-cell transcriptomics data","@type":"WebPage","headline":"Code of conduct","url":"https://chanzuckerberg.github.io/cellxgene/posts/contribute.html","@context":"https://schema.org"}</script>
+{"url":"https://chanzuckerberg.github.io/cellxgene/posts/contribute.html","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"headline":"Code of conduct","description":"An interactive explorer for single-cell transcriptomics data","@type":"WebPage","@context":"https://schema.org"}</script>
 <!-- End Jekyll SEO tag -->
 
-    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=6bda27f5542fb7f469425e1cd99f2f37268b095f">
+    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=3718e894edc8a8f6e7776946695ab37c5c96ec9f">
     <!--[if lt IE 9]>
     <script src="https://cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv.min.js"></script>
     <![endif]-->
@@ -46,6 +46,10 @@
         
         
         
+        <a href="https://cellxgene.cziscience.com/" class="btn">Cellxgene data portal</a><br>
+        
+        
+        
         <a href="/cellxgene/posts/demo-data" class="btn">Demo datasets</a><br>
         
         
@@ -85,10 +89,6 @@
         <a href="/cellxgene/posts/contact" class="btn">Contact & finding help</a><br>
         
         
-        
-        <a href="/cellxgene/posts/cellxgene_cziscience_com" class="btn">cellxgene.cziscience.com</a><br>
-        
-        
 
         <a href="https://github.com/chanzuckerberg/cellxgene" class="btn" target="_blank">Code</a>
     </p>

docs/_site/posts/demo-data.html

@@ -7,7 +7,7 @@
 
 <!-- Begin Jekyll SEO tag v2.6.1 -->
 <title>demo-data | cellxgene</title>
-<meta name="generator" content="Jekyll v3.8.7" />
+<meta name="generator" content="Jekyll v3.9.0" />
 <meta property="og:title" content="demo-data" />
 <meta property="og:locale" content="en_US" />
 <meta name="description" content="Demo datasets" />
@@ -16,10 +16,10 @@
 <meta property="og:url" content="https://chanzuckerberg.github.io/cellxgene/posts/demo-data.html" />
 <meta property="og:site_name" content="cellxgene" />
 <script type="application/ld+json">
-{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"Demo datasets","@type":"WebPage","headline":"demo-data","url":"https://chanzuckerberg.github.io/cellxgene/posts/demo-data.html","@context":"https://schema.org"}</script>
+{"url":"https://chanzuckerberg.github.io/cellxgene/posts/demo-data.html","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"headline":"demo-data","description":"Demo datasets","@type":"WebPage","@context":"https://schema.org"}</script>
 <!-- End Jekyll SEO tag -->
 
-    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=6bda27f5542fb7f469425e1cd99f2f37268b095f">
+    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=3718e894edc8a8f6e7776946695ab37c5c96ec9f">
     <!--[if lt IE 9]>
     <script src="https://cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv.min.js"></script>
     <![endif]-->
@@ -46,6 +46,10 @@
         
         
         
+        <a href="https://cellxgene.cziscience.com/" class="btn">Cellxgene data portal</a><br>
+        
+        
+        
         <a href="/cellxgene/posts/demo-data" class="btn"><b>Demo datasets</b></a><br>
         
         
@@ -85,10 +89,6 @@
         <a href="/cellxgene/posts/contact" class="btn">Contact & finding help</a><br>
         
         
-        
-        <a href="/cellxgene/posts/cellxgene_cziscience_com" class="btn">cellxgene.cziscience.com</a><br>
-        
-        
 
         <a href="https://github.com/chanzuckerberg/cellxgene" class="btn" target="_blank">Code</a>
     </p>

docs/_site/posts/gallery.html

@@ -7,7 +7,7 @@
 
 <!-- Begin Jekyll SEO tag v2.6.1 -->
 <title>Gallery | cellxgene</title>
-<meta name="generator" content="Jekyll v3.8.7" />
+<meta name="generator" content="Jekyll v3.9.0" />
 <meta property="og:title" content="Gallery" />
 <meta property="og:locale" content="en_US" />
 <meta name="description" content="An interactive explorer for single-cell transcriptomics data" />
@@ -16,10 +16,10 @@
 <meta property="og:url" content="https://chanzuckerberg.github.io/cellxgene/posts/gallery.html" />
 <meta property="og:site_name" content="cellxgene" />
 <script type="application/ld+json">
-{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"An interactive explorer for single-cell transcriptomics data","@type":"WebPage","headline":"Gallery","url":"https://chanzuckerberg.github.io/cellxgene/posts/gallery.html","@context":"https://schema.org"}</script>
+{"url":"https://chanzuckerberg.github.io/cellxgene/posts/gallery.html","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"headline":"Gallery","description":"An interactive explorer for single-cell transcriptomics data","@type":"WebPage","@context":"https://schema.org"}</script>
 <!-- End Jekyll SEO tag -->
 
-    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=6bda27f5542fb7f469425e1cd99f2f37268b095f">
+    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=3718e894edc8a8f6e7776946695ab37c5c96ec9f">
     <!--[if lt IE 9]>
     <script src="https://cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv.min.js"></script>
     <![endif]-->
@@ -46,6 +46,10 @@
         
         
         
+        <a href="https://cellxgene.cziscience.com/" class="btn">Cellxgene data portal</a><br>
+        
+        
+        
         <a href="/cellxgene/posts/demo-data" class="btn">Demo datasets</a><br>
         
         
@@ -85,10 +89,6 @@
         <a href="/cellxgene/posts/contact" class="btn">Contact & finding help</a><br>
         
         
-        
-        <a href="/cellxgene/posts/cellxgene_cziscience_com" class="btn">cellxgene.cziscience.com</a><br>
-        
-        
 
         <a href="https://github.com/chanzuckerberg/cellxgene" class="btn" target="_blank">Code</a>
     </p>
@@ -130,7 +130,7 @@ Check out the cool data that our users are using cellxgene to explore!</p>
 
 <h3 id="melanoma"><a href="https://melanoma.cellgeni.sanger.ac.uk/">Melanoma</a></h3>
 
-<h3 id="czis-own-cellxgene-site"><a href="cellxgene_cziscience_com">CZI’s own cellxgene site</a></h3>
+<h3 id="czis-own-cellxgene-site"><a href="https://cellxgene.cziscience.com/">CZI’s own cellxgene site</a></h3>
 
 <p><em>Want us to link to your dataset here? <a href="contact">Just send us a note!</a></em></p>
 

docs/_site/posts/hosted.html

@@ -7,7 +7,7 @@
 
 <!-- Begin Jekyll SEO tag v2.6.1 -->
 <title>Hosting cellxgene on the web | cellxgene</title>
-<meta name="generator" content="Jekyll v3.8.7" />
+<meta name="generator" content="Jekyll v3.9.0" />
 <meta property="og:title" content="Hosting cellxgene on the web" />
 <meta property="og:locale" content="en_US" />
 <meta name="description" content="An interactive explorer for single-cell transcriptomics data" />
@@ -16,10 +16,10 @@
 <meta property="og:url" content="https://chanzuckerberg.github.io/cellxgene/posts/hosted.html" />
 <meta property="og:site_name" content="cellxgene" />
 <script type="application/ld+json">
-{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"An interactive explorer for single-cell transcriptomics data","@type":"WebPage","headline":"Hosting cellxgene on the web","url":"https://chanzuckerberg.github.io/cellxgene/posts/hosted.html","@context":"https://schema.org"}</script>
+{"url":"https://chanzuckerberg.github.io/cellxgene/posts/hosted.html","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"headline":"Hosting cellxgene on the web","description":"An interactive explorer for single-cell transcriptomics data","@type":"WebPage","@context":"https://schema.org"}</script>
 <!-- End Jekyll SEO tag -->
 
-    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=6bda27f5542fb7f469425e1cd99f2f37268b095f">
+    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=3718e894edc8a8f6e7776946695ab37c5c96ec9f">
     <!--[if lt IE 9]>
     <script src="https://cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv.min.js"></script>
     <![endif]-->
@@ -46,6 +46,10 @@
         
         
         
+        <a href="https://cellxgene.cziscience.com/" class="btn">Cellxgene data portal</a><br>
+        
+        
+        
         <a href="/cellxgene/posts/demo-data" class="btn">Demo datasets</a><br>
         
         
@@ -85,10 +89,6 @@
         <a href="/cellxgene/posts/contact" class="btn">Contact & finding help</a><br>
         
         
-        
-        <a href="/cellxgene/posts/cellxgene_cziscience_com" class="btn">cellxgene.cziscience.com</a><br>
-        
-        
 
         <a href="https://github.com/chanzuckerberg/cellxgene" class="btn" target="_blank">Code</a>
     </p>
@@ -139,35 +139,36 @@
 
 <h1 id="deploying-cellxgene-with-heroku">Deploying cellxgene with Heroku</h1>
 
-<h2 id="quickstart">Quickstart</h2>
+<h2 id="heroku-support">Heroku Support</h2>
 
-<p>Clicking on the following button will forward you to Heroku to begin the deployment process:</p>
+<p>The cellxgene team has decided to end our support for our experimental deploy to Heroku button as we move towards providing a supported method of hosted cellxgene.</p>
 
-<p><a href="https://heroku.com/deploy?template=https://github.com/chanzuckerberg/cellxgene">
- <img src="https://www.herokucdn.com/deploy/button.svg" alt="Deploy" />
-</a></p>
+<p>While we no longer directly support Heroku, it is still possible to create a Heroku app via <a href="https://github.com/chanzuckerberg/cellxgene/blob/main/Dockerfile">our provided Dockerfile here</a> and <a href="https://devcenter.heroku.com/articles/build-docker-images-heroku-yml">Heroku’s documentation</a>.</p>
 
-<p>If not already logged in to Heroku, there you will be prompted to log in or sign up for an account.</p>
+<p>You may have to tweak the <code class="language-plaintext highlighter-rouge">Dockerfile</code> like so:</p>
 
-<p>Once logged in you will be sent to the setup page. Here you can set some of the basic settings for the app:</p>
+<pre><code class="language-Dockerfile">FROM ubuntu:bionic
 
-<h3 id="default-settings">Default settings</h3>
+ENV LC_ALL=C.UTF-8
+ENV LANG=C.UTF-8
 
-<ul>
-  <li><code class="language-plaintext highlighter-rouge">App name</code>: the unique name for your deployment</li>
-  <li>This will also serve as the default URL (e.g. https://cellxgene.herokapp.com/)</li>
-  <li><code class="language-plaintext highlighter-rouge">App owner</code>: Who will own this app. Either you personally or an organization/team</li>
-  <li><code class="language-plaintext highlighter-rouge">Region</code>: Location of the server where the app will be deployed (EU or US)</li>
-</ul>
+RUN apt-get update &amp;&amp; \
+    apt-get install -y build-essential libxml2-dev python3-dev python3-pip zlib1g-dev python3-requests &amp;&amp; \
+    pip3 install cellxgene
 
-<h3 id="configuration">Configuration</h3>
+# ENTRYPOINT ["cellxgene"]  # Heroku doesn't work well with ENTRYPOINT
+</code></pre>
 
-<ul>
-  <li><code class="language-plaintext highlighter-rouge">DATASET</code>: A <em>publicly</em> accessible URL pointing to a .h5ad file to view</li>
-  <li>This defaults to pbm3k.h5ad</li>
-</ul>
+<p>and provide a <code class="language-plaintext highlighter-rouge">heroku.yml</code> file similar to this:</p>
 
-<p>After filling out the settings and pressing the <code class="language-plaintext highlighter-rouge">Deploy app</code> button Heroku will begin building your deployment. This process will take a few minutes, but once completed you will have a personal free hosted version of cellxgene!</p>
+<div class="language-yml highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="na">build</span><span class="pi">:</span>
+  <span class="na">docker</span><span class="pi">:</span>
+    <span class="na">web</span><span class="pi">:</span> <span class="s">Dockerfile</span>
+<span class="na">run</span><span class="pi">:</span>
+  <span class="na">web</span><span class="pi">:</span>
+    <span class="na">command</span><span class="pi">:</span>
+      <span class="pi">-</span> <span class="s">cellxgene launch --host 0.0.0.0 --port $PORT $DATASET</span> <span class="c1"># the DATATSET config var must be defined in your dashboard settings.</span>
+</code></pre></div></div>
 
 <h2 id="what-is-heroku">What is Heroku?</h2>
 

docs/_site/posts/hosted.md

@@ -38,31 +38,38 @@ If you know of other solutions, drop us a note and we'll add to this list.
 
 # Deploying cellxgene with Heroku
 
-## Quickstart
+## Heroku Support
 
-Clicking on the following button will forward you to Heroku to begin the deployment process:
+The cellxgene team has decided to end our support for our experimental deploy to Heroku button as we move towards providing a supported method of hosted cellxgene.
 
-<a href="https://heroku.com/deploy?template=https://github.com/chanzuckerberg/cellxgene">
- <img src="https://www.herokucdn.com/deploy/button.svg" alt="Deploy">
-</a>
+While we no longer directly support Heroku, it is still possible to create a Heroku app via [our provided Dockerfile here](https://github.com/chanzuckerberg/cellxgene/blob/main/Dockerfile) and [Heroku's documentation](https://devcenter.heroku.com/articles/build-docker-images-heroku-yml).
 
-If not already logged in to Heroku, there you will be prompted to log in or sign up for an account.
+You may have to tweak the `Dockerfile` like so:
 
-Once logged in you will be sent to the setup page. Here you can set some of the basic settings for the app:
+```Dockerfile
+FROM ubuntu:bionic
 
-### Default settings
+ENV LC_ALL=C.UTF-8
+ENV LANG=C.UTF-8
 
-- `App name`: the unique name for your deployment
-- This will also serve as the default URL (e.g. https://cellxgene.herokapp.com/)
-- `App owner`: Who will own this app. Either you personally or an organization/team
-- `Region`: Location of the server where the app will be deployed (EU or US)
+RUN apt-get update && \
+    apt-get install -y build-essential libxml2-dev python3-dev python3-pip zlib1g-dev python3-requests && \
+    pip3 install cellxgene
 
-### Configuration
+# ENTRYPOINT ["cellxgene"]  # Heroku doesn't work well with ENTRYPOINT
+```
 
-- `DATASET`: A _publicly_ accessible URL pointing to a .h5ad file to view
-- This defaults to pbm3k.h5ad
+and provide a `heroku.yml` file similar to this:
 
-After filling out the settings and pressing the `Deploy app` button Heroku will begin building your deployment. This process will take a few minutes, but once completed you will have a personal free hosted version of cellxgene!
+```yml
+build:
+  docker:
+    web: Dockerfile
+run:
+  web:
+    command:
+      - cellxgene launch --host 0.0.0.0 --port $PORT $DATASET # the DATATSET config var must be defined in your dashboard settings.
+```
 
 ## What is Heroku?
 

docs/_site/posts/install.html

@@ -7,7 +7,7 @@
 
 <!-- Begin Jekyll SEO tag v2.6.1 -->
 <title>Install | cellxgene</title>
-<meta name="generator" content="Jekyll v3.8.7" />
+<meta name="generator" content="Jekyll v3.9.0" />
 <meta property="og:title" content="Install" />
 <meta property="og:locale" content="en_US" />
 <meta name="description" content="An interactive explorer for single-cell transcriptomics data" />
@@ -16,10 +16,10 @@
 <meta property="og:url" content="https://chanzuckerberg.github.io/cellxgene/posts/install.html" />
 <meta property="og:site_name" content="cellxgene" />
 <script type="application/ld+json">
-{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"An interactive explorer for single-cell transcriptomics data","@type":"WebPage","headline":"Install","url":"https://chanzuckerberg.github.io/cellxgene/posts/install.html","@context":"https://schema.org"}</script>
+{"url":"https://chanzuckerberg.github.io/cellxgene/posts/install.html","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"headline":"Install","description":"An interactive explorer for single-cell transcriptomics data","@type":"WebPage","@context":"https://schema.org"}</script>
 <!-- End Jekyll SEO tag -->
 
-    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=6bda27f5542fb7f469425e1cd99f2f37268b095f">
+    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=3718e894edc8a8f6e7776946695ab37c5c96ec9f">
     <!--[if lt IE 9]>
     <script src="https://cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv.min.js"></script>
     <![endif]-->
@@ -46,6 +46,10 @@
         
         
         
+        <a href="https://cellxgene.cziscience.com/" class="btn">Cellxgene data portal</a><br>
+        
+        
+        
         <a href="/cellxgene/posts/demo-data" class="btn">Demo datasets</a><br>
         
         
@@ -85,10 +89,6 @@
         <a href="/cellxgene/posts/contact" class="btn">Contact & finding help</a><br>
         
         
-        
-        <a href="/cellxgene/posts/cellxgene_cziscience_com" class="btn">cellxgene.cziscience.com</a><br>
-        
-        
 
         <a href="https://github.com/chanzuckerberg/cellxgene" class="btn" target="_blank">Code</a>
     </p>

docs/_site/posts/launch.html

@@ -7,7 +7,7 @@
 
 <!-- Begin Jekyll SEO tag v2.6.1 -->
 <title>demo-data | cellxgene</title>
-<meta name="generator" content="Jekyll v3.8.7" />
+<meta name="generator" content="Jekyll v3.9.0" />
 <meta property="og:title" content="demo-data" />
 <meta property="og:locale" content="en_US" />
 <meta name="description" content="Demo datasets" />
@@ -16,10 +16,10 @@
 <meta property="og:url" content="https://chanzuckerberg.github.io/cellxgene/posts/launch.html" />
 <meta property="og:site_name" content="cellxgene" />
 <script type="application/ld+json">
-{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"Demo datasets","@type":"WebPage","headline":"demo-data","url":"https://chanzuckerberg.github.io/cellxgene/posts/launch.html","@context":"https://schema.org"}</script>
+{"url":"https://chanzuckerberg.github.io/cellxgene/posts/launch.html","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"headline":"demo-data","description":"Demo datasets","@type":"WebPage","@context":"https://schema.org"}</script>
 <!-- End Jekyll SEO tag -->
 
-    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=6bda27f5542fb7f469425e1cd99f2f37268b095f">
+    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=3718e894edc8a8f6e7776946695ab37c5c96ec9f">
     <!--[if lt IE 9]>
     <script src="https://cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv.min.js"></script>
     <![endif]-->
@@ -46,6 +46,10 @@
         
         
         
+        <a href="https://cellxgene.cziscience.com/" class="btn">Cellxgene data portal</a><br>
+        
+        
+        
         <a href="/cellxgene/posts/demo-data" class="btn">Demo datasets</a><br>
         
         
@@ -85,10 +89,6 @@
         <a href="/cellxgene/posts/contact" class="btn">Contact & finding help</a><br>
         
         
-        
-        <a href="/cellxgene/posts/cellxgene_cziscience_com" class="btn">cellxgene.cziscience.com</a><br>
-        
-        
 
         <a href="https://github.com/chanzuckerberg/cellxgene" class="btn" target="_blank">Code</a>
     </p>

docs/_site/posts/methods.html

@@ -7,7 +7,7 @@
 
 <!-- Begin Jekyll SEO tag v2.6.1 -->
 <title>Methods | cellxgene</title>
-<meta name="generator" content="Jekyll v3.8.7" />
+<meta name="generator" content="Jekyll v3.9.0" />
 <meta property="og:title" content="Methods" />
 <meta property="og:locale" content="en_US" />
 <meta name="description" content="An interactive explorer for single-cell transcriptomics data" />
@@ -16,10 +16,10 @@
 <meta property="og:url" content="https://chanzuckerberg.github.io/cellxgene/posts/methods.html" />
 <meta property="og:site_name" content="cellxgene" />
 <script type="application/ld+json">
-{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"An interactive explorer for single-cell transcriptomics data","@type":"WebPage","headline":"Methods","url":"https://chanzuckerberg.github.io/cellxgene/posts/methods.html","@context":"https://schema.org"}</script>
+{"url":"https://chanzuckerberg.github.io/cellxgene/posts/methods.html","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"headline":"Methods","description":"An interactive explorer for single-cell transcriptomics data","@type":"WebPage","@context":"https://schema.org"}</script>
 <!-- End Jekyll SEO tag -->
 
-    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=6bda27f5542fb7f469425e1cd99f2f37268b095f">
+    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=3718e894edc8a8f6e7776946695ab37c5c96ec9f">
     <!--[if lt IE 9]>
     <script src="https://cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv.min.js"></script>
     <![endif]-->
@@ -46,6 +46,10 @@
         
         
         
+        <a href="https://cellxgene.cziscience.com/" class="btn">Cellxgene data portal</a><br>
+        
+        
+        
         <a href="/cellxgene/posts/demo-data" class="btn">Demo datasets</a><br>
         
         
@@ -85,10 +89,6 @@
         <a href="/cellxgene/posts/contact" class="btn">Contact & finding help</a><br>
         
         
-        
-        <a href="/cellxgene/posts/cellxgene_cziscience_com" class="btn">cellxgene.cziscience.com</a><br>
-        
-        
 
         <a href="https://github.com/chanzuckerberg/cellxgene" class="btn" target="_blank">Code</a>
     </p>

docs/_site/posts/prepare.html

@@ -7,7 +7,7 @@
 
 <!-- Begin Jekyll SEO tag v2.6.1 -->
 <title>prepare | cellxgene</title>
-<meta name="generator" content="Jekyll v3.8.7" />
+<meta name="generator" content="Jekyll v3.9.0" />
 <meta property="og:title" content="prepare" />
 <meta property="og:locale" content="en_US" />
 <meta name="description" content="Preparing your data" />
@@ -16,10 +16,10 @@
 <meta property="og:url" content="https://chanzuckerberg.github.io/cellxgene/posts/prepare.html" />
 <meta property="og:site_name" content="cellxgene" />
 <script type="application/ld+json">
-{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"Preparing your data","@type":"WebPage","headline":"prepare","url":"https://chanzuckerberg.github.io/cellxgene/posts/prepare.html","@context":"https://schema.org"}</script>
+{"url":"https://chanzuckerberg.github.io/cellxgene/posts/prepare.html","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"headline":"prepare","description":"Preparing your data","@type":"WebPage","@context":"https://schema.org"}</script>
 <!-- End Jekyll SEO tag -->
 
-    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=6bda27f5542fb7f469425e1cd99f2f37268b095f">
+    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=3718e894edc8a8f6e7776946695ab37c5c96ec9f">
     <!--[if lt IE 9]>
     <script src="https://cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv.min.js"></script>
     <![endif]-->
@@ -46,6 +46,10 @@
         
         
         
+        <a href="https://cellxgene.cziscience.com/" class="btn">Cellxgene data portal</a><br>
+        
+        
+        
         <a href="/cellxgene/posts/demo-data" class="btn">Demo datasets</a><br>
         
         
@@ -85,10 +89,6 @@
         <a href="/cellxgene/posts/contact" class="btn">Contact & finding help</a><br>
         
         
-        
-        <a href="/cellxgene/posts/cellxgene_cziscience_com" class="btn">cellxgene.cziscience.com</a><br>
-        
-        
 
         <a href="https://github.com/chanzuckerberg/cellxgene" class="btn" target="_blank">Code</a>
     </p>

docs/_site/posts/roadmap.html

@@ -7,7 +7,7 @@
 
 <!-- Begin Jekyll SEO tag v2.6.1 -->
 <title>roadmap | cellxgene</title>
-<meta name="generator" content="Jekyll v3.8.7" />
+<meta name="generator" content="Jekyll v3.9.0" />
 <meta property="og:title" content="roadmap" />
 <meta property="og:locale" content="en_US" />
 <meta name="description" content="Roadmap" />
@@ -16,10 +16,10 @@
 <meta property="og:url" content="https://chanzuckerberg.github.io/cellxgene/posts/roadmap.html" />
 <meta property="og:site_name" content="cellxgene" />
 <script type="application/ld+json">
-{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"Roadmap","@type":"WebPage","headline":"roadmap","url":"https://chanzuckerberg.github.io/cellxgene/posts/roadmap.html","@context":"https://schema.org"}</script>
+{"url":"https://chanzuckerberg.github.io/cellxgene/posts/roadmap.html","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"headline":"roadmap","description":"Roadmap","@type":"WebPage","@context":"https://schema.org"}</script>
 <!-- End Jekyll SEO tag -->
 
-    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=6bda27f5542fb7f469425e1cd99f2f37268b095f">
+    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=3718e894edc8a8f6e7776946695ab37c5c96ec9f">
     <!--[if lt IE 9]>
     <script src="https://cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv.min.js"></script>
     <![endif]-->
@@ -46,6 +46,10 @@
         
         
         
+        <a href="https://cellxgene.cziscience.com/" class="btn">Cellxgene data portal</a><br>
+        
+        
+        
         <a href="/cellxgene/posts/demo-data" class="btn">Demo datasets</a><br>
         
         
@@ -85,10 +89,6 @@
         <a href="/cellxgene/posts/contact" class="btn">Contact & finding help</a><br>
         
         
-        
-        <a href="/cellxgene/posts/cellxgene_cziscience_com" class="btn">cellxgene.cziscience.com</a><br>
-        
-        
 
         <a href="https://github.com/chanzuckerberg/cellxgene" class="btn" target="_blank">Code</a>
     </p>

docs/_site/posts/troubleshooting.html

@@ -7,7 +7,7 @@
 
 <!-- Begin Jekyll SEO tag v2.6.1 -->
 <title>Troubleshooting | cellxgene</title>
-<meta name="generator" content="Jekyll v3.8.7" />
+<meta name="generator" content="Jekyll v3.9.0" />
 <meta property="og:title" content="Troubleshooting" />
 <meta property="og:locale" content="en_US" />
 <meta name="description" content="Troubleshooting" />
@@ -16,10 +16,10 @@
 <meta property="og:url" content="https://chanzuckerberg.github.io/cellxgene/posts/troubleshooting.html" />
 <meta property="og:site_name" content="cellxgene" />
 <script type="application/ld+json">
-{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"Troubleshooting","@type":"WebPage","headline":"Troubleshooting","url":"https://chanzuckerberg.github.io/cellxgene/posts/troubleshooting.html","@context":"https://schema.org"}</script>
+{"url":"https://chanzuckerberg.github.io/cellxgene/posts/troubleshooting.html","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"headline":"Troubleshooting","description":"Troubleshooting","@type":"WebPage","@context":"https://schema.org"}</script>
 <!-- End Jekyll SEO tag -->
 
-    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=6bda27f5542fb7f469425e1cd99f2f37268b095f">
+    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=3718e894edc8a8f6e7776946695ab37c5c96ec9f">
     <!--[if lt IE 9]>
     <script src="https://cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv.min.js"></script>
     <![endif]-->
@@ -46,6 +46,10 @@
         
         
         
+        <a href="https://cellxgene.cziscience.com/" class="btn">Cellxgene data portal</a><br>
+        
+        
+        
         <a href="/cellxgene/posts/demo-data" class="btn">Demo datasets</a><br>
         
         
@@ -85,10 +89,6 @@
         <a href="/cellxgene/posts/contact" class="btn">Contact & finding help</a><br>
         
         
-        
-        <a href="/cellxgene/posts/cellxgene_cziscience_com" class="btn">cellxgene.cziscience.com</a><br>
-        
-        
 
         <a href="https://github.com/chanzuckerberg/cellxgene" class="btn" target="_blank">Code</a>
     </p>

docs/posts/gallery.md

@@ -39,6 +39,6 @@ Check out the cool data that our users are using cellxgene to explore!
 
 ### [Melanoma](https://melanoma.cellgeni.sanger.ac.uk/)
 
-### [CZI's own cellxgene site](cellxgene_cziscience_com)
+### [CZI's own cellxgene site](https://cellxgene.cziscience.com/)
 
 _Want us to link to your dataset here? [Just send us a note!](contact)_

server/Makefile

@@ -39,3 +39,11 @@ create-test-db:
 clean-test-db:
 	-docker stop test_db
 	-docker rm test_db
+
+.PHONY: test-annotations-performance
+test-annotations-performance:
+	python test/performance/performance_test_annotations_backend.py
+
+.PHONY: test-annotations-scale
+test-annotations-scale:
+	locust -f test/performance/scale_test_annotations.py --headless -u 30 -r 10 --host https://api.cellxgene.dev.single-cell.czi.technology/cellxgene/e/ --run-time 5m 2>&1 | tee locust_dev_stats.txt

server/__init__.py

@@ -1,9 +1,8 @@
 import logging
 import sys
-
 from server.common.utils.utils import import_plugins
 
-__version__ = "0.16.2"
+__version__ = "0.16.6"
 display_version = "cellxgene v" + __version__
 
 try:

server/app/app.py

@@ -2,8 +2,21 @@ import datetime
 import logging
 from functools import wraps
 from http import HTTPStatus
+from urllib.parse import urlparse
+import hashlib
+import os
 
-from flask import Flask, redirect, current_app, make_response, render_template, abort, Blueprint, request
+from flask import (
+    Flask,
+    redirect,
+    current_app,
+    make_response,
+    render_template,
+    abort,
+    Blueprint,
+    request,
+    send_from_directory,
+)
 from flask_restful import Api, Resource
 from server_timing import Timing as ServerTiming
 
@@ -83,16 +96,19 @@ def dataset_index(url_dataroot=None, dataset=None):
         cache_manager = current_app.matrix_data_cache_manager
         with cache_manager.data_adaptor(url_dataroot, location, app_config) as data_adaptor:
             data_adaptor.set_uri_path(f"{url_dataroot}/{dataset}")
-            dataset_title = app_config.get_title(data_adaptor)
-            return render_template(
-                "index.html", datasetTitle=dataset_title, SCRIPTS=scripts, INLINE_SCRIPTS=inline_scripts
-            )
+            args = {"SCRIPTS": scripts, "INLINE_SCRIPTS": inline_scripts}
+            return render_template("index.html", **args)
+
     except DatasetAccessError as e:
         return common_rest.abort_and_log(
             e.status_code, f"Invalid dataset {dataset}: {e.message}", loglevel=logging.INFO, include_exc_info=True
         )
 
 
+# TODO:  This route will be deprecated, but needs to be left for a short time until all the
+# deployments are upgraded to the new location for the health check (or else the upgrade will
+# fail).  Once the upgrade is complete, the deployments can move to the new health check URL
+# and this route will be removed.
 @webbp.route("/health", methods=["GET"])
 @cache_control_always(no_store=True)
 def health():
@@ -178,9 +194,9 @@ def dataroot_test_index():
             data += f"<p>Logged in as {auth.get_user_id()} / {auth.get_user_name()} / {auth.get_user_email()}</p>"
         if auth.requires_client_login():
             if server_config.auth.is_user_authenticated():
-                data += "<p><a href='/logout'>Logout</a></p>"
+                data += f"<p><a href='{auth.get_logout_url(None)}'>Logout</a></p>"
             else:
-                data += "<p><a href='/login'>Login</a></p>"
+                data += f"<p><a href='{auth.get_login_url(None)}'>Login</a></p>"
 
     datasets = []
     for dataroot_dict in server_config.multi_dataset__dataroot.values():
@@ -218,6 +234,13 @@ def dataroot_index():
         return redirect(config.server_config.multi_dataset__index)
 
 
+class HealthAPI(Resource):
+    @cache_control(no_store=True)
+    def get(self):
+        config = current_app.app_config
+        return health_check(config)
+
+
 class DatasetResource(Resource):
     """Base class for all Resources that act on datasets."""
 
@@ -227,7 +250,8 @@ class DatasetResource(Resource):
 
 
 class SchemaAPI(DatasetResource):
-    @cache_control(public=True, max_age=ONE_WEEK)
+    # TODO @mdunitz separate dataset schema and user schema
+    @cache_control(no_store=True)
     @rest_get_data_adaptor
     def get(self, data_adaptor):
         return common_rest.schema_get(data_adaptor)
@@ -240,8 +264,15 @@ class ConfigAPI(DatasetResource):
         return common_rest.config_get(current_app.app_config, data_adaptor)
 
 
+class UserInfoAPI(DatasetResource):
+    @cache_control_always(no_store=True)
+    @rest_get_data_adaptor
+    def get(self, data_adaptor):
+        return common_rest.userinfo_get(current_app.app_config, data_adaptor)
+
+
 class AnnotationsObsAPI(DatasetResource):
-    @cache_control(public=True, max_age=ONE_WEEK)
+    @cache_control(public=True, no_store=True)
     @rest_get_data_adaptor
     def get(self, data_adaptor):
         return common_rest.annotations_obs_get(request, data_adaptor)
@@ -298,8 +329,18 @@ class LayoutObsAPI(DatasetResource):
         return common_rest.layout_obs_put(request, data_adaptor)
 
 
-def get_api_resources(bp_api, url_dataroot=None):
-    api = Api(bp_api)
+def get_api_base_resources(bp_base):
+    """Add resources that are accessed from the api_base_url"""
+    api = Api(bp_base)
+
+    # Diagnostics routes
+    api.add_resource(HealthAPI, "/health")
+    return api
+
+
+def get_api_dataroot_resources(bp_dataroot, url_dataroot=None):
+    """Add resources that refer to a dataset"""
+    api = Api(bp_dataroot)
 
     def add_resource(resource, url):
         """convenience function to make the outer function less verbose"""
@@ -308,6 +349,7 @@ def get_api_resources(bp_api, url_dataroot=None):
     # Initialization routes
     add_resource(SchemaAPI, "/schema")
     add_resource(ConfigAPI, "/config")
+    add_resource(UserInfoAPI, "/userinfo")
     # Data routes
     add_resource(AnnotationsObsAPI, "/annotations/obs")
     add_resource(AnnotationsVarAPI, "/annotations/var")
@@ -320,6 +362,25 @@ def get_api_resources(bp_api, url_dataroot=None):
     return api
 
 
+def handle_api_base_url(app, app_config):
+    """If an api_base_url is provided, then an inline script is generated to
+    handle the new API prefix"""
+    api_base_url = app_config.server_config.get_api_base_url()
+    if not api_base_url:
+        return
+
+    sha256 = hashlib.sha256(api_base_url.encode()).hexdigest()
+    script_name = f"api_base_url-{sha256}.js"
+    script_path = os.path.join(app.root_path, "../common/web/templates", script_name)
+    with open(script_path, "w") as fout:
+        fout.write("window.CELLXGENE.API.prefix = `" + api_base_url + "${location.pathname}api/`;\n")
+
+    dataset_configs = [app_config.default_dataset_config] + list(app_config.dataroot_config.values())
+    for dataset_config in dataset_configs:
+        inline_scripts = dataset_config.app__inline_scripts
+        inline_scripts.append(script_name)
+
+
 class Server:
     @staticmethod
     def _before_adding_routes(app, app_config):
@@ -327,7 +388,8 @@ class Server:
         pass
 
     def __init__(self, app_config):
-        self.app = Flask(__name__, static_folder="../common/web/static")
+        self.app = Flask(__name__, static_folder=None)
+        handle_api_base_url(self.app, app_config)
         self._before_adding_routes(self.app, app_config)
         self.app.json_encoder = Float32JSONEncoder
         server_config = app_config.server_config
@@ -344,28 +406,53 @@ class Server:
         self.app.register_blueprint(webbp)
 
         api_version = "/api/v0.2"
+        api_base_url = server_config.get_api_base_url()
+        api_path = "/"
+        if api_base_url:
+            parse = urlparse(api_base_url)
+            api_path = parse.path
+
+        bp_base = Blueprint("bp_base", __name__, url_prefix=api_path)
+        base_resources = get_api_base_resources(bp_base)
+        self.app.register_blueprint(base_resources.blueprint)
+
         if app_config.is_multi_dataset():
             # NOTE:  These routes only allow the dataset to be in the directory
             # of the dataroot, and not a subdirectory.  We may want to change
             # the route format at some point
             for dataroot_dict in server_config.multi_dataset__dataroot.values():
                 url_dataroot = dataroot_dict["base_url"]
-                bp_api = Blueprint(
-                    f"api_dataset_{url_dataroot}", __name__, url_prefix=f"/{url_dataroot}/<dataset>" + api_version
+                bp_dataroot = Blueprint(
+                    f"api_dataset_{url_dataroot}",
+                    __name__,
+                    url_prefix=f"{api_path}/{url_dataroot}/<dataset>" + api_version,
                 )
-                resources = get_api_resources(bp_api, url_dataroot)
-                self.app.register_blueprint(resources.blueprint)
+                dataroot_resources = get_api_dataroot_resources(bp_dataroot, url_dataroot)
+                self.app.register_blueprint(dataroot_resources.blueprint)
+
                 self.app.add_url_rule(
                     f"/{url_dataroot}/<dataset>/",
                     f"dataset_index_{url_dataroot}",
                     lambda dataset, url_dataroot=url_dataroot: dataset_index(url_dataroot, dataset),
                     methods=["GET"],
                 )
+                self.app.add_url_rule(
+                    f"/{url_dataroot}/<dataset>/static/<path:filename>",
+                    f"static_assets_{url_dataroot}",
+                    view_func=lambda dataset, filename: send_from_directory("../common/web/static", filename),
+                    methods=["GET"],
+                )
 
         else:
-            bp_api = Blueprint("api", __name__, url_prefix=api_version)
-            resources = get_api_resources(bp_api)
+            bp_api = Blueprint("api", __name__, url_prefix=f"{api_path}{api_version}")
+            resources = get_api_dataroot_resources(bp_api)
             self.app.register_blueprint(resources.blueprint)
+            self.app.add_url_rule(
+                "/static/<path:filename>",
+                "static_assets",
+                view_func=lambda filename: send_from_directory("../common/web/static", filename),
+                methods=["GET"],
+            )
 
         self.app.matrix_data_cache_manager = server_config.matrix_data_cache_manager
         self.app.app_config = app_config

server/auth/__init__.py

@@ -1,4 +1,3 @@
-
 # import the built in auth types so they can be registered
 
 import server.auth.auth_none  # noqa: F401

server/auth/auth.py

@@ -43,6 +43,10 @@ class AuthTypeBase(ABC):
         """Return the name of the user (string)"""
         pass
 
+    def get_user_picture(self):
+        """Return the location to the user's picture"""
+        return None
+
 
 class AuthTypeClientBase(AuthTypeBase):
     """Base type for all authentication types that require the client to login"""
@@ -76,7 +80,7 @@ class AuthTypeFactory:
 
     @staticmethod
     def register(name, auth_type):
-        assert(issubclass(auth_type, AuthTypeBase))
+        assert issubclass(auth_type, AuthTypeBase)
         AuthTypeFactory.auth_types[name] = auth_type
 
     @staticmethod

server/auth/auth_none.py

@@ -2,7 +2,6 @@ from server.auth.auth import AuthTypeBase, AuthTypeFactory
 
 
 class AuthTypeNone(AuthTypeBase):
-
     def __init__(self, app_config):
         super().__init__()
 

server/auth/auth_oauth.py

@@ -1,9 +1,10 @@
 from flask import session, request, redirect, current_app, after_this_request, has_request_context, g
 from server.auth.auth import AuthTypeClientBase, AuthTypeFactory
 from server.common.errors import AuthenticationError, ConfigurationError
-from urllib.parse import urlencode
-from urllib.request import urlopen
+from urllib.parse import urlencode, urlparse
 import json
+import requests
+import base64
 
 # It is not required to have authlib or jose.
 # However, it is a configuration error to use this auth type if they are not installed.
@@ -15,27 +16,49 @@ except ModuleNotFoundError:
 
 try:
     from jose import jwt
+    from jose.exceptions import ExpiredSignatureError, JWTError, JWTClaimsError
 except ModuleNotFoundError:
     missingimport.append("jose")
 
 
+class Tokens:
+    """Simple class to represent the tokens that are saved/restored from the cookie"""
+
+    def __init__(self, access_token, id_token, refresh_token, expires_at):
+        self.access_token = access_token
+        self.id_token = id_token
+        self.refresh_token = refresh_token
+        self.expires_at = expires_at
+
+        # expires_at may be None after a token refresh, and so it is not checked here
+        if not (access_token and id_token and refresh_token):
+            raise KeyError(str(self.__dict__))
+
+
 class AuthTypeOAuth(AuthTypeClientBase):
     """An authentication type for oauth2 logins."""
 
-    CXG_ID_TOKEN = "id_token"
+    CXG_TOKENS = "auth_tokens"
 
     def __init__(self, server_config):
         super().__init__()
         if missingimport:
             raise ConfigurationError(f"oauth requires these modules: {', '.join(missingimport)}")
         self.algorithms = ["RS256"]
-        self.api_base_url = server_config.authentication__params_oauth__api_base_url
+        self.oauth_api_base_url = server_config.authentication__params_oauth__oauth_api_base_url
         self.client_id = server_config.authentication__params_oauth__client_id
         self.client_secret = server_config.authentication__params_oauth__client_secret
-        self.callback_base_url = server_config.authentication__params_oauth__callback_base_url
         self.session_cookie = server_config.authentication__params_oauth__session_cookie
         self.cookie_params = server_config.authentication__params_oauth__cookie
+        self.jwt_decode_options = server_config.authentication__params_oauth__jwt_decode_options
+
         self._validate_cookie_params()
+        self._validate_jwt_decode_options()
+
+        self.api_base_url = server_config.get_api_base_url()
+        self.web_base_url = server_config.get_web_base_url()
+        if self.api_base_url is None:
+            raise ConfigurationError("oauth requires the app__api_base_url to be set")
 
         # set the audience
         self.audience = self.client_id
@@ -44,11 +67,13 @@ class AuthTypeOAuth(AuthTypeClientBase):
         # The JSON Web Key Set (JWKS) is a set of keys which contains the public keys used to verify
         # any JSON Web Token (JWT) issued by the authorization server and signed using the RS256
         try:
-            jwksloc = f"{self.api_base_url}/.well-known/jwks.json"
-            jwksurl = urlopen(jwksloc)
-            self.jwks = json.loads(jwksurl.read())
+            jwksloc = f"{self.oauth_api_base_url}/.well-known/jwks.json"
+            jwksurl = requests.get(jwksloc)
+            self.jwks = jwksurl.json()
         except Exception:
-            raise ConfigurationError(f"error in oauth, api_url_base: {self.api_base_url}, cannot access {jwksloc}")
+            raise ConfigurationError(
+                f"error in oauth, api_url_base: {self.oauth_api_base_url}, cannot access {jwksloc}"
+            )
 
     def _validate_cookie_params(self):
         """check the cookie_params, and raise a ConfigurationError if there is something wrong"""
@@ -65,6 +90,29 @@ class AuthTypeOAuth(AuthTypeClientBase):
         if "key" not in keys:
             raise ConfigurationError("must have a key (name) in the cookie params")
 
+    def _validate_jwt_decode_options(self):
+        """check the jwt_decode_options, and raise a ConfigurationError if there is something wrong"""
+        if self.jwt_decode_options is None:
+            self.jwt_decode_options = {}
+            return
+
+        valid_keys = {
+            "verify_signature",
+            "verify_aud",
+            "verify_iat",
+            "verify_exp",
+            "verify_nbf",
+            "verify_iss",
+            "verify_sub",
+            "verify_jti",
+            "verify_at_hash",
+            "leeway",
+        }
+        keys = set(self.jwt_decode_options.keys())
+        unknown = keys - valid_keys
+        if unknown:
+            raise ConfigurationError(f"unexpected key in jwt_decode_options: {', '.join(unknown)}")
+
     def is_valid_authentication_type(self):
         return True
 
@@ -72,64 +120,53 @@ class AuthTypeOAuth(AuthTypeClientBase):
         return True
 
     def add_url_rules(self, app):
-        app.add_url_rule("/login", "login", self.login, methods=["GET"])
-        app.add_url_rule("/logout", "logout", self.logout, methods=["GET"])
-        app.add_url_rule("/oauth2/callback", "callback", self.callback, methods=["GET"])
+        parse = urlparse(self.api_base_url)
+        app.add_url_rule(f"{parse.path}/login", "login", self.login, methods=["GET"])
+        app.add_url_rule(f"{parse.path}/logout", "logout", self.logout, methods=["GET"])
+        app.add_url_rule(f"{parse.path}/logout_redirect", "logout_redirect", self.logout_redirect, methods=["GET"])
+        app.add_url_rule(f"{parse.path}/oauth2/callback", "callback", self.callback, methods=["GET"])
 
     def complete_setup(self, flask_app):
         self.oauth = OAuth(flask_app)
-        if self.callback_base_url is None:
-            # In this case, assume the server is running on the same host as the client,
-            # and the oauth provider has been configured
-            # with a callback that understands a localhost callback (e.g. A http://localhost:5005).
-            server_config = flask_app.app_config.server_config
-            self.callback_base_url = f"http://{server_config.app__host}:{server_config.app__port}"
 
         self.client = self.oauth.register(
-            "oauth",
+            "auth0",
             client_id=self.client_id,
             client_secret=self.client_secret,
-            api_base_url=self.api_base_url,
-            access_token_url=f"{self.api_base_url}/oauth/token",
-            authorize_url=f"{self.api_base_url}/authorize",
-            client_kwargs={
-                "scope" : "openid profile email",
-            }
+            api_base_url=self.oauth_api_base_url,
+            refresh_token_url=f"{self.oauth_api_base_url}/oauth/token",
+            access_token_url=f"{self.oauth_api_base_url}/oauth/token",
+            authorize_url=f"{self.oauth_api_base_url}/authorize",
+            client_kwargs={"scope": "openid profile email offline_access"},
         )
 
     def is_user_authenticated(self):
-        try:
-            payload = self.get_jwt_payload()
-            return payload is not None
-        except AuthenticationError:
-            return False
+        payload = self.get_userinfo()
+        return payload is not None
 
     def get_user_id(self):
-        payload = self.get_jwt_payload()
-        if payload and payload.get("sub"):
-            return payload.get("sub")
-        return None
+        payload = self.get_userinfo()
+        return payload.get("sub") if payload else None
 
     def get_user_name(self):
-        payload = self.get_jwt_payload()
-        if payload and payload.get("name"):
-            return payload.get("name")
-        return None
+        payload = self.get_userinfo()
+        return payload.get("name") if payload else None
 
     def get_user_email(self):
-        payload = self.get_jwt_payload()
-        if payload and payload.get("email"):
-            return payload.get("email")
-        return None
+        payload = self.get_userinfo()
+        return payload.get("email") if payload else None
+
+    def get_user_picture(self):
+        payload = self.get_userinfo()
+        return payload.get("picture") if payload else None
 
     def update_response(self, response):
-        response.cache_control.update(
-            dict(public=True, max_age=0, no_store=True, no_cache=True, must_revalidate=True))
+        response.cache_control.update(dict(public=True, max_age=0, no_store=True, no_cache=True, must_revalidate=True))
 
     def login(self):
-        callbackurl = f'{self.callback_base_url}/oauth2/callback'
+        callbackurl = f"{self.api_base_url}/oauth2/callback"
         return_path = request.args.get("dataset", "")
-        return_to = f"{self.callback_base_url}/{return_path}"
+        return_to = f"{self.web_base_url}/{return_path}"
         # save the return path in the session cookie, accessed in the callback function
         session["oauth_callback_redirect"] = return_to
         response = self.client.authorize_redirect(redirect_uri=callbackurl)
@@ -137,104 +174,209 @@ class AuthTypeOAuth(AuthTypeClientBase):
         return response
 
     def logout(self):
+        """
+        We would like for the user to remain on the same dataset after logout.  oauth requires that
+        the redirect `returnTo` path be whitelisted by the oauth server, therefore a level of
+        indirection is used.  We first redirect to a single path "logout_redirect", and logout_redirect
+        will redirect the user's browser back to the current page.
+        """
+        self.remove_tokens()
+        redirect_path = request.args.get("dataset", "")
+        redirect_to = f"{self.web_base_url}/{redirect_path}"
+        session["oauth_logout_redirect"] = redirect_to
+
+        return_to = f"{self.api_base_url}/logout_redirect"
+        params = {"returnTo": return_to, "client_id": self.client_id}
+        response = redirect(self.client.api_base_url + "/v2/logout?" + urlencode(params))
+        self.update_response(response)
+        return response
+
+    def logout_redirect(self):
+        oauth_logout_redirect = session.pop("oauth_logout_redirect", "/")
+        response = redirect(oauth_logout_redirect)
+        self.update_response(response)
+        return response
+
+    def callback(self):
+        data = self.client.authorize_access_token()
+        tokens = Tokens(
+            access_token=data.get("access_token"),
+            id_token=data.get("id_token"),
+            refresh_token=data.get("refresh_token"),
+            expires_at=data.get("expires_at"),
+        )
+        self.save_tokens(tokens)
+        oauth_callback_redirect = session.pop("oauth_callback_redirect", "/")
+        response = redirect(oauth_callback_redirect)
+        self.update_response(response)
+        return response
+
+    def get_tokens(self):
+        """Extract the tokens from the cookie, and store them in the flask global context"""
+        if "tokens" in g:
+            return g.tokens
+
+        try:
+            if self.session_cookie:
+                tokensdict = session.get(self.CXG_TOKENS)
+                if tokensdict:
+                    g.tokens = Tokens(**tokensdict)
+                else:
+                    return None
+            else:
+                value = request.cookies.get(self.cookie_params["key"])
+                value = base64.b64decode(value)
+                try:
+                    tokensdict = json.loads(value)
+                    g.tokens = Tokens(**tokensdict)
+                except (TypeError, KeyError, json.decoder.JSONDecodeError):
+                    g.pop("tokens", None)
+                    return None
+
+        except (TypeError, KeyError):
+            g.pop("tokens", None)
+            return None
+
+        return g.tokens
+
+    def save_tokens(self, tokens):
+        g.tokens = tokens
         if self.session_cookie:
-            if self.CXG_ID_TOKEN in session:
-                del session[self.CXG_ID_TOKEN]
+            session[self.CXG_TOKENS] = tokens.__dict__
         else:
+
+            @after_this_request
+            def set_cookie(response):
+                args = self.cookie_params.copy()
+                value = base64.b64encode(json.dumps(tokens.__dict__).encode("utf-8"))
+                del args["key"]
+                try:
+                    response.set_cookie(self.cookie_params["key"], value, **args)
+                except Exception as e:
+                    raise AuthenticationError(f"unable to set_cookie {self.cookie_params}") from e
+                return response
+
+    def remove_tokens(self):
+        g.pop("tokens", None)
+        if self.session_cookie:
+            if self.CXG_TOKENS in session:
+                del session[self.CXG_TOKENS]
+        else:
+
             @after_this_request
             def remove_cookie(response):
                 response.set_cookie(self.cookie_params["key"], "", expires=0)
                 self.update_response(response)
                 return response
 
-        params = {'returnTo' : self.callback_base_url, 'client_id' : self.client_id}
-        response = redirect(self.client.api_base_url + '/v2/logout?' + urlencode(params))
-        self.update_response(response)
-        return response
-
-    def callback(self):
-        token = self.client.authorize_access_token()
-        id_token = token.get("id_token")
-        oauth_callback_redirect = session.pop("oauth_callback_redirect", "/")
-        resp = redirect(oauth_callback_redirect)
-
-        if self.session_cookie:
-            session[self.CXG_ID_TOKEN] = id_token
-        else:
-            args = self.cookie_params.copy()
-            del args["key"]
-            try:
-                resp.set_cookie(
-                    self.cookie_params["key"],
-                    id_token,
-                    **args)
-                g.token = id_token
-            except Exception as e:
-                raise AuthenticationError(f"unable to set_cookie {self.cookie_params}") from e
-
-        self.update_response(resp)
-        return resp
-
     def get_login_url(self, data_adaptor):
         """Return the url for the login route"""
-        if current_app.app_config.is_multi_dataset():
-            return f"/login?dataset={data_adaptor.uri_path}"
+        if data_adaptor and current_app.app_config.is_multi_dataset():
+            return f"{self.api_base_url}/login?dataset={data_adaptor.uri_path}/"
         else:
-            return "/login"
+            return f"{self.api_base_url}/login"
 
     def get_logout_url(self, data_adaptor):
         """Return the url for the logout route"""
-        return "/logout"
-
-    def get_token(self):
-        """Function to return the token"""
-        if "token" in g:
-            return g.token
-        if self.session_cookie:
-            g.token = session.get(self.CXG_ID_TOKEN)
+        if data_adaptor and current_app.app_config.is_multi_dataset():
+            return f"{self.api_base_url}/logout?dataset={data_adaptor.uri_path}/"
         else:
-            g.token = request.cookies.get(self.cookie_params["key"])
+            return f"{self.api_base_url}/logout"
 
-        return g.token
-
-    def get_jwt_payload(self):
-        if not has_request_context():
+    def check_jwt_payload(self, id_token):
+        try:
+            unverified_header = jwt.get_unverified_header(id_token)
+        except JWTError:
             return None
 
-        token = self.get_token()
-        if token is None:
-            return None
-
-        unverified_header = jwt.get_unverified_header(token)
         rsa_key = {}
-        for key in self.jwks['keys']:
-            if key['kid'] == unverified_header['kid']:
+        for key in self.jwks["keys"]:
+            if key["kid"] == unverified_header["kid"]:
                 rsa_key = {
-                    'kty': key['kty'],
-                    'kid': key['kid'],
-                    'use': key['use'],
-                    'n': key['n'],
-                    'e': key['e']
+                    "kty": key["kty"],
+                    "kid": key["kid"],
+                    "use": key["use"],
+                    "n": key.get("n"),
+                    "e": key.get("e"),
                 }
         if rsa_key:
             try:
                 payload = jwt.decode(
-                    token,
+                    id_token,
                     rsa_key,
                     algorithms=self.algorithms,
                     audience=self.audience,
-                    issuer=self.api_base_url + "/"
+                    issuer=self.oauth_api_base_url + "/",
+                    options=self.jwt_decode_options,
                 )
                 return payload
 
-            except jwt.JWTError as e:
-                raise AuthenticationError(f"invalid signature: {str(e)}")
-            except jwt.ExpiredSignatureError as e:
-                raise AuthenticationError(f"token expired: {str(e)}")
-            except jwt.JWTClaimsError as e:
-                raise AuthenticationError(f"invalid claims {str(e)}")
+            except ExpiredSignatureError:
+                # This exception is handled in get_userinfo
+                raise
+            except JWTClaimsError as e:
+                raise AuthenticationError(f"invalid claims {str(e)}") from e
+            except JWTError as e:
+                raise AuthenticationError(f"invalid signature: {str(e)}") from e
 
         raise AuthenticationError("Unable to find the appropriate key")
 
+    def get_userinfo(self):
+        if not has_request_context():
+            return None
+
+        # check if the userinfo has been retrieved already in this request
+        if "userinfo" in g:
+            return g.get("userinfo")
+
+        # if there is no id_token, return None (user is not authenticated)
+        tokens = self.get_tokens()
+        if tokens is None or tokens.id_token is None:
+            return None
+
+        try:
+            # check the jwt payload.  This raises an AuthenticationError if the token is not valid.
+            # It the token has expired, we attempt to refresh the token
+            g.userinfo = self.check_jwt_payload(tokens.id_token)
+            return g.userinfo
+
+        except ExpiredSignatureError:
+            tokens = self.refresh_expired_token(tokens.refresh_token)
+            if tokens is None or tokens.id_token is None:
+                return None
+            else:
+                try:
+                    g.userinfo = self.check_jwt_payload(tokens.id_token)
+                    return g.userinfo
+                except JWTError as e:
+                    raise AuthenticationError(f"error during token refresh: {str(e)}") from e
+
+        except AuthenticationError:
+            self.remove_tokens()
+            raise
+
+    def refresh_expired_token(self, refresh_token):
+        params = {
+            "grant_type": "refresh_token",
+            "client_id": self.client_id,
+            "refresh_token": refresh_token,
+            "client_secret": self.client_secret,
+        }
+        headers = {"content-type": "application/x-www-form-urlencoded"}
+        request = requests.post(f"{self.oauth_api_base_url}/oauth/token", urlencode(params), headers=headers)
+        if request.status_code != 200:
+            # unable to refresh the token, log the user out
+            self.remove_tokens()
+            return None
+        data = request.json()
+        tokens = Tokens(
+            access_token=data.get("access_token"),
+            id_token=data.get("id_token"),
+            refresh_token=data.get("refresh_token", refresh_token),
+            expires_at=data.get("expires_at"),
+        )
+        self.save_tokens(tokens)
+        return tokens
+
 
 AuthTypeFactory.register("oauth", AuthTypeOAuth)

server/auth/auth_test.py

@@ -10,12 +10,14 @@ class AuthTypeTest(AuthTypeClientBase):
     CXGUID = "cxguid_test"
     CXGUNAME = "cxguname_test"
     CXGUEMAIL = "cxguemail_test"
+    CXGUPICTURE = "cxgupicture_test"
 
     def __init__(self, app_config):
         super().__init__()
         self.user_name = "test_account"
         self.user_id = "id0001"
         self.user_email = "test_account@test.com"
+        self.user_picture = None
 
     def is_valid_authentication_type(self):
         return True
@@ -42,11 +44,16 @@ class AuthTypeTest(AuthTypeClientBase):
     def get_user_email(self):
         return session.get(self.CXGUEMAIL)
 
+    def get_user_picture(self):
+        return session.get(self.CXGUPICTURE)
+
     def login(self):
         args = request.args
         return_to = args.get("dataset", "/")
         session[self.CXGUID] = args.get("userid", self.user_id)
         session[self.CXGUNAME] = args.get("username", self.user_name)
+        session[self.CXGUEMAIL] = args.get("email", self.user_email)
+        session[self.CXGUPICTURE] = args.get("picture", self.user_picture)
         return redirect(return_to)
 
     def logout(self):

server/cli/cli.py

@@ -1,9 +1,10 @@
 import click
 
-from .. import __version__
+from .convert_to_cxg import convert_to_cxg
 from .launch import launch
 from .prepare import prepare
 from .upgrade import log_upgrade_check
+from .. import __version__
 
 
 @click.group(
@@ -29,3 +30,4 @@ def cli(upgrade_check):
 
 cli.add_command(launch)
 cli.add_command(prepare)
+cli.add_command(convert_to_cxg)

server/cli/convert_to_cxg.py

@@ -0,0 +1,133 @@
+from os import path
+
+import click
+
+from server.converters.h5ad_data_file import H5ADDataFile
+
+
+@click.command(
+    name="convert",
+    short_help="Converts an H5AD dataset to the CXG format.",
+    help="Converts an H5AD dataset to the CXG format. The CXG format is a cellxgene-private data format "
+    "that has performance and access characteristics amenable to a multi-dataset, multi-user serving "
+    "environment. You will be able to launch the cellxgene using the `cellxgene launch` command as "
+    "usually with the generated CXG file.",
+)
+@click.argument(
+    "input-file", nargs=1, type=click.Path(exists=True, dir_okay=False),
+)
+@click.option(
+    "-o",
+    "--output-directory",
+    help="Name of the output CXG directory. If not provided, will default to be the input filename with a "
+    "CXG extension.",
+)
+@click.option(
+    "-b",
+    "--backed",
+    help="When true, loads the H5AD in file backed mode. This will cause the conversion to be slower, "
+    "but will use less memory.",
+    default=False,
+    show_default=True,
+    is_flag=True,
+)
+@click.option(
+    "-t",
+    "--title",
+    help="Human readable dataset title that will be included as metadata about the CXG file. If omitted, "
+    "the dataset title will be the filename.",
+)
+@click.option(
+    "-a",
+    "--about",
+    help="A fully qualified URL that provides more information about the dataset and will be included as "
+    "metadata about the CXG file.",
+)
+@click.option(
+    "-s",
+    "--sparse-threshold",
+    help="If the dataset's percent of non-zero values falls belows the specified threshold, then the X "
+    "array of the dataset will be sparse. Since the default value is 0.0, the default will be to "
+    "convert to dense array.",
+    default=0.0,
+    show_default=True,
+)
+@click.option(
+    "--obs-names",
+    help="Name to a column in the obs dataframe that will be used as the index for the dataframe instead of "
+    "the one designated by the dataframe generated-index.",
+)
+@click.option(
+    "--var-names",
+    help="Name to a column in the var dataframe that will be used as the index for the dataframe instead of "
+    "the one designated by the dataframe generated-index.",
+)
+@click.option(
+    "--disable-custom-colors",
+    help="When set, conversion process will not extract scanpy-compatible category colors from the H5AD file.",
+    default=False,
+    show_default=True,
+    is_flag=True,
+)
+@click.option(
+    "--disable-corpora-schema",
+    help="When set, conversion process will neither extract nor store Corpora schema information. See "
+    "https://github.com/chanzuckerberg/corpora-data-portal/blob/main/backend/schema/corpora_schema.md for more "
+    "information.",
+    default=False,
+    show_default=True,
+    is_flag=True,
+)
+@click.option(
+    "--overwrite",
+    help="When set to true, will overwrite the output file if the output file already exists.",
+    default=False,
+    show_default=True,
+    is_flag=True,
+)
+@click.help_option("--help", "-h", help="Show this message and exit.")
+def convert_to_cxg(
+    input_file,
+    output_directory,
+    backed,
+    title,
+    about,
+    sparse_threshold,
+    obs_names,
+    var_names,
+    disable_custom_colors,
+    disable_corpora_schema,
+    overwrite,
+):
+    """
+    Convert a dataset file into CXG.
+    """
+
+    h5ad_data_file = H5ADDataFile(
+        input_file, backed, title, about, obs_names, var_names, use_corpora_schema=not disable_corpora_schema
+    )
+
+    # Get the directory that will hold all the CXG files
+    cxg_output_container = get_output_directory(input_file, output_directory, overwrite)
+
+    h5ad_data_file.to_cxg(
+        cxg_output_container, sparse_threshold, convert_anndata_colors_to_cxg_colors=not disable_custom_colors
+    )
+
+
+def get_output_directory(input_filename, output_directory, should_overwrite):
+    """
+    Get the name of the CXG output directory to be created/populated during the dataset conversion.
+    """
+
+    if output_directory and (not path.isdir(output_directory) or (path.isdir(output_directory) and should_overwrite)):
+        if output_directory.endswith(".cxg"):
+            return output_directory
+        return output_directory + ".cxg"
+    if output_directory and path.isdir(output_directory) and not should_overwrite:
+        raise click.BadParameter(
+            f"Output directory {output_directory} already exists. If you'd like to overwrite, then run the command "
+            f"with the --overwrite flag."
+        )
+
+    return path.splitext(input_filename)[0] + ".cxg"

server/cli/launch.py

@@ -3,15 +3,14 @@ import functools
 import logging
 import sys
 import webbrowser
-from os import devnull
-
+import os
 import click
 from flask_compress import Compress
 from flask_cors import CORS
 
+from server.default_config import default_config
 from server.app.app import Server
-from server.common.app_config import AppConfig
-from server.common.default_config import default_config
+from server.common.config.app_config import AppConfig
 from server.common.errors import DatasetAccessError, ConfigurationError
 from server.common.utils.utils import sort_options
 
@@ -33,7 +32,7 @@ def annotation_args(func):
         multiple=False,
         metavar="<path>",
         help="CSV file to initialize editing of existing annotations; will be altered in-place. "
-             "Incompatible with --annotations-dir.",
+        "Incompatible with --annotations-dir.",
     )
     @click.option(
         "--annotations-dir",
@@ -42,7 +41,7 @@ def annotation_args(func):
         multiple=False,
         metavar="<directory path>",
         help="Directory of where to save output annotations; filename will be specified in the application. "
-             "Incompatible with --annotations-file.",
+        "Incompatible with --annotations-file.",
     )
     @click.option(
         "--experimental-annotations-ontology",
@@ -170,7 +169,7 @@ def server_args(func):
         default=DEFAULT_CONFIG.server_config.app__debug,
         show_default=True,
         help="Run in debug mode. This is helpful for cellxgene developers, "
-             "or when you want more information about an error condition.",
+        "or when you want more information about an error condition.",
     )
     @click.option(
         "--verbose",
@@ -203,7 +202,7 @@ def server_args(func):
         multiple=True,
         metavar="<text>",
         help="Additional script files to include in HTML page. If not specified, "
-             "no additional script files will be included.",
+        "no additional script files will be included.",
         show_default=False,
     )
     @functools.wraps(func)
@@ -223,7 +222,7 @@ def launch_args(func):
         default=DEFAULT_CONFIG.server_config.multi_dataset__dataroot,
         metavar="<data directory>",
         help="Enable cellxgene to serve multiple files. Supply path (local directory or URL)"
-             " to folder containing H5AD and/or CXG datasets.",
+        " to folder containing H5AD and/or CXG datasets.",
         hidden=True,
     )  # TODO, unhide when dataroot is supported)
     @click.argument("datapath", required=False, metavar="<path to data file>")
@@ -307,32 +306,32 @@ class CliLaunchServer(Server):
 )
 @launch_args
 def launch(
-        datapath,
-        dataroot,
-        verbose,
-        debug,
-        open_browser,
-        port,
-        host,
-        embedding,
-        obs_names,
-        var_names,
-        max_category_items,
-        disable_custom_colors,
-        diffexp_lfc_cutoff,
-        title,
-        scripts,
-        about,
-        disable_annotations,
-        annotations_file,
-        annotations_dir,
-        backed,
-        disable_diffexp,
-        experimental_annotations_ontology,
-        experimental_annotations_ontology_obo,
-        experimental_enable_reembedding,
-        config_file,
-        dump_default_config,
+    datapath,
+    dataroot,
+    verbose,
+    debug,
+    open_browser,
+    port,
+    host,
+    embedding,
+    obs_names,
+    var_names,
+    max_category_items,
+    disable_custom_colors,
+    diffexp_lfc_cutoff,
+    title,
+    scripts,
+    about,
+    disable_annotations,
+    annotations_file,
+    annotations_dir,
+    backed,
+    disable_diffexp,
+    experimental_annotations_ontology,
+    experimental_annotations_ontology_obo,
+    experimental_enable_reembedding,
+    config_file,
+    dump_default_config,
 ):
     """Launch the cellxgene data viewer.
     This web app lets you explore single-cell expression data.
@@ -443,7 +442,7 @@ def launch(
     click.echo("[cellxgene] Type CTRL-C at any time to exit.")
 
     if not server_config.app__verbose:
-        f = open(devnull, "w")
+        f = open(os.devnull, "w")
         sys.stdout = f
 
     try:

server/cli/prepare.py

@@ -37,7 +37,7 @@ from server.common.utils.utils import sort_options
     default=False,
     is_flag=True,
     help="Do not run quality control metrics. By default cellxgene runs them "
-         "(saved to adata.obs and adata.var; see scanpy.pp.calculate_qc_metrics for details).",
+    "(saved to adata.obs and adata.var; see scanpy.pp.calculate_qc_metrics for details).",
 )
 @click.option(
     "--make-obs-names-unique/--no-make-obs-names-unique",
@@ -53,18 +53,18 @@ from server.common.utils.utils import sort_options
 )
 @click.help_option("--help", "-h", help="Show this message and exit.")
 def prepare(
-        data,
-        embedding,
-        recipe,
-        output,
-        plotting,
-        sparse,
-        overwrite,
-        set_obs_names,
-        set_var_names,
-        skip_qc,
-        make_obs_names_unique,
-        make_var_names_unique,
+    data,
+    embedding,
+    recipe,
+    output,
+    plotting,
+    sparse,
+    overwrite,
+    set_obs_names,
+    set_var_names,
+    skip_qc,
+    make_obs_names_unique,
+    make_var_names_unique,
 ):
     """
     Preprocess data for use with cellxgene.

server/cli/upgrade.py

@@ -1,16 +1,16 @@
-import click
 import re
-import requests
 
+import click
+import requests
 from requests.exceptions import ConnectionError
+
 from .. import __version__
 
 # Official SemVer regex: https://semver.org/
 SEMVER_FORMAT = re.compile(
-    r"^(?P<major>0|[1-9]\d*)\.(?P<minor>0|[1-9]\d*)\.(?P<patch>0|[1-9]\d*)"
-    + r"(?:-(?P<prerelease>(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)"
-    + r"(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?"
-    + r"(?:\+(?P<buildmetadata>[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$"
+    r"^(?P<major>0|[1-9]\d*)\.(?P<minor>0|[1-9]\d*)\.(?P<patch>0|[1-9]\d*)(?:-(?P<prerelease>(?:0|[1-9]\d*|\d*["
+    r"a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?P<buildmetadata>[0-9a-zA-Z-]+("
+    r"?:\.[0-9a-zA-Z-]+)*))?$"
 )
 
 
@@ -23,8 +23,8 @@ def log_upgrade_check():
         release_tag_generator = (r["tag_name"] for r in _request_cellxgene_releases())
         latest_release = next(release_tag_generator, lambda tag_name: validate_version_str(tag_name))
         if version_gt(latest_release, __version__):
-            click.echo(f"There's a new version of cellxgene available ({latest_release})!")
-            click.echo("To upgrade, run the following: pip install --upgrade cellxgene\n")
+            click.echo(f"There's a new version of cellxgene available ({latest_release})!", err=True)
+            click.echo("To upgrade, run the following: pip install --upgrade cellxgene\n", err=True)
     except (ConnectionError, RateLimitException):
         click.echo("Upgrade check failed.\n")
 

server/common/annotations/annotations.py

@@ -64,15 +64,7 @@ class Annotations(metaclass=ABCMeta):
         """Write the labels (df) to a persistent storage such that it can later be read"""
         pass
 
+    @abstractmethod
     def update_parameters(self, parameters, data_adaptor):
         """Update configuration parameters that describe information about the annotations feature"""
-        params = {}
-        params["annotations"] = True
-
-        if self.ontology_data:
-            params["annotations_cell_ontology_enabled"] = True
-            params["annotations_cell_ontology_terms"] = self.ontology_data
-        else:
-            params["annotations_cell_ontology_enabled"] = False
-
-        parameters.update(params)
+        pass

server/common/annotations/hosted_tiledb.py

@@ -8,8 +8,10 @@ import tiledb
 from flask import current_app
 
 from server.common.annotations.annotations import Annotations
-from server.converters.cxgtool import sanitize_keys, generate_schema_hints_and_convert_value_types, cxg_dtype
-from server.db.cellxgene_orm import CellxGeneDataset, Annotation
+from server.common.errors import AnnotationCategoryNameError
+from server.common.utils.sanitization_utils import sanitize_values_in_list
+from server.common.utils.type_conversion_utils import get_dtypes_and_schemas_of_dataframe, get_dtype_of_array
+from server.db.cellxgene_orm import Annotation
 
 
 class AnnotationsHostedTileDB(Annotations):
@@ -18,10 +20,25 @@ class AnnotationsHostedTileDB(Annotations):
     def __init__(self, directory_path, db):
         super().__init__()
         self.db = db
-        self.directory_path = directory_path
+        if directory_path[-1] == "/":
+            self.directory_path = directory_path
+        else:
+            self.directory_path = directory_path + "/"
 
     def check_category_names(self, df):
-        sanitize_keys(df.keys().to_list(), False)
+        original_category_names = df.keys().to_list()
+        sanitized_category_names = set(sanitize_values_in_list(original_category_names).values())
+        unsanitary_original_category_names = set(original_category_names).difference(sanitized_category_names)
+        if unsanitary_original_category_names:
+            raise AnnotationCategoryNameError(
+                f"{unsanitary_original_category_names} are not valid category names, please resubmit"
+            )
+
+    def get_user_name(self):
+        return current_app.auth.get_user_name()
+
+    def get_user_id(self):
+        return current_app.auth.get_user_id()
 
     def is_safe_collection_name(self, name):
         """
@@ -37,37 +54,47 @@ class AnnotationsHostedTileDB(Annotations):
         self.CXG_ANNO_COLLECTION = name
 
     def read_labels(self, data_adaptor):
-        user_id = current_app.auth.get_user_id()
+        user_id = self.get_user_id()
+        if user_id is None:
+            return
         dataset_name = data_adaptor.get_location()
-        dataset_id = str(self.db.query(
-            table_args=[CellxGeneDataset],
-            filter_args=[CellxGeneDataset.name == dataset_name]
-        )[0].id)
+        dataset_id = self.db.get_or_create_dataset(dataset_name)
 
         annotation_object = self.db.query_for_most_recent(
             Annotation, [Annotation.user_id == user_id, Annotation.dataset_id == dataset_id]
         )
         if annotation_object:
             df = tiledb.open(annotation_object.tiledb_uri)
-            pandas_df = self.convert_to_pandas_df(df)
+            pandas_df = self.convert_to_pandas_df(df, annotation_object.schema_hints)
             return pandas_df
         else:
             return None
 
-    def convert_to_pandas_df(self, tileDBArray):
+    def convert_to_pandas_df(self, tileDBArray, schema_hints):
         repr_meta = None
         index_dims = None
-        if '__pandas_attribute_repr' in tileDBArray.meta:
+        schema_hints = json.loads(schema_hints)
+
+        if "__pandas_attribute_repr" in tileDBArray.meta:
             # backwards compatibility... unsure if necessary at this point
-            repr_meta = json.loads(tileDBArray.meta['__pandas_attribute_repr'])
-        if '__pandas_index_dims' in tileDBArray.meta:
-            index_dims = json.loads(tileDBArray.meta['__pandas_index_dims'])
+            repr_meta = json.loads(tileDBArray.meta["__pandas_attribute_repr"])
+        if "__pandas_index_dims" in tileDBArray.meta:
+            index_dims = json.loads(tileDBArray.meta["__pandas_index_dims"])
 
         data = tileDBArray[:]
         indexes = list()
 
         for col_name, col_val in data.items():
-            if repr_meta and col_name in repr_meta:
+            # If the column values are byte literals, decode them
+            if isinstance(col_val[0], bytes):
+                col_val = [value.decode("utf-8") for value in col_val]
+
+            if schema_hints and col_name in schema_hints:
+                type = schema_hints.get(col_name).get("type")
+                if type and type == "categorical":
+                    new_col = pd.Series(col_val, dtype="category")
+                    data[col_name] = new_col
+            elif repr_meta and col_name in repr_meta:
                 new_col = pd.Series(col_val, dtype=repr_meta[col_name])
                 data[col_name] = new_col
             elif index_dims and col_name in index_dims:
@@ -82,32 +109,53 @@ class AnnotationsHostedTileDB(Annotations):
         return new_df
 
     def write_labels(self, df, data_adaptor):
-
-        user_id = current_app.auth.get_user_id()
+        auth_user_id = self.get_user_id()
+        user_name = self.get_user_name()
         timestamp = time.time()
-        dataset_name = data_adaptor.get_location()
-        dataset_id = self.db.get_or_create_dataset(dataset_name)
-        user_id = self.db.get_or_create_user(user_id)
+        dataset_location = data_adaptor.get_location()
+        dataset_id = self.db.get_or_create_dataset(dataset_location)
+        dataset_name = data_adaptor.get_title()
+        user_id = self.db.get_or_create_user(auth_user_id)
+        """
+        NOTE: The uri contains the dataset name, user name and a timestamp as a convenience for debugging purposes.
+        People may have the same name and time.time() can be server dependent.
+        See - https://docs.python.org/2/library/time.html#time.time
 
-        uri = f"{self.directory_path}-{dataset_name}-{user_id}-{timestamp}"
+        The annotations objects in the database should be used as the source of truth about who an annotation belongs
+        to (for authorization purposes) and what time it was created (for garbage collection).
+        """
+        uri = f"{self.directory_path}{dataset_name}/{user_name}/{timestamp}"
         if uri.startswith("s3://"):
             pass
         else:
             os.makedirs(uri, exist_ok=True)
-        schema_hints, values = generate_schema_hints_and_convert_value_types(df)
-
+        _, dataframe_schema_type_hints = get_dtypes_and_schemas_of_dataframe(df)
         annotation = Annotation(
             tiledb_uri=uri,
             user_id=user_id,
             dataset_id=str(dataset_id),
-            schema_hints=json.dumps(schema_hints)
+            schema_hints=json.dumps(dataframe_schema_type_hints),
         )
         if not df.empty:
             self.check_category_names(df)
             # convert to tiledb datatypes
+
             for col in df:
-                df[col] = df[col].astype(cxg_dtype(df[col]))
+                df[col] = df[col].astype(get_dtype_of_array(df[col]))
             tiledb.from_pandas(uri, df)
 
         self.db.session.add(annotation)
         self.db.session.commit()
+
+    def update_parameters(self, parameters, data_adaptor):
+        params = {}
+        params["annotations"] = True
+        params["user_annotation_collection_name_enabled"] = False
+
+        if self.ontology_data:
+            params["annotations_cell_ontology_enabled"] = True
+            params["annotations_cell_ontology_terms"] = self.ontology_data
+        else:
+            params["annotations_cell_ontology_enabled"] = False
+
+        parameters.update(params)

server/common/annotations/local_file_csv.py

@@ -171,6 +171,7 @@ class AnnotationsLocalFile(Annotations):
     def update_parameters(self, parameters, data_adaptor):
         params = {}
         params["annotations"] = True
+        params["user_annotation_collection_name_enabled"] = True
 
         if self.ontology_data:
             params["annotations_cell_ontology_enabled"] = True

server/common/app_config.py

@@ -1,905 +0,0 @@
-import copy
-import os
-import sys
-import warnings
-from os.path import splitext, basename, isdir
-from urllib.parse import urlparse, quote_plus
-
-import yaml
-from flatten_dict import flatten, unflatten
-
-import server.compute.diffexp_cxg as diffexp_tiledb
-from server import display_version as cellxgene_display_version
-from server.auth.auth import AuthTypeFactory
-from server.common.annotations.hosted_tiledb import AnnotationsHostedTileDB
-from server.common.annotations.local_file_csv import AnnotationsLocalFile
-from server.common.data_locator import discover_s3_region_name
-from server.common.default_config import get_default_config
-from server.common.errors import ConfigurationError, DatasetAccessError, OntologyLoadFailure
-from server.common.utils.utils import custom_format_warning, find_available_port, is_port_available
-from server.data_common.matrix_loader import MatrixDataLoader, MatrixDataCacheManager, MatrixDataType
-from server.db.db_utils import DbUtils
-
-DEFAULT_SERVER_PORT = 5005
-# anything bigger than this will generate a special message
-BIG_FILE_SIZE_THRESHOLD = 100 * 2 ** 20  # 100MB
-
-
-class AppFeature(object):
-    def __init__(self, path, available=False, method="POST", extra={}):
-        self.path = path
-        self.available = available
-        self.method = method
-        self.extra = extra
-        for k, v in extra.items():
-            setattr(self, k, v)
-
-    def todict(self):
-        d = dict(available=self.available, method=self.method, path=self.path)
-        d.update(self.extra)
-        return d
-
-
-class AppConfig(object):
-    """AppConfig stores all the configuration for cellxgene.  The configuration is divided into two main parts:
-    server attributes, and dataset attributes. The server_config contains attributes that refer to the server process
-    as a whole.   The default_dataset_config referes to attributes that are associated with the features and
-    presentations of a dataset.  The dataset config attributes can be overridden depending on the url by which the
-    dataset was accessed.  These are stored in dataroot_config.
-    AppConfig has methods to initialize, modify, and access the configuration.
-    """
-
-    def __init__(self):
-
-        # the default configuration (see default_config.py)
-        self.default_config = get_default_config()
-        # the server configuration
-        self.server_config = ServerConfig(self, self.default_config["server"])
-        # the dataset config, unless overridden by an entry in dataroot_config
-        self.default_dataset_config = DatasetConfig(None, self, self.default_config["dataset"])
-        # a dictionary of keys to DatasetConfig objects.  Each key must exist in the multi_dataset__dataroot
-        # attribute of the server_config.
-        self.dataroot_config = {}
-
-        # Set to true when config_completed is called
-        self.is_completed = False
-
-    def get_dataset_config(self, dataroot_key):
-        if self.server_config.single_dataset__datapath:
-            return self.default_dataset_config
-        else:
-            return self.dataroot_config.get(dataroot_key, self.default_dataset_config)
-
-    def check_config(self):
-        """Verify all the attributes have been checked"""
-        if not self.is_completed:
-            raise ConfigurationError("The configuration has not been completed")
-        self.server_config.check_config()
-        self.default_dataset_config.check_config()
-        for dataset_config in self.dataroot_config.values():
-            dataset_config.check_config()
-
-    def update_server_config(self, **kw):
-        self.server_config.update(**kw)
-        self.is_complete = False
-
-    def update_default_dataset_config(self, **kw):
-        self.default_dataset_config.update(**kw)
-        # update all the other dataset configs, if any
-        for value in self.dataroot_config.values():
-            value.update(**kw)
-        self.is_complete = False
-
-    def update_from_config_file(self, config_file):
-        with open(config_file) as fyaml:
-            config = yaml.load(fyaml, Loader=yaml.FullLoader)
-
-        self.server_config.update_from_config(config["server"], "server")
-        self.default_dataset_config.update_from_config(config["dataset"], "dataset")
-
-        per_dataset_config = config.get("per_dataset_config", {})
-        for key, dataroot_config in per_dataset_config.items():
-            self.add_dataroot_config(key, **dataroot_config)
-
-        self.is_complete = False
-
-    def write_config(self, config_file):
-        """output the config to a yaml file"""
-        server = self.server_config.create_mapping(self.server_config.default_config)
-        dataset = self.default_dataset_config.create_mapping(self.default_dataset_config.default_config)
-        config = dict(server={}, dataset={})
-        for attrname in server.keys():
-            config["server__" + attrname] = getattr(self.server_config, attrname)
-        for attrname in dataset.keys():
-            config["dataset__" + attrname] = getattr(self.default_dataset_config, attrname)
-        if self.dataroot_config:
-            config["per_dataset_config"] = {}
-        for dataroot_tag, dataroot_config in self.dataroot_config.items():
-            dataset = dataroot_config.create_mapping(dataroot_config.default_config)
-            for attrname in dataset.keys():
-                config[f"per_dataset_config__{dataroot_tag}__" + attrname] = getattr(dataroot_config, attrname)
-
-        config = unflatten(config, splitter=lambda key: key.split("__"))
-        yaml.dump(config, open(config_file, "w"))
-
-    def changes_from_default(self):
-        """Return all the attribute that are different from the default"""
-        diff_server = self.server_config.changes_from_default()
-        diff_dataset = self.default_dataset_config.changes_from_default()
-        diff = dict(server=diff_server, dataset=diff_dataset)
-        return diff
-
-    def add_dataroot_config(self, dataroot_tag, **kw):
-        """Create a new dataset config object based on the default dataset config, and kw parameters"""
-        if dataroot_tag in self.dataroot_config:
-            raise ConfigurationError(f"dataroot config already exists: {dataroot_tag}")
-        if type(self.server_config.multi_dataset__dataroot) != dict:
-            raise ConfigurationError("The server__multi_dataset__dataroot must be a dictionary")
-        if dataroot_tag not in self.server_config.multi_dataset__dataroot:
-            raise ConfigurationError(f"The dataroot_tag ({dataroot_tag}) not found in server__multi_dataset__dataroot")
-
-        self.is_completed = False
-        self.dataroot_config[dataroot_tag] = DatasetConfig(dataroot_tag, self, self.default_config["dataset"])
-        flat_config = self.default_dataset_config.create_mapping(self.default_dataset_config.default_config)
-        config = {key: value[1] for key, value in flat_config.items()}
-        self.dataroot_config[dataroot_tag].update(**config)
-        self.dataroot_config[dataroot_tag].update_from_config(kw, dataroot_tag)
-
-    def complete_config(self, messagefn=None):
-        """The configure options are checked, and any additional setup based on the config
-        parameters is done"""
-
-        if messagefn is None:
-            def noop(message):
-                pass
-
-            messagefn = noop
-
-        # TODO: to give better error messages we can add a mapping between where each config
-        # attribute originated (e.g. command line argument or config file), then in the error
-        # messages we can give correct context for attributes with bad value.
-        context = dict(messagefn=messagefn)
-
-        self.server_config.complete_config(context)
-        self.default_dataset_config.complete_config(context)
-        for dataroot_config in self.dataroot_config.values():
-            dataroot_config.complete_config(context)
-
-        self.is_completed = True
-        self.check_config()
-
-    def get_matrix_data_cache_manager(self):
-        return self.server_config.matrix_data_cache_manager
-
-    def is_multi_dataset(self):
-        return self.server_config.multi_dataset__dataroot is not None
-
-    def get_title(self, data_adaptor):
-        return (
-            self.server_config.single_dataset__title
-            if self.server_config.single_dataset__title
-            else data_adaptor.get_title()
-        )
-
-    def get_about(self, data_adaptor):
-        return (
-            self.server_config.single_dataset__about
-            if self.server_config.single_dataset__about
-            else data_adaptor.get_about()
-        )
-
-    def get_client_config(self, data_adaptor):
-        """
-        Return the configuration as required by the /config REST route
-        """
-
-        server_config = self.server_config
-        dataset_config = data_adaptor.dataset_config
-        annotation = dataset_config.user_annotations
-        auth = server_config.auth
-
-        # FIXME The current set of config is not consistently presented:
-        # we have camalCase, hyphen-text, and underscore_text
-
-        # make sure the configuration has been checked.
-        self.check_config()
-
-        # features
-        features = [f.todict() for f in data_adaptor.get_features(annotation)]
-
-        # display_names
-        title = self.get_title(data_adaptor)
-        about = self.get_about(data_adaptor)
-
-        display_names = dict(engine=data_adaptor.get_name(), dataset=title)
-
-        # library_versions
-        library_versions = {}
-        library_versions.update(data_adaptor.get_library_versions())
-        library_versions["cellxgene"] = cellxgene_display_version
-
-        # links
-        links = {"about-dataset": about}
-
-        # parameters
-        parameters = {
-            "layout": dataset_config.embeddings__names,
-            "max-category-items": dataset_config.presentation__max_categories,
-            "obs_names": server_config.single_dataset__obs_names,
-            "var_names": server_config.single_dataset__var_names,
-            "diffexp_lfc_cutoff": dataset_config.diffexp__lfc_cutoff,
-            "backed": server_config.adaptor__anndata_adaptor__backed,
-            "disable-diffexp": not dataset_config.diffexp__enable,
-            "enable-reembedding": dataset_config.embeddings__enable_reembedding,
-            "annotations": False,
-            "annotations_file": None,
-            "annotations_dir": None,
-            "annotations_cell_ontology_enabled": False,
-            "annotations_cell_ontology_obopath": None,
-            "annotations_cell_ontology_terms": None,
-            "custom_colors": dataset_config.presentation__custom_colors,
-            "diffexp-may-be-slow": False,
-            "about_legal_tos": dataset_config.app__about_legal_tos,
-            "about_legal_privacy": dataset_config.app__about_legal_privacy,
-        }
-
-        # corpora dataset_props
-        # TODO/Note: putting info from the dataset into the /config is not ideal.
-        # However, it is definitely not part of /schema, and we do not have a top-level
-        # route for data properties.  Consider creating one at some point.
-        corpora_props = data_adaptor.get_corpora_props()
-        if corpora_props and "default_embedding" in corpora_props:
-            default_embedding = corpora_props["default_embedding"]
-            if isinstance(default_embedding, str) and default_embedding.startswith("X_"):
-                default_embedding = default_embedding[2:]  # drop X_ prefix
-            if default_embedding in data_adaptor.get_embedding_names():
-                parameters["default_embedding"] = default_embedding
-
-        data_adaptor.update_parameters(parameters)
-        if annotation:
-            annotation.update_parameters(parameters, data_adaptor)
-
-        # gather it all together
-        c = {}
-        config = c["config"] = {}
-        config["features"] = features
-        config["displayNames"] = display_names
-        config["library_versions"] = library_versions
-        config["links"] = links
-        config["parameters"] = parameters
-        config["corpora_props"] = corpora_props
-        config["limits"] = {
-            "column_request_max": server_config.limits__column_request_max,
-            "diffexp_cellcount_max": server_config.limits__diffexp_cellcount_max,
-        }
-
-        if dataset_config.app__authentication_enable and auth.is_valid_authentication_type():
-            config["authentication"] = {
-                "is_authenticated": auth.is_user_authenticated(),
-                "requires_client_login": auth.requires_client_login(),
-                "username": auth.get_user_name(),
-                "user_id": auth.get_user_id()
-            }
-            if auth.requires_client_login():
-                config["authentication"].update({
-                    "login": auth.get_login_url(data_adaptor),
-                    "logout": auth.get_logout_url(data_adaptor),
-                })
-
-        return c
-
-
-class BaseConfig(object):
-    """This class handles the mechanics of updating and checking attributes.
-    Derived classes are expected to store the actual attributes"""
-
-    def __init__(self, app_config, default_config, dictval_cases={}):
-        # reference back to the app_config
-        self.app_config = app_config
-        # the complete set of attribute and their default values (unflattened)
-        self.default_config = default_config
-        # attributes where the value may be a dict (and therefore are not flattened)
-        self.dictval_cases = dictval_cases
-        # used to make sure every attribute value is checked
-        self.attr_checked = {k: False for k in self.create_mapping(default_config).keys()}
-
-    def create_mapping(self, config):
-        """Create a mapping from attribute names to (location in the config tree, value)"""
-        dc = copy.deepcopy(config)
-        mapping = {}
-
-        # special cases where the value could be a dict.
-        # If its value is not None, the entry is added to the mapping, and not included
-        # in the flattening below.
-        for dictval_case in self.dictval_cases:
-            cur = dc
-            for part in dictval_case[:-1]:
-                cur = cur.get(part, {})
-            val = cur.get(dictval_case[-1])
-            if val is not None:
-                key = "__".join(dictval_case)
-                mapping[key] = (dictval_case, val)
-                del cur[dictval_case[-1]]
-
-        flat_config = flatten(dc)
-        for key, value in flat_config.items():
-            # name of the attribute
-            attr = "__".join(key)
-            mapping[attr] = (key, value)
-
-        return mapping
-
-    def check_attr(self, attrname, vtype):
-        val = getattr(self, attrname)
-        if type(vtype) in (list, tuple):
-            if type(val) not in vtype:
-                tnames = ",".join([x.__name__ for x in vtype])
-                raise ConfigurationError(
-                    f"Invalid type for attribute: {attrname}, expected types ({tnames}), got {type(val).__name__}"
-                )
-        else:
-            if type(val) != vtype:
-                raise ConfigurationError(
-                    f"Invalid type for attribute: {attrname}, "
-                    f"expected type {vtype.__name__}, got {type(val).__name__}"
-                )
-
-        self.attr_checked[attrname] = True
-
-    def check_config(self):
-        mapping = self.create_mapping(self.default_config)
-        for key in mapping.keys():
-            if not self.attr_checked[key]:
-                raise ConfigurationError(f"The attr '{key}' has not been checked")
-
-    def update(self, **kw):
-        for key, value in kw.items():
-            if not hasattr(self, key):
-                raise ConfigurationError(f"unknown config parameter {key}.")
-            try:
-                if type(value) == tuple:
-                    # convert tuple values to list values
-                    value = list(value)
-                setattr(self, key, value)
-            except KeyError:
-                raise ConfigurationError(f"Unable to set config parameter {key}.")
-
-            self.attr_checked[key] = False
-
-    def update_from_config(self, config, prefix):
-        mapping = self.create_mapping(config)
-        for attr, (key, value) in mapping.items():
-            if not hasattr(self, attr):
-                raise ConfigurationError(f"Unknown key from config file: {prefix}__{attr}")
-            try:
-                setattr(self, attr, value)
-            except KeyError:
-                raise ConfigurationError(f"Unable to set config attribute: {prefix}__{attr}")
-
-            self.attr_checked[attr] = False
-
-    def changes_from_default(self):
-        """Return all the attribute that are different from the default"""
-        mapping = self.create_mapping(self.default_config)
-        diff = []
-        for attrname, (key, defval) in mapping.items():
-            curval = getattr(self, attrname)
-            if curval != defval:
-                diff.append((attrname, curval, defval))
-        return diff
-
-
-class ServerConfig(BaseConfig):
-    """Manages the config attribute associated with the server."""
-
-    def __init__(self, app_config, default_config):
-        dictval_cases = [
-            ("app", "csp_directives"),
-            ("authentication", "params_oauth", "cookie"),
-            ("adaptor", "cxg_adaptor", "tiledb_ctx"),
-            ("multi_dataset", "dataroot"),
-        ]
-        super().__init__(app_config, default_config, dictval_cases)
-
-        dc = default_config
-        try:
-            self.app__verbose = dc["app"]["verbose"]
-            self.app__debug = dc["app"]["debug"]
-            self.app__host = dc["app"]["host"]
-            self.app__port = dc["app"]["port"]
-            self.app__open_browser = dc["app"]["open_browser"]
-            self.app__force_https = dc["app"]["force_https"]
-            self.app__flask_secret_key = dc["app"]["flask_secret_key"]
-            self.app__generate_cache_control_headers = dc["app"]["generate_cache_control_headers"]
-            self.app__server_timing_headers = dc["app"]["server_timing_headers"]
-            self.app__csp_directives = dc["app"]["csp_directives"]
-
-            self.authentication__type = dc["authentication"]["type"]
-            self.authentication__params_oauth__api_base_url = dc["authentication"]["params_oauth"]["api_base_url"]
-            self.authentication__params_oauth__client_id = dc["authentication"]["params_oauth"]["client_id"]
-            self.authentication__params_oauth__client_secret = dc["authentication"]["params_oauth"]["client_secret"]
-            self.authentication__params_oauth__callback_base_url = \
-                dc["authentication"]["params_oauth"]["callback_base_url"]
-            self.authentication__params_oauth__session_cookie = dc["authentication"]["params_oauth"]["session_cookie"]
-            self.authentication__params_oauth__cookie = dc["authentication"]["params_oauth"]["cookie"]
-
-            self.multi_dataset__dataroot = dc["multi_dataset"]["dataroot"]
-            self.multi_dataset__index = dc["multi_dataset"]["index"]
-            self.multi_dataset__allowed_matrix_types = dc["multi_dataset"]["allowed_matrix_types"]
-            self.multi_dataset__matrix_cache__max_datasets = dc["multi_dataset"]["matrix_cache"]["max_datasets"]
-            self.multi_dataset__matrix_cache__timelimit_s = dc["multi_dataset"]["matrix_cache"]["timelimit_s"]
-
-            self.single_dataset__datapath = dc["single_dataset"]["datapath"]
-            self.single_dataset__obs_names = dc["single_dataset"]["obs_names"]
-            self.single_dataset__var_names = dc["single_dataset"]["var_names"]
-            self.single_dataset__about = dc["single_dataset"]["about"]
-            self.single_dataset__title = dc["single_dataset"]["title"]
-
-            self.diffexp__alg_cxg__max_workers = dc["diffexp"]["alg_cxg"]["max_workers"]
-            self.diffexp__alg_cxg__cpu_multiplier = dc["diffexp"]["alg_cxg"]["cpu_multiplier"]
-            self.diffexp__alg_cxg__target_workunit = dc["diffexp"]["alg_cxg"]["target_workunit"]
-
-            self.data_locator__s3__region_name = dc["data_locator"]["s3"]["region_name"]
-
-            self.adaptor__cxg_adaptor__tiledb_ctx = dc["adaptor"]["cxg_adaptor"]["tiledb_ctx"]
-            self.adaptor__anndata_adaptor__backed = dc["adaptor"]["anndata_adaptor"]["backed"]
-
-            self.limits__diffexp_cellcount_max = dc["limits"]["diffexp_cellcount_max"]
-            self.limits__column_request_max = dc["limits"]["column_request_max"]
-
-        except KeyError as e:
-            raise ConfigurationError(f"Unexpected config: {str(e)}")
-
-        # The matrix data cache manager is created during the complete_config and stored here.
-        self.matrix_data_cache_manager = None
-
-        # The authentication object
-        self.auth = None
-
-    def complete_config(self, context):
-        self.handle_app(context)
-        self.handle_data_source(context)
-        self.handle_authentication(context)
-        self.handle_data_locator(context)
-        self.handle_adaptor(context)  # may depend on data_locator
-        self.handle_single_dataset(context)  # may depend on adaptor
-        self.handle_multi_dataset(context)  # may depend on adaptor
-        self.handle_diffexp(context)
-        self.handle_limits(context)
-
-        self.check_config()
-
-    def handle_app(self, context):
-        self.check_attr("app__verbose", bool)
-        self.check_attr("app__debug", bool)
-        self.check_attr("app__host", str)
-        self.check_attr("app__port", (type(None), int))
-        self.check_attr("app__open_browser", bool)
-        self.check_attr("app__force_https", bool)
-        self.check_attr("app__flask_secret_key", (type(None), str))
-        self.check_attr("app__generate_cache_control_headers", bool)
-        self.check_attr("app__server_timing_headers", bool)
-        self.check_attr("app__csp_directives", (type(None), dict))
-
-        if self.app__port:
-            try:
-                if not is_port_available(self.app__host, self.app__port):
-                    raise ConfigurationError(
-                        f"The port selected {self.app__port} is in use, please configure an open port."
-                    )
-            except OverflowError:
-                raise ConfigurationError(f"Invalid port: {self.app__port}")
-        else:
-            try:
-                default_server_port = int(os.environ.get("CXG_SERVER_PORT", DEFAULT_SERVER_PORT))
-            except ValueError:
-                raise ConfigurationError(
-                    "Invalid port from environment variable CXG_SERVER_PORT: " + os.environ.get("CXG_SERVER_PORT")
-                )
-            try:
-                self.app__port = find_available_port(self.app__host, default_server_port)
-            except OverflowError:
-                raise ConfigurationError(f"Invalid port: {default_server_port}")
-
-        if self.app__debug:
-            context["messagefn"]("in debug mode, setting verbose=True and open_browser=False")
-            self.app__verbose = True
-            self.app__open_browser = False
-        else:
-            warnings.formatwarning = custom_format_warning
-
-        if not self.app__verbose:
-            sys.tracebacklimit = 0
-
-        # secret key:
-        #   first, from CXG_SECRET_KEY environment variable
-        #   second, from config file
-        self.app__flask_secret_key = os.environ.get("CXG_SECRET_KEY", self.app__flask_secret_key)
-
-        # CSP Directives are a dict of string: list(string) or string: string
-        if self.app__csp_directives is not None:
-            for k, v in self.app__csp_directives.items():
-                if not isinstance(k, str):
-                    raise ConfigurationError("CSP directive names must be a string.")
-                if isinstance(v, list):
-                    for policy in v:
-                        if not isinstance(policy, str):
-                            raise ConfigurationError("CSP directive value must be a string or list of strings.")
-                elif not isinstance(v, str):
-                    raise ConfigurationError("CSP directive value must be a string or list of strings.")
-
-    def handle_authentication(self, context):
-        self.check_attr("authentication__type", (type(None), str))
-
-        # oauth
-        ptypes = str if self.authentication__type == "oauth" else (type(None), str)
-        self.check_attr("authentication__params_oauth__api_base_url", ptypes)
-        self.check_attr("authentication__params_oauth__client_id", ptypes)
-        self.check_attr("authentication__params_oauth__client_secret", ptypes)
-        self.check_attr("authentication__params_oauth__callback_base_url", (type(None), str))
-        self.check_attr("authentication__params_oauth__session_cookie", bool)
-
-        if self.authentication__params_oauth__session_cookie:
-            self.check_attr("authentication__params_oauth__cookie", (type(None), dict))
-        else:
-            self.check_attr("authentication__params_oauth__cookie", dict)
-        #   secret key: first, from CXG_OAUTH_CLIENT_SECRET environment variable
-        #   second, from config file
-        self.authentication__params__oauth__client_secret = os.environ.get(
-            "CXG_OAUTH_CLIENT_SECRET", self.authentication__params_oauth__client_secret)
-
-        self.auth = AuthTypeFactory.create(self.authentication__type, self)
-        if self.auth is None:
-            raise ConfigurationError(f"Unknown authentication type: {self.authentication__type}")
-
-    def handle_data_locator(self, context):
-        self.check_attr("data_locator__s3__region_name", (type(None), bool, str))
-        if self.data_locator__s3__region_name is True:
-            path = self.single_dataset__datapath or self.multi_dataset__dataroot
-            if type(path) == dict:
-                # if multi_dataset__dataroot is a dict, then use the first key
-                # that is in s3.   NOTE:  it is not supported to have dataroots
-                # in different regions.
-                paths = [val.get("dataroot") for val in path.values()]
-                for path in paths:
-                    if path.startswith("s3://"):
-                        break
-            if path.startswith("s3://"):
-                region_name = discover_s3_region_name(path)
-                if region_name is None:
-                    raise ConfigurationError(f"Unable to discover s3 region name from {path}")
-            else:
-                region_name = None
-            self.data_locator__s3__region_name = region_name
-
-    def handle_data_source(self, context):
-        self.check_attr("single_dataset__datapath", (str, type(None)))
-        self.check_attr("multi_dataset__dataroot", (type(None), dict, str))
-
-        if self.single_dataset__datapath is None:
-            if self.multi_dataset__dataroot is None:
-                # TODO:  change the error message once dataroot is fully supported
-                raise ConfigurationError("missing datapath")
-            return
-        else:
-            if self.multi_dataset__dataroot is not None:
-                raise ConfigurationError("must supply only one of datapath or dataroot")
-
-    def handle_single_dataset(self, context):
-        self.check_attr("single_dataset__datapath", (str, type(None)))
-        self.check_attr("single_dataset__title", (str, type(None)))
-        self.check_attr("single_dataset__about", (str, type(None)))
-        self.check_attr("single_dataset__obs_names", (str, type(None)))
-        self.check_attr("single_dataset__var_names", (str, type(None)))
-
-        if self.single_dataset__datapath is None:
-            return
-
-        # create the matrix data cache manager:
-        if self.matrix_data_cache_manager is None:
-            self.matrix_data_cache_manager = MatrixDataCacheManager(max_cached=1, timelimit_s=None)
-
-        # preload this data set
-        matrix_data_loader = MatrixDataLoader(self.single_dataset__datapath, app_config=self.app_config)
-        try:
-            matrix_data_loader.pre_load_validation()
-        except DatasetAccessError as e:
-            raise ConfigurationError(str(e))
-
-        file_size = matrix_data_loader.file_size()
-        file_basename = basename(self.single_dataset__datapath)
-        if file_size > BIG_FILE_SIZE_THRESHOLD:
-            context["messagefn"](f"Loading data from {file_basename}, this may take a while...")
-        else:
-            context["messagefn"](f"Loading data from {file_basename}.")
-
-        if self.single_dataset__about:
-
-            def url_check(url):
-                try:
-                    result = urlparse(url)
-                    if all([result.scheme, result.netloc]):
-                        return True
-                    else:
-                        return False
-                except ValueError:
-                    return False
-
-            if not url_check(self.single_dataset__about):
-                raise ConfigurationError(
-                    "Must provide an absolute URL for --about. (Example format: http://example.com)"
-                )
-
-    def handle_multi_dataset(self, context):
-        self.check_attr("multi_dataset__dataroot", (type(None), dict, str))
-        self.check_attr("multi_dataset__index", (type(None), bool, str))
-        self.check_attr("multi_dataset__allowed_matrix_types", list)
-        self.check_attr("multi_dataset__matrix_cache__max_datasets", int)
-        self.check_attr("multi_dataset__matrix_cache__timelimit_s", (type(None), int, float))
-
-        if self.multi_dataset__dataroot is None:
-            return
-
-        if type(self.multi_dataset__dataroot) == str:
-            default_dict = dict(base_url="d", dataroot=self.multi_dataset__dataroot)
-            self.multi_dataset__dataroot = dict(d=default_dict)
-
-        for tag, dataroot_dict in self.multi_dataset__dataroot.items():
-            if "base_url" not in dataroot_dict:
-                raise ConfigurationError(f"error in multi_dataset__dataroot: missing base_url for tag {tag}")
-            if "dataroot" not in dataroot_dict:
-                raise ConfigurationError(f"error in multi_dataset__dataroot: missing dataroot, for tag {tag}")
-
-            base_url = dataroot_dict["base_url"]
-
-            # sanity check for well formed base urls
-            bad = False
-            if type(base_url) != str:
-                bad = True
-            elif os.path.normpath(base_url) != base_url:
-                bad = True
-            else:
-                base_url_parts = base_url.split("/")
-                if [quote_plus(part) for part in base_url_parts] != base_url_parts:
-                    bad = True
-                if ".." in base_url_parts:
-                    bad = True
-            if bad:
-                raise ConfigurationError(f"error in multi_dataset__dataroot base_url {base_url} for tag {tag}")
-
-        # verify all the base_urls are unique
-        base_urls = [d["base_url"] for d in self.multi_dataset__dataroot.values()]
-        if len(base_urls) > len(set(base_urls)):
-            raise ConfigurationError("error in multi_dataset__dataroot:  base_urls must be unique")
-
-        # error checking
-        for mtype in self.multi_dataset__allowed_matrix_types:
-            try:
-                MatrixDataType(mtype)
-            except ValueError:
-                raise ConfigurationError(f'Invalid matrix type in "allowed_matrix_types": {mtype}')
-
-        # create the matrix data cache manager:
-        if self.matrix_data_cache_manager is None:
-            self.matrix_data_cache_manager = MatrixDataCacheManager(
-                max_cached=self.multi_dataset__matrix_cache__max_datasets,
-                timelimit_s=self.multi_dataset__matrix_cache__timelimit_s,
-            )
-
-    def handle_diffexp(self, context):
-        self.check_attr("diffexp__alg_cxg__max_workers", (str, int))
-        self.check_attr("diffexp__alg_cxg__cpu_multiplier", int)
-        self.check_attr("diffexp__alg_cxg__target_workunit", int)
-
-        max_workers = self.diffexp__alg_cxg__max_workers
-        cpu_multiplier = self.diffexp__alg_cxg__cpu_multiplier
-        cpu_count = os.cpu_count()
-        max_workers = min(max_workers, cpu_multiplier * cpu_count)
-        diffexp_tiledb.set_config(max_workers, self.diffexp__alg_cxg__target_workunit)
-
-    def handle_adaptor(self, context):
-        # cxg
-        self.check_attr("adaptor__cxg_adaptor__tiledb_ctx", dict)
-        regionkey = "vfs.s3.region"
-        if regionkey not in self.adaptor__cxg_adaptor__tiledb_ctx:
-            if type(self.data_locator__s3__region_name) == str:
-                self.adaptor__cxg_adaptor__tiledb_ctx[regionkey] = self.data_locator__s3__region_name
-
-        from server.data_cxg.cxg_adaptor import CxgAdaptor
-
-        CxgAdaptor.set_tiledb_context(self.adaptor__cxg_adaptor__tiledb_ctx)
-
-        # anndata
-        self.check_attr("adaptor__anndata_adaptor__backed", bool)
-
-    def handle_limits(self, context):
-        self.check_attr("limits__diffexp_cellcount_max", (type(None), int))
-        self.check_attr("limits__column_request_max", (type(None), int))
-
-    def exceeds_limit(self, limit_name, value):
-        limit_value = getattr(self, "limits__" + limit_name, None)
-        if limit_value is None:  # disabled
-            return False
-        return value > limit_value
-
-
-class DatasetConfig(BaseConfig):
-    """Manages the config attribute associated with a dataset."""
-
-    def __init__(self, tag, app_config, default_config):
-        super().__init__(app_config, default_config)
-        self.tag = tag
-        dc = default_config
-        try:
-            self.app__scripts = dc["app"]["scripts"]
-            self.app__inline_scripts = dc["app"]["inline_scripts"]
-            self.app__about_legal_tos = dc["app"]["about_legal_tos"]
-            self.app__about_legal_privacy = dc["app"]["about_legal_privacy"]
-            self.app__authentication_enable = dc["app"]["authentication_enable"]
-
-            self.presentation__max_categories = dc["presentation"]["max_categories"]
-            self.presentation__custom_colors = dc["presentation"]["custom_colors"]
-
-            self.user_annotations__enable = dc["user_annotations"]["enable"]
-            self.user_annotations__type = dc["user_annotations"]["type"]
-            self.user_annotations__local_file_csv__directory = dc["user_annotations"]["local_file_csv"]["directory"]
-            self.user_annotations__local_file_csv__file = dc["user_annotations"]["local_file_csv"]["file"]
-            self.user_annotations__ontology__enable = dc["user_annotations"]["ontology"]["enable"]
-            self.user_annotations__ontology__obo_location = dc["user_annotations"]["ontology"]["obo_location"]
-            self.user_annotations__hosted_tiledb_array__db_uri = dc["user_annotations"]["hosted_tiledb_array"]["db_uri"]
-            self.user_annotations__hosted_tiledb_array__hosted_file_directory = \
-            dc["user_annotations"]["hosted_tiledb_array"]["hosted_file_directory"]  # noqa E501
-
-            self.embeddings__names = dc["embeddings"]["names"]
-            self.embeddings__enable_reembedding = dc["embeddings"]["enable_reembedding"]
-
-            self.diffexp__enable = dc["diffexp"]["enable"]
-            self.diffexp__lfc_cutoff = dc["diffexp"]["lfc_cutoff"]
-            self.diffexp__top_n = dc["diffexp"]["top_n"]
-
-        except KeyError as e:
-            raise ConfigurationError(f"Unexpected config: {str(e)}")
-
-        # The annotation object is created during complete_config and stored here.
-        self.user_annotations = None
-
-    def complete_config(self, context):
-        self.handle_app(context)
-        self.handle_presentation(context)
-        self.handle_user_annotations(context)
-        self.handle_embeddings(context)
-        self.handle_diffexp(context)
-
-    def handle_app(self, context):
-        self.check_attr("app__scripts", list)
-        self.check_attr("app__inline_scripts", list)
-        self.check_attr("app__about_legal_tos", (type(None), str))
-        self.check_attr("app__about_legal_privacy", (type(None), str))
-        self.check_attr("app__authentication_enable", bool)
-
-        # scripts can be string (filename) or dict (attributes).   Convert string to dict.
-        scripts = []
-        for s in self.app__scripts:
-            if isinstance(s, str):
-                scripts.append({"src": s})
-            elif isinstance(s, dict) and isinstance(s["src"], str):
-                scripts.append(s)
-            else:
-                raise ConfigurationError("Scripts must be string or dict")
-        self.app__scripts = scripts
-
-    def handle_presentation(self, context):
-        self.check_attr("presentation__max_categories", int)
-        self.check_attr("presentation__custom_colors", bool)
-
-    def handle_user_annotations(self, context):
-        self.check_attr("user_annotations__enable", bool)
-        self.check_attr("user_annotations__type", str)
-        self.check_attr("user_annotations__local_file_csv__directory", (type(None), str))
-        self.check_attr("user_annotations__local_file_csv__file", (type(None), str))
-        self.check_attr("user_annotations__ontology__enable", bool)
-        self.check_attr("user_annotations__ontology__obo_location", (type(None), str))
-        self.check_attr("user_annotations__hosted_tiledb_array__db_uri", (type(None), str))
-        self.check_attr("user_annotations__hosted_tiledb_array__hosted_file_directory", (type(None), str))
-
-        if self.user_annotations__enable:
-            server_config = self.app_config.server_config
-            if not self.app__authentication_enable:
-                raise ConfigurationError("user annotations requires authentication to be enabled")
-            if not server_config.auth.is_valid_authentication_type():
-                auth_type = server_config.authentication__type
-                raise ConfigurationError(f"authentication method {auth_type} is not compatible with user annotations")
-
-            # TODO, replace this with a factory pattern once we have more than one way
-            # to do annotations.  currently only local_file_csv
-            if self.user_annotations__type == "local_file_csv":
-                dirname = self.user_annotations__local_file_csv__directory
-                filename = self.user_annotations__local_file_csv__file
-
-                if filename is not None and dirname is not None:
-                    raise ConfigurationError("'annotations-file' and 'annotations-dir' may not be used together.")
-
-                if filename is not None:
-                    lf_name, lf_ext = splitext(filename)
-                    if lf_ext and lf_ext != ".csv":
-                        raise ConfigurationError(f"annotation file type must be .csv: {filename}")
-
-                if dirname is not None and not isdir(dirname):
-                    try:
-                        os.mkdir(dirname)
-                    except OSError:
-                        raise ConfigurationError("Unable to create directory specified by --annotations-dir")
-
-                self.user_annotations = AnnotationsLocalFile(dirname, filename)
-
-                # if the user has specified a fixed label file, go ahead and validate it
-                # so that we can remove errors early in the process.
-                server_config = self.app_config.server_config
-                if server_config.single_dataset__datapath and self.user_annotations__local_file_csv__file:
-                    with server_config.matrix_data_cache_manager.data_adaptor(
-                            self.tag, server_config.single_dataset__datapath, self.app_config
-                    ) as data_adaptor:
-                        data_adaptor.check_new_labels(self.user_annotations.read_labels(data_adaptor))
-
-                if self.user_annotations__ontology__enable or self.user_annotations__ontology__obo_location:
-                    try:
-                        self.user_annotations.load_ontology(self.user_annotations__ontology__obo_location)
-                    except OntologyLoadFailure as e:
-                        raise ConfigurationError("Unable to load ontology terms\n" + str(e))
-            elif self.user_annotations__type == "hosted_tiledb_array":
-                self.check_attr("user_annotations__hosted_tiledb_array__db_uri", str)
-                self.check_attr("user_annotations__hosted_tiledb_array__hosted_file_directory", str)
-                self.user_annotations = AnnotationsHostedTileDB(
-                    directory_path=self.user_annotations__hosted_tiledb_array__hosted_file_directory,
-                    db=DbUtils(self.user_annotations__hosted_tiledb_array__db_uri),
-                )
-            else:
-                raise ConfigurationError('The only annotation type support is "local_file_csv" or "hosted_tiledb_array')
-        else:
-            if self.user_annotations__type == "local_file_csv":
-                dirname = self.user_annotations__local_file_csv__directory
-                filename = self.user_annotations__local_file_csv__file
-                if filename is not None:
-                    context["messsagefn"]("Warning: --annotations-file ignored as annotations are disabled.")
-                if dirname is not None:
-                    context["messagefn"]("Warning: --annotations-dir ignored as annotations are disabled.")
-
-            if self.user_annotations__ontology__enable:
-                context["messagefn"](
-                    "Warning: --experimental-annotations-ontology" " ignored as annotations are disabled."
-                )
-            if self.user_annotations__ontology__obo_location is not None:
-                context["messagefn"](
-                    "Warning: --experimental-annotations-ontology-obo" " ignored as annotations are disabled."
-                )
-
-    def handle_embeddings(self, context):
-        self.check_attr("embeddings__names", list)
-        self.check_attr("embeddings__enable_reembedding", bool)
-
-        server_config = self.app_config.server_config
-        if server_config.single_dataset__datapath:
-            if self.embeddings__enable_reembedding:
-                matrix_data_loader = MatrixDataLoader(
-                    server_config.single_dataset__datapath, app_config=self.app_config
-                )
-                if matrix_data_loader.matrix_data_type != MatrixDataType.H5AD:
-                    raise ConfigurationError("'enable-reembedding is only supported with H5AD files.")
-                if server_config.adaptor__anndata_adaptor__backed:
-                    raise ConfigurationError("enable-reembedding is not supported when run in --backed mode.")
-
-    def handle_diffexp(self, context):
-        self.check_attr("diffexp__enable", bool)
-        self.check_attr("diffexp__lfc_cutoff", float)
-        self.check_attr("diffexp__top_n", int)
-
-        server_config = self.app_config.server_config
-        if server_config.single_dataset__datapath:
-            with server_config.matrix_data_cache_manager.data_adaptor(
-                    self.tag, server_config.single_dataset__datapath, self.app_config
-            ) as data_adaptor:
-                if self.diffexp__enable and data_adaptor.parameters.get("diffexp_may_be_slow", False):
-                    context["messagefn"](
-                        "CAUTION: due to the size of your dataset, "
-                        "running differential expression may take longer or fail."
-                    )

server/common/aws_secret_utils.py

@@ -1,72 +1,11 @@
 import logging
-import os
-import sys
 
 import boto3
 from flask import json
 
-from server.common.data_locator import discover_s3_region_name
 from server.common.errors import SecretKeyRetrievalError
 
 
-def handle_config_from_secret(app_config):
-    """Update configuration from the secret manager"""
-    secret_name = os.getenv("CXG_AWS_SECRET_NAME")
-    if not secret_name:
-        return
-
-    # need to find the secret manager region.
-    #  1. from CXG_AWS_SECRET_REGION_NAME
-    #  2. discover from dataroot location (if on s3)
-    #  3. discover from config file location (if on s3)
-    secret_region_name = os.getenv("CXG_AWS_SECRET_REGION_NAME")
-    if secret_region_name is None:
-        secret_region_name = discover_s3_region_name(app_config.multi_dataset__dataroot)
-        if not secret_region_name:
-            from server.eb.app import config_file
-            secret_region_name = discover_s3_region_name(config_file)
-    if not secret_region_name:
-        logging.error("Could not determine the AWS Secret Manager region")
-        sys.exit(1)
-
-    secrets = get_secret_key(secret_region_name, secret_name)
-
-    if not secrets:
-        return
-
-    server_attrs = (
-        ("flask_secret_key", "app__flask_secret_key"),
-        ("oauth_client_secret", "authentication__params_oauth__client_secret"),
-    )
-    default_dataset_attrs = (
-        ("db_uri", "user_annotations__hosted_tiledb_array__db_uri"),
-    )
-
-    # update server configuration attributes
-    for key, attr in server_attrs:
-        cur_val = getattr(app_config.server_config, attr)
-        if cur_val:
-            continue
-
-        # replace the attr with the secret if it is not set
-        val = secrets.get(key)
-        if val:
-            logging.info(f"set {attr} from secret")
-            app_config.update_server_config(**{attr : val})
-
-    # update default dataset configuration attributes
-    for key, attr in default_dataset_attrs:
-        cur_val = getattr(app_config.default_dataset_config, attr)
-        if cur_val:
-            continue
-
-        # replace the attr with the secret if it is not set
-        val = secrets.get(key)
-        if val:
-            logging.info(f"set {attr} from secret")
-            app_config.update_default_dataset_config(**{attr : val})
-
-
 def get_secret_key(region_name, secret_name):
     session = boto3.session.Session()
     client = session.client(service_name="secretsmanager", region_name=region_name)
@@ -79,6 +18,6 @@ def get_secret_key(region_name, secret_name):
             return secret
     except Exception as e:
         logging.critical(f"Caught exception during get_secret_key, {e}", exc_info=True)
-        raise SecretKeyRetrievalError
+        raise SecretKeyRetrievalError(str(e))
 
     return None

server/common/config/__init__.py

@@ -0,0 +1,4 @@
+from server.common.aws_secret_utils import get_secret_key  # noqa F504
+
+DEFAULT_SERVER_PORT = 5005
+BIG_FILE_SIZE_THRESHOLD = 100 * 2 ** 20  # 100MB

server/common/config/app_config.py

@@ -0,0 +1,247 @@
+import yaml
+from flatten_dict import unflatten
+
+from server.default_config import get_default_config
+from server.common.config.dataset_config import DatasetConfig
+from server.common.config.server_config import ServerConfig
+from server.common.config.external_config import ExternalConfig
+from server.common.errors import ConfigurationError
+
+
+class AppConfig(object):
+    """
+    AppConfig stores all the configuration for cellxgene.
+    AppConfig contains one or more DatasetConfig(s) and one ServerConfig.
+    The server_config contains attributes that refer to the server process as a whole.
+    The default_dataset_config refers to attributes that are associated with the features and
+    presentations of a dataset.
+    The dataset config attributes can be overridden depending on the url by which the
+    dataset was accessed.  These are stored in dataroot_config.
+    AppConfig has methods to initialize, modify, and access the configuration.
+    """
+
+    def __init__(self):
+
+        # the default configuration (see default_config.py)
+        # TODO @madison -- if we always read from the default config (hard coded path) can we set those values as
+        #  defaults within the config class?
+        self.default_config = get_default_config()
+        # the server configuration
+        self.server_config = ServerConfig(self, self.default_config["server"])
+        # the dataset config, unless overridden by an entry in dataroot_config
+        self.default_dataset_config = DatasetConfig(None, self, self.default_config["dataset"])
+        # a dictionary of keys to DatasetConfig objects.  Each key must exist in the multi_dataset__dataroot
+        # attribute of the server_config. The default dataset config will apply to all datasets unless a different set
+        # of config vars was passed for a specific dataset under the multidataset config. For example:
+        """
+        per_dataset_config:
+            d1:
+               user_annotations:
+                   enable:  false
+            d2:
+               user_annotations:
+                   enable:  true
+         """
+        #   dataroot config
+        self.dataroot_config = {}
+
+        #  external config
+        self.external_config = ExternalConfig(self, self.default_config["external"])
+
+        # Set to true when config_completed is called
+        self.is_completed = False
+
+    def get_dataset_config(self, dataroot_key):
+        if self.server_config.single_dataset__datapath:
+            return self.default_dataset_config
+        else:
+            return self.dataroot_config.get(dataroot_key, self.default_dataset_config)
+
+    def check_config(self):
+        """Verify all the attributes in the config have been type checked"""
+        if not self.is_completed:
+            raise ConfigurationError("The configuration has not been completed")
+        self.server_config.check_config()
+        self.default_dataset_config.check_config()
+        for dataset_config in self.dataroot_config.values():
+            dataset_config.check_config()
+        self.external_config.check_config()
+
+    def update_server_config(self, **kw):
+        self.server_config.update(**kw)
+        self.is_complete = False
+
+    def update_default_dataset_config(self, **kw):
+        self.default_dataset_config.update(**kw)
+        # update all the other dataset configs, if any
+        for value in self.dataroot_config.values():
+            value.update(**kw)
+        self.is_complete = False
+
+    def update_single_config_from_path_and_value(self, path, value):
+        """Update a single config parameter with the value.
+        Path is a list of string, that gives a path to the config parameter to be updated.
+        For example, path may be ["server","app","port"].
+        """
+        self.is_complete = False
+        if not isinstance(path, list):
+            raise ConfigurationError(f"path must be a list of strings, got '{str(path)}'")
+        for part in path:
+            if not isinstance(part, str):
+                raise ConfigurationError(f"path must be a list of strings, got '{str(path)}'")
+
+        if len(path) < 1 or path[0] not in ("server", "dataset", "per_dataset_config"):
+            raise ConfigurationError("path must start with 'server', 'dataset', or 'per_dataset_config'")
+
+        if path[0] == "server":
+            attr = "__".join(path[1:])
+            try:
+                self.update_server_config(**{attr: value})
+            except ConfigurationError:
+                raise ConfigurationError(f"unknown config parameter at path: '{str(path)}'")
+        elif path[0] == "dataset":
+            attr = "__".join(path[1:])
+            try:
+                self.update_default_dataset_config(**{attr: value})
+            except ConfigurationError:
+                raise ConfigurationError(f"unknown config parameter at path: '{str(path)}'")
+
+        elif path[0] == "per_dataset_config":
+            if len(path) < 2:
+                raise ConfigurationError(f"missing dataroot when using per_dataset_config: got '{path}'")
+            dataroot = path[1]
+            if dataroot not in self.dataroot_config:
+                dataroots = str(list(self.dataroot_config.keys()))
+                raise ConfigurationError(
+                    f"unknown dataroot when using per_dataset_config: got '{path}',"
+                    f" dataroots specified in config are {dataroots}"
+                )
+
+            attr = "__".join(path[2:])
+            try:
+                self.dataroot_config[dataroot].update(**{attr: value})
+            except ConfigurationError:
+                raise ConfigurationError(f"unknown config parameter at path: '{str(path)}'")
+
+    def update_from_config_file(self, config_file):
+        try:
+            with open(config_file) as yml_file:
+                config = yaml.safe_load(yml_file)
+        except yaml.YAMLError as e:
+            raise ConfigurationError(f"The specified config file contained an error: {e}")
+        except OSError as e:
+            raise ConfigurationError(f"Issue retrieving the specified config file: {e}")
+
+        if config.get("server"):
+            self.server_config.update_from_config(config["server"], "server")
+        if config.get("dataset"):
+            self.default_dataset_config.update_from_config(config["dataset"], "dataset")
+
+        per_dataset_config = config.get("per_dataset_config", {})
+        for key, dataroot_config in per_dataset_config.items():
+            # first create and initialize the dataroot with the default config
+            self.add_dataroot_config(key, **config["dataset"])
+            # then apply the per dataset configuration
+            self.dataroot_config[key].update_from_config(dataroot_config, f"per_dataset_config__{key}")
+
+        if config.get("external"):
+            self.external_config.update_from_config(config["external"], "external")
+
+        self.is_complete = False
+
+    def config_to_dict(self):
+        """return the configuration as an unflattened dict"""
+        server = self.server_config.create_mapping(self.server_config.default_config)
+        dataset = self.default_dataset_config.create_mapping(self.default_dataset_config.default_config)
+        external = self.external_config.create_mapping(self.external_config.default_config)
+        config = dict(server={}, dataset={})
+        for attrname in server.keys():
+            config["server__" + attrname] = getattr(self.server_config, attrname)
+        for attrname in dataset.keys():
+            config["dataset__" + attrname] = getattr(self.default_dataset_config, attrname)
+        if self.dataroot_config:
+            config["per_dataset_config"] = {}
+        for dataroot_tag, dataroot_config in self.dataroot_config.items():
+            dataset = dataroot_config.create_mapping(dataroot_config.default_config)
+            for attrname in dataset.keys():
+                config[f"per_dataset_config__{dataroot_tag}__" + attrname] = getattr(dataroot_config, attrname)
+        for attrname in external.keys():
+            config["external__" + attrname] = getattr(self.external_config, attrname)
+
+        config = unflatten(config, splitter=lambda key: key.split("__"))
+        return config
+
+    def write_config(self, config_file):
+        """output the config to a yaml file"""
+        config = self.config_to_dict()
+        yaml.dump(config, open(config_file, "w"))
+
+    def changes_from_default(self):
+        """Return all the attribute that are different from the default"""
+        diff_server = self.server_config.changes_from_default()
+        diff_dataset = self.default_dataset_config.changes_from_default()
+        diff_external = self.external.changes_from_default()
+        diff = dict(server=diff_server, dataset=diff_dataset, external=diff_external)
+        return diff
+
+    def add_dataroot_config(self, dataroot_tag, **kw):
+        """Create a new dataset config object based on the default dataset config, and kw parameters"""
+        if dataroot_tag in self.dataroot_config:
+            raise ConfigurationError(f"dataroot config already exists: {dataroot_tag}")
+        if type(self.server_config.multi_dataset__dataroot) != dict:
+            raise ConfigurationError("The server__multi_dataset__dataroot must be a dictionary")
+        if dataroot_tag not in self.server_config.multi_dataset__dataroot:
+            raise ConfigurationError(f"The dataroot_tag ({dataroot_tag}) not found in server__multi_dataset__dataroot")
+
+        self.is_completed = False
+        self.dataroot_config[dataroot_tag] = DatasetConfig(dataroot_tag, self, self.default_config["dataset"])
+        flat_config = self.default_dataset_config.create_mapping(self.default_dataset_config.default_config)
+        config = {key: value[1] for key, value in flat_config.items()}
+        self.dataroot_config[dataroot_tag].update(**config)
+        self.dataroot_config[dataroot_tag].update_from_config(kw, dataroot_tag)
+
+    def complete_config(self, messagefn=None):
+        """The configure options are checked, and any additional setup based on the config
+        parameters is done"""
+
+        if messagefn is None:
+
+            def noop(message):
+                pass
+
+            messagefn = noop
+
+        # TODO: to give better error messages we can add a mapping between where each config
+        # attribute originated (e.g. command line argument or config file), then in the error
+        # messages we can give correct context for attributes with bad value.
+        context = dict(messagefn=messagefn)
+
+        # complete config for external_config first, since this may update values in the other sections
+        self.external_config.complete_config(context)
+        self.server_config.complete_config(context)
+        self.default_dataset_config.complete_config(context)
+        for dataroot_config in self.dataroot_config.values():
+            dataroot_config.complete_config(context)
+
+        self.is_completed = True
+        self.check_config()
+
+    def get_matrix_data_cache_manager(self):
+        return self.server_config.matrix_data_cache_manager
+
+    def is_multi_dataset(self):
+        return self.server_config.multi_dataset__dataroot is not None
+
+    def get_title(self, data_adaptor):
+        return (
+            self.server_config.single_dataset__title
+            if self.server_config.single_dataset__title
+            else data_adaptor.get_title()
+        )
+
+    def get_about(self, data_adaptor):
+        return (
+            self.server_config.single_dataset__about
+            if self.server_config.single_dataset__about
+            else data_adaptor.get_about()
+        )

server/common/config/base_config.py

@@ -0,0 +1,132 @@
+import copy
+
+from flatten_dict import flatten
+from server.common.errors import ConfigurationError
+
+
+class BaseConfig(object):
+    """
+    This class handles the mechanics of updating and checking attributes.
+    Derived classes are expected to store the actual attributes
+    Currently DatasetConfig and ServerConfig both inherit from BaseConfig.
+    """
+
+    def __init__(self, app_config, default_config, dictval_cases={}):
+        # reference back to the app_config
+        self.app_config = app_config
+        # the complete set of attributes and their default values (unflattened)
+        self.default_config = default_config
+        # attributes where the value may be a dict (and therefore are not flattened)
+        self.dictval_cases = dictval_cases
+        # used to make sure every attribute value is checked
+        self.attr_checked = {key_name: False for key_name in self.create_mapping(default_config).keys()}
+
+    def create_mapping(self, config):
+        """
+        Create a dictionary where the keys are the name of attributes (using double underscore convention)
+        For example: authentication__type
+
+        The values are a tuple,
+        - the first item of the tuple is a tuple of path elements (location in config 'tree')
+        - the second item is the value of the config parameter
+
+        For example: (('authentication', 'type'), 'session'))
+        """
+        config_copy = copy.deepcopy(config)
+        mapping = {}
+
+        # special cases where the value could be a dict.
+        # If its value is not None, the entry is added to the mapping, and not included
+        # in the flattening below.
+        for dictval_case in self.dictval_cases:
+            cur = config_copy
+            for part in dictval_case[:-1]:
+                cur = cur.get(part, {})
+            val = cur.get(dictval_case[-1])
+            if val is not None:
+                key = "__".join(dictval_case)
+                mapping[key] = (dictval_case, val)
+                del cur[dictval_case[-1]]
+
+        flat_config = flatten(config_copy)
+        for key, value in flat_config.items():
+            # name of the attribute
+            attr = "__".join(key)
+            mapping[attr] = (key, value)
+
+        return mapping
+
+    def validate_correct_type_of_configuration_attribute(self, attrname, vtype):
+        val = getattr(self, attrname)
+        if type(vtype) in (list, tuple):
+            if type(val) not in vtype:
+                tnames = ",".join([x.__name__ for x in vtype])
+                raise ConfigurationError(
+                    f"Invalid type for attribute: {attrname}, expected types ({tnames}), got {type(val).__name__}"
+                )
+        else:
+            if type(val) != vtype:
+                raise ConfigurationError(
+                    f"Invalid type for attribute: {attrname}, "
+                    f"expected type {vtype.__name__}, got {type(val).__name__}"
+                )
+
+        self.attr_checked[attrname] = True
+
+    def check_config(self):
+        mapping = self.create_mapping(self.default_config)
+        for key in mapping.keys():
+            if not self.attr_checked[key]:
+                raise ConfigurationError(f"The attr '{key}' has not been checked")
+
+    def update(self, **kw):
+        """Update the attributes defined in kw with their new values."""
+        for key, value in kw.items():
+            if not hasattr(self, key):
+
+                # check if the key is setting into a dictval entry.
+                found_dictval = False
+                for dictval in self.dictval_cases:
+                    dictvalname = "__".join(dictval)
+                    if dictvalname + "__" in key:
+                        dictkey = key[len(dictvalname) + 2 :]
+                        curdictval = getattr(self, dictvalname)
+                        if curdictval is None:
+                            setattr(self, dictvalname, dict(dictkey=value))
+                        else:
+                            curdictval[dictkey] = value
+
+                        found_dictval = True
+                        break
+
+                if found_dictval:
+                    continue
+                raise ConfigurationError(f"unknown config parameter {key}.")
+            try:
+                if type(value) == tuple:
+                    # convert tuple values to list values
+                    value = list(value)
+                setattr(self, key, value)
+            except KeyError:
+                raise ConfigurationError(f"Unable to set config parameter {key}.")
+
+            self.attr_checked[key] = False
+
+    def update_from_config(self, config, prefix):
+        mapping = self.create_mapping(config)
+        for attr, (key, value) in mapping.items():
+            if not hasattr(self, attr):
+                raise ConfigurationError(f"Unknown key from config file: {prefix}__{attr}")
+            setattr(self, attr, value)
+
+            self.attr_checked[attr] = False
+
+    def changes_from_default(self):
+        """Return all the attribute that are different from the default"""
+        mapping = self.create_mapping(self.default_config)
+        diff = []
+        for attrname, (key, defval) in mapping.items():
+            curval = getattr(self, attrname)
+            if curval != defval:
+                diff.append((attrname, curval, defval))
+        return diff

server/common/config/client_config.py

@@ -0,0 +1,122 @@
+from server import display_version as cellxgene_display_version
+
+
+def get_client_config(app_config, data_adaptor):
+    """
+    Return the configuration as required by the /config REST route
+    """
+
+    server_config = app_config.server_config
+    dataset_config = data_adaptor.dataset_config
+    annotation = dataset_config.user_annotations
+    auth = server_config.auth
+
+    # FIXME The current set of config is not consistently presented:
+    # we have camalCase, hyphen-text, and underscore_text
+
+    # make sure the configuration has been checked.
+    app_config.check_config()
+
+    # display_names
+    title = app_config.get_title(data_adaptor)
+    about = app_config.get_about(data_adaptor)
+
+    display_names = dict(engine=data_adaptor.get_name(), dataset=title)
+
+    # library_versions
+    library_versions = {}
+    library_versions.update(data_adaptor.get_library_versions())
+    library_versions["cellxgene"] = cellxgene_display_version
+
+    # links
+    links = {"about-dataset": about}
+
+    # parameters
+    parameters = {
+        "layout": dataset_config.embeddings__names,
+        "max-category-items": dataset_config.presentation__max_categories,
+        "obs_names": server_config.single_dataset__obs_names,
+        "var_names": server_config.single_dataset__var_names,
+        "diffexp_lfc_cutoff": dataset_config.diffexp__lfc_cutoff,
+        "backed": server_config.adaptor__anndata_adaptor__backed,
+        "disable-diffexp": not dataset_config.diffexp__enable,
+        "enable-reembedding": dataset_config.embeddings__enable_reembedding,
+        "annotations": False,
+        "annotations_file": None,
+        "annotations_dir": None,
+        "annotations_cell_ontology_enabled": False,
+        "annotations_cell_ontology_obopath": None,
+        "annotations_cell_ontology_terms": None,
+        "custom_colors": dataset_config.presentation__custom_colors,
+        "diffexp-may-be-slow": False,
+        "about_legal_tos": dataset_config.app__about_legal_tos,
+        "about_legal_privacy": dataset_config.app__about_legal_privacy,
+    }
+
+    # corpora dataset_props
+    # TODO/Note: putting info from the dataset into the /config is not ideal.
+    # However, it is definitely not part of /schema, and we do not have a top-level
+    # route for data properties.  Consider creating one at some point.
+    corpora_props = data_adaptor.get_corpora_props()
+    if corpora_props and "default_embedding" in corpora_props:
+        default_embedding = corpora_props["default_embedding"]
+        if isinstance(default_embedding, str) and default_embedding.startswith("X_"):
+            default_embedding = default_embedding[2:]  # drop X_ prefix
+        if default_embedding in data_adaptor.get_embedding_names():
+            parameters["default_embedding"] = default_embedding
+
+    data_adaptor.update_parameters(parameters)
+    if annotation:
+        annotation.update_parameters(parameters, data_adaptor)
+
+    # gather it all together
+    client_config = {}
+    config = client_config["config"] = {}
+    config["displayNames"] = display_names
+    config["library_versions"] = library_versions
+    config["links"] = links
+    config["parameters"] = parameters
+    config["corpora_props"] = corpora_props
+    config["limits"] = {
+        "column_request_max": server_config.limits__column_request_max,
+        "diffexp_cellcount_max": server_config.limits__diffexp_cellcount_max,
+    }
+
+    if dataset_config.app__authentication_enable and auth.is_valid_authentication_type():
+        config["authentication"] = {
+            "requires_client_login": auth.requires_client_login(),
+        }
+        if auth.requires_client_login():
+            config["authentication"].update(
+                {
+                    # Todo why are these stored on the data_adaptor?
+                    "login": auth.get_login_url(data_adaptor),
+                    "logout": auth.get_logout_url(data_adaptor),
+                }
+            )
+
+    return client_config
+
+
+def get_client_userinfo(app_config, data_adaptor):
+    """
+    Return the userinfo as required by the /userinfo REST route
+    """
+
+    server_config = app_config.server_config
+    dataset_config = data_adaptor.dataset_config
+    auth = server_config.auth
+
+    # make sure the configuration has been checked.
+    app_config.check_config()
+
+    if dataset_config.app__authentication_enable and auth.is_valid_authentication_type():
+        userinfo = {}
+        userinfo["userinfo"] = {
+            "is_authenticated": auth.is_user_authenticated(),
+            "username": auth.get_user_name(),
+            "user_id": auth.get_user_id(),
+            "email": auth.get_user_email(),
+            "picture": auth.get_user_picture(),
+        }
+        return userinfo

server/common/config/dataset_config.py

@@ -0,0 +1,230 @@
+import os
+from os.path import splitext, isdir
+
+from server.common.annotations.hosted_tiledb import AnnotationsHostedTileDB
+from server.common.annotations.local_file_csv import AnnotationsLocalFile
+from server.common.config.base_config import BaseConfig
+from server.common.errors import ConfigurationError, OntologyLoadFailure
+from server.compute.scanpy import get_scanpy_module
+from server.data_common.matrix_loader import MatrixDataLoader, MatrixDataType
+from server.db.db_utils import DbUtils
+
+
+class DatasetConfig(BaseConfig):
+    """Manages the config attribute associated with a dataset."""
+
+    def __init__(self, tag, app_config, default_config):
+        super().__init__(app_config, default_config)
+        self.tag = tag
+        try:
+            self.app__scripts = default_config["app"]["scripts"]
+            self.app__inline_scripts = default_config["app"]["inline_scripts"]
+            self.app__about_legal_tos = default_config["app"]["about_legal_tos"]
+            self.app__about_legal_privacy = default_config["app"]["about_legal_privacy"]
+            self.app__authentication_enable = default_config["app"]["authentication_enable"]
+
+            self.presentation__max_categories = default_config["presentation"]["max_categories"]
+            self.presentation__custom_colors = default_config["presentation"]["custom_colors"]
+
+            self.user_annotations__enable = default_config["user_annotations"]["enable"]
+            self.user_annotations__type = default_config["user_annotations"]["type"]
+            self.user_annotations__local_file_csv__directory = default_config["user_annotations"]["local_file_csv"][
+                "directory"
+            ]
+            self.user_annotations__local_file_csv__file = default_config["user_annotations"]["local_file_csv"]["file"]
+            self.user_annotations__ontology__enable = default_config["user_annotations"]["ontology"]["enable"]
+            self.user_annotations__ontology__obo_location = default_config["user_annotations"]["ontology"][
+                "obo_location"
+            ]
+            self.user_annotations__hosted_tiledb_array__db_uri = default_config["user_annotations"][
+                "hosted_tiledb_array"
+            ]["db_uri"]
+            self.user_annotations__hosted_tiledb_array__hosted_file_directory = default_config["user_annotations"][
+                "hosted_tiledb_array"
+            ]["hosted_file_directory"]
+
+            self.embeddings__names = default_config["embeddings"]["names"]
+            self.embeddings__enable_reembedding = default_config["embeddings"]["enable_reembedding"]
+
+            self.diffexp__enable = default_config["diffexp"]["enable"]
+            self.diffexp__lfc_cutoff = default_config["diffexp"]["lfc_cutoff"]
+            self.diffexp__top_n = default_config["diffexp"]["top_n"]
+
+        except KeyError as e:
+            raise ConfigurationError(f"Unexpected config: {str(e)}")
+
+        # The annotation object is created during complete_config and stored here.
+        self.user_annotations = None
+
+    def complete_config(self, context):
+        self.handle_app()
+        self.handle_presentation()
+        self.handle_user_annotations(context)
+        self.handle_embeddings()
+        self.handle_diffexp(context)
+
+    def handle_app(self):
+        self.validate_correct_type_of_configuration_attribute("app__scripts", list)
+        self.validate_correct_type_of_configuration_attribute("app__inline_scripts", list)
+        self.validate_correct_type_of_configuration_attribute("app__about_legal_tos", (type(None), str))
+        self.validate_correct_type_of_configuration_attribute("app__about_legal_privacy", (type(None), str))
+        self.validate_correct_type_of_configuration_attribute("app__authentication_enable", bool)
+
+        # scripts can be string (filename) or dict (attributes). Convert string to dict.
+        scripts = []
+        for script in self.app__scripts:
+            try:
+                if isinstance(script, str):
+                    scripts.append({"src": script})
+                elif isinstance(script, dict) and isinstance(script["src"], str):
+                    scripts.append(script)
+                else:
+                    raise Exception
+            except Exception as e:
+                raise ConfigurationError(f"Scripts must be string or a dict containing an src key: {e}")
+
+        self.app__scripts = scripts
+
+    def handle_presentation(self):
+        self.validate_correct_type_of_configuration_attribute("presentation__max_categories", int)
+        self.validate_correct_type_of_configuration_attribute("presentation__custom_colors", bool)
+
+    def handle_user_annotations(self, context):
+        self.validate_correct_type_of_configuration_attribute("user_annotations__enable", bool)
+        self.validate_correct_type_of_configuration_attribute("user_annotations__type", str)
+        self.validate_correct_type_of_configuration_attribute(
+            "user_annotations__local_file_csv__directory", (type(None), str)
+        )
+        self.validate_correct_type_of_configuration_attribute(
+            "user_annotations__local_file_csv__file", (type(None), str)
+        )
+        self.validate_correct_type_of_configuration_attribute("user_annotations__ontology__enable", bool)
+        self.validate_correct_type_of_configuration_attribute(
+            "user_annotations__ontology__obo_location", (type(None), str)
+        )
+        self.validate_correct_type_of_configuration_attribute(
+            "user_annotations__hosted_tiledb_array__db_uri", (type(None), str)
+        )
+        self.validate_correct_type_of_configuration_attribute(
+            "user_annotations__hosted_tiledb_array__hosted_file_directory", (type(None), str)
+        )
+        if self.user_annotations__enable:
+            server_config = self.app_config.server_config
+            if not self.app__authentication_enable:
+                raise ConfigurationError("user annotations requires authentication to be enabled")
+            if not server_config.auth.is_valid_authentication_type():
+                auth_type = server_config.authentication__type
+                raise ConfigurationError(f"authentication method {auth_type} is not compatible with user annotations")
+
+            if self.user_annotations__type == "local_file_csv":
+                self.handle_local_file_csv_annotations()
+            elif self.user_annotations__type == "hosted_tiledb_array":
+                self.handle_hosted_tiledb_annotations()
+            else:
+                raise ConfigurationError('The only annotation type support is "local_file_csv" or "hosted_tiledb_array')
+            if self.user_annotations__ontology__enable or self.user_annotations__ontology__obo_location:
+                try:
+                    self.user_annotations.load_ontology(self.user_annotations__ontology__obo_location)
+                except OntologyLoadFailure as e:
+                    raise ConfigurationError("Unable to load ontology terms\n" + str(e))
+        else:
+            self.check_annotation_config_vars_not_set(context)
+
+    def handle_local_file_csv_annotations(self):
+        dirname = self.user_annotations__local_file_csv__directory
+        filename = self.user_annotations__local_file_csv__file
+        if filename is not None and dirname is not None:
+            raise ConfigurationError("'annotations-file' and 'annotations-dir' may not be used together.")
+
+        if filename is not None:
+            lf_name, lf_ext = splitext(filename)
+            if lf_ext and lf_ext != ".csv":
+                raise ConfigurationError(f"annotation file type must be .csv: {filename}")
+
+        if dirname is not None and not isdir(dirname):
+            try:
+                os.mkdir(dirname)
+            except OSError:
+                raise ConfigurationError("Unable to create directory specified by --annotations-dir")
+
+        self.user_annotations = AnnotationsLocalFile(dirname, filename)
+
+        # if the user has specified a fixed label file, go ahead and validate it
+        # so that we can remove errors early in the process.
+        server_config = self.app_config.server_config
+        if server_config.single_dataset__datapath and self.user_annotations__local_file_csv__file:
+            with server_config.matrix_data_cache_manager.data_adaptor(
+                self.tag, server_config.single_dataset__datapath, self.app_config
+            ) as data_adaptor:
+                data_adaptor.check_new_labels(self.user_annotations.read_labels(data_adaptor))
+
+    def handle_hosted_tiledb_annotations(self):
+        self.validate_correct_type_of_configuration_attribute("user_annotations__hosted_tiledb_array__db_uri", str)
+        self.validate_correct_type_of_configuration_attribute(
+            "user_annotations__hosted_tiledb_array__hosted_file_directory", str
+        )
+        self.user_annotations = AnnotationsHostedTileDB(
+            directory_path=self.user_annotations__hosted_tiledb_array__hosted_file_directory,
+            db=DbUtils(self.user_annotations__hosted_tiledb_array__db_uri),
+        )
+
+    def check_annotation_config_vars_not_set(self, context):
+        if self.user_annotations__type is not None:
+            dirname = self.user_annotations__local_file_csv__directory
+            filename = self.user_annotations__local_file_csv__file
+            db_uri = self.user_annotations__hosted_tiledb_array__db_uri
+            hosted_file_dirname = self.user_annotations__hosted_tiledb_array__hosted_file_directory
+            if filename is not None:
+                context["messagefn"]("Warning: --annotations-file ignored as annotations are disabled.")
+            if dirname is not None:
+                context["messagefn"]("Warning: --annotations-dir ignored as annotations are disabled.")
+            if db_uri is not None:
+                context["messagefn"]("Warning: db_uri ignored as annotations are disabled.")
+            if hosted_file_dirname is not None:
+                context["messagefn"](
+                    "Warning: hosted_file_directory for hosted_tiledb_array ignored as annotations are disabled."
+                )
+
+        if self.user_annotations__ontology__enable:
+            context["messagefn"]("Warning: --experimental-annotations-ontology ignored as annotations are disabled.")
+        if self.user_annotations__ontology__obo_location is not None:
+            context["messagefn"](
+                "Warning: --experimental-annotations-ontology-obo ignored as annotations are disabled."
+            )
+
+    def handle_embeddings(self):
+        self.validate_correct_type_of_configuration_attribute("embeddings__names", list)
+        self.validate_correct_type_of_configuration_attribute("embeddings__enable_reembedding", bool)
+
+        server_config = self.app_config.server_config
+        if self.embeddings__enable_reembedding:
+            if server_config.single_dataset__datapath:
+                matrix_data_loader = MatrixDataLoader(
+                    server_config.single_dataset__datapath, app_config=self.app_config
+                )
+                if matrix_data_loader.matrix_data_type != MatrixDataType.H5AD:
+                    raise ConfigurationError("enable-reembedding is only supported with H5AD files.")
+                if server_config.adaptor__anndata_adaptor__backed:
+                    raise ConfigurationError("enable-reembedding is not supported when run in --backed mode.")
+
+            try:
+                get_scanpy_module()
+            except NotImplementedError:
+                # Todo add scanpy to requirements.txt and remove this check once re-embeddings is fully supported
+                raise ConfigurationError("Please install scanpy to enable UMAP re-embedding")
+
+    def handle_diffexp(self, context):
+        self.validate_correct_type_of_configuration_attribute("diffexp__enable", bool)
+        self.validate_correct_type_of_configuration_attribute("diffexp__lfc_cutoff", float)
+        self.validate_correct_type_of_configuration_attribute("diffexp__top_n", int)
+
+        server_config = self.app_config.server_config
+        if server_config.single_dataset__datapath:
+            with server_config.matrix_data_cache_manager.data_adaptor(
+                self.tag, server_config.single_dataset__datapath, self.app_config
+            ) as data_adaptor:
+                if self.diffexp__enable and data_adaptor.parameters.get("diffexp_may_be_slow", False):
+                    context["messagefn"](
+                        "CAUTION: due to the size of your dataset, "
+                        "running differential expression may take longer or fail."
+                    )

server/common/config/external_config.py

@@ -0,0 +1,96 @@
+import os
+
+from server.common.config.base_config import BaseConfig
+from server.common.errors import ConfigurationError
+from server.common.config import get_secret_key
+from server.common.errors import SecretKeyRetrievalError
+from server.common.utils.type_conversion_utils import convert_string_to_value
+
+
+class ExternalConfig(BaseConfig):
+    """Manages the config attribute associated with external configuration sources, such as
+    environment variables or the AWS Secrets Manager."""
+
+    def __init__(self, app_config, default_config):
+        super().__init__(app_config, default_config)
+        try:
+            self.environment = default_config["environment"]
+            self.aws_secrets_manager__region = default_config["aws_secrets_manager"]["region"]
+            self.aws_secrets_manager__secrets = default_config["aws_secrets_manager"]["secrets"]
+
+        except KeyError as e:
+            raise ConfigurationError(f"Unexpected config: {str(e)}")
+
+    def complete_config(self, context):
+        self.handle_environment(context)
+        self.handle_aws_secrets_manager(context)
+
+    def handle_environment(self, context):
+        """For each environment variable defined, get the value (if it is set),
+        and set the specified config parameter"""
+        self.validate_correct_type_of_configuration_attribute("environment", list)
+        for envdict in self.environment:
+            name = envdict.get("name")
+            if name is None:
+                raise ConfigurationError("environment: 'name' is missing")
+            required = envdict.get("required", False)
+            if type(required) != bool:
+                raise ConfigurationError("environment: 'required' must be a bool")
+            path = envdict.get("path")
+            if path is None:
+                raise ConfigurationError("environment: 'path' is missing")
+
+            value = os.environ.get(name)
+            if value is None:
+                if required:
+                    raise ConfigurationError(f"required environment variable '{name}' not set")
+            else:
+                value = convert_string_to_value(value)
+                self.app_config.update_single_config_from_path_and_value(path, value)
+
+    def handle_aws_secrets_manager(self, context):
+        """For each aws secret defined, get the key/values, and set the specified config parameter"""
+        self.validate_correct_type_of_configuration_attribute("aws_secrets_manager__region", (type(None), str))
+        self.validate_correct_type_of_configuration_attribute("aws_secrets_manager__secrets", list)
+
+        if not self.aws_secrets_manager__secrets:
+            return
+
+        self.validate_correct_type_of_configuration_attribute("aws_secrets_manager__region", str)
+
+        for secret in self.aws_secrets_manager__secrets:
+            secret_name = secret.get("name")
+            if secret_name is None:
+                raise ConfigurationError("aws_secrets_manager: 'name' is missing")
+            if not isinstance(secret_name, str):
+                raise ConfigurationError("aws_secrets_manager: 'name' must be a string")
+
+            try:
+                secret_dict = get_secret_key(self.aws_secrets_manager__region, secret_name)
+            except SecretKeyRetrievalError as e:
+                raise ConfigurationError(f"Unable to retrieve secret {secret_name}: {str(e)}")
+
+            values = secret.get("values")
+            if values is None:
+                raise ConfigurationError("aws_secrets_manager: 'values' is missing")
+            if not isinstance(values, list):
+                raise ConfigurationError("aws_secrets_manager: 'values' must be a list")
+
+            for value in values:
+                key = value.get("key")
+                if key is None:
+                    raise ConfigurationError(f"missing 'key' in secret values: {secret_name}")
+                path = value.get("path")
+                if path is None:
+                    raise ConfigurationError(f"missing 'path' in secret values: {secret_name}")
+                required = value.get("required", False)
+                if type(required) != bool:
+                    raise ConfigurationError(f"wrong type for 'required' in secret values: {secret_name}")
+
+                secret_value = secret_dict.get(key)
+                if secret_value is None:
+                    if required:
+                        raise ConfigurationError(f"required secret '{secret_name}:{key}' not set")
+                else:
+                    secret_value = convert_string_to_value(secret_value)
+                    self.app_config.update_single_config_from_path_and_value(path, secret_value)

server/common/config/server_config.py

@@ -0,0 +1,380 @@
+import os
+import sys
+import warnings
+from os.path import basename
+from urllib.parse import urlparse, quote_plus
+
+from server.auth.auth import AuthTypeFactory
+from server.common.config.base_config import BaseConfig
+from server.common.config import DEFAULT_SERVER_PORT, BIG_FILE_SIZE_THRESHOLD
+from server.common.errors import ConfigurationError, DatasetAccessError
+from server.common.data_locator import discover_s3_region_name
+from server.common.utils.utils import is_port_available, find_available_port, custom_format_warning
+from server.compute import diffexp_cxg as diffexp_tiledb
+from server.data_common.matrix_loader import MatrixDataCacheManager, MatrixDataLoader, MatrixDataType
+
+
+class ServerConfig(BaseConfig):
+    """Manages the config attribute associated with the server."""
+
+    def __init__(self, app_config, default_config):
+        dictval_cases = [
+            ("app", "csp_directives"),
+            ("authentication", "params_oauth", "cookie"),
+            ("authentication", "params_oauth", "jwt_decode_options"),
+            ("adaptor", "cxg_adaptor", "tiledb_ctx"),
+            ("multi_dataset", "dataroot"),
+        ]
+        super().__init__(app_config, default_config, dictval_cases)
+
+        try:
+            self.app__verbose = default_config["app"]["verbose"]
+            self.app__debug = default_config["app"]["debug"]
+            self.app__host = default_config["app"]["host"]
+            self.app__port = default_config["app"]["port"]
+            self.app__open_browser = default_config["app"]["open_browser"]
+            self.app__force_https = default_config["app"]["force_https"]
+            self.app__flask_secret_key = default_config["app"]["flask_secret_key"]
+            self.app__generate_cache_control_headers = default_config["app"]["generate_cache_control_headers"]
+            self.app__server_timing_headers = default_config["app"]["server_timing_headers"]
+            self.app__csp_directives = default_config["app"]["csp_directives"]
+            self.app__api_base_url = default_config["app"]["api_base_url"]
+            self.app__web_base_url = default_config["app"]["web_base_url"]
+
+            self.authentication__type = default_config["authentication"]["type"]
+            self.authentication__params_oauth__oauth_api_base_url = default_config["authentication"]["params_oauth"][
+                "oauth_api_base_url"
+            ]
+            self.authentication__params_oauth__client_id = default_config["authentication"]["params_oauth"]["client_id"]
+            self.authentication__params_oauth__client_secret = default_config["authentication"]["params_oauth"][
+                "client_secret"
+            ]
+            self.authentication__params_oauth__jwt_decode_options = default_config["authentication"]["params_oauth"][
+                "jwt_decode_options"
+            ]
+            self.authentication__params_oauth__session_cookie = default_config["authentication"]["params_oauth"][
+                "session_cookie"
+            ]
+            self.authentication__params_oauth__cookie = default_config["authentication"]["params_oauth"]["cookie"]
+
+            self.multi_dataset__dataroot = default_config["multi_dataset"]["dataroot"]
+            self.multi_dataset__index = default_config["multi_dataset"]["index"]
+            self.multi_dataset__allowed_matrix_types = default_config["multi_dataset"]["allowed_matrix_types"]
+            self.multi_dataset__matrix_cache__max_datasets = default_config["multi_dataset"]["matrix_cache"][
+                "max_datasets"
+            ]
+            self.multi_dataset__matrix_cache__timelimit_s = default_config["multi_dataset"]["matrix_cache"][
+                "timelimit_s"
+            ]
+
+            self.single_dataset__datapath = default_config["single_dataset"]["datapath"]
+            self.single_dataset__obs_names = default_config["single_dataset"]["obs_names"]
+            self.single_dataset__var_names = default_config["single_dataset"]["var_names"]
+            self.single_dataset__about = default_config["single_dataset"]["about"]
+            self.single_dataset__title = default_config["single_dataset"]["title"]
+
+            self.diffexp__alg_cxg__max_workers = default_config["diffexp"]["alg_cxg"]["max_workers"]
+            self.diffexp__alg_cxg__cpu_multiplier = default_config["diffexp"]["alg_cxg"]["cpu_multiplier"]
+            self.diffexp__alg_cxg__target_workunit = default_config["diffexp"]["alg_cxg"]["target_workunit"]
+
+            self.data_locator__s3__region_name = default_config["data_locator"]["s3"]["region_name"]
+
+            self.adaptor__cxg_adaptor__tiledb_ctx = default_config["adaptor"]["cxg_adaptor"]["tiledb_ctx"]
+            self.adaptor__anndata_adaptor__backed = default_config["adaptor"]["anndata_adaptor"]["backed"]
+
+            self.limits__diffexp_cellcount_max = default_config["limits"]["diffexp_cellcount_max"]
+            self.limits__column_request_max = default_config["limits"]["column_request_max"]
+
+        except KeyError as e:
+            raise ConfigurationError(f"Unexpected config: {str(e)}")
+
+        # The matrix data cache manager is created during the complete_config and stored here.
+        self.matrix_data_cache_manager = None
+
+        # The authentication object
+        self.auth = None
+
+    def complete_config(self, context):
+        self.handle_app(context)
+        self.handle_data_source()
+        self.handle_authentication()
+        self.handle_data_locator()
+        self.handle_adaptor()  # may depend on data_locator
+        self.handle_single_dataset(context)  # may depend on adaptor
+        self.handle_multi_dataset()  # may depend on adaptor
+        self.handle_diffexp()
+        self.handle_limits()
+
+        self.check_config()
+
+    def handle_app(self, context):
+        self.validate_correct_type_of_configuration_attribute("app__verbose", bool)
+        self.validate_correct_type_of_configuration_attribute("app__debug", bool)
+        self.validate_correct_type_of_configuration_attribute("app__host", str)
+        self.validate_correct_type_of_configuration_attribute("app__port", (type(None), int))
+        self.validate_correct_type_of_configuration_attribute("app__open_browser", bool)
+        self.validate_correct_type_of_configuration_attribute("app__force_https", bool)
+        self.validate_correct_type_of_configuration_attribute("app__flask_secret_key", str)
+        self.validate_correct_type_of_configuration_attribute("app__generate_cache_control_headers", bool)
+        self.validate_correct_type_of_configuration_attribute("app__server_timing_headers", bool)
+        self.validate_correct_type_of_configuration_attribute("app__csp_directives", (type(None), dict))
+        self.validate_correct_type_of_configuration_attribute("app__api_base_url", (type(None), str))
+        self.validate_correct_type_of_configuration_attribute("app__web_base_url", (type(None), str))
+
+        if self.app__port:
+            try:
+                if not is_port_available(self.app__host, self.app__port):
+                    raise ConfigurationError(
+                        f"The port selected {self.app__port} is in use, please configure an open port."
+                    )
+            except OverflowError:
+                raise ConfigurationError(f"Invalid port: {self.app__port}")
+        else:
+            try:
+                default_server_port = int(os.environ.get("CXG_SERVER_PORT", DEFAULT_SERVER_PORT))
+            except ValueError:
+                raise ConfigurationError(
+                    "Invalid port from environment variable CXG_SERVER_PORT: " + os.environ.get("CXG_SERVER_PORT")
+                )
+            try:
+                self.app__port = find_available_port(self.app__host, default_server_port)
+            except OverflowError:
+                raise ConfigurationError(f"Invalid port: {default_server_port}")
+
+        if self.app__debug:
+            context["messagefn"]("in debug mode, setting verbose=True and open_browser=False")
+            self.app__verbose = True
+            self.app__open_browser = False
+        else:
+            warnings.formatwarning = custom_format_warning
+
+        if not self.app__verbose:
+            sys.tracebacklimit = 0
+
+        # CSP Directives are a dict of string: list(string) or string: string
+        if self.app__csp_directives is not None:
+            for k, v in self.app__csp_directives.items():
+                if not isinstance(k, str):
+                    raise ConfigurationError("CSP directive names must be a string.")
+                if isinstance(v, list):
+                    for policy in v:
+                        if not isinstance(policy, str):
+                            raise ConfigurationError("CSP directive value must be a string or list of strings.")
+                elif not isinstance(v, str):
+                    raise ConfigurationError("CSP directive value must be a string or list of strings.")
+
+        if self.app__web_base_url is None:
+            self.app__web_base_url = self.app__api_base_url
+
+    def handle_authentication(self):
+        self.validate_correct_type_of_configuration_attribute("authentication__type", (type(None), str))
+
+        # oauth
+        ptypes = str if self.authentication__type == "oauth" else (type(None), str)
+        self.validate_correct_type_of_configuration_attribute(
+            "authentication__params_oauth__oauth_api_base_url", ptypes
+        )
+        self.validate_correct_type_of_configuration_attribute("authentication__params_oauth__client_id", ptypes)
+        self.validate_correct_type_of_configuration_attribute("authentication__params_oauth__client_secret", ptypes)
+        self.validate_correct_type_of_configuration_attribute(
+            "authentication__params_oauth__jwt_decode_options", (type(None), dict)
+        )
+        self.validate_correct_type_of_configuration_attribute("authentication__params_oauth__session_cookie", bool)
+
+        if self.authentication__params_oauth__session_cookie:
+            self.validate_correct_type_of_configuration_attribute(
+                "authentication__params_oauth__cookie", (type(None), dict)
+            )
+        else:
+            self.validate_correct_type_of_configuration_attribute("authentication__params_oauth__cookie", dict)
+
+        self.auth = AuthTypeFactory.create(self.authentication__type, self)
+        if self.auth is None:
+            raise ConfigurationError(f"Unknown authentication type: {self.authentication__type}")
+
+    def handle_data_locator(self):
+        self.validate_correct_type_of_configuration_attribute("data_locator__s3__region_name", (type(None), bool, str))
+        if self.data_locator__s3__region_name is True:
+            path = self.single_dataset__datapath or self.multi_dataset__dataroot
+
+            if type(path) == dict:
+                # if multi_dataset__dataroot is a dict, then use the first key
+                # that is in s3.   NOTE:  it is not supported to have dataroots
+                # in different regions.
+                paths = [val.get("dataroot") for val in path.values()]
+                for path in paths:
+                    if path.startswith("s3://"):
+                        break
+            if path.startswith("s3://"):
+                region_name = discover_s3_region_name(path)
+                if region_name is None:
+                    raise ConfigurationError(f"Unable to discover s3 region name from {path}")
+            else:
+                region_name = None
+            self.data_locator__s3__region_name = region_name
+
+    def handle_data_source(self):
+        self.validate_correct_type_of_configuration_attribute("single_dataset__datapath", (str, type(None)))
+        self.validate_correct_type_of_configuration_attribute("multi_dataset__dataroot", (type(None), dict, str))
+
+        if self.single_dataset__datapath and self.multi_dataset__dataroot:
+            raise ConfigurationError(
+                "You must supply either a datapath (for single datasets) or a dataroot (for multidatasets). Not both"
+            )
+        if self.single_dataset__datapath is None and self.multi_dataset__dataroot is None:
+            raise ConfigurationError("You must specify a datapath for a single dataset or a dataroot for multidatasets")
+
+    def handle_single_dataset(self, context):
+        self.validate_correct_type_of_configuration_attribute("single_dataset__datapath", (str, type(None)))
+        self.validate_correct_type_of_configuration_attribute("single_dataset__title", (str, type(None)))
+        self.validate_correct_type_of_configuration_attribute("single_dataset__about", (str, type(None)))
+        self.validate_correct_type_of_configuration_attribute("single_dataset__obs_names", (str, type(None)))
+        self.validate_correct_type_of_configuration_attribute("single_dataset__var_names", (str, type(None)))
+
+        if self.single_dataset__datapath is None:
+            return
+
+        # create the matrix data cache manager:
+        if self.matrix_data_cache_manager is None:
+            self.matrix_data_cache_manager = MatrixDataCacheManager(max_cached=1, timelimit_s=None)
+
+        # preload this data set
+        matrix_data_loader = MatrixDataLoader(self.single_dataset__datapath, app_config=self.app_config)
+        try:
+            matrix_data_loader.pre_load_validation()
+        except DatasetAccessError as e:
+            raise ConfigurationError(str(e))
+
+        file_size = matrix_data_loader.file_size()
+        file_basename = basename(self.single_dataset__datapath)
+        if file_size > BIG_FILE_SIZE_THRESHOLD:
+            context["messagefn"](f"Loading data from {file_basename}, this may take a while...")
+        else:
+            context["messagefn"](f"Loading data from {file_basename}.")
+
+        if self.single_dataset__about:
+
+            def url_check(url):
+                try:
+                    result = urlparse(url)
+                    if all([result.scheme, result.netloc]):
+                        return True
+                    else:
+                        return False
+                except ValueError:
+                    return False
+
+            if not url_check(self.single_dataset__about):
+                raise ConfigurationError(
+                    "Must provide an absolute URL for --about. (Example format: http://example.com)"
+                )
+
+    def handle_multi_dataset(self):
+        self.validate_correct_type_of_configuration_attribute("multi_dataset__dataroot", (type(None), dict, str))
+        self.validate_correct_type_of_configuration_attribute("multi_dataset__index", (type(None), bool, str))
+        self.validate_correct_type_of_configuration_attribute("multi_dataset__allowed_matrix_types", list)
+        self.validate_correct_type_of_configuration_attribute("multi_dataset__matrix_cache__max_datasets", int)
+        self.validate_correct_type_of_configuration_attribute(
+            "multi_dataset__matrix_cache__timelimit_s", (type(None), int, float)
+        )
+
+        if self.multi_dataset__dataroot is None:
+            return
+
+        if type(self.multi_dataset__dataroot) == str:
+            default_dict = dict(base_url="d", dataroot=self.multi_dataset__dataroot)
+            self.multi_dataset__dataroot = dict(d=default_dict)
+
+        for tag, dataroot_dict in self.multi_dataset__dataroot.items():
+            if "base_url" not in dataroot_dict:
+                raise ConfigurationError(f"error in multi_dataset__dataroot: missing base_url for tag {tag}")
+            if "dataroot" not in dataroot_dict:
+                raise ConfigurationError(f"error in multi_dataset__dataroot: missing dataroot, for tag {tag}")
+
+            base_url = dataroot_dict["base_url"]
+
+            # sanity check for well formed base urls
+            bad = False
+            if type(base_url) != str:
+                bad = True
+            elif os.path.normpath(base_url) != base_url:
+                bad = True
+            else:
+                base_url_parts = base_url.split("/")
+                if [quote_plus(part) for part in base_url_parts] != base_url_parts:
+                    bad = True
+                if ".." in base_url_parts:
+                    bad = True
+            if bad:
+                raise ConfigurationError(f"error in multi_dataset__dataroot base_url {base_url} for tag {tag}")
+
+        # verify all the base_urls are unique
+        base_urls = [d["base_url"] for d in self.multi_dataset__dataroot.values()]
+        if len(base_urls) > len(set(base_urls)):
+            raise ConfigurationError("error in multi_dataset__dataroot:  base_urls must be unique")
+
+        # error checking
+        for mtype in self.multi_dataset__allowed_matrix_types:
+            try:
+                MatrixDataType(mtype)
+            except ValueError:
+                raise ConfigurationError(f'Invalid matrix type in "allowed_matrix_types": {mtype}')
+
+        # create the matrix data cache manager:
+        if self.matrix_data_cache_manager is None:
+            self.matrix_data_cache_manager = MatrixDataCacheManager(
+                max_cached=self.multi_dataset__matrix_cache__max_datasets,
+                timelimit_s=self.multi_dataset__matrix_cache__timelimit_s,
+            )
+
+    def handle_diffexp(self):
+        self.validate_correct_type_of_configuration_attribute("diffexp__alg_cxg__max_workers", (str, int))
+        self.validate_correct_type_of_configuration_attribute("diffexp__alg_cxg__cpu_multiplier", int)
+        self.validate_correct_type_of_configuration_attribute("diffexp__alg_cxg__target_workunit", int)
+
+        max_workers = self.diffexp__alg_cxg__max_workers
+        cpu_multiplier = self.diffexp__alg_cxg__cpu_multiplier
+        cpu_count = os.cpu_count()
+        max_workers = min(max_workers, cpu_multiplier * cpu_count)
+        diffexp_tiledb.set_config(max_workers, self.diffexp__alg_cxg__target_workunit)
+
+    def handle_adaptor(self):
+        # cxg
+        self.validate_correct_type_of_configuration_attribute("adaptor__cxg_adaptor__tiledb_ctx", dict)
+        regionkey = "vfs.s3.region"
+        if regionkey not in self.adaptor__cxg_adaptor__tiledb_ctx:
+            if type(self.data_locator__s3__region_name) == str:
+                self.adaptor__cxg_adaptor__tiledb_ctx[regionkey] = self.data_locator__s3__region_name
+
+        from server.data_cxg.cxg_adaptor import CxgAdaptor
+
+        CxgAdaptor.set_tiledb_context(self.adaptor__cxg_adaptor__tiledb_ctx)
+
+        # anndata
+        self.validate_correct_type_of_configuration_attribute("adaptor__anndata_adaptor__backed", bool)
+
+    def handle_limits(self):
+        self.validate_correct_type_of_configuration_attribute("limits__diffexp_cellcount_max", (type(None), int))
+        self.validate_correct_type_of_configuration_attribute("limits__column_request_max", (type(None), int))
+
+    def exceeds_limit(self, limit_name, value):
+        limit_value = getattr(self, "limits__" + limit_name, None)
+        if limit_value is None:  # disabled
+            return False
+        return value > limit_value
+
+    def get_api_base_url(self):
+        if self.app__api_base_url == "local":
+            return f"http://{self.app__host}:{self.app__port}"
+        if self.app__api_base_url and self.app__api_base_url.endswith("/"):
+            return self.app__api_base_url[:-1]
+        return self.app__api_base_url
+
+    def get_web_base_url(self):
+        if self.app__web_base_url == "local":
+            return f"http://{self.app__host}:{self.app__port}"
+        if self.app__web_base_url is None:
+            return self.get_api_base_url()
+        if self.app__web_base_url.endswith("/"):
+            return self.app__web_base_url[:-1]
+        return self.app__web_base_url

server/common/corpora.py

@@ -9,6 +9,7 @@ import collections
 import json
 
 from server.cli.upgrade import validate_version_str
+from server.common.utils.corpora_constants import CorporaConstants
 
 
 def corpora_get_versions_from_anndata(adata):
@@ -56,26 +57,13 @@ def corpora_get_props_from_anndata(adata):
     if not version_is_supported:
         raise ValueError("Unsupported Corpora schema version")
 
-    required_simple_fields = [
-        "version",
-        "title",
-        "layer_descriptions",
-        "organism",
-        "organism_ontology_term_id",
-        "project_name",
-        "project_description",
-    ]
-    # Spec says some values encoded as JSON due to the inability of AnnData to store complex types.
-    required_json_fields = ["contributors", "project_links"]
-    optional_simple_fields = ["preprint_doi", "publication_doi", "default_embedding", "default_field", "tags"]
-
     corpora_props = {}
-    for key in required_simple_fields:
+    for key in CorporaConstants.REQUIRED_SIMPLE_METADATA_FIELDS:
         if key not in adata.uns:
             raise KeyError(f"missing Corpora schema field {key}")
         corpora_props[key] = adata.uns[key]
 
-    for key in required_json_fields:
+    for key in CorporaConstants.REQUIRED_JSON_ENCODED_METADATA_FIELD:
         if key not in adata.uns:
             raise KeyError(f"missing Corpora schema field {key}")
         try:
@@ -83,7 +71,7 @@ def corpora_get_props_from_anndata(adata):
         except json.JSONDecodeError:
             raise json.JSONDecodeError(f"Corpora schema field {key} is expected to be a valid JSON string")
 
-    for key in optional_simple_fields:
+    for key in CorporaConstants.OPTIONAL_SIMPLE_METADATA_FIELDS:
         if key in adata.uns:
             corpora_props[key] = adata.uns[key]
 

server/common/errors.py

@@ -42,14 +42,14 @@ define_request_exception(
 define_request_exception("ExceedsLimitError", "Raised when an HTTP request exceeds a limit/quota")
 define_request_exception("ColorFormatException", "Raised when color helper functions encounter an unknown color format")
 define_request_exception(
-    "AuthenticationError",
-    "Raised when there is an authentication error",
-    default_status_code=HTTPStatus.UNAUTHORIZED)
+    "AuthenticationError", "Raised when there is an authentication error", default_status_code=HTTPStatus.UNAUTHORIZED
+)
 
 define_request_exception(
     "AnnotationCategoryNameError",
     "Raised when an annotation category name cant be saved",
-    default_status_code=HTTPStatus.UNPROCESSABLE_ENTITY)
+    default_status_code=HTTPStatus.UNPROCESSABLE_ENTITY,
+)
 
 define_exception("OntologyLoadFailure", "Raised when reading the ontology file fails")
 define_exception("ConfigurationError", "Raised when checking configuration errors")