Merge remote-tracking branch 'origin/main' into release-version-0.16.5

.github/workflows/push_tests.yml

@@ -31,9 +31,10 @@ jobs:
       - name: Install dependencies
         run: |
           pip install flake8
+          pip install black
           cd client
           npm install
-      - name: Lint with flake8
+      - name: Format with black and lint with flake8
         run: |
           make lint-server
       - name: Lint src with eslint

LICENSE.txt

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2013
+Copyright (c) 2017-2020 Chan Zuckerberg Initiative
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of
 this software and associated documentation files (the "Software"), to deal in

Makefile

@@ -82,8 +82,9 @@ fmt-py:
 lint: lint-server lint-client
 
 .PHONY: lint-server
-lint-server:
-	flake8 server
+lint-server: fmt-py
+	flake8 server --per-file-ignores='server/test/fixtures/dataset_config_outline.py:F821 server/test/fixtures/server_config_outline.py:F821'
+
 
 .PHONY: lint-client
 lint-client:

README.md

@@ -81,7 +81,7 @@ If you believe you have found a security issue, we would appreciate notification
 
 # Inspiration
 
-We've been heavily inspired by several other related single-cell visualization projects, including the [UCSC Cell Browswer](http://cells.ucsc.edu/), [Cytoscape](http://www.cytoscape.org/), [Xena](https://xena.ucsc.edu/), [ASAP](https://asap.epfl.ch/), [Gene Pattern](http://genepattern-notebook.org/), and many others. We hope to explore collaborations where useful as this community works together on improving interactive visualization for single-cell data.
+We've been heavily inspired by several other related single-cell visualization projects, including the [UCSC Cell Browser](http://cells.ucsc.edu/), [Cytoscape](http://www.cytoscape.org/), [Xena](https://xena.ucsc.edu/), [ASAP](https://asap.epfl.ch/), [GenePattern](http://genepattern-notebook.org/), and many others. We hope to explore collaborations where useful as this community works together on improving interactive visualization for single-cell data.
 
 We were inspired by Mike Bostock and the [crossfilter](https://github.com/crossfilter) team for the design of our filtering implementation.
 

client/__tests__/e2e/__snapshots__/e2eAnnotations.test.js.snap

@@ -3,14 +3,14 @@
 exports[`annotations stacked bar graph renders 1`] = `
 Array [
   "<div class=\\"categorical__value___2RKaC\\" data-testclass=\\"categorical-row\\" style=\\"padding: 4px 0px 4px 7px; display: flex; align-items: baseline; justify-content: space-between; margin-bottom: 2px; border-radius: 2px;\\"><div style=\\"margin: 0px; padding: 0px; user-select: none; width: 220px; display: flex; justify-content: space-between;\\"><div style=\\"display: flex; align-items: baseline;\\"><label for=\\"value-toggle-checkbox-TEST-CATEGORY-TEST-LABEL\\" class=\\"bp3-control bp3-checkbox\\" style=\\"margin: 0px;\\"><input id=\\"value-toggle-checkbox-TEST-CATEGORY-TEST-LABEL\\" data-testclass=\\"categorical-value-select\\" data-testid=\\"categorical-value-select-TEST-CATEGORY-TEST-LABEL\\" type=\\"checkbox\\" checked=\\"\\"><span class=\\"bp3-control-indicator\\"></span></label><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\" style=\\"width: 63px; color: black; font-style: normal; display: inline-block; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px;\\"><span data-testid=\\"categorical-value-TEST-CATEGORY-TEST-LABEL\\" data-testclass=\\"categorical-value\\" aria-label=\\"TEST-LABEL\\" class=\\"\\" tabindex=\\"0\\" style=\\"width: 63px; color: black; font-style: normal; display: inline-block; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px;\\"><span style=\\"width: 100%; color: black; font-style: normal; display: flex; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px; justify-content: flex-start; padding: 0px;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">TEST-</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">LABEL</span><span style=\\"position: absolute; right: 0px; color: black;\\">LABEL</span></span></span></span></span></span></div><span style=\\"flex-shrink: 0;\\"></span></div><div><span><span data-testclass=\\"categorical-value-count\\" data-testid=\\"categorical-value-count-TEST-CATEGORY-TEST-LABEL\\" style=\\"color: black;\\">0</span><svg display=\\"none\\" style=\\"margin-left: 5px; width: 11px; height: 11px; background-color: inherit;\\"></svg><span><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\"><button type=\\"button\\" data-testclass=\\"seeActions\\" data-testid=\\"TEST-CATEGORY:TEST-LABEL:see-actions\\" class=\\"bp3-button bp3-minimal bp3-small\\" tabindex=\\"0\\" style=\\"margin-left: 2px; position: relative; top: -1px; min-height: 16px;\\"><span icon=\\"more\\" class=\\"bp3-icon bp3-icon-more\\"><svg data-icon=\\"more\\" width=\\"10\\" height=\\"10\\" viewBox=\\"0 0 16 16\\"><desc>more</desc><path d=\\"M2 6.03a2 2 0 100 4 2 2 0 100-4zM14 6.03a2 2 0 100 4 2 2 0 100-4zM8 6.03a2 2 0 100 4 2 2 0 100-4z\\" fill-rule=\\"evenodd\\"></path></svg></span></button></span></span></span></span></div></div>",
-  "<div class=\\"categorical__value___2RKaC\\" data-testclass=\\"categorical-row\\" style=\\"padding: 4px 0px 4px 7px; display: flex; align-items: baseline; justify-content: space-between; margin-bottom: 2px; border-radius: 2px;\\"><div style=\\"margin: 0px; padding: 0px; user-select: none; width: 220px; display: flex; justify-content: space-between;\\"><div style=\\"display: flex; align-items: baseline;\\"><label for=\\"value-toggle-checkbox-TEST-CATEGORY-unassigned\\" class=\\"bp3-control bp3-checkbox\\" style=\\"margin: 0px;\\"><input id=\\"value-toggle-checkbox-TEST-CATEGORY-unassigned\\" data-testclass=\\"categorical-value-select\\" data-testid=\\"categorical-value-select-TEST-CATEGORY-unassigned\\" type=\\"checkbox\\" checked=\\"\\"><span class=\\"bp3-control-indicator\\"></span></label><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\" style=\\"width: 63px; color: rgb(171, 171, 171); font-style: italic; display: inline-block; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px;\\"><span data-testid=\\"categorical-value-TEST-CATEGORY-unassigned\\" data-testclass=\\"categorical-value\\" aria-label=\\"unassigned\\" class=\\"\\" tabindex=\\"0\\" style=\\"width: 63px; color: rgb(171, 171, 171); font-style: italic; display: inline-block; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px;\\"><span style=\\"width: 100%; color: rgb(171, 171, 171); font-style: italic; display: flex; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px; justify-content: flex-start; padding: 0px;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">unass</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">igned</span><span style=\\"position: absolute; right: 0px; color: rgb(171, 171, 171);\\">igned</span></span></span></span></span></span></div><span style=\\"flex-shrink: 0;\\"><canvas class=\\"bp3-popover-targer\\" width=\\"100\\" height=\\"11\\" style=\\"margin-right: 5px; width: 100px; height: 11px;\\"></canvas></span></div><div><span><span data-testclass=\\"categorical-value-count\\" data-testid=\\"categorical-value-count-TEST-CATEGORY-unassigned\\" style=\\"color: rgb(171, 171, 171); font-style: italic;\\">2133</span><svg display=\\"none\\" style=\\"margin-left: 5px; width: 11px; height: 11px; background-color: inherit;\\"></svg><span><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\"><button type=\\"button\\" data-testclass=\\"seeActions\\" data-testid=\\"TEST-CATEGORY:unassigned:see-actions\\" class=\\"bp3-button bp3-minimal bp3-small\\" tabindex=\\"0\\" style=\\"margin-left: 2px; position: relative; top: -1px; min-height: 16px;\\"><span icon=\\"more\\" class=\\"bp3-icon bp3-icon-more\\"><svg data-icon=\\"more\\" width=\\"10\\" height=\\"10\\" viewBox=\\"0 0 16 16\\"><desc>more</desc><path d=\\"M2 6.03a2 2 0 100 4 2 2 0 100-4zM14 6.03a2 2 0 100 4 2 2 0 100-4zM8 6.03a2 2 0 100 4 2 2 0 100-4z\\" fill-rule=\\"evenodd\\"></path></svg></span></button></span></span></span></span></div></div>",
+  "<div class=\\"categorical__value___2RKaC\\" data-testclass=\\"categorical-row\\" style=\\"padding: 4px 0px 4px 7px; display: flex; align-items: baseline; justify-content: space-between; margin-bottom: 2px; border-radius: 2px;\\"><div style=\\"margin: 0px; padding: 0px; user-select: none; width: 220px; display: flex; justify-content: space-between;\\"><div style=\\"display: flex; align-items: baseline;\\"><label for=\\"value-toggle-checkbox-TEST-CATEGORY-unassigned\\" class=\\"bp3-control bp3-checkbox\\" style=\\"margin: 0px;\\"><input id=\\"value-toggle-checkbox-TEST-CATEGORY-unassigned\\" data-testclass=\\"categorical-value-select\\" data-testid=\\"categorical-value-select-TEST-CATEGORY-unassigned\\" type=\\"checkbox\\" checked=\\"\\"><span class=\\"bp3-control-indicator\\"></span></label><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\" style=\\"width: 63px; color: rgb(171, 171, 171); font-style: italic; display: inline-block; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px;\\"><span data-testid=\\"categorical-value-TEST-CATEGORY-unassigned\\" data-testclass=\\"categorical-value\\" aria-label=\\"unassigned\\" class=\\"\\" tabindex=\\"0\\" style=\\"width: 63px; color: rgb(171, 171, 171); font-style: italic; display: inline-block; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px;\\"><span style=\\"width: 100%; color: rgb(171, 171, 171); font-style: italic; display: flex; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px; justify-content: flex-start; padding: 0px;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">unass</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">igned</span><span style=\\"position: absolute; right: 0px; color: rgb(171, 171, 171);\\">igned</span></span></span></span></span></span></div><span style=\\"flex-shrink: 0;\\"><canvas width=\\"100\\" height=\\"11\\" style=\\"margin-right: 5px; width: 100px; height: 11px;\\"></canvas></span></div><div><span><span data-testclass=\\"categorical-value-count\\" data-testid=\\"categorical-value-count-TEST-CATEGORY-unassigned\\" style=\\"color: rgb(171, 171, 171); font-style: italic;\\">2133</span><svg display=\\"none\\" style=\\"margin-left: 5px; width: 11px; height: 11px; background-color: inherit;\\"></svg><span><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\"><button type=\\"button\\" data-testclass=\\"seeActions\\" data-testid=\\"TEST-CATEGORY:unassigned:see-actions\\" class=\\"bp3-button bp3-minimal bp3-small\\" tabindex=\\"0\\" style=\\"margin-left: 2px; position: relative; top: -1px; min-height: 16px;\\"><span icon=\\"more\\" class=\\"bp3-icon bp3-icon-more\\"><svg data-icon=\\"more\\" width=\\"10\\" height=\\"10\\" viewBox=\\"0 0 16 16\\"><desc>more</desc><path d=\\"M2 6.03a2 2 0 100 4 2 2 0 100-4zM14 6.03a2 2 0 100 4 2 2 0 100-4zM8 6.03a2 2 0 100 4 2 2 0 100-4z\\" fill-rule=\\"evenodd\\"></path></svg></span></button></span></span></span></span></div></div>",
 ]
 `;
 
 exports[`annotations stacked bar graph renders 2`] = `
 Array [
   "<div class=\\"categorical__value___2RKaC\\" data-testclass=\\"categorical-row\\" style=\\"padding: 4px 0px 4px 7px; display: flex; align-items: baseline; justify-content: space-between; margin-bottom: 2px; border-radius: 2px;\\"><div style=\\"margin: 0px; padding: 0px; user-select: none; width: 220px; display: flex; justify-content: space-between;\\"><div style=\\"display: flex; align-items: baseline;\\"><label for=\\"value-toggle-checkbox-TEST-CATEGORY-TEST-LABEL\\" class=\\"bp3-control bp3-checkbox\\" style=\\"margin: 0px;\\"><input id=\\"value-toggle-checkbox-TEST-CATEGORY-TEST-LABEL\\" data-testclass=\\"categorical-value-select\\" data-testid=\\"categorical-value-select-TEST-CATEGORY-TEST-LABEL\\" type=\\"checkbox\\" checked=\\"\\"><span class=\\"bp3-control-indicator\\"></span></label><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\" style=\\"width: 63px; color: black; font-style: normal; display: inline-block; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px;\\"><span data-testid=\\"categorical-value-TEST-CATEGORY-TEST-LABEL\\" data-testclass=\\"categorical-value\\" aria-label=\\"TEST-LABEL\\" class=\\"\\" tabindex=\\"0\\" style=\\"width: 63px; color: black; font-style: normal; display: inline-block; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px;\\"><span style=\\"width: 100%; color: black; font-style: normal; display: flex; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px; justify-content: flex-start; padding: 0px;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">TEST-</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">LABEL</span><span style=\\"position: absolute; right: 0px; color: black;\\">LABEL</span></span></span></span></span></span></div><span style=\\"flex-shrink: 0;\\"></span></div><div><span><span data-testclass=\\"categorical-value-count\\" data-testid=\\"categorical-value-count-TEST-CATEGORY-TEST-LABEL\\" style=\\"color: black;\\">0</span><svg display=\\"none\\" style=\\"margin-left: 5px; width: 11px; height: 11px; background-color: inherit;\\"></svg><span><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\"><button type=\\"button\\" data-testclass=\\"seeActions\\" data-testid=\\"TEST-CATEGORY:TEST-LABEL:see-actions\\" class=\\"bp3-button bp3-minimal bp3-small\\" tabindex=\\"0\\" style=\\"margin-left: 2px; position: relative; top: -1px; min-height: 16px;\\"><span icon=\\"more\\" class=\\"bp3-icon bp3-icon-more\\"><svg data-icon=\\"more\\" width=\\"10\\" height=\\"10\\" viewBox=\\"0 0 16 16\\"><desc>more</desc><path d=\\"M2 6.03a2 2 0 100 4 2 2 0 100-4zM14 6.03a2 2 0 100 4 2 2 0 100-4zM8 6.03a2 2 0 100 4 2 2 0 100-4z\\" fill-rule=\\"evenodd\\"></path></svg></span></button></span></span></span></span></div></div>",
-  "<div class=\\"categorical__value___2RKaC\\" data-testclass=\\"categorical-row\\" style=\\"padding: 4px 0px 4px 7px; display: flex; align-items: baseline; justify-content: space-between; margin-bottom: 2px; border-radius: 2px;\\"><div style=\\"margin: 0px; padding: 0px; user-select: none; width: 220px; display: flex; justify-content: space-between;\\"><div style=\\"display: flex; align-items: baseline;\\"><label for=\\"value-toggle-checkbox-TEST-CATEGORY-unassigned\\" class=\\"bp3-control bp3-checkbox\\" style=\\"margin: 0px;\\"><input id=\\"value-toggle-checkbox-TEST-CATEGORY-unassigned\\" data-testclass=\\"categorical-value-select\\" data-testid=\\"categorical-value-select-TEST-CATEGORY-unassigned\\" type=\\"checkbox\\" checked=\\"\\"><span class=\\"bp3-control-indicator\\"></span></label><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\" style=\\"width: 63px; color: rgb(171, 171, 171); font-style: italic; display: inline-block; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px;\\"><span data-testid=\\"categorical-value-TEST-CATEGORY-unassigned\\" data-testclass=\\"categorical-value\\" aria-label=\\"unassigned\\" class=\\"\\" tabindex=\\"0\\" style=\\"width: 63px; color: rgb(171, 171, 171); font-style: italic; display: inline-block; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px;\\"><span style=\\"width: 100%; color: rgb(171, 171, 171); font-style: italic; display: flex; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px; justify-content: flex-start; padding: 0px;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">unass</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">igned</span><span style=\\"position: absolute; right: 0px; color: rgb(171, 171, 171);\\">igned</span></span></span></span></span></span></div><span style=\\"flex-shrink: 0;\\"><canvas class=\\"bp3-popover-targer\\" width=\\"100\\" height=\\"11\\" style=\\"margin-right: 5px; width: 100px; height: 11px;\\"></canvas></span></div><div><span><span data-testclass=\\"categorical-value-count\\" data-testid=\\"categorical-value-count-TEST-CATEGORY-unassigned\\" style=\\"color: rgb(171, 171, 171); font-style: italic;\\">2638</span><svg display=\\"none\\" style=\\"margin-left: 5px; width: 11px; height: 11px; background-color: inherit;\\"></svg><span><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\"><button type=\\"button\\" data-testclass=\\"seeActions\\" data-testid=\\"TEST-CATEGORY:unassigned:see-actions\\" class=\\"bp3-button bp3-minimal bp3-small\\" tabindex=\\"0\\" style=\\"margin-left: 2px; position: relative; top: -1px; min-height: 16px;\\"><span icon=\\"more\\" class=\\"bp3-icon bp3-icon-more\\"><svg data-icon=\\"more\\" width=\\"10\\" height=\\"10\\" viewBox=\\"0 0 16 16\\"><desc>more</desc><path d=\\"M2 6.03a2 2 0 100 4 2 2 0 100-4zM14 6.03a2 2 0 100 4 2 2 0 100-4zM8 6.03a2 2 0 100 4 2 2 0 100-4z\\" fill-rule=\\"evenodd\\"></path></svg></span></button></span></span></span></span></div></div>",
+  "<div class=\\"categorical__value___2RKaC\\" data-testclass=\\"categorical-row\\" style=\\"padding: 4px 0px 4px 7px; display: flex; align-items: baseline; justify-content: space-between; margin-bottom: 2px; border-radius: 2px;\\"><div style=\\"margin: 0px; padding: 0px; user-select: none; width: 220px; display: flex; justify-content: space-between;\\"><div style=\\"display: flex; align-items: baseline;\\"><label for=\\"value-toggle-checkbox-TEST-CATEGORY-unassigned\\" class=\\"bp3-control bp3-checkbox\\" style=\\"margin: 0px;\\"><input id=\\"value-toggle-checkbox-TEST-CATEGORY-unassigned\\" data-testclass=\\"categorical-value-select\\" data-testid=\\"categorical-value-select-TEST-CATEGORY-unassigned\\" type=\\"checkbox\\" checked=\\"\\"><span class=\\"bp3-control-indicator\\"></span></label><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\" style=\\"width: 63px; color: rgb(171, 171, 171); font-style: italic; display: inline-block; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px;\\"><span data-testid=\\"categorical-value-TEST-CATEGORY-unassigned\\" data-testclass=\\"categorical-value\\" aria-label=\\"unassigned\\" class=\\"\\" tabindex=\\"0\\" style=\\"width: 63px; color: rgb(171, 171, 171); font-style: italic; display: inline-block; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px;\\"><span style=\\"width: 100%; color: rgb(171, 171, 171); font-style: italic; display: flex; overflow: hidden; line-height: 1.1em; height: 1.1em; vertical-align: middle; margin-right: 16px; justify-content: flex-start; padding: 0px;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">unass</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">igned</span><span style=\\"position: absolute; right: 0px; color: rgb(171, 171, 171);\\">igned</span></span></span></span></span></span></div><span style=\\"flex-shrink: 0;\\"><canvas width=\\"100\\" height=\\"11\\" style=\\"margin-right: 5px; width: 100px; height: 11px;\\"></canvas></span></div><div><span><span data-testclass=\\"categorical-value-count\\" data-testid=\\"categorical-value-count-TEST-CATEGORY-unassigned\\" style=\\"color: rgb(171, 171, 171); font-style: italic;\\">2638</span><svg display=\\"none\\" style=\\"margin-left: 5px; width: 11px; height: 11px; background-color: inherit;\\"></svg><span><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\"><button type=\\"button\\" data-testclass=\\"seeActions\\" data-testid=\\"TEST-CATEGORY:unassigned:see-actions\\" class=\\"bp3-button bp3-minimal bp3-small\\" tabindex=\\"0\\" style=\\"margin-left: 2px; position: relative; top: -1px; min-height: 16px;\\"><span icon=\\"more\\" class=\\"bp3-icon bp3-icon-more\\"><svg data-icon=\\"more\\" width=\\"10\\" height=\\"10\\" viewBox=\\"0 0 16 16\\"><desc>more</desc><path d=\\"M2 6.03a2 2 0 100 4 2 2 0 100-4zM14 6.03a2 2 0 100 4 2 2 0 100-4zM8 6.03a2 2 0 100 4 2 2 0 100-4z\\" fill-rule=\\"evenodd\\"></path></svg></span></button></span></span></span></span></div></div>",
 ]
 `;
 

client/__tests__/e2e/cellxgeneActions.js

@@ -13,8 +13,11 @@ import {
   getTestClass,
   getTestId,
   isElementPresent,
+  goToPage,
 } from "./puppeteerUtils";
 
+import { appUrlBase } from "./config";
+
 export async function drag(testId, start, end, lasso = false) {
   const layout = await waitByID(testId);
   const elBox = await layout.boxModel();
@@ -312,4 +315,75 @@ export async function assertCategoryDoesNotExist(categoryName) {
 
   await expect(result).toBe(false);
 }
+
+export async function login() {
+  const email = `cellxgene-smoke-test+${process.env.DEPLOYMENT_STAGE}@chanzuckerberg.com`;
+  const password = "Test1111";
+
+  await goToPage(appUrlBase);
+
+  await clickOn("auth-button");
+
+  // (thuang): Auth0 form is unstable and unsafe for input until verified
+  await waitUntilFormFieldStable('[name="email"]');
+
+  await expect(page).toFillForm("form", {
+    email,
+    password,
+  });
+
+  await Promise.all([
+    page.waitForNavigation({ waitUntil: "networkidle0" }),
+    expect(page).toClick('[name="submit"]'),
+  ]);
+
+  expect(page.url()).toContain(appUrlBase);
+}
+
+export async function logout() {
+  await clickOnUntil("menu", async () => {
+    await expect(page).toMatch("Log Out");
+
+    await Promise.all([
+      page.waitForNavigation({ waitUntil: "networkidle0" }),
+      expect(page).toClick("a", { text: "Log Out" }),
+    ]);
+  });
+
+  await expect(page).toMatch("Log In");
+}
+
+async function waitUntilFormFieldStable(selector) {
+  const MAX_RETRY = 10;
+  const WAIT_FOR_MS = 200;
+
+  const EXPECTED_VALUE = "aaa";
+
+  let retry = 0;
+
+  while (retry < MAX_RETRY) {
+    try {
+      await expect(page).toFill(selector, EXPECTED_VALUE);
+
+      const fieldHandle = await expect(page).toMatchElement(selector);
+
+      const fieldValue = await page.evaluate(
+        (input) => input.value,
+        fieldHandle
+      );
+
+      expect(fieldValue).toBe(EXPECTED_VALUE);
+
+      break;
+    } catch (error) {
+      retry += 1;
+
+      await page.waitFor(WAIT_FOR_MS);
+    }
+  }
+
+  if (retry === MAX_RETRY) {
+    throw Error("clickOnUntil() assertion failed!");
+  }
+}
 /* eslint-enable no-await-in-loop -- await in loop is needed to emulate sequential user actions */

client/__tests__/e2e/e2e.test.js

@@ -31,6 +31,8 @@ import {
   runDiffExp,
   selectCategory,
   subset,
+  login,
+  logout,
 } from "./cellxgeneActions";
 
 const data = datasets[DATASET];
@@ -518,4 +520,18 @@ test("lasso moves after pan", async () => {
 
   expect(panCount).toBe(initialCount);
 });
+
+const conditionalDescribe =
+  process.env.TEST_AUTH_INTEGRATION === "true" ? describe : describe.skip;
+
+conditionalDescribe("AuthN Integration", () => {
+  it("logs in", async () => {
+    await login();
+  });
+
+  it("logs out", async () => {
+    await login();
+    await logout();
+  });
+});
 /* eslint-enable no-await-in-loop -- await in loop is needed to emulate sequential user actions */

client/__tests__/e2e/puppeteer.setup.js

@@ -17,7 +17,9 @@ setDefaultOptions({ timeout: 20 * 1000 });
 
 jest.retryTimes(ENV_DEFAULT.RETRY_ATTEMPTS);
 
-(async () => {
+beforeEach(async () => {
+  await jestPuppeteer.resetBrowser();
+
   const userAgent = await browser.userAgent();
   await page.setUserAgent(`${userAgent}bot`);
 
@@ -53,6 +55,4 @@ jest.retryTimes(ENV_DEFAULT.RETRY_ATTEMPTS);
       }
     }
   });
-})().catch((error) => {
-  console.error("puppeteer.setup.js error", error);
 });

client/configuration/eslint/eslint.js

@@ -4,6 +4,7 @@ module.exports = {
   extends: [
     "airbnb",
     "plugin:eslint-comments/recommended",
+    "plugin:@blueprintjs/recommended",
     "plugin:compat/recommended",
     "plugin:prettier/recommended",
     "prettier/react",

client/package-lock.json

@@ -4156,6 +4156,216 @@
         "tslib": "~1.10.0"
       }
     },
+    "@blueprintjs/eslint-plugin": {
+      "version": "0.3.0",
+      "resolved": "https://registry.npmjs.org/@blueprintjs/eslint-plugin/-/eslint-plugin-0.3.0.tgz",
+      "integrity": "sha512-bQEdE4ApEHxCDV8hT9uIxeRbDFKOtRLBT3/Zy3Ku+nowDAYl/8jwZKp6lJuR/nqvsfuIXTnVef6ivwdBEieQfA==",
+      "dev": true,
+      "requires": {
+        "@typescript-eslint/experimental-utils": "^4.2.0",
+        "eslint": "^7.9.0"
+      },
+      "dependencies": {
+        "@typescript-eslint/experimental-utils": {
+          "version": "4.3.0",
+          "resolved": "https://registry.npmjs.org/@typescript-eslint/experimental-utils/-/experimental-utils-4.3.0.tgz",
+          "integrity": "sha512-cmmIK8shn3mxmhpKfzMMywqiEheyfXLV/+yPDnOTvQX/ztngx7Lg/OD26J8gTZfkLKUmaEBxO2jYP3keV7h2OQ==",
+          "dev": true,
+          "requires": {
+            "@types/json-schema": "^7.0.3",
+            "@typescript-eslint/scope-manager": "4.3.0",
+            "@typescript-eslint/types": "4.3.0",
+            "@typescript-eslint/typescript-estree": "4.3.0",
+            "eslint-scope": "^5.0.0",
+            "eslint-utils": "^2.0.0"
+          }
+        },
+        "@typescript-eslint/typescript-estree": {
+          "version": "4.3.0",
+          "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-4.3.0.tgz",
+          "integrity": "sha512-ZAI7xjkl+oFdLV/COEz2tAbQbR3XfgqHEGy0rlUXzfGQic6EBCR4s2+WS3cmTPG69aaZckEucBoTxW9PhzHxxw==",
+          "dev": true,
+          "requires": {
+            "@typescript-eslint/types": "4.3.0",
+            "@typescript-eslint/visitor-keys": "4.3.0",
+            "debug": "^4.1.1",
+            "globby": "^11.0.1",
+            "is-glob": "^4.0.1",
+            "lodash": "^4.17.15",
+            "semver": "^7.3.2",
+            "tsutils": "^3.17.1"
+          }
+        },
+        "acorn": {
+          "version": "7.4.0",
+          "resolved": "https://registry.npmjs.org/acorn/-/acorn-7.4.0.tgz",
+          "integrity": "sha512-+G7P8jJmCHr+S+cLfQxygbWhXy+8YTVGzAkpEbcLo2mLoL7tij/VG41QSHACSf5QgYRhMZYHuNc6drJaO0Da+w==",
+          "dev": true
+        },
+        "cross-spawn": {
+          "version": "7.0.3",
+          "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz",
+          "integrity": "sha512-iRDPJKUPVEND7dHPO8rkbOnPpyDygcDFtWjpeWNCgy8WP2rXcxXL8TskReQl6OrB2G7+UJrags1q15Fudc7G6w==",
+          "dev": true,
+          "requires": {
+            "path-key": "^3.1.0",
+            "shebang-command": "^2.0.0",
+            "which": "^2.0.1"
+          }
+        },
+        "eslint": {
+          "version": "7.10.0",
+          "resolved": "https://registry.npmjs.org/eslint/-/eslint-7.10.0.tgz",
+          "integrity": "sha512-BDVffmqWl7JJXqCjAK6lWtcQThZB/aP1HXSH1JKwGwv0LQEdvpR7qzNrUT487RM39B5goWuboFad5ovMBmD8yA==",
+          "dev": true,
+          "requires": {
+            "@babel/code-frame": "^7.0.0",
+            "@eslint/eslintrc": "^0.1.3",
+            "ajv": "^6.10.0",
+            "chalk": "^4.0.0",
+            "cross-spawn": "^7.0.2",
+            "debug": "^4.0.1",
+            "doctrine": "^3.0.0",
+            "enquirer": "^2.3.5",
+            "eslint-scope": "^5.1.1",
+            "eslint-utils": "^2.1.0",
+            "eslint-visitor-keys": "^1.3.0",
+            "espree": "^7.3.0",
+            "esquery": "^1.2.0",
+            "esutils": "^2.0.2",
+            "file-entry-cache": "^5.0.1",
+            "functional-red-black-tree": "^1.0.1",
+            "glob-parent": "^5.0.0",
+            "globals": "^12.1.0",
+            "ignore": "^4.0.6",
+            "import-fresh": "^3.0.0",
+            "imurmurhash": "^0.1.4",
+            "is-glob": "^4.0.0",
+            "js-yaml": "^3.13.1",
+            "json-stable-stringify-without-jsonify": "^1.0.1",
+            "levn": "^0.4.1",
+            "lodash": "^4.17.19",
+            "minimatch": "^3.0.4",
+            "natural-compare": "^1.4.0",
+            "optionator": "^0.9.1",
+            "progress": "^2.0.0",
+            "regexpp": "^3.1.0",
+            "semver": "^7.2.1",
+            "strip-ansi": "^6.0.0",
+            "strip-json-comments": "^3.1.0",
+            "table": "^5.2.3",
+            "text-table": "^0.2.0",
+            "v8-compile-cache": "^2.0.3"
+          },
+          "dependencies": {
+            "eslint-scope": {
+              "version": "5.1.1",
+              "resolved": "https://registry.npmjs.org/eslint-scope/-/eslint-scope-5.1.1.tgz",
+              "integrity": "sha512-2NxwbF/hZ0KpepYN0cNbo+FN6XoK7GaHlQhgx/hIZl6Va0bF45RQOOwhLIy8lQDbuCiadSLCBnH2CFYquit5bw==",
+              "dev": true,
+              "requires": {
+                "esrecurse": "^4.3.0",
+                "estraverse": "^4.1.1"
+              }
+            }
+          }
+        },
+        "eslint-visitor-keys": {
+          "version": "1.3.0",
+          "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-1.3.0.tgz",
+          "integrity": "sha512-6J72N8UNa462wa/KFODt/PJ3IU60SDpC3QXC1Hjc1BXXpfL2C9R5+AU7jhe0F6GREqVMh4Juu+NY7xn+6dipUQ==",
+          "dev": true
+        },
+        "espree": {
+          "version": "7.3.0",
+          "resolved": "https://registry.npmjs.org/espree/-/espree-7.3.0.tgz",
+          "integrity": "sha512-dksIWsvKCixn1yrEXO8UosNSxaDoSYpq9reEjZSbHLpT5hpaCAKTLBwq0RHtLrIr+c0ByiYzWT8KTMRzoRCNlw==",
+          "dev": true,
+          "requires": {
+            "acorn": "^7.4.0",
+            "acorn-jsx": "^5.2.0",
+            "eslint-visitor-keys": "^1.3.0"
+          }
+        },
+        "esrecurse": {
+          "version": "4.3.0",
+          "resolved": "https://registry.npmjs.org/esrecurse/-/esrecurse-4.3.0.tgz",
+          "integrity": "sha512-KmfKL3b6G+RXvP8N1vr3Tq1kL/oCFgn2NYXEtqP8/L3pKapUA4G8cFVaoF3SU323CD4XypR/ffioHmkti6/Tag==",
+          "dev": true,
+          "requires": {
+            "estraverse": "^5.2.0"
+          },
+          "dependencies": {
+            "estraverse": {
+              "version": "5.2.0",
+              "resolved": "https://registry.npmjs.org/estraverse/-/estraverse-5.2.0.tgz",
+              "integrity": "sha512-BxbNGGNm0RyRYvUdHpIwv9IWzeM9XClbOxwoATuFdOE7ZE6wHL+HQ5T8hoPM+zHvmKzzsEqhgy0GrQ5X13afiQ==",
+              "dev": true
+            }
+          }
+        },
+        "glob-parent": {
+          "version": "5.1.1",
+          "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.1.tgz",
+          "integrity": "sha512-FnI+VGOpnlGHWZxthPGR+QhR78fuiK0sNLkHQv+bL9fQi57lNNdquIbna/WrfROrolq8GK5Ek6BiMwqL/voRYQ==",
+          "dev": true,
+          "requires": {
+            "is-glob": "^4.0.1"
+          }
+        },
+        "globals": {
+          "version": "12.4.0",
+          "resolved": "https://registry.npmjs.org/globals/-/globals-12.4.0.tgz",
+          "integrity": "sha512-BWICuzzDvDoH54NHKCseDanAhE3CeDorgDL5MT6LMXXj2WCnd9UC2szdk4AWLfjdgNBCXLUanXYcpBBKOSWGwg==",
+          "dev": true,
+          "requires": {
+            "type-fest": "^0.8.1"
+          }
+        },
+        "ignore": {
+          "version": "4.0.6",
+          "resolved": "https://registry.npmjs.org/ignore/-/ignore-4.0.6.tgz",
+          "integrity": "sha512-cyFDKrqc/YdcWFniJhzI42+AzS+gNwmUzOSFcRCQYwySuBBBy/KjuxWLZ/FHEH6Moq1NizMOBWyTcv8O4OZIMg==",
+          "dev": true
+        },
+        "path-key": {
+          "version": "3.1.1",
+          "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz",
+          "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==",
+          "dev": true
+        },
+        "semver": {
+          "version": "7.3.2",
+          "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.2.tgz",
+          "integrity": "sha512-OrOb32TeeambH6UrhtShmF7CRDqhL6/5XpPNp2DuRH6+9QLw/orhp72j87v8Qa1ScDkvrrBNpZcDejAirJmfXQ==",
+          "dev": true
+        },
+        "shebang-command": {
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz",
+          "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==",
+          "dev": true,
+          "requires": {
+            "shebang-regex": "^3.0.0"
+          }
+        },
+        "shebang-regex": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz",
+          "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==",
+          "dev": true
+        },
+        "which": {
+          "version": "2.0.2",
+          "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz",
+          "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==",
+          "dev": true,
+          "requires": {
+            "isexe": "^2.0.0"
+          }
+        }
+      }
+    },
     "@blueprintjs/icons": {
       "version": "3.19.0",
       "resolved": "https://registry.npmjs.org/@blueprintjs/icons/-/icons-3.19.0.tgz",
@@ -4184,6 +4394,76 @@
         "minimist": "^1.2.0"
       }
     },
+    "@eslint/eslintrc": {
+      "version": "0.1.3",
+      "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-0.1.3.tgz",
+      "integrity": "sha512-4YVwPkANLeNtRjMekzux1ci8hIaH5eGKktGqR0d3LWsKNn5B2X/1Z6Trxy7jQXl9EBGE6Yj02O+t09FMeRllaA==",
+      "dev": true,
+      "requires": {
+        "ajv": "^6.12.4",
+        "debug": "^4.1.1",
+        "espree": "^7.3.0",
+        "globals": "^12.1.0",
+        "ignore": "^4.0.6",
+        "import-fresh": "^3.2.1",
+        "js-yaml": "^3.13.1",
+        "lodash": "^4.17.19",
+        "minimatch": "^3.0.4",
+        "strip-json-comments": "^3.1.1"
+      },
+      "dependencies": {
+        "acorn": {
+          "version": "7.4.0",
+          "resolved": "https://registry.npmjs.org/acorn/-/acorn-7.4.0.tgz",
+          "integrity": "sha512-+G7P8jJmCHr+S+cLfQxygbWhXy+8YTVGzAkpEbcLo2mLoL7tij/VG41QSHACSf5QgYRhMZYHuNc6drJaO0Da+w==",
+          "dev": true
+        },
+        "ajv": {
+          "version": "6.12.5",
+          "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.5.tgz",
+          "integrity": "sha512-lRF8RORchjpKG50/WFf8xmg7sgCLFiYNNnqdKflk63whMQcWR5ngGjiSXkL9bjxy6B2npOK2HSMN49jEBMSkag==",
+          "dev": true,
+          "requires": {
+            "fast-deep-equal": "^3.1.1",
+            "fast-json-stable-stringify": "^2.0.0",
+            "json-schema-traverse": "^0.4.1",
+            "uri-js": "^4.2.2"
+          }
+        },
+        "eslint-visitor-keys": {
+          "version": "1.3.0",
+          "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-1.3.0.tgz",
+          "integrity": "sha512-6J72N8UNa462wa/KFODt/PJ3IU60SDpC3QXC1Hjc1BXXpfL2C9R5+AU7jhe0F6GREqVMh4Juu+NY7xn+6dipUQ==",
+          "dev": true
+        },
+        "espree": {
+          "version": "7.3.0",
+          "resolved": "https://registry.npmjs.org/espree/-/espree-7.3.0.tgz",
+          "integrity": "sha512-dksIWsvKCixn1yrEXO8UosNSxaDoSYpq9reEjZSbHLpT5hpaCAKTLBwq0RHtLrIr+c0ByiYzWT8KTMRzoRCNlw==",
+          "dev": true,
+          "requires": {
+            "acorn": "^7.4.0",
+            "acorn-jsx": "^5.2.0",
+            "eslint-visitor-keys": "^1.3.0"
+          }
+        },
+        "globals": {
+          "version": "12.4.0",
+          "resolved": "https://registry.npmjs.org/globals/-/globals-12.4.0.tgz",
+          "integrity": "sha512-BWICuzzDvDoH54NHKCseDanAhE3CeDorgDL5MT6LMXXj2WCnd9UC2szdk4AWLfjdgNBCXLUanXYcpBBKOSWGwg==",
+          "dev": true,
+          "requires": {
+            "type-fest": "^0.8.1"
+          }
+        },
+        "ignore": {
+          "version": "4.0.6",
+          "resolved": "https://registry.npmjs.org/ignore/-/ignore-4.0.6.tgz",
+          "integrity": "sha512-cyFDKrqc/YdcWFniJhzI42+AzS+gNwmUzOSFcRCQYwySuBBBy/KjuxWLZ/FHEH6Moq1NizMOBWyTcv8O4OZIMg==",
+          "dev": true
+        }
+      }
+    },
     "@hapi/address": {
       "version": "2.1.4",
       "resolved": "https://registry.npmjs.org/@hapi/address/-/address-2.1.4.tgz",
@@ -5137,6 +5417,32 @@
         }
       }
     },
+    "@nodelib/fs.scandir": {
+      "version": "2.1.3",
+      "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.3.tgz",
+      "integrity": "sha512-eGmwYQn3gxo4r7jdQnkrrN6bY478C3P+a/y72IJukF8LjB6ZHeB3c+Ehacj3sYeSmUXGlnA67/PmbM9CVwL7Dw==",
+      "dev": true,
+      "requires": {
+        "@nodelib/fs.stat": "2.0.3",
+        "run-parallel": "^1.1.9"
+      }
+    },
+    "@nodelib/fs.stat": {
+      "version": "2.0.3",
+      "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.3.tgz",
+      "integrity": "sha512-bQBFruR2TAwoevBEd/NWMoAAtNGzTRgdrqnYCc7dhzfoNvqPzLyqlEQnzZ3kVnNrSp25iyxE00/3h2fqGAGArA==",
+      "dev": true
+    },
+    "@nodelib/fs.walk": {
+      "version": "1.2.4",
+      "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.4.tgz",
+      "integrity": "sha512-1V9XOY4rDW0rehzbrcqAmHnz8e7SKvX27gh8Gt2WgB0+pdzdiLV83p72kZPU+jvMbS1qU5mauP2iOvO8rhmurQ==",
+      "dev": true,
+      "requires": {
+        "@nodelib/fs.scandir": "2.1.3",
+        "fastq": "^1.6.0"
+      }
+    },
     "@npmcli/move-file": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/@npmcli/move-file/-/move-file-1.0.1.tgz",
@@ -5489,6 +5795,22 @@
         "eslint-utils": "^2.0.0"
       }
     },
+    "@typescript-eslint/scope-manager": {
+      "version": "4.3.0",
+      "resolved": "https://registry.npmjs.org/@typescript-eslint/scope-manager/-/scope-manager-4.3.0.tgz",
+      "integrity": "sha512-cTeyP5SCNE8QBRfc+Lgh4Xpzje46kNUhXYfc3pQWmJif92sjrFuHT9hH4rtOkDTo/si9Klw53yIr+djqGZS1ig==",
+      "dev": true,
+      "requires": {
+        "@typescript-eslint/types": "4.3.0",
+        "@typescript-eslint/visitor-keys": "4.3.0"
+      }
+    },
+    "@typescript-eslint/types": {
+      "version": "4.3.0",
+      "resolved": "https://registry.npmjs.org/@typescript-eslint/types/-/types-4.3.0.tgz",
+      "integrity": "sha512-Cx9TpRvlRjOppGsU6Y6KcJnUDOelja2NNCX6AZwtVHRzaJkdytJWMuYiqi8mS35MRNA3cJSwDzXePfmhU6TANw==",
+      "dev": true
+    },
     "@typescript-eslint/typescript-estree": {
       "version": "2.34.0",
       "resolved": "https://registry.npmjs.org/@typescript-eslint/typescript-estree/-/typescript-estree-2.34.0.tgz",
@@ -5512,6 +5834,24 @@
         }
       }
     },
+    "@typescript-eslint/visitor-keys": {
+      "version": "4.3.0",
+      "resolved": "https://registry.npmjs.org/@typescript-eslint/visitor-keys/-/visitor-keys-4.3.0.tgz",
+      "integrity": "sha512-xZxkuR7XLM6RhvLkgv9yYlTcBHnTULzfnw4i6+z2TGBLy9yljAypQaZl9c3zFvy7PNI7fYWyvKYtohyF8au3cw==",
+      "dev": true,
+      "requires": {
+        "@typescript-eslint/types": "4.3.0",
+        "eslint-visitor-keys": "^2.0.0"
+      },
+      "dependencies": {
+        "eslint-visitor-keys": {
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/eslint-visitor-keys/-/eslint-visitor-keys-2.0.0.tgz",
+          "integrity": "sha512-QudtT6av5WXels9WjIM7qz1XD1cWGvX4gGXvp/zBn9nXG02D0utdU3Em2m/QjTnrsk6bBjmCygl3rmj118msQQ==",
+          "dev": true
+        }
+      }
+    },
     "@webassemblyjs/ast": {
       "version": "1.9.0",
       "resolved": "https://registry.npmjs.org/@webassemblyjs/ast/-/ast-1.9.0.tgz",
@@ -6387,9 +6727,9 @@
       }
     },
     "bl": {
-      "version": "4.0.2",
-      "resolved": "https://registry.npmjs.org/bl/-/bl-4.0.2.tgz",
-      "integrity": "sha512-j4OH8f6Qg2bGuWfRiltT2HYGx0e1QcBTrK9KAHNMwMZdQnDZFk0ZSYIpADjYCB3U12nicC5tVJwSIhwOWjb4RQ==",
+      "version": "4.0.3",
+      "resolved": "https://registry.npmjs.org/bl/-/bl-4.0.3.tgz",
+      "integrity": "sha512-fs4G6/Hu4/EE+F75J8DuN/0IpQqNjAdC7aEQv7Qt8MHGUH7Ckv2MwTEEeN9QehD0pfIDkMI1bkHYkKy7xHyKIg==",
       "dev": true,
       "requires": {
         "buffer": "^5.5.0",
@@ -6397,16 +6737,6 @@
         "readable-stream": "^3.4.0"
       },
       "dependencies": {
-        "buffer": {
-          "version": "5.5.0",
-          "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.5.0.tgz",
-          "integrity": "sha512-9FTEDjLjwoAkEwyMGDjYJQN2gfRgOKBKRfiglhvibGbpeeU/pQn1bJxQqm32OD/AIeEuHxU9roxXxg34Byp/Ww==",
-          "dev": true,
-          "requires": {
-            "base64-js": "^1.0.2",
-            "ieee754": "^1.1.4"
-          }
-        },
         "readable-stream": {
           "version": "3.6.0",
           "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz",
@@ -8796,6 +9126,15 @@
         }
       }
     },
+    "dir-glob": {
+      "version": "3.0.1",
+      "resolved": "https://registry.npmjs.org/dir-glob/-/dir-glob-3.0.1.tgz",
+      "integrity": "sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA==",
+      "dev": true,
+      "requires": {
+        "path-type": "^4.0.0"
+      }
+    },
     "doctrine": {
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-3.0.0.tgz",
@@ -10187,6 +10526,31 @@
       "integrity": "sha512-xJuoT5+L99XlZ8twedaRf6Ax2TgQVxvgZOYoPKqZufmJib0tL2tegPBOZb1pVNgIhlqDlA0eO0c3wBvQcmzx4w==",
       "dev": true
     },
+    "fast-glob": {
+      "version": "3.2.4",
+      "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.2.4.tgz",
+      "integrity": "sha512-kr/Oo6PX51265qeuCYsyGypiO5uJFgBS0jksyG7FUeCyQzNwYnzrNIMR1NXfkZXsMYXYLRAHgISHBz8gQcxKHQ==",
+      "dev": true,
+      "requires": {
+        "@nodelib/fs.stat": "^2.0.2",
+        "@nodelib/fs.walk": "^1.2.3",
+        "glob-parent": "^5.1.0",
+        "merge2": "^1.3.0",
+        "micromatch": "^4.0.2",
+        "picomatch": "^2.2.1"
+      },
+      "dependencies": {
+        "glob-parent": {
+          "version": "5.1.1",
+          "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.1.tgz",
+          "integrity": "sha512-FnI+VGOpnlGHWZxthPGR+QhR78fuiK0sNLkHQv+bL9fQi57lNNdquIbna/WrfROrolq8GK5Ek6BiMwqL/voRYQ==",
+          "dev": true,
+          "requires": {
+            "is-glob": "^4.0.1"
+          }
+        }
+      }
+    },
     "fast-json-stable-stringify": {
       "version": "2.1.0",
       "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz",
@@ -10202,6 +10566,15 @@
       "resolved": "https://registry.npmjs.org/fastestsmallesttextencoderdecoder/-/fastestsmallesttextencoderdecoder-1.0.22.tgz",
       "integrity": "sha512-Pb8d48e+oIuY4MaM64Cd7OW1gt4nxCHs7/ddPPZ/Ic3sg8yVGM7O9wDvZ7us6ScaUupzM+pfBolwtYhN1IxBIw=="
     },
+    "fastq": {
+      "version": "1.8.0",
+      "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.8.0.tgz",
+      "integrity": "sha512-SMIZoZdLh/fgofivvIkmknUXyPnvxRE3DhtZ5Me3Mrsk5gyPL42F0xr51TdRXskBxHfMp+07bcYzfsYEsSQA9Q==",
+      "dev": true,
+      "requires": {
+        "reusify": "^1.0.4"
+      }
+    },
     "favicons": {
       "version": "5.5.0",
       "resolved": "https://registry.npmjs.org/favicons/-/favicons-5.5.0.tgz",
@@ -11159,6 +11532,28 @@
       "resolved": "https://registry.npmjs.org/globals/-/globals-11.12.0.tgz",
       "integrity": "sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA=="
     },
+    "globby": {
+      "version": "11.0.1",
+      "resolved": "https://registry.npmjs.org/globby/-/globby-11.0.1.tgz",
+      "integrity": "sha512-iH9RmgwCmUJHi2z5o2l3eTtGBtXek1OYlHrbcxOYugyHLmAsZrPj43OtHThd62Buh/Vv6VyCBD2bdyWcGNQqoQ==",
+      "dev": true,
+      "requires": {
+        "array-union": "^2.1.0",
+        "dir-glob": "^3.0.1",
+        "fast-glob": "^3.1.1",
+        "ignore": "^5.1.4",
+        "merge2": "^1.3.0",
+        "slash": "^3.0.0"
+      },
+      "dependencies": {
+        "array-union": {
+          "version": "2.1.0",
+          "resolved": "https://registry.npmjs.org/array-union/-/array-union-2.1.0.tgz",
+          "integrity": "sha512-HGyxoOTYUyCM6stUe6EJgnd4EoewAI7zMdfqO+kGjnlZmBDz/cR5pf8r/cR4Wq60sL/p0IkcjUEEPwS3GFrIyw==",
+          "dev": true
+        }
+      }
+    },
     "got": {
       "version": "6.7.1",
       "resolved": "https://registry.npmjs.org/got/-/got-6.7.1.tgz",
@@ -11691,6 +12086,12 @@
       "integrity": "sha1-xg7taebY/bazEEofy8ocGS3FtQE=",
       "dev": true
     },
+    "ignore": {
+      "version": "5.1.8",
+      "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.1.8.tgz",
+      "integrity": "sha512-BMpfD7PpiETpBl/A6S498BaIJ6Y/ABT93ETbby2fP00v4EbvPBXWEoaR1UBPKs3iR53pJY7EtZk5KACI57i1Uw==",
+      "dev": true
+    },
     "ignore-walk": {
       "version": "3.0.3",
       "resolved": "https://registry.npmjs.org/ignore-walk/-/ignore-walk-3.0.3.tgz",
@@ -14189,6 +14590,12 @@
       "resolved": "https://registry.npmjs.org/merge-stream/-/merge-stream-2.0.0.tgz",
       "integrity": "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w=="
     },
+    "merge2": {
+      "version": "1.4.1",
+      "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz",
+      "integrity": "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==",
+      "dev": true
+    },
     "methods": {
       "version": "1.1.2",
       "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz",
@@ -15251,6 +15658,12 @@
       "integrity": "sha1-32BBeABfUi8V60SQ5yR6G/qmf4w=",
       "dev": true
     },
+    "path-type": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/path-type/-/path-type-4.0.0.tgz",
+      "integrity": "sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==",
+      "dev": true
+    },
     "pbkdf2": {
       "version": "3.1.1",
       "resolved": "https://registry.npmjs.org/pbkdf2/-/pbkdf2-3.1.1.tgz",
@@ -17065,6 +17478,12 @@
       "resolved": "https://registry.npmjs.org/ret/-/ret-0.1.15.tgz",
       "integrity": "sha512-TTlYpa+OL+vMMNG24xSlQGEJ3B/RzEfUlLct7b5G/ytav+wPrplCpVMFuwzXbkecJrb6IYo1iFb0S9v37754mg=="
     },
+    "reusify": {
+      "version": "1.0.4",
+      "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.0.4.tgz",
+      "integrity": "sha512-U9nH88a3fc/ekCF1l0/UP1IosiuIjyTh7hBvXVMHYgVcfGvt897Xguj2UOLDeI5BG2m7/uwyaLVT6fbtCwTyzw==",
+      "dev": true
+    },
     "rgb-regex": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/rgb-regex/-/rgb-regex-1.0.1.tgz",
@@ -17100,6 +17519,12 @@
       "resolved": "https://registry.npmjs.org/rsvp/-/rsvp-4.8.5.tgz",
       "integrity": "sha512-nfMOlASu9OnRJo1mbEk2cz0D56a1MBNrJ7orjRZQG10XDyuvwksKbuXNp6qa+kbn839HwjwhBzhFmdsaEAfauA=="
     },
+    "run-parallel": {
+      "version": "1.1.9",
+      "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.1.9.tgz",
+      "integrity": "sha512-DEqnSRTDw/Tc3FXf49zedI638Z9onwUotBMiUFKmrO2sdFKIbXamXGQ3Axd4qgphxKB4kw/qP1w5kTxnfU1B9Q==",
+      "dev": true
+    },
     "run-queue": {
       "version": "1.0.3",
       "resolved": "https://registry.npmjs.org/run-queue/-/run-queue-1.0.3.tgz",

client/package.json

@@ -84,6 +84,7 @@
     "@babel/preset-react": "^7.10.4",
     "@babel/register": "^7.10.5",
     "@babel/runtime": "^7.10.5",
+    "@blueprintjs/eslint-plugin": "^0.3.0",
     "@sentry/webpack-plugin": "^1.12.0",
     "babel-eslint": "^10.1.0",
     "babel-jest": "^26.1.0",

client/src/actions/embedding.js

@@ -5,20 +5,23 @@ action creators related to embeddings choice
 import { AnnoMatrixObsCrossfilter } from "../annoMatrix";
 import { _setEmbeddingSubset } from "../util/stateManager/viewStackHelpers";
 
-export async function _switchEmbedding(prevAnnoMatrix, prevCrossfilter, newEmbeddingName) {
+export async function _switchEmbedding(
+  prevAnnoMatrix,
+  prevCrossfilter,
+  newEmbeddingName
+) {
   /*
   DRY helper used by this and reembedding action creators
   */
   const base = prevAnnoMatrix.base();
   const embeddingDf = await base.fetch("emb", newEmbeddingName);
   const annoMatrix = _setEmbeddingSubset(prevAnnoMatrix, embeddingDf);
-  const obsCrossfilter = await new AnnoMatrixObsCrossfilter(annoMatrix, prevCrossfilter.obsCrossfilter).select(
-    "emb",
-    newEmbeddingName,
-    {
-      mode: "all",
-    }
-  );
+  const obsCrossfilter = await new AnnoMatrixObsCrossfilter(
+    annoMatrix,
+    prevCrossfilter.obsCrossfilter
+  ).select("emb", newEmbeddingName, {
+    mode: "all",
+  });
   return [annoMatrix, obsCrossfilter];
 }
 
@@ -30,7 +33,10 @@ export const layoutChoiceAction = (newLayoutChoice) => async (
   On layout choice, make sure we have selected all on the previous layout, AND the new
   layout.
   */
-  const { annoMatrix: prevAnnoMatrix, obsCrossfilter: prevCrossfilter } = getState();
+  const {
+    annoMatrix: prevAnnoMatrix,
+    obsCrossfilter: prevCrossfilter,
+  } = getState();
   const [annoMatrix, obsCrossfilter] = await _switchEmbedding(
     prevAnnoMatrix,
     prevCrossfilter,

client/src/components/autosave/filenameDialog.js

@@ -3,11 +3,12 @@ import { connect } from "react-redux";
 
 import {
   Button,
-  Tooltip,
-  InputGroup,
-  Dialog,
   Classes,
+  Code,
   Colors,
+  Dialog,
+  InputGroup,
+  Tooltip,
 } from "@blueprintjs/core";
 
 @connect((state) => ({
@@ -145,9 +146,9 @@ class FilenameDialog extends React.Component {
             <div>
               <p>
                 Your annotations are stored in this file:
-                <code className="bp3-code">
+                <Code>
                   {filenameText}-{idhash}.csv
-                </code>
+                </Code>
               </p>
               <p style={{ fontStyle: "italic" }}>
                 (We added a unique ID to your filename)

client/src/components/brushableHistogram/index.js

@@ -5,12 +5,13 @@ https://bl.ocks.org/SpaceActuary/2f004899ea1b2bd78d6f1dbb2febf771
 https://bl.ocks.org/mbostock/3019563
 */
 import React, { useEffect, useRef, useState, useCallback } from "react";
-import { Button, ButtonGroup, Tooltip } from "@blueprintjs/core";
+import { Button, ButtonGroup, Icon, Tooltip } from "@blueprintjs/core";
 import { connect } from "react-redux";
 import * as d3 from "d3";
 import { interpolateCool } from "d3-scale-chromatic";
 import Async from "react-async";
 import memoize from "memoize-one";
+import { IconNames } from "@blueprintjs/icons";
 import * as globals from "../../globals";
 import actions from "../../actions";
 import { histogramContinuous } from "../../util/dataframe/histogram";
@@ -214,10 +215,7 @@ const HistogramHeader = React.memo(
       >
         {onScatterPlotXClick && onScatterPlotYClick ? (
           <span>
-            <span
-              style={{ marginRight: 7 }}
-              className="bp3-icon-standard bp3-icon-scatter-plot"
-            />
+            <Icon icon={IconNames.SCATTER_PLOT} style={{ marginRight: 7 }} />
             <ButtonGroup style={{ marginRight: 7 }}>
               <Button
                 data-testid={`plot-x-${fieldId}`}

client/src/components/categorical/category/index.js

@@ -1,7 +1,13 @@
 import React, { useRef, useEffect } from "react";
 import { connect, shallowEqual } from "react-redux";
 import { FaChevronRight, FaChevronDown } from "react-icons/fa";
-import { AnchorButton, Button, Tooltip, Position } from "@blueprintjs/core";
+import {
+  AnchorButton,
+  Button,
+  Classes,
+  Position,
+  Tooltip,
+} from "@blueprintjs/core";
 import { Flipper, Flipped } from "react-flip-toolkit";
 import Async from "react-async";
 import memoize from "memoize-one";
@@ -301,9 +307,12 @@ const StillLoading = ({ metadataField, checkboxID }) => {
             alignItems: "flex-start",
           }}
         >
-          <label htmlFor={checkboxID} className="bp3-control bp3-checkbox">
+          <label
+            htmlFor={checkboxID}
+            className={`${Classes.CONTROL} ${Classes.CHECKBOX}`}
+          >
             <input disabled id={checkboxID} checked type="checkbox" />
-            <span className="bp3-control-indicator" />
+            <span className={Classes.CONTROL_INDICATOR} />
           </label>
           <Truncate>
             <span
@@ -375,7 +384,10 @@ const CategoryHeader = React.memo(
             alignItems: "flex-start",
           }}
         >
-          <label className="bp3-control bp3-checkbox" htmlFor={checkboxID}>
+          <label
+            className={`${Classes.CONTROL} ${Classes.CHECKBOX}`}
+            htmlFor={checkboxID}
+          >
             <input
               id={checkboxID}
               data-testclass="category-select"
@@ -385,7 +397,7 @@ const CategoryHeader = React.memo(
               checked={selectionState === "all"}
               type="checkbox"
             />
-            <span className="bp3-control-indicator" />
+            <span className={Classes.CONTROL_INDICATOR} />
           </label>
           <span
             role="menuitem"

client/src/components/categorical/index.js

@@ -185,7 +185,9 @@ class Categories extends React.Component {
         {/* READ ONLY CATEGORICAL FIELDS */}
         {/* this is duplicative but flat, could be abstracted */}
         {allCategoryNames.map((catName) =>
-          !schema.annotations.obsByName[catName].writable ? (
+          !schema.annotations.obsByName[catName].writable &&
+          (schema.annotations.obsByName[catName].categories?.length > 1 ||
+            !schema.annotations.obsByName[catName].categories) ? (
             <Category
               key={catName}
               metadataField={catName}

client/src/components/categorical/value/index.js

@@ -4,12 +4,13 @@ import * as d3 from "d3";
 
 import {
   Button,
+  Classes,
+  Icon,
   Menu,
   MenuItem,
   Popover,
-  Position,
-  Icon,
   PopoverInteractionKind,
+  Position,
 } from "@blueprintjs/core";
 import * as globals from "../../../globals";
 import styles from "../categorical.css";
@@ -410,7 +411,6 @@ class CategoryValue extends React.Component {
 
     return (
       <MiniStackedBar
-        /* eslint-disable react/jsx-props-no-spreading -- Disable unneeded on next release of eslint-config-airbnb */
         {...{
           colorTable,
           domainValues,
@@ -418,7 +418,6 @@ class CategoryValue extends React.Component {
           domain,
           occupancy,
         }}
-        /* eslint-enable react/jsx-props-no-spreading -- enable */
         height={VALUE_HEIGHT}
         width={CHART_WIDTH}
       />
@@ -461,14 +460,12 @@ class CategoryValue extends React.Component {
 
     return (
       <MiniHistogram
-        /* eslint-disable react/jsx-props-no-spreading -- Disable unneeded on next release of eslint-config-airbnb */
         {...{
           colorScale,
           xScale,
           yScale,
           bins,
         }}
-        /* eslint-enable react/jsx-props-no-spreading -- enable */
         obsOrVarContinuousFieldDisplayName={colorAccessor}
         domainLabel={label}
         height={VALUE_HEIGHT}
@@ -564,7 +561,7 @@ class CategoryValue extends React.Component {
           <div style={{ display: "flex", alignItems: "baseline" }}>
             <label
               htmlFor={valueToggleLabel}
-              className="bp3-control bp3-checkbox"
+              className={`${Classes.CONTROL} ${Classes.CHECKBOX}`}
               style={{ margin: 0 }}
             >
               <input
@@ -576,7 +573,7 @@ class CategoryValue extends React.Component {
                 type="checkbox"
               />
               <span
-                className="bp3-control-indicator"
+                className={Classes.CONTROL_INDICATOR}
                 onMouseEnter={this.handleMouseExit}
                 onMouseLeave={this.handleMouseEnter}
               />

client/src/components/categorical/value/occupancy.js

@@ -3,10 +3,10 @@ import React from "react";
 import { connect } from "react-redux";
 import * as d3 from "d3";
 import {
+  Classes,
   Popover,
   PopoverInteractionKind,
   Position,
-  Classes,
 } from "@blueprintjs/core";
 
 @connect((state) => ({
@@ -155,7 +155,7 @@ class Occupancy extends React.PureComponent {
         popoverClassName={Classes.POPOVER_CONTENT_SIZING}
       >
         <canvas
-          className="bp3-popover-targer"
+          className={Classes.POPOVER_TARGET}
           style={{
             marginRight: 5,
             width: this._WIDTH,

client/src/components/embedding/index.js

@@ -2,13 +2,14 @@ import React from "react";
 import { connect } from "react-redux";
 import { useAsync } from "react-async";
 import {
-  ButtonGroup,
-  Popover,
   Button,
+  ButtonGroup,
+  H4,
+  Popover,
+  Position,
   Radio,
   RadioGroup,
   Tooltip,
-  Position,
 } from "@blueprintjs/core";
 import * as globals from "../../globals";
 import actions from "../../actions";
@@ -80,7 +81,7 @@ class Embedding extends React.PureComponent {
                 width: 400,
               }}
             >
-              <h1>Embedding Choice</h1>
+              <H4>Embedding Choice</H4>
               <p style={{ fontStyle: "italic" }}>
                 There are {schema?.dataframe?.nObs} cells in the entire dataset.
               </p>

client/src/components/geneExpression/addGenes.js

@@ -6,11 +6,12 @@ import fuzzysort from "fuzzysort";
 import { connect } from "react-redux";
 import { Suggest } from "@blueprintjs/select";
 import {
-  MenuItem,
   Button,
+  ControlGroup,
   FormGroup,
   InputGroup,
-  ControlGroup,
+  Intent,
+  MenuItem,
 } from "@blueprintjs/core";
 import * as globals from "../../globals";
 import actions from "../../actions";
@@ -278,7 +279,7 @@ class AddGenes extends React.Component {
               popoverProps={{ minimal: true }}
             />
             <Button
-              className="bp3-button bp3-intent-primary"
+              intent={Intent.PRIMARY}
               data-testid="add-gene"
               loading={userDefinedGenesLoading}
               onClick={() => this.handleClick(activeItem)}

client/src/components/graph/setupLasso.js

@@ -10,6 +10,7 @@ const Lasso = () => {
     let lassoPolygon;
     let lassoPath;
     let closePath;
+    let lassoInProgress;
 
     const polygonToPath = (polygon) =>
       `M${polygon.map((d) => d.join(",")).join("L")}`;
@@ -25,8 +26,18 @@ const Lasso = () => {
       lassoPolygon = [d3.mouse(svg.node())]; // current x y of mouse within element
 
       if (lassoPath) {
+        // If the existing path is in progress
+        if (lassoInProgress) {
+          // cancel the existing lasso
+          handleCancel();
+          // Don't continue with current drag start
+          return;
+        }
+
         lassoPath.remove();
       }
+      // We're starting a new drag
+      lassoInProgress = true;
 
       lassoPath = g
         .append("path")
@@ -67,25 +78,33 @@ const Lasso = () => {
       }
     };
 
+    const handleCancel = () => {
+      lassoPath.remove();
+      closePath = closePath?.remove();
+      lassoPath = null;
+      lassoPolygon = null;
+      closePath = null;
+      dispatch.call("cancel");
+    };
+
     const handleDragEnd = () => {
       // remove the close path
       closePath.remove();
       closePath = null;
 
-      // succesfully closed
+      // successfully closed
       if (
         distance(lassoPolygon[0], lassoPolygon[lassoPolygon.length - 1]) <
         closeDistance
       ) {
+        lassoInProgress = false;
+
         lassoPath.attr("d", `${polygonToPath(lassoPolygon)}Z`);
         dispatch.call("end", lasso, lassoPolygon);
 
         // otherwise cancel
       } else {
-        lassoPath.remove();
-        lassoPath = null;
-        lassoPolygon = null;
-        dispatch.call("cancel");
+        handleCancel();
       }
     };
 

client/src/components/infoDrawer/infoDrawer.js

@@ -38,6 +38,8 @@ class InfoDrawer extends PureComponent {
     const singleValueCategories = (
       await Promise.all(nonUserAnnoCategories)
     ).reduce((acc, categoryData, i) => {
+      // Actually check to see if it is null(user anno)
+      if (!categoryData) return acc;
       const catName = allCategoryNames[i];
 
       const column = categoryData.icol(0);

client/src/components/leftSidebar/topLeftLogoAndTitle.js

@@ -7,11 +7,19 @@ import * as globals from "../../globals";
 import Logo from "../framework/logo";
 import Truncate from "../util/truncate";
 import InfoDrawer from "../infoDrawer/infoDrawer";
+import AuthButtons from "../menubar/authButtons";
+import InformationMenu from "../menubar/infoMenu";
 
 const DATASET_TITLE_FONT_SIZE = 14;
 
 @connect((state) => ({
   datasetTitle: state.config?.displayNames?.dataset ?? "",
+  auth: state.config?.authentication,
+  userinfo: state.userinfo,
+  libraryVersions: state.config?.["library_versions"],
+  aboutLink: state.config?.links?.["about-dataset"],
+  tosURL: state.config?.parameters?.["about_legal_tos"],
+  privacyURL: state.config?.parameters?.["about_legal_privacy"],
 }))
 class LeftSideBar extends React.Component {
   handleClick = () => {
@@ -20,7 +28,16 @@ class LeftSideBar extends React.Component {
   };
 
   render() {
-    const { datasetTitle } = this.props;
+    const {
+      datasetTitle,
+      auth,
+      userinfo,
+      libraryVersions,
+      aboutLink,
+      privacyURL,
+      tosURL,
+      dispatch,
+    } = this.props;
 
     return (
       <div
@@ -30,49 +47,70 @@ class LeftSideBar extends React.Component {
           width: globals.leftSidebarWidth,
           zIndex: 1,
           borderBottom: `1px solid ${globals.lighterGrey}`,
+          display: "flex",
+          justifyContent: "space-between",
+          alignItems: "center",
         }}
       >
-        <Logo size={30} />
-        <span
-          style={{
-            fontSize: 28,
-            position: "relative",
-            top: -6,
-            fontWeight: "bold",
-            marginLeft: 5,
-            color: globals.logoColor,
-            userSelect: "none",
-          }}
-        >
-          cell
+        <div>
+          <Logo size={28} />
           <span
             style={{
-              position: "relative",
-              top: 1,
-              fontWeight: 300,
               fontSize: 24,
+              position: "relative",
+              top: -6,
+              fontWeight: "bold",
+              marginLeft: 5,
+              color: globals.logoColor,
+              userSelect: "none",
             }}
           >
-            ×
-          </span>
-          gene
-        </span>
-        <Button
-          minimal
-          style={{
-            fontSize: DATASET_TITLE_FONT_SIZE,
-            position: "absolute",
-            right: 10,
-          }}
-          onClick={this.handleClick}
-        >
-          <Truncate>
-            <span style={{ maxWidth: 155 }} data-testid="header">
-              {datasetTitle}
+            cell
+            <span
+              style={{
+                position: "relative",
+                top: 1,
+                fontWeight: 300,
+                fontSize: 24,
+              }}
+            >
+              ×
             </span>
-          </Truncate>
-        </Button>
-        <InfoDrawer />
+            gene
+          </span>
+        </div>
+        <div style={{ marginRight: 5, position: "relative", top: -7 }}>
+          <Button
+            minimal
+            style={{
+              fontSize: DATASET_TITLE_FONT_SIZE,
+              position: "relative",
+              top: -1,
+            }}
+            onClick={this.handleClick}
+          >
+            <Truncate>
+              <span style={{ maxWidth: 155 }} data-testid="header">
+                {datasetTitle}
+              </span>
+            </Truncate>
+          </Button>
+          <InfoDrawer />
+          <InformationMenu
+            {...{
+              libraryVersions,
+              aboutLink,
+              tosURL,
+              privacyURL,
+              auth,
+              dispatch,
+              userinfo,
+            }}
+          />
+          {!userinfo.is_authenticated ? (
+            <AuthButtons auth={auth} userinfo={userinfo} />
+          ) : null}
+        </div>
       </div>
     );
   }

client/src/components/menubar/authButtons.js

@@ -1,5 +1,5 @@
 import React from "react";
-import { AnchorButton, Tooltip } from "@blueprintjs/core";
+import { AnchorButton, ButtonGroup, Tooltip } from "@blueprintjs/core";
 import * as globals from "../../globals";
 import styles from "./menubar.css";
 
@@ -9,9 +9,9 @@ const Auth = React.memo((props) => {
   if (!auth || (auth && !auth.requires_client_login)) return null;
 
   return (
-    <div className={`bp3-button-group ${styles.menubarButton}`}>
+    <ButtonGroup className={styles.menubarButton}>
       <Tooltip
-        content="Log in or log out of cellxgene"
+        content="Log in to cellxgene"
         position="bottom"
         hoverOpenDelay={globals.tooltipHoverOpenDelay}
       >
@@ -19,13 +19,12 @@ const Auth = React.memo((props) => {
           type="button"
           data-testid="auth-button"
           disabled={false}
-          icon={!userinfo.is_authenticated ? "log-in" : "log-out"}
           href={!userinfo.is_authenticated ? auth.login : auth.logout}
         >
           {!userinfo.is_authenticated ? "Log In" : "Log Out"}
         </AnchorButton>
       </Tooltip>
-    </div>
+    </ButtonGroup>
   );
 });
 

client/src/components/menubar/clip.js

@@ -1,12 +1,16 @@
 import React from "react";
 import {
-  Position,
   Button,
-  Popover,
-  NumericInput,
+  ButtonGroup,
   Icon,
+  Intent,
+  NumericInput,
+  Popover,
+  Position,
   Tooltip,
 } from "@blueprintjs/core";
+import { IconNames } from "@blueprintjs/icons";
+
 import { tooltipHoverOpenDelay } from "../../globals";
 import styles from "./menubar.css";
 
@@ -28,13 +32,13 @@ const Clip = React.memo((props) => {
     pendingClipPercentiles?.clipPercentileMin ?? clipPercentileMin;
   const clipMax =
     pendingClipPercentiles?.clipPercentileMax ?? clipPercentileMax;
-  const activeClipClass =
+  const intent =
     clipPercentileMin > 0 || clipPercentileMax < 100
-      ? " bp3-intent-warning"
-      : "";
+      ? Intent.INTENT_WARNING
+      : Intent.NONE;
 
   return (
-    <div className={`bp3-button-group ${styles.menubarButton}`}>
+    <ButtonGroup className={`${styles.menubarButton}`}>
       <Popover
         target={
           <Tooltip
@@ -45,7 +49,8 @@ const Clip = React.memo((props) => {
             <Button
               type="button"
               data-testid="visualization-settings"
-              className={`bp3-button bp3-icon-timeline-bar-chart ${activeClipClass}`}
+              intent={intent}
+              icon={IconNames.TIMELINE_BAR_CHART}
               style={{
                 cursor: "pointer",
               }}
@@ -126,7 +131,7 @@ const Clip = React.memo((props) => {
           </div>
         }
       />
-    </div>
+    </ButtonGroup>
   );
 });
 

client/src/components/menubar/index.js

@@ -6,8 +6,7 @@ import * as globals from "../../globals";
 import styles from "./menubar.css";
 import actions from "../../actions";
 import Clip from "./clip";
-import AuthButtons from "./authButtons";
-import InformationMenu from "./infoMenu";
+
 import Subset from "./subset";
 import UndoRedoReset from "./undoRedo";
 import DiffexpButtons from "./diffexpButtons";
@@ -204,7 +203,6 @@ class MenuBar extends React.PureComponent {
   render() {
     const {
       dispatch,
-      libraryVersions,
       disableDiffexp,
       undoDisabled,
       redoDisabled,
@@ -212,17 +210,12 @@ class MenuBar extends React.PureComponent {
       clipPercentileMin,
       clipPercentileMax,
       graphInteractionMode,
-      aboutLink,
       showCentroidLabels,
-      privacyURL,
-      tosURL,
       categoricalSelection,
       colorAccessor,
       subsetPossible,
       subsetResetPossible,
       enableReembedding,
-      auth,
-      userinfo,
     } = this.props;
     const { pendingClipPercentiles } = this.state;
 
@@ -248,10 +241,6 @@ class MenuBar extends React.PureComponent {
           zIndex: 3,
         }}
       >
-        <AuthButtons auth={auth} userinfo={userinfo} />
-        <InformationMenu
-          {...{ libraryVersions, aboutLink, tosURL, privacyURL, dispatch }}
-        />
         <UndoRedoReset
           dispatch={dispatch}
           undoDisabled={undoDisabled}

client/src/components/menubar/infoMenu.js

@@ -1,7 +1,14 @@
 // jshint esversion: 6
 import React from "react";
-import { Button, Popover, Menu, MenuItem, Position } from "@blueprintjs/core";
-import { IconNames } from "@blueprintjs/icons";
+import {
+  Button,
+  ButtonGroup,
+  Classes,
+  Menu,
+  MenuItem,
+  Popover,
+  Position,
+} from "@blueprintjs/core";
 import styles from "./menubar.css";
 
 const handleClick = (dispatch) => {
@@ -9,35 +16,45 @@ const handleClick = (dispatch) => {
 };
 
 const InformationMenu = React.memo((props) => {
-  const { libraryVersions, tosURL, privacyURL, dispatch } = props;
+  const {
+    libraryVersions,
+    tosURL,
+    privacyURL,
+    auth,
+    userinfo,
+    dispatch,
+  } = props;
   return (
-    <div className={`bp3-button-group ${styles.menubarButton}`}>
+    <ButtonGroup className={`${styles.menubarButton}`}>
       <Popover
         content={
           <Menu>
             <MenuItem
               onClick={() => handleClick(dispatch)}
-              icon={IconNames.INFO_SIGN}
+              icon="info-sign"
               text="Dataset Overview"
             />
 
             <MenuItem
               href="https://chanzuckerberg.github.io/cellxgene/"
               target="_blank"
-              icon="help"
-              text="Help"
+              icon="book"
+              text="Documentation"
+              rel="noopener"
             />
             <MenuItem
               href="https://join-cellxgene-users.herokuapp.com/"
               target="_blank"
               icon="chat"
               text="Chat"
+              rel="noopener"
             />
             <MenuItem
               href="https://github.com/chanzuckerberg/cellxgene"
               target="_blank"
               icon="git-branch"
               text="Github"
+              rel="noopener"
             />
             <MenuItem
               target="_blank"
@@ -56,21 +73,35 @@ const InformationMenu = React.memo((props) => {
                 href={privacyURL}
                 target="_blank"
                 text="Privacy Policy"
+                rel="noopener"
               />
             ) : null}
+
+            {auth?.["requires_client_login"] &&
+            userinfo?.["is_authenticated"] ? (
+              <>
+                <MenuItem text={`Logged in as: ${userinfo.email}`} />
+                <MenuItem text="Log Out" href={auth.logout} />
+              </>
+            ) : null}
           </Menu>
         }
         position={Position.BOTTOM_RIGHT}
+        modifiers={{
+          preventOverflow: { enabled: false },
+          hide: { enabled: false },
+        }}
       >
         <Button
+          data-testid="menu"
           type="button"
-          className="bp3-button bp3-icon-info-sign"
+          className={`${Classes.BUTTON} bp3-icon-info-sign`}
           style={{
             cursor: "pointer",
           }}
         />
       </Popover>
-    </div>
+    </ButtonGroup>
   );
 });
 

client/src/components/menubar/undoRedo.js

@@ -1,12 +1,13 @@
 import React from "react";
-import { AnchorButton, Tooltip } from "@blueprintjs/core";
+import { AnchorButton, ButtonGroup, Tooltip } from "@blueprintjs/core";
+import { IconNames } from "@blueprintjs/icons";
 import { tooltipHoverOpenDelay } from "../../globals";
 import styles from "./menubar.css";
 
 const UndoRedo = React.memo((props) => {
   const { undoDisabled, redoDisabled, dispatch } = props;
   return (
-    <div className={`bp3-button-group ${styles.menubarButton}`}>
+    <ButtonGroup className={`${styles.menubarButton}`}>
       <Tooltip
         content="Undo"
         position="bottom"
@@ -14,7 +15,7 @@ const UndoRedo = React.memo((props) => {
       >
         <AnchorButton
           type="button"
-          className="bp3-button bp3-icon-undo"
+          icon={IconNames.UNDO}
           disabled={undoDisabled}
           onClick={() => {
             dispatch({ type: "@@undoable/undo" });
@@ -32,7 +33,7 @@ const UndoRedo = React.memo((props) => {
       >
         <AnchorButton
           type="button"
-          className="bp3-button bp3-icon-redo"
+          icon={IconNames.REDO}
           disabled={redoDisabled}
           onClick={() => {
             dispatch({ type: "@@undoable/redo" });
@@ -43,7 +44,7 @@ const UndoRedo = React.memo((props) => {
           data-testid="redo"
         />
       </Tooltip>
-    </div>
+    </ButtonGroup>
   );
 });
 

client/src/components/miniHistogram/index.js

@@ -72,7 +72,6 @@ export default class MiniHistogram extends React.PureComponent {
         popoverClassName={Classes.POPOVER_CONTENT_SIZING}
       >
         <canvas
-          className="bp3-popover-targer"
           style={{
             marginRight: 5,
             width,

client/src/components/miniStackedBar/index.js

@@ -1,4 +1,3 @@
-// jshint esversion: 6
 import React from "react";
 
 export default class MiniStackedBar extends React.PureComponent {
@@ -59,7 +58,6 @@ export default class MiniStackedBar extends React.PureComponent {
 
     return (
       <canvas
-        className="bp3-popover-targer"
         style={{
           marginRight: 5,
           width,

dev_docs/cxg.md

@@ -1,3 +1,5 @@
+## UPDATE (9/30/2020): Starting today, the name Corpora will only be used as the internal project name, with cellxgene Data Portal being the official product name
+
 # CXG Data Format Specification
 
 Document Status: _draft_

docs/_config.yml

@@ -15,6 +15,8 @@ nav:
     url: posts/gallery
   - title: Demo datasets
     url: posts/demo-data
+  - title: All other datasets
+    url: https://cellxgene.cziscience.com/
   - title: Preparing your data
     url: posts/prepare
   - title: Launching cellxgene
@@ -33,5 +35,3 @@ nav:
     url: posts/contribute
   - title: Contact & finding help
     url: posts/contact
-  - title: cellxgene.cziscience.com
-    url: posts/cellxgene_cziscience_com

docs/_site/index.html

@@ -7,19 +7,19 @@
 
 <!-- Begin Jekyll SEO tag v2.6.1 -->
 <title>Index | cellxgene</title>
-<meta name="generator" content="Jekyll v3.8.7" />
+<meta name="generator" content="Jekyll v3.9.0" />
 <meta property="og:title" content="Index" />
 <meta property="og:locale" content="en_US" />
 <meta name="description" content="An interactive explorer for single-cell transcriptomics data" />
 <meta property="og:description" content="An interactive explorer for single-cell transcriptomics data" />
-<link rel="canonical" href="https://chanzuckerberg.github.io/cellxgene/" />
-<meta property="og:url" content="https://chanzuckerberg.github.io/cellxgene/" />
+<link rel="canonical" href="http://localhost:4000/cellxgene/" />
+<meta property="og:url" content="http://localhost:4000/cellxgene/" />
 <meta property="og:site_name" content="cellxgene" />
 <script type="application/ld+json">
-{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"An interactive explorer for single-cell transcriptomics data","@type":"WebSite","headline":"Index","url":"https://chanzuckerberg.github.io/cellxgene/","name":"cellxgene","@context":"https://schema.org"}</script>
+{"url":"http://localhost:4000/cellxgene/","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"http://localhost:4000/cellxgene/cellxgene-logo.png"}},"headline":"Index","name":"cellxgene","description":"An interactive explorer for single-cell transcriptomics data","@type":"WebSite","@context":"https://schema.org"}</script>
 <!-- End Jekyll SEO tag -->
 
-    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=6bda27f5542fb7f469425e1cd99f2f37268b095f">
+    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=365253953c0e46d0d4d50d9072d6ae60a12b4eb7">
     <!--[if lt IE 9]>
     <script src="https://cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv.min.js"></script>
     <![endif]-->
@@ -50,6 +50,10 @@
         
         
         
+        <a href="https://cellxgene.cziscience.com/" class="btn">All other datasets</a><br>
+        
+        
+        
         <a href="/cellxgene/posts/prepare" class="btn">Preparing your data</a><br>
         
         
@@ -86,10 +90,6 @@
         
         
 
-        <a href="/cellxgene/posts/cellxgene_cziscience_com" class="btn">cellxgene.cziscience.com</a><br>
-        
-        
-
         <a href="https://github.com/chanzuckerberg/cellxgene" class="btn" target="_blank">Code</a>
     </p>
     </header>

docs/_site/posts/annotations.html

@@ -7,19 +7,19 @@
 
 <!-- Begin Jekyll SEO tag v2.6.1 -->
 <title>annotations | cellxgene</title>
-<meta name="generator" content="Jekyll v3.8.7" />
+<meta name="generator" content="Jekyll v3.9.0" />
 <meta property="og:title" content="annotations" />
 <meta property="og:locale" content="en_US" />
 <meta name="description" content="Creating annotations" />
 <meta property="og:description" content="Creating annotations" />
-<link rel="canonical" href="https://chanzuckerberg.github.io/cellxgene/posts/annotations.html" />
-<meta property="og:url" content="https://chanzuckerberg.github.io/cellxgene/posts/annotations.html" />
+<link rel="canonical" href="http://localhost:4000/cellxgene/posts/annotations.html" />
+<meta property="og:url" content="http://localhost:4000/cellxgene/posts/annotations.html" />
 <meta property="og:site_name" content="cellxgene" />
 <script type="application/ld+json">
-{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"Creating annotations","@type":"WebPage","headline":"annotations","url":"https://chanzuckerberg.github.io/cellxgene/posts/annotations.html","@context":"https://schema.org"}</script>
+{"url":"http://localhost:4000/cellxgene/posts/annotations.html","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"http://localhost:4000/cellxgene/cellxgene-logo.png"}},"headline":"annotations","description":"Creating annotations","@type":"WebPage","@context":"https://schema.org"}</script>
 <!-- End Jekyll SEO tag -->
 
-    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=6bda27f5542fb7f469425e1cd99f2f37268b095f">
+    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=365253953c0e46d0d4d50d9072d6ae60a12b4eb7">
     <!--[if lt IE 9]>
     <script src="https://cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv.min.js"></script>
     <![endif]-->
@@ -50,6 +50,10 @@
         
         
         
+        <a href="https://cellxgene.cziscience.com/" class="btn">All other datasets</a><br>
+        
+        
+        
         <a href="/cellxgene/posts/prepare" class="btn">Preparing your data</a><br>
         
         
@@ -86,10 +90,6 @@
         
         
 
-        <a href="/cellxgene/posts/cellxgene_cziscience_com" class="btn">cellxgene.cziscience.com</a><br>
-        
-        
-
         <a href="https://github.com/chanzuckerberg/cellxgene" class="btn" target="_blank">Code</a>
     </p>
     </header>

docs/_site/posts/contact.html

@@ -7,19 +7,19 @@
 
 <!-- Begin Jekyll SEO tag v2.6.1 -->
 <title>Contact | cellxgene</title>
-<meta name="generator" content="Jekyll v3.8.7" />
+<meta name="generator" content="Jekyll v3.9.0" />
 <meta property="og:title" content="Contact" />
 <meta property="og:locale" content="en_US" />
 <meta name="description" content="Contact" />
 <meta property="og:description" content="Contact" />
-<link rel="canonical" href="https://chanzuckerberg.github.io/cellxgene/posts/contact.html" />
-<meta property="og:url" content="https://chanzuckerberg.github.io/cellxgene/posts/contact.html" />
+<link rel="canonical" href="http://localhost:4000/cellxgene/posts/contact.html" />
+<meta property="og:url" content="http://localhost:4000/cellxgene/posts/contact.html" />
 <meta property="og:site_name" content="cellxgene" />
 <script type="application/ld+json">
-{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"Contact","@type":"WebPage","headline":"Contact","url":"https://chanzuckerberg.github.io/cellxgene/posts/contact.html","@context":"https://schema.org"}</script>
+{"url":"http://localhost:4000/cellxgene/posts/contact.html","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"http://localhost:4000/cellxgene/cellxgene-logo.png"}},"headline":"Contact","description":"Contact","@type":"WebPage","@context":"https://schema.org"}</script>
 <!-- End Jekyll SEO tag -->
 
-    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=6bda27f5542fb7f469425e1cd99f2f37268b095f">
+    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=365253953c0e46d0d4d50d9072d6ae60a12b4eb7">
     <!--[if lt IE 9]>
     <script src="https://cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv.min.js"></script>
     <![endif]-->
@@ -50,6 +50,10 @@
         
         
         
+        <a href="https://cellxgene.cziscience.com/" class="btn">All other datasets</a><br>
+        
+        
+        
         <a href="/cellxgene/posts/prepare" class="btn">Preparing your data</a><br>
         
         
@@ -86,10 +90,6 @@
         
         
 
-        <a href="/cellxgene/posts/cellxgene_cziscience_com" class="btn">cellxgene.cziscience.com</a><br>
-        
-        
-
         <a href="https://github.com/chanzuckerberg/cellxgene" class="btn" target="_blank">Code</a>
     </p>
     </header>

docs/_site/posts/contribute.html

@@ -7,19 +7,19 @@
 
 <!-- Begin Jekyll SEO tag v2.6.1 -->
 <title>Code of conduct | cellxgene</title>
-<meta name="generator" content="Jekyll v3.8.7" />
+<meta name="generator" content="Jekyll v3.9.0" />
 <meta property="og:title" content="Code of conduct" />
 <meta property="og:locale" content="en_US" />
 <meta name="description" content="An interactive explorer for single-cell transcriptomics data" />
 <meta property="og:description" content="An interactive explorer for single-cell transcriptomics data" />
-<link rel="canonical" href="https://chanzuckerberg.github.io/cellxgene/posts/contribute.html" />
-<meta property="og:url" content="https://chanzuckerberg.github.io/cellxgene/posts/contribute.html" />
+<link rel="canonical" href="http://localhost:4000/cellxgene/posts/contribute.html" />
+<meta property="og:url" content="http://localhost:4000/cellxgene/posts/contribute.html" />
 <meta property="og:site_name" content="cellxgene" />
 <script type="application/ld+json">
-{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"An interactive explorer for single-cell transcriptomics data","@type":"WebPage","headline":"Code of conduct","url":"https://chanzuckerberg.github.io/cellxgene/posts/contribute.html","@context":"https://schema.org"}</script>
+{"url":"http://localhost:4000/cellxgene/posts/contribute.html","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"http://localhost:4000/cellxgene/cellxgene-logo.png"}},"headline":"Code of conduct","description":"An interactive explorer for single-cell transcriptomics data","@type":"WebPage","@context":"https://schema.org"}</script>
 <!-- End Jekyll SEO tag -->
 
-    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=6bda27f5542fb7f469425e1cd99f2f37268b095f">
+    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=365253953c0e46d0d4d50d9072d6ae60a12b4eb7">
     <!--[if lt IE 9]>
     <script src="https://cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv.min.js"></script>
     <![endif]-->
@@ -50,6 +50,10 @@
         
         
         
+        <a href="https://cellxgene.cziscience.com/" class="btn">All other datasets</a><br>
+        
+        
+        
         <a href="/cellxgene/posts/prepare" class="btn">Preparing your data</a><br>
         
         
@@ -86,10 +90,6 @@
         
         
 
-        <a href="/cellxgene/posts/cellxgene_cziscience_com" class="btn">cellxgene.cziscience.com</a><br>
-        
-        
-
         <a href="https://github.com/chanzuckerberg/cellxgene" class="btn" target="_blank">Code</a>
     </p>
     </header>

docs/_site/posts/demo-data.html

@@ -7,19 +7,19 @@
 
 <!-- Begin Jekyll SEO tag v2.6.1 -->
 <title>demo-data | cellxgene</title>
-<meta name="generator" content="Jekyll v3.8.7" />
+<meta name="generator" content="Jekyll v3.9.0" />
 <meta property="og:title" content="demo-data" />
 <meta property="og:locale" content="en_US" />
 <meta name="description" content="Demo datasets" />
 <meta property="og:description" content="Demo datasets" />
-<link rel="canonical" href="https://chanzuckerberg.github.io/cellxgene/posts/demo-data.html" />
-<meta property="og:url" content="https://chanzuckerberg.github.io/cellxgene/posts/demo-data.html" />
+<link rel="canonical" href="http://localhost:4000/cellxgene/posts/demo-data.html" />
+<meta property="og:url" content="http://localhost:4000/cellxgene/posts/demo-data.html" />
 <meta property="og:site_name" content="cellxgene" />
 <script type="application/ld+json">
-{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"Demo datasets","@type":"WebPage","headline":"demo-data","url":"https://chanzuckerberg.github.io/cellxgene/posts/demo-data.html","@context":"https://schema.org"}</script>
+{"url":"http://localhost:4000/cellxgene/posts/demo-data.html","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"http://localhost:4000/cellxgene/cellxgene-logo.png"}},"headline":"demo-data","description":"Demo datasets","@type":"WebPage","@context":"https://schema.org"}</script>
 <!-- End Jekyll SEO tag -->
 
-    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=6bda27f5542fb7f469425e1cd99f2f37268b095f">
+    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=365253953c0e46d0d4d50d9072d6ae60a12b4eb7">
     <!--[if lt IE 9]>
     <script src="https://cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv.min.js"></script>
     <![endif]-->
@@ -50,6 +50,10 @@
         
         
         
+        <a href="https://cellxgene.cziscience.com/" class="btn">All other datasets</a><br>
+        
+        
+        
         <a href="/cellxgene/posts/prepare" class="btn">Preparing your data</a><br>
         
         
@@ -86,10 +90,6 @@
         
         
 
-        <a href="/cellxgene/posts/cellxgene_cziscience_com" class="btn">cellxgene.cziscience.com</a><br>
-        
-        
-
         <a href="https://github.com/chanzuckerberg/cellxgene" class="btn" target="_blank">Code</a>
     </p>
     </header>

docs/_site/posts/gallery.html

@@ -7,19 +7,19 @@
 
 <!-- Begin Jekyll SEO tag v2.6.1 -->
 <title>Gallery | cellxgene</title>
-<meta name="generator" content="Jekyll v3.8.7" />
+<meta name="generator" content="Jekyll v3.9.0" />
 <meta property="og:title" content="Gallery" />
 <meta property="og:locale" content="en_US" />
 <meta name="description" content="An interactive explorer for single-cell transcriptomics data" />
 <meta property="og:description" content="An interactive explorer for single-cell transcriptomics data" />
-<link rel="canonical" href="https://chanzuckerberg.github.io/cellxgene/posts/gallery.html" />
-<meta property="og:url" content="https://chanzuckerberg.github.io/cellxgene/posts/gallery.html" />
+<link rel="canonical" href="http://localhost:4000/cellxgene/posts/gallery.html" />
+<meta property="og:url" content="http://localhost:4000/cellxgene/posts/gallery.html" />
 <meta property="og:site_name" content="cellxgene" />
 <script type="application/ld+json">
-{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"An interactive explorer for single-cell transcriptomics data","@type":"WebPage","headline":"Gallery","url":"https://chanzuckerberg.github.io/cellxgene/posts/gallery.html","@context":"https://schema.org"}</script>
+{"url":"http://localhost:4000/cellxgene/posts/gallery.html","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"http://localhost:4000/cellxgene/cellxgene-logo.png"}},"headline":"Gallery","description":"An interactive explorer for single-cell transcriptomics data","@type":"WebPage","@context":"https://schema.org"}</script>
 <!-- End Jekyll SEO tag -->
 
-    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=6bda27f5542fb7f469425e1cd99f2f37268b095f">
+    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=365253953c0e46d0d4d50d9072d6ae60a12b4eb7">
     <!--[if lt IE 9]>
     <script src="https://cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv.min.js"></script>
     <![endif]-->
@@ -50,6 +50,10 @@
         
         
         
+        <a href="https://cellxgene.cziscience.com/" class="btn">All other datasets</a><br>
+        
+        
+        
         <a href="/cellxgene/posts/prepare" class="btn">Preparing your data</a><br>
         
         
@@ -86,10 +90,6 @@
         
         
 
-        <a href="/cellxgene/posts/cellxgene_cziscience_com" class="btn">cellxgene.cziscience.com</a><br>
-        
-        
-
         <a href="https://github.com/chanzuckerberg/cellxgene" class="btn" target="_blank">Code</a>
     </p>
     </header>
@@ -130,7 +130,7 @@ Check out the cool data that our users are using cellxgene to explore!</p>
 
 <h3 id="melanoma"><a href="https://melanoma.cellgeni.sanger.ac.uk/">Melanoma</a></h3>
 
-<h3 id="czis-own-cellxgene-site"><a href="cellxgene_cziscience_com">CZI’s own cellxgene site</a></h3>
+<h3 id="czis-own-cellxgene-site"><a href="https://cellxgene.cziscience.com/">CZI’s own cellxgene site</a></h3>
 
 <p><em>Want us to link to your dataset here? <a href="contact">Just send us a note!</a></em></p>
 

docs/_site/posts/hosted.html

@@ -7,19 +7,19 @@
 
 <!-- Begin Jekyll SEO tag v2.6.1 -->
 <title>Hosting cellxgene on the web | cellxgene</title>
-<meta name="generator" content="Jekyll v3.8.7" />
+<meta name="generator" content="Jekyll v3.9.0" />
 <meta property="og:title" content="Hosting cellxgene on the web" />
 <meta property="og:locale" content="en_US" />
 <meta name="description" content="An interactive explorer for single-cell transcriptomics data" />
 <meta property="og:description" content="An interactive explorer for single-cell transcriptomics data" />
-<link rel="canonical" href="https://chanzuckerberg.github.io/cellxgene/posts/hosted.html" />
-<meta property="og:url" content="https://chanzuckerberg.github.io/cellxgene/posts/hosted.html" />
+<link rel="canonical" href="http://localhost:4000/cellxgene/posts/hosted.html" />
+<meta property="og:url" content="http://localhost:4000/cellxgene/posts/hosted.html" />
 <meta property="og:site_name" content="cellxgene" />
 <script type="application/ld+json">
-{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"An interactive explorer for single-cell transcriptomics data","@type":"WebPage","headline":"Hosting cellxgene on the web","url":"https://chanzuckerberg.github.io/cellxgene/posts/hosted.html","@context":"https://schema.org"}</script>
+{"url":"http://localhost:4000/cellxgene/posts/hosted.html","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"http://localhost:4000/cellxgene/cellxgene-logo.png"}},"headline":"Hosting cellxgene on the web","description":"An interactive explorer for single-cell transcriptomics data","@type":"WebPage","@context":"https://schema.org"}</script>
 <!-- End Jekyll SEO tag -->
 
-    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=6bda27f5542fb7f469425e1cd99f2f37268b095f">
+    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=365253953c0e46d0d4d50d9072d6ae60a12b4eb7">
     <!--[if lt IE 9]>
     <script src="https://cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv.min.js"></script>
     <![endif]-->
@@ -50,6 +50,10 @@
         
         
         
+        <a href="https://cellxgene.cziscience.com/" class="btn">All other datasets</a><br>
+        
+        
+        
         <a href="/cellxgene/posts/prepare" class="btn">Preparing your data</a><br>
         
         
@@ -86,10 +90,6 @@
         
         
 
-        <a href="/cellxgene/posts/cellxgene_cziscience_com" class="btn">cellxgene.cziscience.com</a><br>
-        
-        
-
         <a href="https://github.com/chanzuckerberg/cellxgene" class="btn" target="_blank">Code</a>
     </p>
     </header>
@@ -139,35 +139,36 @@
 
 <h1 id="deploying-cellxgene-with-heroku">Deploying cellxgene with Heroku</h1>
 
-<h2 id="quickstart">Quickstart</h2>
+<h2 id="heroku-support">Heroku Support</h2>
 
-<p>Clicking on the following button will forward you to Heroku to begin the deployment process:</p>
+<p>The cellxgene team has decided to end our support for our experimental deploy to Heroku button as we move towards providing a supported method of hosted cellxgene.</p>
 
-<p><a href="https://heroku.com/deploy?template=https://github.com/chanzuckerberg/cellxgene">
- <img src="https://www.herokucdn.com/deploy/button.svg" alt="Deploy" />
-</a></p>
+<p>While we no longer directly support Heroku, it is still possible to create a Heroku app via <a href="https://github.com/chanzuckerberg/cellxgene/blob/main/Dockerfile">our provided Dockerfile here</a> and <a href="https://devcenter.heroku.com/articles/build-docker-images-heroku-yml">Heroku’s documentation</a>.</p>
 
-<p>If not already logged in to Heroku, there you will be prompted to log in or sign up for an account.</p>
+<p>You may have to tweak the <code class="language-plaintext highlighter-rouge">Dockerfile</code> like so:</p>
 
-<p>Once logged in you will be sent to the setup page. Here you can set some of the basic settings for the app:</p>
+<pre><code class="language-Dockerfile">FROM ubuntu:bionic
 
-<h3 id="default-settings">Default settings</h3>
+ENV LC_ALL=C.UTF-8
+ENV LANG=C.UTF-8
 
-<ul>
-  <li><code class="language-plaintext highlighter-rouge">App name</code>: the unique name for your deployment</li>
-  <li>This will also serve as the default URL (e.g. https://cellxgene.herokapp.com/)</li>
-  <li><code class="language-plaintext highlighter-rouge">App owner</code>: Who will own this app. Either you personally or an organization/team</li>
-  <li><code class="language-plaintext highlighter-rouge">Region</code>: Location of the server where the app will be deployed (EU or US)</li>
-</ul>
+RUN apt-get update &amp;&amp; \
+    apt-get install -y build-essential libxml2-dev python3-dev python3-pip zlib1g-dev python3-requests &amp;&amp; \
+    pip3 install cellxgene
 
-<h3 id="configuration">Configuration</h3>
+# ENTRYPOINT ["cellxgene"]  # Heroku doesn't work well with ENTRYPOINT
+</code></pre>
 
-<ul>
-  <li><code class="language-plaintext highlighter-rouge">DATASET</code>: A <em>publicly</em> accessible URL pointing to a .h5ad file to view</li>
-  <li>This defaults to pbm3k.h5ad</li>
-</ul>
+<p>and provide a <code class="language-plaintext highlighter-rouge">heroku.yml</code> file similar to this:</p>
 
-<p>After filling out the settings and pressing the <code class="language-plaintext highlighter-rouge">Deploy app</code> button Heroku will begin building your deployment. This process will take a few minutes, but once completed you will have a personal free hosted version of cellxgene!</p>
+<div class="language-yml highlighter-rouge"><div class="highlight"><pre class="highlight"><code><span class="na">build</span><span class="pi">:</span>
+  <span class="na">docker</span><span class="pi">:</span>
+    <span class="na">web</span><span class="pi">:</span> <span class="s">Dockerfile</span>
+<span class="na">run</span><span class="pi">:</span>
+  <span class="na">web</span><span class="pi">:</span>
+    <span class="na">command</span><span class="pi">:</span>
+      <span class="pi">-</span> <span class="s">cellxgene launch --host 0.0.0.0 --port $PORT $DATASET</span> <span class="c1"># the DATATSET config var must be defined in your dashboard settings.</span>
+</code></pre></div></div>
 
 <h2 id="what-is-heroku">What is Heroku?</h2>
 

docs/_site/posts/hosted.md

@@ -38,31 +38,38 @@ If you know of other solutions, drop us a note and we'll add to this list.
 
 # Deploying cellxgene with Heroku
 
-## Quickstart
+## Heroku Support
 
-Clicking on the following button will forward you to Heroku to begin the deployment process:
+The cellxgene team has decided to end our support for our experimental deploy to Heroku button as we move towards providing a supported method of hosted cellxgene.
 
-<a href="https://heroku.com/deploy?template=https://github.com/chanzuckerberg/cellxgene">
- <img src="https://www.herokucdn.com/deploy/button.svg" alt="Deploy">
-</a>
+While we no longer directly support Heroku, it is still possible to create a Heroku app via [our provided Dockerfile here](https://github.com/chanzuckerberg/cellxgene/blob/main/Dockerfile) and [Heroku's documentation](https://devcenter.heroku.com/articles/build-docker-images-heroku-yml).
 
-If not already logged in to Heroku, there you will be prompted to log in or sign up for an account.
+You may have to tweak the `Dockerfile` like so:
 
-Once logged in you will be sent to the setup page. Here you can set some of the basic settings for the app:
+```Dockerfile
+FROM ubuntu:bionic
 
-### Default settings
+ENV LC_ALL=C.UTF-8
+ENV LANG=C.UTF-8
 
-- `App name`: the unique name for your deployment
-- This will also serve as the default URL (e.g. https://cellxgene.herokapp.com/)
-- `App owner`: Who will own this app. Either you personally or an organization/team
-- `Region`: Location of the server where the app will be deployed (EU or US)
+RUN apt-get update && \
+    apt-get install -y build-essential libxml2-dev python3-dev python3-pip zlib1g-dev python3-requests && \
+    pip3 install cellxgene
 
-### Configuration
+# ENTRYPOINT ["cellxgene"]  # Heroku doesn't work well with ENTRYPOINT
+```
 
-- `DATASET`: A _publicly_ accessible URL pointing to a .h5ad file to view
-- This defaults to pbm3k.h5ad
+and provide a `heroku.yml` file similar to this:
 
-After filling out the settings and pressing the `Deploy app` button Heroku will begin building your deployment. This process will take a few minutes, but once completed you will have a personal free hosted version of cellxgene!
+```yml
+build:
+  docker:
+    web: Dockerfile
+run:
+  web:
+    command:
+      - cellxgene launch --host 0.0.0.0 --port $PORT $DATASET # the DATATSET config var must be defined in your dashboard settings.
+```
 
 ## What is Heroku?
 

docs/_site/posts/install.html

@@ -7,19 +7,19 @@
 
 <!-- Begin Jekyll SEO tag v2.6.1 -->
 <title>Install | cellxgene</title>
-<meta name="generator" content="Jekyll v3.8.7" />
+<meta name="generator" content="Jekyll v3.9.0" />
 <meta property="og:title" content="Install" />
 <meta property="og:locale" content="en_US" />
 <meta name="description" content="An interactive explorer for single-cell transcriptomics data" />
 <meta property="og:description" content="An interactive explorer for single-cell transcriptomics data" />
-<link rel="canonical" href="https://chanzuckerberg.github.io/cellxgene/posts/install.html" />
-<meta property="og:url" content="https://chanzuckerberg.github.io/cellxgene/posts/install.html" />
+<link rel="canonical" href="http://localhost:4000/cellxgene/posts/install.html" />
+<meta property="og:url" content="http://localhost:4000/cellxgene/posts/install.html" />
 <meta property="og:site_name" content="cellxgene" />
 <script type="application/ld+json">
-{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"An interactive explorer for single-cell transcriptomics data","@type":"WebPage","headline":"Install","url":"https://chanzuckerberg.github.io/cellxgene/posts/install.html","@context":"https://schema.org"}</script>
+{"url":"http://localhost:4000/cellxgene/posts/install.html","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"http://localhost:4000/cellxgene/cellxgene-logo.png"}},"headline":"Install","description":"An interactive explorer for single-cell transcriptomics data","@type":"WebPage","@context":"https://schema.org"}</script>
 <!-- End Jekyll SEO tag -->
 
-    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=6bda27f5542fb7f469425e1cd99f2f37268b095f">
+    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=365253953c0e46d0d4d50d9072d6ae60a12b4eb7">
     <!--[if lt IE 9]>
     <script src="https://cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv.min.js"></script>
     <![endif]-->
@@ -50,6 +50,10 @@
         
         
         
+        <a href="https://cellxgene.cziscience.com/" class="btn">All other datasets</a><br>
+        
+        
+        
         <a href="/cellxgene/posts/prepare" class="btn">Preparing your data</a><br>
         
         
@@ -86,10 +90,6 @@
         
         
 
-        <a href="/cellxgene/posts/cellxgene_cziscience_com" class="btn">cellxgene.cziscience.com</a><br>
-        
-        
-
         <a href="https://github.com/chanzuckerberg/cellxgene" class="btn" target="_blank">Code</a>
     </p>
     </header>

docs/_site/posts/launch.html

@@ -7,19 +7,19 @@
 
 <!-- Begin Jekyll SEO tag v2.6.1 -->
 <title>demo-data | cellxgene</title>
-<meta name="generator" content="Jekyll v3.8.7" />
+<meta name="generator" content="Jekyll v3.9.0" />
 <meta property="og:title" content="demo-data" />
 <meta property="og:locale" content="en_US" />
 <meta name="description" content="Demo datasets" />
 <meta property="og:description" content="Demo datasets" />
-<link rel="canonical" href="https://chanzuckerberg.github.io/cellxgene/posts/launch.html" />
-<meta property="og:url" content="https://chanzuckerberg.github.io/cellxgene/posts/launch.html" />
+<link rel="canonical" href="http://localhost:4000/cellxgene/posts/launch.html" />
+<meta property="og:url" content="http://localhost:4000/cellxgene/posts/launch.html" />
 <meta property="og:site_name" content="cellxgene" />
 <script type="application/ld+json">
-{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"Demo datasets","@type":"WebPage","headline":"demo-data","url":"https://chanzuckerberg.github.io/cellxgene/posts/launch.html","@context":"https://schema.org"}</script>
+{"url":"http://localhost:4000/cellxgene/posts/launch.html","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"http://localhost:4000/cellxgene/cellxgene-logo.png"}},"headline":"demo-data","description":"Demo datasets","@type":"WebPage","@context":"https://schema.org"}</script>
 <!-- End Jekyll SEO tag -->
 
-    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=6bda27f5542fb7f469425e1cd99f2f37268b095f">
+    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=365253953c0e46d0d4d50d9072d6ae60a12b4eb7">
     <!--[if lt IE 9]>
     <script src="https://cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv.min.js"></script>
     <![endif]-->
@@ -50,6 +50,10 @@
         
         
         
+        <a href="https://cellxgene.cziscience.com/" class="btn">All other datasets</a><br>
+        
+        
+        
         <a href="/cellxgene/posts/prepare" class="btn">Preparing your data</a><br>
         
         
@@ -86,10 +90,6 @@
         
         
 
-        <a href="/cellxgene/posts/cellxgene_cziscience_com" class="btn">cellxgene.cziscience.com</a><br>
-        
-        
-
         <a href="https://github.com/chanzuckerberg/cellxgene" class="btn" target="_blank">Code</a>
     </p>
     </header>

docs/_site/posts/methods.html

@@ -7,19 +7,19 @@
 
 <!-- Begin Jekyll SEO tag v2.6.1 -->
 <title>Methods | cellxgene</title>
-<meta name="generator" content="Jekyll v3.8.7" />
+<meta name="generator" content="Jekyll v3.9.0" />
 <meta property="og:title" content="Methods" />
 <meta property="og:locale" content="en_US" />
 <meta name="description" content="An interactive explorer for single-cell transcriptomics data" />
 <meta property="og:description" content="An interactive explorer for single-cell transcriptomics data" />
-<link rel="canonical" href="https://chanzuckerberg.github.io/cellxgene/posts/methods.html" />
-<meta property="og:url" content="https://chanzuckerberg.github.io/cellxgene/posts/methods.html" />
+<link rel="canonical" href="http://localhost:4000/cellxgene/posts/methods.html" />
+<meta property="og:url" content="http://localhost:4000/cellxgene/posts/methods.html" />
 <meta property="og:site_name" content="cellxgene" />
 <script type="application/ld+json">
-{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"An interactive explorer for single-cell transcriptomics data","@type":"WebPage","headline":"Methods","url":"https://chanzuckerberg.github.io/cellxgene/posts/methods.html","@context":"https://schema.org"}</script>
+{"url":"http://localhost:4000/cellxgene/posts/methods.html","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"http://localhost:4000/cellxgene/cellxgene-logo.png"}},"headline":"Methods","description":"An interactive explorer for single-cell transcriptomics data","@type":"WebPage","@context":"https://schema.org"}</script>
 <!-- End Jekyll SEO tag -->
 
-    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=6bda27f5542fb7f469425e1cd99f2f37268b095f">
+    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=365253953c0e46d0d4d50d9072d6ae60a12b4eb7">
     <!--[if lt IE 9]>
     <script src="https://cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv.min.js"></script>
     <![endif]-->
@@ -50,6 +50,10 @@
         
         
         
+        <a href="https://cellxgene.cziscience.com/" class="btn">All other datasets</a><br>
+        
+        
+        
         <a href="/cellxgene/posts/prepare" class="btn">Preparing your data</a><br>
         
         
@@ -86,10 +90,6 @@
         
         
 
-        <a href="/cellxgene/posts/cellxgene_cziscience_com" class="btn">cellxgene.cziscience.com</a><br>
-        
-        
-
         <a href="https://github.com/chanzuckerberg/cellxgene" class="btn" target="_blank">Code</a>
     </p>
     </header>

docs/_site/posts/prepare.html

@@ -7,19 +7,19 @@
 
 <!-- Begin Jekyll SEO tag v2.6.1 -->
 <title>prepare | cellxgene</title>
-<meta name="generator" content="Jekyll v3.8.7" />
+<meta name="generator" content="Jekyll v3.9.0" />
 <meta property="og:title" content="prepare" />
 <meta property="og:locale" content="en_US" />
 <meta name="description" content="Preparing your data" />
 <meta property="og:description" content="Preparing your data" />
-<link rel="canonical" href="https://chanzuckerberg.github.io/cellxgene/posts/prepare.html" />
-<meta property="og:url" content="https://chanzuckerberg.github.io/cellxgene/posts/prepare.html" />
+<link rel="canonical" href="http://localhost:4000/cellxgene/posts/prepare.html" />
+<meta property="og:url" content="http://localhost:4000/cellxgene/posts/prepare.html" />
 <meta property="og:site_name" content="cellxgene" />
 <script type="application/ld+json">
-{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"Preparing your data","@type":"WebPage","headline":"prepare","url":"https://chanzuckerberg.github.io/cellxgene/posts/prepare.html","@context":"https://schema.org"}</script>
+{"url":"http://localhost:4000/cellxgene/posts/prepare.html","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"http://localhost:4000/cellxgene/cellxgene-logo.png"}},"headline":"prepare","description":"Preparing your data","@type":"WebPage","@context":"https://schema.org"}</script>
 <!-- End Jekyll SEO tag -->
 
-    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=6bda27f5542fb7f469425e1cd99f2f37268b095f">
+    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=365253953c0e46d0d4d50d9072d6ae60a12b4eb7">
     <!--[if lt IE 9]>
     <script src="https://cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv.min.js"></script>
     <![endif]-->
@@ -50,6 +50,10 @@
         
         
         
+        <a href="https://cellxgene.cziscience.com/" class="btn">All other datasets</a><br>
+        
+        
+        
         <a href="/cellxgene/posts/prepare" class="btn"><b>Preparing your data</b></a><br>
         
         
@@ -86,10 +90,6 @@
         
         
 
-        <a href="/cellxgene/posts/cellxgene_cziscience_com" class="btn">cellxgene.cziscience.com</a><br>
-        
-        
-
         <a href="https://github.com/chanzuckerberg/cellxgene" class="btn" target="_blank">Code</a>
     </p>
     </header>

docs/_site/posts/roadmap.html

@@ -7,19 +7,19 @@
 
 <!-- Begin Jekyll SEO tag v2.6.1 -->
 <title>roadmap | cellxgene</title>
-<meta name="generator" content="Jekyll v3.8.7" />
+<meta name="generator" content="Jekyll v3.9.0" />
 <meta property="og:title" content="roadmap" />
 <meta property="og:locale" content="en_US" />
 <meta name="description" content="Roadmap" />
 <meta property="og:description" content="Roadmap" />
-<link rel="canonical" href="https://chanzuckerberg.github.io/cellxgene/posts/roadmap.html" />
-<meta property="og:url" content="https://chanzuckerberg.github.io/cellxgene/posts/roadmap.html" />
+<link rel="canonical" href="http://localhost:4000/cellxgene/posts/roadmap.html" />
+<meta property="og:url" content="http://localhost:4000/cellxgene/posts/roadmap.html" />
 <meta property="og:site_name" content="cellxgene" />
 <script type="application/ld+json">
-{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"Roadmap","@type":"WebPage","headline":"roadmap","url":"https://chanzuckerberg.github.io/cellxgene/posts/roadmap.html","@context":"https://schema.org"}</script>
+{"url":"http://localhost:4000/cellxgene/posts/roadmap.html","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"http://localhost:4000/cellxgene/cellxgene-logo.png"}},"headline":"roadmap","description":"Roadmap","@type":"WebPage","@context":"https://schema.org"}</script>
 <!-- End Jekyll SEO tag -->
 
-    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=6bda27f5542fb7f469425e1cd99f2f37268b095f">
+    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=365253953c0e46d0d4d50d9072d6ae60a12b4eb7">
     <!--[if lt IE 9]>
     <script src="https://cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv.min.js"></script>
     <![endif]-->
@@ -50,6 +50,10 @@
         
         
         
+        <a href="https://cellxgene.cziscience.com/" class="btn">All other datasets</a><br>
+        
+        
+        
         <a href="/cellxgene/posts/prepare" class="btn">Preparing your data</a><br>
         
         
@@ -86,10 +90,6 @@
         
         
 
-        <a href="/cellxgene/posts/cellxgene_cziscience_com" class="btn">cellxgene.cziscience.com</a><br>
-        
-        
-
         <a href="https://github.com/chanzuckerberg/cellxgene" class="btn" target="_blank">Code</a>
     </p>
     </header>

docs/_site/posts/troubleshooting.html

@@ -7,19 +7,19 @@
 
 <!-- Begin Jekyll SEO tag v2.6.1 -->
 <title>Troubleshooting | cellxgene</title>
-<meta name="generator" content="Jekyll v3.8.7" />
+<meta name="generator" content="Jekyll v3.9.0" />
 <meta property="og:title" content="Troubleshooting" />
 <meta property="og:locale" content="en_US" />
 <meta name="description" content="Troubleshooting" />
 <meta property="og:description" content="Troubleshooting" />
-<link rel="canonical" href="https://chanzuckerberg.github.io/cellxgene/posts/troubleshooting.html" />
-<meta property="og:url" content="https://chanzuckerberg.github.io/cellxgene/posts/troubleshooting.html" />
+<link rel="canonical" href="http://localhost:4000/cellxgene/posts/troubleshooting.html" />
+<meta property="og:url" content="http://localhost:4000/cellxgene/posts/troubleshooting.html" />
 <meta property="og:site_name" content="cellxgene" />
 <script type="application/ld+json">
-{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"Troubleshooting","@type":"WebPage","headline":"Troubleshooting","url":"https://chanzuckerberg.github.io/cellxgene/posts/troubleshooting.html","@context":"https://schema.org"}</script>
+{"url":"http://localhost:4000/cellxgene/posts/troubleshooting.html","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"http://localhost:4000/cellxgene/cellxgene-logo.png"}},"headline":"Troubleshooting","description":"Troubleshooting","@type":"WebPage","@context":"https://schema.org"}</script>
 <!-- End Jekyll SEO tag -->
 
-    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=6bda27f5542fb7f469425e1cd99f2f37268b095f">
+    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=365253953c0e46d0d4d50d9072d6ae60a12b4eb7">
     <!--[if lt IE 9]>
     <script src="https://cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv.min.js"></script>
     <![endif]-->
@@ -50,6 +50,10 @@
         
         
         
+        <a href="https://cellxgene.cziscience.com/" class="btn">All other datasets</a><br>
+        
+        
+        
         <a href="/cellxgene/posts/prepare" class="btn">Preparing your data</a><br>
         
         
@@ -86,10 +90,6 @@
         
         
 
-        <a href="/cellxgene/posts/cellxgene_cziscience_com" class="btn">cellxgene.cziscience.com</a><br>
-        
-        
-
         <a href="https://github.com/chanzuckerberg/cellxgene" class="btn" target="_blank">Code</a>
     </p>
     </header>

docs/posts/gallery.md

@@ -39,6 +39,6 @@ Check out the cool data that our users are using cellxgene to explore!
 
 ### [Melanoma](https://melanoma.cellgeni.sanger.ac.uk/)
 
-### [CZI's own cellxgene site](cellxgene_cziscience_com)
+### [CZI's own cellxgene site](https://cellxgene.cziscience.com/)
 
 _Want us to link to your dataset here? [Just send us a note!](contact)_

server/__init__.py

@@ -1,6 +1,5 @@
 import logging
 import sys
-
 from server.common.utils.utils import import_plugins
 
 __version__ = "0.16.5-rc.0"

server/app/app.py

@@ -6,8 +6,17 @@ from urllib.parse import urlparse
 import hashlib
 import os
 
-from flask import Flask, redirect, current_app, make_response, render_template, abort, Blueprint, request, \
-    send_from_directory
+from flask import (
+    Flask,
+    redirect,
+    current_app,
+    make_response,
+    render_template,
+    abort,
+    Blueprint,
+    request,
+    send_from_directory,
+)
 from flask_restful import Api, Resource
 from server_timing import Timing as ServerTiming
 
@@ -87,10 +96,7 @@ def dataset_index(url_dataroot=None, dataset=None):
         cache_manager = current_app.matrix_data_cache_manager
         with cache_manager.data_adaptor(url_dataroot, location, app_config) as data_adaptor:
             data_adaptor.set_uri_path(f"{url_dataroot}/{dataset}")
-            args = {
-                "SCRIPTS" : scripts,
-                "INLINE_SCRIPTS" : inline_scripts
-            }
+            args = {"SCRIPTS": scripts, "INLINE_SCRIPTS": inline_scripts}
             return render_template("index.html", **args)
 
     except DatasetAccessError as e:
@@ -417,8 +423,9 @@ class Server:
             for dataroot_dict in server_config.multi_dataset__dataroot.values():
                 url_dataroot = dataroot_dict["base_url"]
                 bp_dataroot = Blueprint(
-                    f"api_dataset_{url_dataroot}", __name__,
-                    url_prefix=f"{api_path}/{url_dataroot}/<dataset>" + api_version
+                    f"api_dataset_{url_dataroot}",
+                    __name__,
+                    url_prefix=f"{api_path}/{url_dataroot}/<dataset>" + api_version,
                 )
                 dataroot_resources = get_api_dataroot_resources(bp_dataroot, url_dataroot)
                 self.app.register_blueprint(dataroot_resources.blueprint)
@@ -433,7 +440,7 @@ class Server:
                     f"/{url_dataroot}/<dataset>/static/<path:filename>",
                     f"static_assets_{url_dataroot}",
                     view_func=lambda dataset, filename: send_from_directory("../common/web/static", filename),
-                    methods=["GET"]
+                    methods=["GET"],
                 )
 
         else:
@@ -444,7 +451,7 @@ class Server:
                 "/static/<path:filename>",
                 "static_assets",
                 view_func=lambda filename: send_from_directory("../common/web/static", filename),
-                methods=["GET"]
+                methods=["GET"],
             )
 
         self.app.matrix_data_cache_manager = server_config.matrix_data_cache_manager

server/auth/__init__.py

@@ -1,4 +1,3 @@
-
 # import the built in auth types so they can be registered
 
 import server.auth.auth_none  # noqa: F401

server/auth/auth.py

@@ -76,7 +76,7 @@ class AuthTypeFactory:
 
     @staticmethod
     def register(name, auth_type):
-        assert(issubclass(auth_type, AuthTypeBase))
+        assert issubclass(auth_type, AuthTypeBase)
         AuthTypeFactory.auth_types[name] = auth_type
 
     @staticmethod

server/auth/auth_none.py

@@ -2,7 +2,6 @@ from server.auth.auth import AuthTypeBase, AuthTypeFactory
 
 
 class AuthTypeNone(AuthTypeBase):
-
     def __init__(self, app_config):
         super().__init__()
 

server/auth/auth_oauth.py

@@ -97,8 +97,17 @@ class AuthTypeOAuth(AuthTypeClientBase):
             return
 
         valid_keys = {
-            "verify_signature", "verify_aud", "verify_iat", "verify_exp", "verify_nbf", "verify_iss",
-            "verify_sub", "verify_jti", "verify_at_hash", "leeway"}
+            "verify_signature",
+            "verify_aud",
+            "verify_iat",
+            "verify_exp",
+            "verify_nbf",
+            "verify_iss",
+            "verify_sub",
+            "verify_jti",
+            "verify_at_hash",
+            "leeway",
+        }
         keys = set(self.jwt_decode_options.keys())
         unknown = keys - valid_keys
         if unknown:
@@ -114,6 +123,7 @@ class AuthTypeOAuth(AuthTypeClientBase):
         parse = urlparse(self.api_base_url)
         app.add_url_rule(f"{parse.path}/login", "login", self.login, methods=["GET"])
         app.add_url_rule(f"{parse.path}/logout", "logout", self.logout, methods=["GET"])
+        app.add_url_rule(f"{parse.path}/logout_redirect", "logout_redirect", self.logout_redirect, methods=["GET"])
         app.add_url_rule(f"{parse.path}/oauth2/callback", "callback", self.callback, methods=["GET"])
 
     def complete_setup(self, flask_app):
@@ -166,12 +176,29 @@ class AuthTypeOAuth(AuthTypeClientBase):
         return response
 
     def logout(self):
+        """
+        We would like for the user to remain on the same dataset after logout.  oauth requires that
+        the redirect `returnTo` path be whitelisted by the oauth server, therefore a level of
+        indirection is used.  We first redirect to a single path "logout_redirect", and logout_redirect
+        will redirect the user's browser back to the current page.
+        """
         self.remove_tokens()
-        params = {"returnTo": self.web_base_url, "client_id": self.client_id}
+        redirect_path = request.args.get("dataset", "")
+        redirect_to = f"{self.web_base_url}/{redirect_path}"
+        session["oauth_logout_redirect"] = redirect_to
+
+        return_to = f"{self.api_base_url}/logout_redirect"
+        params = {"returnTo": return_to, "client_id": self.client_id}
         response = redirect(self.client.api_base_url + "/v2/logout?" + urlencode(params))
         self.update_response(response)
         return response
 
+    def logout_redirect(self):
+        oauth_logout_redirect = session.pop("oauth_logout_redirect", "/")
+        response = redirect(oauth_logout_redirect)
+        self.update_response(response)
+        return response
+
     def callback(self):
         data = self.client.authorize_access_token()
         tokens = Tokens(
@@ -253,7 +280,10 @@ class AuthTypeOAuth(AuthTypeClientBase):
 
     def get_logout_url(self, data_adaptor):
         """Return the url for the logout route"""
-        return f"{self.api_base_url}/logout"
+        if data_adaptor and current_app.app_config.is_multi_dataset():
+            return f"{self.api_base_url}/logout?dataset={data_adaptor.uri_path}/"
+        else:
+            return f"{self.api_base_url}/logout"
 
     def check_jwt_payload(self, id_token):
         try:

server/auth/auth_test.py

@@ -47,6 +47,7 @@ class AuthTypeTest(AuthTypeClientBase):
         return_to = args.get("dataset", "/")
         session[self.CXGUID] = args.get("userid", self.user_id)
         session[self.CXGUNAME] = args.get("username", self.user_name)
+        session[self.CXGUEMAIL] = args.get("email", self.user_email)
         return redirect(return_to)
 
     def logout(self):

server/cli/convert_to_cxg.py

@@ -9,26 +9,24 @@ from server.converters.h5ad_data_file import H5ADDataFile
     name="convert",
     short_help="Converts an H5AD dataset to the CXG format.",
     help="Converts an H5AD dataset to the CXG format. The CXG format is a cellxgene-private data format "
-         "that has performance and access characteristics amenable to a multi-dataset, multi-user serving "
-         "environment. You will be able to launch the cellxgene using the `cellxgene launch` command as "
-         "usually with the generated CXG file.",
+    "that has performance and access characteristics amenable to a multi-dataset, multi-user serving "
+    "environment. You will be able to launch the cellxgene using the `cellxgene launch` command as "
+    "usually with the generated CXG file.",
 )
 @click.argument(
-    "input-file",
-    nargs=1,
-    type=click.Path(exists=True, dir_okay=False),
+    "input-file", nargs=1, type=click.Path(exists=True, dir_okay=False),
 )
 @click.option(
     "-o",
     "--output-directory",
     help="Name of the output CXG directory. If not provided, will default to be the input filename with a "
-         "CXG extension.",
+    "CXG extension.",
 )
 @click.option(
     "-b",
     "--backed",
     help="When true, loads the H5AD in file backed mode. This will cause the conversion to be slower, "
-         "but will use less memory.",
+    "but will use less memory.",
     default=False,
     show_default=True,
     is_flag=True,
@@ -37,29 +35,33 @@ from server.converters.h5ad_data_file import H5ADDataFile
     "-t",
     "--title",
     help="Human readable dataset title that will be included as metadata about the CXG file. If omitted, "
-         "the dataset title will be the filename.",
+    "the dataset title will be the filename.",
 )
 @click.option(
     "-a",
     "--about",
     help="A fully qualified URL that provides more information about the dataset and will be included as "
-         "metadata about the CXG file.",
+    "metadata about the CXG file.",
 )
 @click.option(
     "-s",
     "--sparse-threshold",
     help="If the dataset's percent of non-zero values falls belows the specified threshold, then the X "
-         "array of the dataset will be sparse. Since the default value is 0.0, the default will be to "
-         "convert to dense array.",
+    "array of the dataset will be sparse. Since the default value is 0.0, the default will be to "
+    "convert to dense array.",
     default=0.0,
     show_default=True,
 )
-@click.option("--obs-names",
-              help="Name to a column in the obs dataframe that will be used as the index for the dataframe instead of "
-                   "the one designated by the dataframe generated-index.")
-@click.option("--var-names",
-              help="Name to a column in the var dataframe that will be used as the index for the dataframe instead of "
-                   "the one designated by the dataframe generated-index.")
+@click.option(
+    "--obs-names",
+    help="Name to a column in the obs dataframe that will be used as the index for the dataframe instead of "
+    "the one designated by the dataframe generated-index.",
+)
+@click.option(
+    "--var-names",
+    help="Name to a column in the var dataframe that will be used as the index for the dataframe instead of "
+    "the one designated by the dataframe generated-index.",
+)
 @click.option(
     "--disable-custom-colors",
     help="When set, conversion process will not extract scanpy-compatible category colors from the H5AD file.",
@@ -70,8 +72,8 @@ from server.converters.h5ad_data_file import H5ADDataFile
 @click.option(
     "--disable-corpora-schema",
     help="When set, conversion process will neither extract nor store Corpora schema information. See "
-         "https://github.com/chanzuckerberg/corpora-data-portal/blob/main/backend/schema/corpora_schema.md for more "
-         "information.",
+    "https://github.com/chanzuckerberg/corpora-data-portal/blob/main/backend/schema/corpora_schema.md for more "
+    "information.",
     default=False,
     show_default=True,
     is_flag=True,
@@ -85,30 +87,32 @@ from server.converters.h5ad_data_file import H5ADDataFile
 )
 @click.help_option("--help", "-h", help="Show this message and exit.")
 def convert_to_cxg(
-        input_file,
-        output_directory,
-        backed,
-        title,
-        about,
-        sparse_threshold,
-        obs_names,
-        var_names,
-        disable_custom_colors,
-        disable_corpora_schema,
-        overwrite,
+    input_file,
+    output_directory,
+    backed,
+    title,
+    about,
+    sparse_threshold,
+    obs_names,
+    var_names,
+    disable_custom_colors,
+    disable_corpora_schema,
+    overwrite,
 ):
     """
     Convert a dataset file into CXG.
     """
 
-    h5ad_data_file = H5ADDataFile(input_file, backed, title, about, obs_names, var_names,
-                                  use_corpora_schema=not disable_corpora_schema)
+    h5ad_data_file = H5ADDataFile(
+        input_file, backed, title, about, obs_names, var_names, use_corpora_schema=not disable_corpora_schema
+    )
 
     # Get the directory that will hold all the CXG files
     cxg_output_container = get_output_directory(input_file, output_directory, overwrite)
 
-    h5ad_data_file.to_cxg(cxg_output_container, sparse_threshold,
-                          convert_anndata_colors_to_cxg_colors=not disable_custom_colors)
+    h5ad_data_file.to_cxg(
+        cxg_output_container, sparse_threshold, convert_anndata_colors_to_cxg_colors=not disable_custom_colors
+    )
 
 
 def get_output_directory(input_filename, output_directory, should_overwrite):

server/cli/launch.py

@@ -3,15 +3,14 @@ import functools
 import logging
 import sys
 import webbrowser
-from os import devnull
-
+import os
 import click
 from flask_compress import Compress
 from flask_cors import CORS
 
+from server.default_config import default_config
 from server.app.app import Server
-from server.common.app_config import AppConfig
-from server.common.default_config import default_config
+from server.common.config.app_config import AppConfig
 from server.common.errors import DatasetAccessError, ConfigurationError
 from server.common.utils.utils import sort_options
 
@@ -33,7 +32,7 @@ def annotation_args(func):
         multiple=False,
         metavar="<path>",
         help="CSV file to initialize editing of existing annotations; will be altered in-place. "
-             "Incompatible with --annotations-dir.",
+        "Incompatible with --annotations-dir.",
     )
     @click.option(
         "--annotations-dir",
@@ -42,7 +41,7 @@ def annotation_args(func):
         multiple=False,
         metavar="<directory path>",
         help="Directory of where to save output annotations; filename will be specified in the application. "
-             "Incompatible with --annotations-file.",
+        "Incompatible with --annotations-file.",
     )
     @click.option(
         "--experimental-annotations-ontology",
@@ -170,7 +169,7 @@ def server_args(func):
         default=DEFAULT_CONFIG.server_config.app__debug,
         show_default=True,
         help="Run in debug mode. This is helpful for cellxgene developers, "
-             "or when you want more information about an error condition.",
+        "or when you want more information about an error condition.",
     )
     @click.option(
         "--verbose",
@@ -203,7 +202,7 @@ def server_args(func):
         multiple=True,
         metavar="<text>",
         help="Additional script files to include in HTML page. If not specified, "
-             "no additional script files will be included.",
+        "no additional script files will be included.",
         show_default=False,
     )
     @functools.wraps(func)
@@ -223,7 +222,7 @@ def launch_args(func):
         default=DEFAULT_CONFIG.server_config.multi_dataset__dataroot,
         metavar="<data directory>",
         help="Enable cellxgene to serve multiple files. Supply path (local directory or URL)"
-             " to folder containing H5AD and/or CXG datasets.",
+        " to folder containing H5AD and/or CXG datasets.",
         hidden=True,
     )  # TODO, unhide when dataroot is supported)
     @click.argument("datapath", required=False, metavar="<path to data file>")
@@ -307,32 +306,32 @@ class CliLaunchServer(Server):
 )
 @launch_args
 def launch(
-        datapath,
-        dataroot,
-        verbose,
-        debug,
-        open_browser,
-        port,
-        host,
-        embedding,
-        obs_names,
-        var_names,
-        max_category_items,
-        disable_custom_colors,
-        diffexp_lfc_cutoff,
-        title,
-        scripts,
-        about,
-        disable_annotations,
-        annotations_file,
-        annotations_dir,
-        backed,
-        disable_diffexp,
-        experimental_annotations_ontology,
-        experimental_annotations_ontology_obo,
-        experimental_enable_reembedding,
-        config_file,
-        dump_default_config,
+    datapath,
+    dataroot,
+    verbose,
+    debug,
+    open_browser,
+    port,
+    host,
+    embedding,
+    obs_names,
+    var_names,
+    max_category_items,
+    disable_custom_colors,
+    diffexp_lfc_cutoff,
+    title,
+    scripts,
+    about,
+    disable_annotations,
+    annotations_file,
+    annotations_dir,
+    backed,
+    disable_diffexp,
+    experimental_annotations_ontology,
+    experimental_annotations_ontology_obo,
+    experimental_enable_reembedding,
+    config_file,
+    dump_default_config,
 ):
     """Launch the cellxgene data viewer.
     This web app lets you explore single-cell expression data.
@@ -443,7 +442,7 @@ def launch(
     click.echo("[cellxgene] Type CTRL-C at any time to exit.")
 
     if not server_config.app__verbose:
-        f = open(devnull, "w")
+        f = open(os.devnull, "w")
         sys.stdout = f
 
     try:

server/cli/prepare.py

@@ -37,7 +37,7 @@ from server.common.utils.utils import sort_options
     default=False,
     is_flag=True,
     help="Do not run quality control metrics. By default cellxgene runs them "
-         "(saved to adata.obs and adata.var; see scanpy.pp.calculate_qc_metrics for details).",
+    "(saved to adata.obs and adata.var; see scanpy.pp.calculate_qc_metrics for details).",
 )
 @click.option(
     "--make-obs-names-unique/--no-make-obs-names-unique",
@@ -53,18 +53,18 @@ from server.common.utils.utils import sort_options
 )
 @click.help_option("--help", "-h", help="Show this message and exit.")
 def prepare(
-        data,
-        embedding,
-        recipe,
-        output,
-        plotting,
-        sparse,
-        overwrite,
-        set_obs_names,
-        set_var_names,
-        skip_qc,
-        make_obs_names_unique,
-        make_var_names_unique,
+    data,
+    embedding,
+    recipe,
+    output,
+    plotting,
+    sparse,
+    overwrite,
+    set_obs_names,
+    set_var_names,
+    skip_qc,
+    make_obs_names_unique,
+    make_var_names_unique,
 ):
     """
     Preprocess data for use with cellxgene.

server/cli/upgrade.py

@@ -10,7 +10,8 @@ from .. import __version__
 SEMVER_FORMAT = re.compile(
     r"^(?P<major>0|[1-9]\d*)\.(?P<minor>0|[1-9]\d*)\.(?P<patch>0|[1-9]\d*)(?:-(?P<prerelease>(?:0|[1-9]\d*|\d*["
     r"a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?P<buildmetadata>[0-9a-zA-Z-]+("
-    r"?:\.[0-9a-zA-Z-]+)*))?$")
+    r"?:\.[0-9a-zA-Z-]+)*))?$"
+)
 
 
 def log_upgrade_check():

server/common/annotations/hosted_tiledb.py

@@ -31,7 +31,8 @@ class AnnotationsHostedTileDB(Annotations):
         unsanitary_original_category_names = set(original_category_names).difference(sanitized_category_names)
         if unsanitary_original_category_names:
             raise AnnotationCategoryNameError(
-                f"{unsanitary_original_category_names} are not valid category names, please resubmit")
+                f"{unsanitary_original_category_names} are not valid category names, please resubmit"
+            )
 
     def is_safe_collection_name(self, name):
         """
@@ -68,11 +69,11 @@ class AnnotationsHostedTileDB(Annotations):
         index_dims = None
         schema_hints = json.loads(schema_hints)
 
-        if '__pandas_attribute_repr' in tileDBArray.meta:
+        if "__pandas_attribute_repr" in tileDBArray.meta:
             # backwards compatibility... unsure if necessary at this point
-            repr_meta = json.loads(tileDBArray.meta['__pandas_attribute_repr'])
-        if '__pandas_index_dims' in tileDBArray.meta:
-            index_dims = json.loads(tileDBArray.meta['__pandas_index_dims'])
+            repr_meta = json.loads(tileDBArray.meta["__pandas_attribute_repr"])
+        if "__pandas_index_dims" in tileDBArray.meta:
+            index_dims = json.loads(tileDBArray.meta["__pandas_index_dims"])
 
         data = tileDBArray[:]
         indexes = list()
@@ -80,12 +81,12 @@ class AnnotationsHostedTileDB(Annotations):
         for col_name, col_val in data.items():
             # If the column values are byte literals, decode them
             if isinstance(col_val[0], bytes):
-                col_val = [value.decode('utf-8') for value in col_val]
+                col_val = [value.decode("utf-8") for value in col_val]
 
             if schema_hints and col_name in schema_hints:
                 type = schema_hints.get(col_name).get("type")
                 if type and type == "categorical":
-                    new_col = pd.Series(col_val, dtype='category')
+                    new_col = pd.Series(col_val, dtype="category")
                     data[col_name] = new_col
             elif repr_meta and col_name in repr_meta:
                 new_col = pd.Series(col_val, dtype=repr_meta[col_name])
@@ -127,7 +128,7 @@ class AnnotationsHostedTileDB(Annotations):
             tiledb_uri=uri,
             user_id=user_id,
             dataset_id=str(dataset_id),
-            schema_hints=json.dumps(dataframe_schema_type_hints)
+            schema_hints=json.dumps(dataframe_schema_type_hints),
         )
         if not df.empty:
             self.check_category_names(df)

server/common/app_config.py

@@ -1,962 +0,0 @@
-import copy
-import os
-import sys
-import warnings
-from os.path import splitext, basename, isdir
-from urllib.parse import urlparse, quote_plus
-
-import yaml
-from flatten_dict import flatten, unflatten
-
-import server.compute.diffexp_cxg as diffexp_tiledb
-import server.compute.scanpy
-from server import display_version as cellxgene_display_version
-from server.auth.auth import AuthTypeFactory
-from server.common.annotations.hosted_tiledb import AnnotationsHostedTileDB
-from server.common.annotations.local_file_csv import AnnotationsLocalFile
-from server.common.data_locator import discover_s3_region_name
-from server.common.default_config import get_default_config
-from server.common.errors import ConfigurationError, DatasetAccessError, OntologyLoadFailure
-from server.common.utils.utils import custom_format_warning, find_available_port, is_port_available
-from server.data_common.matrix_loader import MatrixDataLoader, MatrixDataCacheManager, MatrixDataType
-from server.db.db_utils import DbUtils
-
-DEFAULT_SERVER_PORT = 5005
-# anything bigger than this will generate a special message
-BIG_FILE_SIZE_THRESHOLD = 100 * 2 ** 20  # 100MB
-
-
-class AppFeature(object):
-    def __init__(self, path, available=False, method="POST", extra={}):
-        self.path = path
-        self.available = available
-        self.method = method
-        self.extra = extra
-        for k, v in extra.items():
-            setattr(self, k, v)
-
-    def todict(self):
-        d = dict(available=self.available, method=self.method, path=self.path)
-        d.update(self.extra)
-        return d
-
-
-class AppConfig(object):
-    """AppConfig stores all the configuration for cellxgene.  The configuration is divided into two main parts:
-    server attributes, and dataset attributes. The server_config contains attributes that refer to the server process
-    as a whole.   The default_dataset_config referes to attributes that are associated with the features and
-    presentations of a dataset.  The dataset config attributes can be overridden depending on the url by which the
-    dataset was accessed.  These are stored in dataroot_config.
-    AppConfig has methods to initialize, modify, and access the configuration.
-    """
-
-    def __init__(self):
-
-        # the default configuration (see default_config.py)
-        self.default_config = get_default_config()
-        # the server configuration
-        self.server_config = ServerConfig(self, self.default_config["server"])
-        # the dataset config, unless overridden by an entry in dataroot_config
-        self.default_dataset_config = DatasetConfig(None, self, self.default_config["dataset"])
-        # a dictionary of keys to DatasetConfig objects.  Each key must exist in the multi_dataset__dataroot
-        # attribute of the server_config.
-        self.dataroot_config = {}
-
-        # Set to true when config_completed is called
-        self.is_completed = False
-
-    def get_dataset_config(self, dataroot_key):
-        if self.server_config.single_dataset__datapath:
-            return self.default_dataset_config
-        else:
-            return self.dataroot_config.get(dataroot_key, self.default_dataset_config)
-
-    def check_config(self):
-        """Verify all the attributes have been checked"""
-        if not self.is_completed:
-            raise ConfigurationError("The configuration has not been completed")
-        self.server_config.check_config()
-        self.default_dataset_config.check_config()
-        for dataset_config in self.dataroot_config.values():
-            dataset_config.check_config()
-
-    def update_server_config(self, **kw):
-        self.server_config.update(**kw)
-        self.is_complete = False
-
-    def update_default_dataset_config(self, **kw):
-        self.default_dataset_config.update(**kw)
-        # update all the other dataset configs, if any
-        for value in self.dataroot_config.values():
-            value.update(**kw)
-        self.is_complete = False
-
-    def update_from_config_file(self, config_file):
-        with open(config_file) as fyaml:
-            config = yaml.load(fyaml, Loader=yaml.FullLoader)
-
-        if config.get("server"):
-            self.server_config.update_from_config(config["server"], "server")
-        if config.get("dataset"):
-            self.default_dataset_config.update_from_config(config["dataset"], "dataset")
-
-        per_dataset_config = config.get("per_dataset_config", {})
-        for key, dataroot_config in per_dataset_config.items():
-            # first create and initialize the dataroot with the default config
-            self.add_dataroot_config(key, **config["dataset"])
-            # then apply the per dataset configuration
-            self.dataroot_config[key].update_from_config(dataroot_config, f"per_dataset_config__{key}")
-
-        self.is_complete = False
-
-    def write_config(self, config_file):
-        """output the config to a yaml file"""
-        server = self.server_config.create_mapping(self.server_config.default_config)
-        dataset = self.default_dataset_config.create_mapping(self.default_dataset_config.default_config)
-        config = dict(server={}, dataset={})
-        for attrname in server.keys():
-            config["server__" + attrname] = getattr(self.server_config, attrname)
-        for attrname in dataset.keys():
-            config["dataset__" + attrname] = getattr(self.default_dataset_config, attrname)
-        if self.dataroot_config:
-            config["per_dataset_config"] = {}
-        for dataroot_tag, dataroot_config in self.dataroot_config.items():
-            dataset = dataroot_config.create_mapping(dataroot_config.default_config)
-            for attrname in dataset.keys():
-                config[f"per_dataset_config__{dataroot_tag}__" + attrname] = getattr(dataroot_config, attrname)
-
-        config = unflatten(config, splitter=lambda key: key.split("__"))
-        yaml.dump(config, open(config_file, "w"))
-
-    def changes_from_default(self):
-        """Return all the attribute that are different from the default"""
-        diff_server = self.server_config.changes_from_default()
-        diff_dataset = self.default_dataset_config.changes_from_default()
-        diff = dict(server=diff_server, dataset=diff_dataset)
-        return diff
-
-    def add_dataroot_config(self, dataroot_tag, **kw):
-        """Create a new dataset config object based on the default dataset config, and kw parameters"""
-        if dataroot_tag in self.dataroot_config:
-            raise ConfigurationError(f"dataroot config already exists: {dataroot_tag}")
-        if type(self.server_config.multi_dataset__dataroot) != dict:
-            raise ConfigurationError("The server__multi_dataset__dataroot must be a dictionary")
-        if dataroot_tag not in self.server_config.multi_dataset__dataroot:
-            raise ConfigurationError(f"The dataroot_tag ({dataroot_tag}) not found in server__multi_dataset__dataroot")
-
-        self.is_completed = False
-        self.dataroot_config[dataroot_tag] = DatasetConfig(dataroot_tag, self, self.default_config["dataset"])
-        flat_config = self.default_dataset_config.create_mapping(self.default_dataset_config.default_config)
-        config = {key: value[1] for key, value in flat_config.items()}
-        self.dataroot_config[dataroot_tag].update(**config)
-        self.dataroot_config[dataroot_tag].update_from_config(kw, dataroot_tag)
-
-    def complete_config(self, messagefn=None):
-        """The configure options are checked, and any additional setup based on the config
-        parameters is done"""
-
-        if messagefn is None:
-            def noop(message):
-                pass
-
-            messagefn = noop
-
-        # TODO: to give better error messages we can add a mapping between where each config
-        # attribute originated (e.g. command line argument or config file), then in the error
-        # messages we can give correct context for attributes with bad value.
-        context = dict(messagefn=messagefn)
-
-        self.server_config.complete_config(context)
-        self.default_dataset_config.complete_config(context)
-        for dataroot_config in self.dataroot_config.values():
-            dataroot_config.complete_config(context)
-
-        self.is_completed = True
-        self.check_config()
-
-    def get_matrix_data_cache_manager(self):
-        return self.server_config.matrix_data_cache_manager
-
-    def is_multi_dataset(self):
-        return self.server_config.multi_dataset__dataroot is not None
-
-    def get_title(self, data_adaptor):
-        return (
-            self.server_config.single_dataset__title
-            if self.server_config.single_dataset__title
-            else data_adaptor.get_title()
-        )
-
-    def get_about(self, data_adaptor):
-        return (
-            self.server_config.single_dataset__about
-            if self.server_config.single_dataset__about
-            else data_adaptor.get_about()
-        )
-
-    def get_client_config(self, data_adaptor):
-        """
-        Return the configuration as required by the /config REST route
-        """
-
-        server_config = self.server_config
-        dataset_config = data_adaptor.dataset_config
-        annotation = dataset_config.user_annotations
-        auth = server_config.auth
-
-        # FIXME The current set of config is not consistently presented:
-        # we have camalCase, hyphen-text, and underscore_text
-
-        # make sure the configuration has been checked.
-        self.check_config()
-
-        # features
-        features = [f.todict() for f in data_adaptor.get_features(annotation)]
-
-        # display_names
-        title = self.get_title(data_adaptor)
-        about = self.get_about(data_adaptor)
-
-        display_names = dict(engine=data_adaptor.get_name(), dataset=title)
-
-        # library_versions
-        library_versions = {}
-        library_versions.update(data_adaptor.get_library_versions())
-        library_versions["cellxgene"] = cellxgene_display_version
-
-        # links
-        links = {"about-dataset": about}
-
-        # parameters
-        parameters = {
-            "layout": dataset_config.embeddings__names,
-            "max-category-items": dataset_config.presentation__max_categories,
-            "obs_names": server_config.single_dataset__obs_names,
-            "var_names": server_config.single_dataset__var_names,
-            "diffexp_lfc_cutoff": dataset_config.diffexp__lfc_cutoff,
-            "backed": server_config.adaptor__anndata_adaptor__backed,
-            "disable-diffexp": not dataset_config.diffexp__enable,
-            "enable-reembedding": dataset_config.embeddings__enable_reembedding,
-            "annotations": False,
-            "annotations_file": None,
-            "annotations_dir": None,
-            "annotations_cell_ontology_enabled": False,
-            "annotations_cell_ontology_obopath": None,
-            "annotations_cell_ontology_terms": None,
-            "custom_colors": dataset_config.presentation__custom_colors,
-            "diffexp-may-be-slow": False,
-            "about_legal_tos": dataset_config.app__about_legal_tos,
-            "about_legal_privacy": dataset_config.app__about_legal_privacy,
-        }
-
-        # corpora dataset_props
-        # TODO/Note: putting info from the dataset into the /config is not ideal.
-        # However, it is definitely not part of /schema, and we do not have a top-level
-        # route for data properties.  Consider creating one at some point.
-        corpora_props = data_adaptor.get_corpora_props()
-        if corpora_props and "default_embedding" in corpora_props:
-            default_embedding = corpora_props["default_embedding"]
-            if isinstance(default_embedding, str) and default_embedding.startswith("X_"):
-                default_embedding = default_embedding[2:]  # drop X_ prefix
-            if default_embedding in data_adaptor.get_embedding_names():
-                parameters["default_embedding"] = default_embedding
-
-        data_adaptor.update_parameters(parameters)
-        if annotation:
-            annotation.update_parameters(parameters, data_adaptor)
-
-        # gather it all together
-        c = {}
-        config = c["config"] = {}
-        config["features"] = features
-        config["displayNames"] = display_names
-        config["library_versions"] = library_versions
-        config["links"] = links
-        config["parameters"] = parameters
-        config["corpora_props"] = corpora_props
-        config["limits"] = {
-            "column_request_max": server_config.limits__column_request_max,
-            "diffexp_cellcount_max": server_config.limits__diffexp_cellcount_max,
-        }
-
-        if dataset_config.app__authentication_enable and auth.is_valid_authentication_type():
-            config["authentication"] = {
-                "requires_client_login": auth.requires_client_login(),
-            }
-            if auth.requires_client_login():
-                config["authentication"].update({
-                    "login": auth.get_login_url(data_adaptor),
-                    "logout": auth.get_logout_url(data_adaptor),
-                })
-
-        return c
-
-    def get_client_userinfo(self, data_adaptor):
-        """
-        Return the userinfo as required by the /userinfo REST route
-        """
-
-        server_config = self.server_config
-        dataset_config = data_adaptor.dataset_config
-        auth = server_config.auth
-
-        # make sure the configuration has been checked.
-        self.check_config()
-
-        if dataset_config.app__authentication_enable and auth.is_valid_authentication_type():
-            userinfo = {}
-            userinfo["userinfo"] = {
-                "is_authenticated": auth.is_user_authenticated(),
-                "username": auth.get_user_name(),
-                "user_id": auth.get_user_id()
-            }
-            return userinfo
-        else:
-            return None
-
-
-class BaseConfig(object):
-    """This class handles the mechanics of updating and checking attributes.
-    Derived classes are expected to store the actual attributes"""
-
-    def __init__(self, app_config, default_config, dictval_cases={}):
-        # reference back to the app_config
-        self.app_config = app_config
-        # the complete set of attribute and their default values (unflattened)
-        self.default_config = default_config
-        # attributes where the value may be a dict (and therefore are not flattened)
-        self.dictval_cases = dictval_cases
-        # used to make sure every attribute value is checked
-        self.attr_checked = {k: False for k in self.create_mapping(default_config).keys()}
-
-    def create_mapping(self, config):
-        """Create a mapping from attribute names to (location in the config tree, value)"""
-        dc = copy.deepcopy(config)
-        mapping = {}
-
-        # special cases where the value could be a dict.
-        # If its value is not None, the entry is added to the mapping, and not included
-        # in the flattening below.
-        for dictval_case in self.dictval_cases:
-            cur = dc
-            for part in dictval_case[:-1]:
-                cur = cur.get(part, {})
-            val = cur.get(dictval_case[-1])
-            if val is not None:
-                key = "__".join(dictval_case)
-                mapping[key] = (dictval_case, val)
-                del cur[dictval_case[-1]]
-
-        flat_config = flatten(dc)
-        for key, value in flat_config.items():
-            # name of the attribute
-            attr = "__".join(key)
-            mapping[attr] = (key, value)
-
-        return mapping
-
-    def check_attr(self, attrname, vtype):
-        val = getattr(self, attrname)
-        if type(vtype) in (list, tuple):
-            if type(val) not in vtype:
-                tnames = ",".join([x.__name__ for x in vtype])
-                raise ConfigurationError(
-                    f"Invalid type for attribute: {attrname}, expected types ({tnames}), got {type(val).__name__}"
-                )
-        else:
-            if type(val) != vtype:
-                raise ConfigurationError(
-                    f"Invalid type for attribute: {attrname}, "
-                    f"expected type {vtype.__name__}, got {type(val).__name__}"
-                )
-
-        self.attr_checked[attrname] = True
-
-    def check_config(self):
-        mapping = self.create_mapping(self.default_config)
-        for key in mapping.keys():
-            if not self.attr_checked[key]:
-                raise ConfigurationError(f"The attr '{key}' has not been checked")
-
-    def update(self, **kw):
-        for key, value in kw.items():
-            if not hasattr(self, key):
-                raise ConfigurationError(f"unknown config parameter {key}.")
-            try:
-                if type(value) == tuple:
-                    # convert tuple values to list values
-                    value = list(value)
-                setattr(self, key, value)
-            except KeyError:
-                raise ConfigurationError(f"Unable to set config parameter {key}.")
-
-            self.attr_checked[key] = False
-
-    def update_from_config(self, config, prefix):
-        mapping = self.create_mapping(config)
-        for attr, (key, value) in mapping.items():
-            if not hasattr(self, attr):
-                raise ConfigurationError(f"Unknown key from config file: {prefix}__{attr}")
-            try:
-                setattr(self, attr, value)
-            except KeyError:
-                raise ConfigurationError(f"Unable to set config attribute: {prefix}__{attr}")
-
-            self.attr_checked[attr] = False
-
-    def changes_from_default(self):
-        """Return all the attribute that are different from the default"""
-        mapping = self.create_mapping(self.default_config)
-        diff = []
-        for attrname, (key, defval) in mapping.items():
-            curval = getattr(self, attrname)
-            if curval != defval:
-                diff.append((attrname, curval, defval))
-        return diff
-
-
-class ServerConfig(BaseConfig):
-    """Manages the config attribute associated with the server."""
-
-    def __init__(self, app_config, default_config):
-        dictval_cases = [
-            ("app", "csp_directives"),
-            ("authentication", "params_oauth", "cookie"),
-            ("authentication", "params_oauth", "jwt_decode_options"),
-            ("adaptor", "cxg_adaptor", "tiledb_ctx"),
-            ("multi_dataset", "dataroot"),
-        ]
-        super().__init__(app_config, default_config, dictval_cases)
-
-        dc = default_config
-        try:
-            self.app__verbose = dc["app"]["verbose"]
-            self.app__debug = dc["app"]["debug"]
-            self.app__host = dc["app"]["host"]
-            self.app__port = dc["app"]["port"]
-            self.app__open_browser = dc["app"]["open_browser"]
-            self.app__force_https = dc["app"]["force_https"]
-            self.app__flask_secret_key = dc["app"]["flask_secret_key"]
-            self.app__generate_cache_control_headers = dc["app"]["generate_cache_control_headers"]
-            self.app__server_timing_headers = dc["app"]["server_timing_headers"]
-            self.app__csp_directives = dc["app"]["csp_directives"]
-            self.app__api_base_url = dc["app"]["api_base_url"]
-            self.app__web_base_url = dc["app"]["web_base_url"]
-
-            self.authentication__type = dc["authentication"]["type"]
-            self.authentication__params_oauth__oauth_api_base_url = dc["authentication"]["params_oauth"][
-                "oauth_api_base_url"
-            ]
-            self.authentication__params_oauth__client_id = dc["authentication"]["params_oauth"]["client_id"]
-            self.authentication__params_oauth__client_secret = dc["authentication"]["params_oauth"]["client_secret"]
-            self.authentication__params_oauth__jwt_decode_options = dc["authentication"]["params_oauth"][
-                "jwt_decode_options"]
-            self.authentication__params_oauth__session_cookie = dc["authentication"]["params_oauth"]["session_cookie"]
-            self.authentication__params_oauth__cookie = dc["authentication"]["params_oauth"]["cookie"]
-
-            self.multi_dataset__dataroot = dc["multi_dataset"]["dataroot"]
-            self.multi_dataset__index = dc["multi_dataset"]["index"]
-            self.multi_dataset__allowed_matrix_types = dc["multi_dataset"]["allowed_matrix_types"]
-            self.multi_dataset__matrix_cache__max_datasets = dc["multi_dataset"]["matrix_cache"]["max_datasets"]
-            self.multi_dataset__matrix_cache__timelimit_s = dc["multi_dataset"]["matrix_cache"]["timelimit_s"]
-
-            self.single_dataset__datapath = dc["single_dataset"]["datapath"]
-            self.single_dataset__obs_names = dc["single_dataset"]["obs_names"]
-            self.single_dataset__var_names = dc["single_dataset"]["var_names"]
-            self.single_dataset__about = dc["single_dataset"]["about"]
-            self.single_dataset__title = dc["single_dataset"]["title"]
-
-            self.diffexp__alg_cxg__max_workers = dc["diffexp"]["alg_cxg"]["max_workers"]
-            self.diffexp__alg_cxg__cpu_multiplier = dc["diffexp"]["alg_cxg"]["cpu_multiplier"]
-            self.diffexp__alg_cxg__target_workunit = dc["diffexp"]["alg_cxg"]["target_workunit"]
-
-            self.data_locator__s3__region_name = dc["data_locator"]["s3"]["region_name"]
-
-            self.adaptor__cxg_adaptor__tiledb_ctx = dc["adaptor"]["cxg_adaptor"]["tiledb_ctx"]
-            self.adaptor__anndata_adaptor__backed = dc["adaptor"]["anndata_adaptor"]["backed"]
-
-            self.limits__diffexp_cellcount_max = dc["limits"]["diffexp_cellcount_max"]
-            self.limits__column_request_max = dc["limits"]["column_request_max"]
-
-        except KeyError as e:
-            raise ConfigurationError(f"Unexpected config: {str(e)}")
-
-        # The matrix data cache manager is created during the complete_config and stored here.
-        self.matrix_data_cache_manager = None
-
-        # The authentication object
-        self.auth = None
-
-    def complete_config(self, context):
-        self.handle_app(context)
-        self.handle_data_source(context)
-        self.handle_authentication(context)
-        self.handle_data_locator(context)
-        self.handle_adaptor(context)  # may depend on data_locator
-        self.handle_single_dataset(context)  # may depend on adaptor
-        self.handle_multi_dataset(context)  # may depend on adaptor
-        self.handle_diffexp(context)
-        self.handle_limits(context)
-
-        self.check_config()
-
-    def handle_app(self, context):
-        self.check_attr("app__verbose", bool)
-        self.check_attr("app__debug", bool)
-        self.check_attr("app__host", str)
-        self.check_attr("app__port", (type(None), int))
-        self.check_attr("app__open_browser", bool)
-        self.check_attr("app__force_https", bool)
-        self.check_attr("app__flask_secret_key", (type(None), str))
-        self.check_attr("app__generate_cache_control_headers", bool)
-        self.check_attr("app__server_timing_headers", bool)
-        self.check_attr("app__csp_directives", (type(None), dict))
-        self.check_attr("app__api_base_url", (type(None), str))
-        self.check_attr("app__web_base_url", (type(None), str))
-
-        if self.app__port:
-            try:
-                if not is_port_available(self.app__host, self.app__port):
-                    raise ConfigurationError(
-                        f"The port selected {self.app__port} is in use, please configure an open port."
-                    )
-            except OverflowError:
-                raise ConfigurationError(f"Invalid port: {self.app__port}")
-        else:
-            try:
-                default_server_port = int(os.environ.get("CXG_SERVER_PORT", DEFAULT_SERVER_PORT))
-            except ValueError:
-                raise ConfigurationError(
-                    "Invalid port from environment variable CXG_SERVER_PORT: " + os.environ.get("CXG_SERVER_PORT")
-                )
-            try:
-                self.app__port = find_available_port(self.app__host, default_server_port)
-            except OverflowError:
-                raise ConfigurationError(f"Invalid port: {default_server_port}")
-
-        if self.app__debug:
-            context["messagefn"]("in debug mode, setting verbose=True and open_browser=False")
-            self.app__verbose = True
-            self.app__open_browser = False
-        else:
-            warnings.formatwarning = custom_format_warning
-
-        if not self.app__verbose:
-            sys.tracebacklimit = 0
-
-        # secret key:
-        #   first, from CXG_SECRET_KEY environment variable
-        #   second, from config file
-        self.app__flask_secret_key = os.environ.get("CXG_SECRET_KEY", self.app__flask_secret_key)
-
-        # CSP Directives are a dict of string: list(string) or string: string
-        if self.app__csp_directives is not None:
-            for k, v in self.app__csp_directives.items():
-                if not isinstance(k, str):
-                    raise ConfigurationError("CSP directive names must be a string.")
-                if isinstance(v, list):
-                    for policy in v:
-                        if not isinstance(policy, str):
-                            raise ConfigurationError("CSP directive value must be a string or list of strings.")
-                elif not isinstance(v, str):
-                    raise ConfigurationError("CSP directive value must be a string or list of strings.")
-
-        if self.app__web_base_url is None:
-            self.app__web_base_url = self.app__api_base_url
-
-    def handle_authentication(self, context):
-        self.check_attr("authentication__type", (type(None), str))
-
-        # oauth
-        ptypes = str if self.authentication__type == "oauth" else (type(None), str)
-        self.check_attr("authentication__params_oauth__oauth_api_base_url", ptypes)
-        self.check_attr("authentication__params_oauth__client_id", ptypes)
-        self.check_attr("authentication__params_oauth__client_secret", ptypes)
-        self.check_attr("authentication__params_oauth__jwt_decode_options", (type(None), dict))
-        self.check_attr("authentication__params_oauth__session_cookie", bool)
-
-        if self.authentication__params_oauth__session_cookie:
-            self.check_attr("authentication__params_oauth__cookie", (type(None), dict))
-        else:
-            self.check_attr("authentication__params_oauth__cookie", dict)
-        #   secret key: first, from CXG_OAUTH_CLIENT_SECRET environment variable
-        #   second, from config file
-        self.authentication__params__oauth__client_secret = os.environ.get(
-            "CXG_OAUTH_CLIENT_SECRET", self.authentication__params_oauth__client_secret)
-
-        self.auth = AuthTypeFactory.create(self.authentication__type, self)
-        if self.auth is None:
-            raise ConfigurationError(f"Unknown authentication type: {self.authentication__type}")
-
-    def handle_data_locator(self, context):
-        self.check_attr("data_locator__s3__region_name", (type(None), bool, str))
-        if self.data_locator__s3__region_name is True:
-            path = self.single_dataset__datapath or self.multi_dataset__dataroot
-            if type(path) == dict:
-                # if multi_dataset__dataroot is a dict, then use the first key
-                # that is in s3.   NOTE:  it is not supported to have dataroots
-                # in different regions.
-                paths = [val.get("dataroot") for val in path.values()]
-                for path in paths:
-                    if path.startswith("s3://"):
-                        break
-            if path.startswith("s3://"):
-                region_name = discover_s3_region_name(path)
-                if region_name is None:
-                    raise ConfigurationError(f"Unable to discover s3 region name from {path}")
-            else:
-                region_name = None
-            self.data_locator__s3__region_name = region_name
-
-    def handle_data_source(self, context):
-        self.check_attr("single_dataset__datapath", (str, type(None)))
-        self.check_attr("multi_dataset__dataroot", (type(None), dict, str))
-
-        if self.single_dataset__datapath is None:
-            if self.multi_dataset__dataroot is None:
-                # TODO:  change the error message once dataroot is fully supported
-                raise ConfigurationError("missing datapath")
-            return
-        else:
-            if self.multi_dataset__dataroot is not None:
-                raise ConfigurationError("must supply only one of datapath or dataroot")
-
-    def handle_single_dataset(self, context):
-        self.check_attr("single_dataset__datapath", (str, type(None)))
-        self.check_attr("single_dataset__title", (str, type(None)))
-        self.check_attr("single_dataset__about", (str, type(None)))
-        self.check_attr("single_dataset__obs_names", (str, type(None)))
-        self.check_attr("single_dataset__var_names", (str, type(None)))
-
-        if self.single_dataset__datapath is None:
-            return
-
-        # create the matrix data cache manager:
-        if self.matrix_data_cache_manager is None:
-            self.matrix_data_cache_manager = MatrixDataCacheManager(max_cached=1, timelimit_s=None)
-
-        # preload this data set
-        matrix_data_loader = MatrixDataLoader(self.single_dataset__datapath, app_config=self.app_config)
-        try:
-            matrix_data_loader.pre_load_validation()
-        except DatasetAccessError as e:
-            raise ConfigurationError(str(e))
-
-        file_size = matrix_data_loader.file_size()
-        file_basename = basename(self.single_dataset__datapath)
-        if file_size > BIG_FILE_SIZE_THRESHOLD:
-            context["messagefn"](f"Loading data from {file_basename}, this may take a while...")
-        else:
-            context["messagefn"](f"Loading data from {file_basename}.")
-
-        if self.single_dataset__about:
-
-            def url_check(url):
-                try:
-                    result = urlparse(url)
-                    if all([result.scheme, result.netloc]):
-                        return True
-                    else:
-                        return False
-                except ValueError:
-                    return False
-
-            if not url_check(self.single_dataset__about):
-                raise ConfigurationError(
-                    "Must provide an absolute URL for --about. (Example format: http://example.com)"
-                )
-
-    def handle_multi_dataset(self, context):
-        self.check_attr("multi_dataset__dataroot", (type(None), dict, str))
-        self.check_attr("multi_dataset__index", (type(None), bool, str))
-        self.check_attr("multi_dataset__allowed_matrix_types", list)
-        self.check_attr("multi_dataset__matrix_cache__max_datasets", int)
-        self.check_attr("multi_dataset__matrix_cache__timelimit_s", (type(None), int, float))
-
-        if self.multi_dataset__dataroot is None:
-            return
-
-        if type(self.multi_dataset__dataroot) == str:
-            default_dict = dict(base_url="d", dataroot=self.multi_dataset__dataroot)
-            self.multi_dataset__dataroot = dict(d=default_dict)
-
-        for tag, dataroot_dict in self.multi_dataset__dataroot.items():
-            if "base_url" not in dataroot_dict:
-                raise ConfigurationError(f"error in multi_dataset__dataroot: missing base_url for tag {tag}")
-            if "dataroot" not in dataroot_dict:
-                raise ConfigurationError(f"error in multi_dataset__dataroot: missing dataroot, for tag {tag}")
-
-            base_url = dataroot_dict["base_url"]
-
-            # sanity check for well formed base urls
-            bad = False
-            if type(base_url) != str:
-                bad = True
-            elif os.path.normpath(base_url) != base_url:
-                bad = True
-            else:
-                base_url_parts = base_url.split("/")
-                if [quote_plus(part) for part in base_url_parts] != base_url_parts:
-                    bad = True
-                if ".." in base_url_parts:
-                    bad = True
-            if bad:
-                raise ConfigurationError(f"error in multi_dataset__dataroot base_url {base_url} for tag {tag}")
-
-        # verify all the base_urls are unique
-        base_urls = [d["base_url"] for d in self.multi_dataset__dataroot.values()]
-        if len(base_urls) > len(set(base_urls)):
-            raise ConfigurationError("error in multi_dataset__dataroot:  base_urls must be unique")
-
-        # error checking
-        for mtype in self.multi_dataset__allowed_matrix_types:
-            try:
-                MatrixDataType(mtype)
-            except ValueError:
-                raise ConfigurationError(f'Invalid matrix type in "allowed_matrix_types": {mtype}')
-
-        # create the matrix data cache manager:
-        if self.matrix_data_cache_manager is None:
-            self.matrix_data_cache_manager = MatrixDataCacheManager(
-                max_cached=self.multi_dataset__matrix_cache__max_datasets,
-                timelimit_s=self.multi_dataset__matrix_cache__timelimit_s,
-            )
-
-    def handle_diffexp(self, context):
-        self.check_attr("diffexp__alg_cxg__max_workers", (str, int))
-        self.check_attr("diffexp__alg_cxg__cpu_multiplier", int)
-        self.check_attr("diffexp__alg_cxg__target_workunit", int)
-
-        max_workers = self.diffexp__alg_cxg__max_workers
-        cpu_multiplier = self.diffexp__alg_cxg__cpu_multiplier
-        cpu_count = os.cpu_count()
-        max_workers = min(max_workers, cpu_multiplier * cpu_count)
-        diffexp_tiledb.set_config(max_workers, self.diffexp__alg_cxg__target_workunit)
-
-    def handle_adaptor(self, context):
-        # cxg
-        self.check_attr("adaptor__cxg_adaptor__tiledb_ctx", dict)
-        regionkey = "vfs.s3.region"
-        if regionkey not in self.adaptor__cxg_adaptor__tiledb_ctx:
-            if type(self.data_locator__s3__region_name) == str:
-                self.adaptor__cxg_adaptor__tiledb_ctx[regionkey] = self.data_locator__s3__region_name
-
-        from server.data_cxg.cxg_adaptor import CxgAdaptor
-
-        CxgAdaptor.set_tiledb_context(self.adaptor__cxg_adaptor__tiledb_ctx)
-
-        # anndata
-        self.check_attr("adaptor__anndata_adaptor__backed", bool)
-
-    def handle_limits(self, context):
-        self.check_attr("limits__diffexp_cellcount_max", (type(None), int))
-        self.check_attr("limits__column_request_max", (type(None), int))
-
-    def exceeds_limit(self, limit_name, value):
-        limit_value = getattr(self, "limits__" + limit_name, None)
-        if limit_value is None:  # disabled
-            return False
-        return value > limit_value
-
-    def get_api_base_url(self):
-        if self.app__api_base_url == "local":
-            return f"http://{self.app__host}:{self.app__port}"
-        if self.app__api_base_url and self.app__api_base_url.endswith("/"):
-            return self.app__api_base_url[:-1]
-        return self.app__api_base_url
-
-    def get_web_base_url(self):
-        if self.app__web_base_url == "local":
-            return f"http://{self.app__host}:{self.app__port}"
-        if self.app__web_base_url is None:
-            return self.get_api_base_url()
-        if self.app__web_base_url.endswith("/"):
-            return self.app__web_base_url[:-1]
-        return self.api__web_base_url
-
-
-class DatasetConfig(BaseConfig):
-    """Manages the config attribute associated with a dataset."""
-
-    def __init__(self, tag, app_config, default_config):
-        super().__init__(app_config, default_config)
-        self.tag = tag
-        dc = default_config
-        try:
-            self.app__scripts = dc["app"]["scripts"]
-            self.app__inline_scripts = dc["app"]["inline_scripts"]
-            self.app__about_legal_tos = dc["app"]["about_legal_tos"]
-            self.app__about_legal_privacy = dc["app"]["about_legal_privacy"]
-            self.app__authentication_enable = dc["app"]["authentication_enable"]
-
-            self.presentation__max_categories = dc["presentation"]["max_categories"]
-            self.presentation__custom_colors = dc["presentation"]["custom_colors"]
-
-            self.user_annotations__enable = dc["user_annotations"]["enable"]
-            self.user_annotations__type = dc["user_annotations"]["type"]
-            self.user_annotations__local_file_csv__directory = dc["user_annotations"]["local_file_csv"]["directory"]
-            self.user_annotations__local_file_csv__file = dc["user_annotations"]["local_file_csv"]["file"]
-            self.user_annotations__ontology__enable = dc["user_annotations"]["ontology"]["enable"]
-            self.user_annotations__ontology__obo_location = dc["user_annotations"]["ontology"]["obo_location"]
-            self.user_annotations__hosted_tiledb_array__db_uri = dc["user_annotations"]["hosted_tiledb_array"]["db_uri"]
-            self.user_annotations__hosted_tiledb_array__hosted_file_directory = \
-                dc["user_annotations"][ "hosted_tiledb_array" ][ "hosted_file_directory" ]  # noqa E501
-
-            self.embeddings__names = dc["embeddings"]["names"]
-            self.embeddings__enable_reembedding = dc["embeddings"]["enable_reembedding"]
-
-            self.diffexp__enable = dc["diffexp"]["enable"]
-            self.diffexp__lfc_cutoff = dc["diffexp"]["lfc_cutoff"]
-            self.diffexp__top_n = dc["diffexp"]["top_n"]
-
-        except KeyError as e:
-            raise ConfigurationError(f"Unexpected config: {str(e)}")
-
-        # The annotation object is created during complete_config and stored here.
-        self.user_annotations = None
-
-    def complete_config(self, context):
-        self.handle_app(context)
-        self.handle_presentation(context)
-        self.handle_user_annotations(context)
-        self.handle_embeddings(context)
-        self.handle_diffexp(context)
-
-    def handle_app(self, context):
-        self.check_attr("app__scripts", list)
-        self.check_attr("app__inline_scripts", list)
-        self.check_attr("app__about_legal_tos", (type(None), str))
-        self.check_attr("app__about_legal_privacy", (type(None), str))
-        self.check_attr("app__authentication_enable", bool)
-
-        # scripts can be string (filename) or dict (attributes).   Convert string to dict.
-        scripts = []
-        for s in self.app__scripts:
-            if isinstance(s, str):
-                scripts.append({"src": s})
-            elif isinstance(s, dict) and isinstance(s["src"], str):
-                scripts.append(s)
-            else:
-                raise ConfigurationError("Scripts must be string or dict")
-        self.app__scripts = scripts
-
-    def handle_presentation(self, context):
-        self.check_attr("presentation__max_categories", int)
-        self.check_attr("presentation__custom_colors", bool)
-
-    def handle_user_annotations(self, context):
-        self.check_attr("user_annotations__enable", bool)
-        self.check_attr("user_annotations__type", str)
-        self.check_attr("user_annotations__local_file_csv__directory", (type(None), str))
-        self.check_attr("user_annotations__local_file_csv__file", (type(None), str))
-        self.check_attr("user_annotations__ontology__enable", bool)
-        self.check_attr("user_annotations__ontology__obo_location", (type(None), str))
-        self.check_attr("user_annotations__hosted_tiledb_array__db_uri", (type(None), str))
-        self.check_attr("user_annotations__hosted_tiledb_array__hosted_file_directory", (type(None), str))
-
-        if self.user_annotations__enable:
-            server_config = self.app_config.server_config
-            if not self.app__authentication_enable:
-                raise ConfigurationError("user annotations requires authentication to be enabled")
-            if not server_config.auth.is_valid_authentication_type():
-                auth_type = server_config.authentication__type
-                raise ConfigurationError(f"authentication method {auth_type} is not compatible with user annotations")
-
-            # TODO, replace this with a factory pattern once we have more than one way
-            # to do annotations.  currently only local_file_csv
-            if self.user_annotations__type == "local_file_csv":
-                dirname = self.user_annotations__local_file_csv__directory
-                filename = self.user_annotations__local_file_csv__file
-
-                if filename is not None and dirname is not None:
-                    raise ConfigurationError("'annotations-file' and 'annotations-dir' may not be used together.")
-
-                if filename is not None:
-                    lf_name, lf_ext = splitext(filename)
-                    if lf_ext and lf_ext != ".csv":
-                        raise ConfigurationError(f"annotation file type must be .csv: {filename}")
-
-                if dirname is not None and not isdir(dirname):
-                    try:
-                        os.mkdir(dirname)
-                    except OSError:
-                        raise ConfigurationError("Unable to create directory specified by --annotations-dir")
-
-                self.user_annotations = AnnotationsLocalFile(dirname, filename)
-
-                # if the user has specified a fixed label file, go ahead and validate it
-                # so that we can remove errors early in the process.
-                server_config = self.app_config.server_config
-                if server_config.single_dataset__datapath and self.user_annotations__local_file_csv__file:
-                    with server_config.matrix_data_cache_manager.data_adaptor(
-                            self.tag, server_config.single_dataset__datapath, self.app_config
-                    ) as data_adaptor:
-                        data_adaptor.check_new_labels(self.user_annotations.read_labels(data_adaptor))
-
-                if self.user_annotations__ontology__enable or self.user_annotations__ontology__obo_location:
-                    try:
-                        self.user_annotations.load_ontology(self.user_annotations__ontology__obo_location)
-                    except OntologyLoadFailure as e:
-                        raise ConfigurationError("Unable to load ontology terms\n" + str(e))
-            elif self.user_annotations__type == "hosted_tiledb_array":
-                self.check_attr("user_annotations__hosted_tiledb_array__db_uri", str)
-                self.check_attr("user_annotations__hosted_tiledb_array__hosted_file_directory", str)
-                self.user_annotations = AnnotationsHostedTileDB(
-                    directory_path=self.user_annotations__hosted_tiledb_array__hosted_file_directory,
-                    db=DbUtils(self.user_annotations__hosted_tiledb_array__db_uri),
-                )
-            else:
-                raise ConfigurationError('The only annotation type support is "local_file_csv" or "hosted_tiledb_array')
-        else:
-            if self.user_annotations__type == "local_file_csv":
-                dirname = self.user_annotations__local_file_csv__directory
-                filename = self.user_annotations__local_file_csv__file
-                if filename is not None:
-                    context["messsagefn"]("Warning: --annotations-file ignored as annotations are disabled.")
-                if dirname is not None:
-                    context["messagefn"]("Warning: --annotations-dir ignored as annotations are disabled.")
-
-            if self.user_annotations__ontology__enable:
-                context["messagefn"](
-                    "Warning: --experimental-annotations-ontology" " ignored as annotations are disabled."
-                )
-            if self.user_annotations__ontology__obo_location is not None:
-                context["messagefn"](
-                    "Warning: --experimental-annotations-ontology-obo" " ignored as annotations are disabled."
-                )
-
-    def handle_embeddings(self, context):
-        self.check_attr("embeddings__names", list)
-        self.check_attr("embeddings__enable_reembedding", bool)
-
-        server_config = self.app_config.server_config
-        if self.embeddings__enable_reembedding:
-            if server_config.single_dataset__datapath:
-                matrix_data_loader = MatrixDataLoader(
-                    server_config.single_dataset__datapath, app_config=self.app_config
-                )
-                if matrix_data_loader.matrix_data_type != MatrixDataType.H5AD:
-                    raise ConfigurationError("enable-reembedding is only supported with H5AD files.")
-                if server_config.adaptor__anndata_adaptor__backed:
-                    raise ConfigurationError("enable-reembedding is not supported when run in --backed mode.")
-
-            try:
-                server.compute.scanpy.get_scanpy_module()
-            except NotImplementedError:
-                raise ConfigurationError("Please install scanpy to enable UMAP re-embedding")
-
-    def handle_diffexp(self, context):
-        self.check_attr("diffexp__enable", bool)
-        self.check_attr("diffexp__lfc_cutoff", float)
-        self.check_attr("diffexp__top_n", int)
-
-        server_config = self.app_config.server_config
-        if server_config.single_dataset__datapath:
-            with server_config.matrix_data_cache_manager.data_adaptor(
-                    self.tag, server_config.single_dataset__datapath, self.app_config
-            ) as data_adaptor:
-                if self.diffexp__enable and data_adaptor.parameters.get("diffexp_may_be_slow", False):
-                    context["messagefn"](
-                        "CAUTION: due to the size of your dataset, "
-                        "running differential expression may take longer or fail."
-                    )

server/common/aws_secret_utils.py

@@ -1,72 +1,11 @@
 import logging
-import os
-import sys
 
 import boto3
 from flask import json
 
-from server.common.data_locator import discover_s3_region_name
 from server.common.errors import SecretKeyRetrievalError
 
 
-def handle_config_from_secret(app_config):
-    """Update configuration from the secret manager"""
-    secret_name = os.getenv("CXG_AWS_SECRET_NAME")
-    if not secret_name:
-        return
-
-    # need to find the secret manager region.
-    #  1. from CXG_AWS_SECRET_REGION_NAME
-    #  2. discover from dataroot location (if on s3)
-    #  3. discover from config file location (if on s3)
-    secret_region_name = os.getenv("CXG_AWS_SECRET_REGION_NAME")
-    if secret_region_name is None:
-        secret_region_name = discover_s3_region_name(app_config.multi_dataset__dataroot)
-        if not secret_region_name:
-            from server.eb.app import config_file
-            secret_region_name = discover_s3_region_name(config_file)
-    if not secret_region_name:
-        logging.error("Could not determine the AWS Secret Manager region")
-        sys.exit(1)
-
-    secrets = get_secret_key(secret_region_name, secret_name)
-
-    if not secrets:
-        return
-
-    server_attrs = (
-        ("flask_secret_key", "app__flask_secret_key"),
-        ("oauth_client_secret", "authentication__params_oauth__client_secret"),
-    )
-    default_dataset_attrs = (
-        ("db_uri", "user_annotations__hosted_tiledb_array__db_uri"),
-    )
-
-    # update server configuration attributes
-    for key, attr in server_attrs:
-        cur_val = getattr(app_config.server_config, attr)
-        if cur_val:
-            continue
-
-        # replace the attr with the secret if it is not set
-        val = secrets.get(key)
-        if val:
-            logging.info(f"set {attr} from secret")
-            app_config.update_server_config(**{attr : val})
-
-    # update default dataset configuration attributes
-    for key, attr in default_dataset_attrs:
-        cur_val = getattr(app_config.default_dataset_config, attr)
-        if cur_val:
-            continue
-
-        # replace the attr with the secret if it is not set
-        val = secrets.get(key)
-        if val:
-            logging.info(f"set {attr} from secret")
-            app_config.update_default_dataset_config(**{attr : val})
-
-
 def get_secret_key(region_name, secret_name):
     session = boto3.session.Session()
     client = session.client(service_name="secretsmanager", region_name=region_name)

server/common/config/__init__.py

@@ -0,0 +1,66 @@
+import logging
+import os
+import sys
+
+from server.common.aws_secret_utils import get_secret_key
+from server.common.data_locator import discover_s3_region_name
+
+DEFAULT_SERVER_PORT = 5005
+BIG_FILE_SIZE_THRESHOLD = 100 * 2 ** 20  # 100MB
+
+
+def handle_config_from_secret(app_config):
+    """Update configuration from the secret manager"""
+    secret_name = os.getenv("CXG_AWS_SECRET_NAME")
+    if not secret_name:
+        return
+
+    # need to find the secret manager region.
+    #  1. from CXG_AWS_SECRET_REGION_NAME
+    #  2. discover from dataroot location (if on s3)
+    #  3. discover from config file location (if on s3)
+    secret_region_name = os.getenv("CXG_AWS_SECRET_REGION_NAME")
+    if secret_region_name is None:
+        secret_region_name = discover_s3_region_name(app_config.multi_dataset__dataroot)
+        if not secret_region_name:
+            from server.eb.app import config_file
+
+            secret_region_name = discover_s3_region_name(config_file)
+    if not secret_region_name:
+        logging.error("Could not determine the AWS Secret Manager region")
+        sys.exit(1)
+
+    secrets = get_secret_key(secret_region_name, secret_name)
+
+    if not secrets:
+        return
+
+    server_attrs = (
+        ("flask_secret_key", "app__flask_secret_key"),
+        ("oauth_client_secret", "authentication__params_oauth__client_secret"),
+    )
+    default_dataset_attrs = (("db_uri", "user_annotations__hosted_tiledb_array__db_uri"),)
+
+    # update server configuration attributes
+    for key, attr in server_attrs:
+        cur_val = getattr(app_config.server_config, attr)
+        if cur_val:
+            continue
+
+        # replace the attr with the secret if it is not set
+        val = secrets.get(key)
+        if val:
+            logging.info(f"set {attr} from secret")
+            app_config.update_server_config(**{attr: val})
+
+    # update default dataset configuration attributes
+    for key, attr in default_dataset_attrs:
+        cur_val = getattr(app_config.default_dataset_config, attr)
+        if cur_val:
+            continue
+
+        # replace the attr with the secret if it is not set
+        val = secrets.get(key)
+        if val:
+            logging.info(f"set {attr} from secret")
+            app_config.update_default_dataset_config(**{attr: val})

server/common/config/app_config.py

@@ -0,0 +1,183 @@
+import yaml
+from flatten_dict import unflatten
+
+from server.default_config import get_default_config
+from server.common.config.dataset_config import DatasetConfig
+from server.common.config.server_config import ServerConfig
+from server.common.errors import ConfigurationError
+
+
+class AppConfig(object):
+    """
+    AppConfig stores all the configuration for cellxgene.
+    AppConfig contains one or more DatasetConfig(s) and one ServerConfig.
+    The server_config contains attributes that refer to the server process as a whole.
+    The default_dataset_config refers to attributes that are associated with the features and
+    presentations of a dataset.
+    The dataset config attributes can be overridden depending on the url by which the
+    dataset was accessed.  These are stored in dataroot_config.
+    AppConfig has methods to initialize, modify, and access the configuration.
+    """
+
+    def __init__(self):
+
+        # the default configuration (see default_config.py)
+        # TODO @madison -- if we always read from the default config (hard coded path) can we set those values as
+        #  defaults within the config class?
+        self.default_config = get_default_config()
+        # the server configuration
+        self.server_config = ServerConfig(self, self.default_config["server"])
+        # the dataset config, unless overridden by an entry in dataroot_config
+        self.default_dataset_config = DatasetConfig(None, self, self.default_config["dataset"])
+        # a dictionary of keys to DatasetConfig objects.  Each key must exist in the multi_dataset__dataroot
+        # attribute of the server_config. The default dataset config will apply to all datasets unless a different set
+        # of config vars was passed for a specific dataset under the multidataset config. For example:
+        """
+        per_dataset_config:
+            d1:
+               user_annotations:
+                   enable:  false
+            d2:
+               user_annotations:
+                   enable:  true
+         """
+        #   dataroot config
+        self.dataroot_config = {}
+
+        # Set to true when config_completed is called
+        self.is_completed = False
+
+    def get_dataset_config(self, dataroot_key):
+        if self.server_config.single_dataset__datapath:
+            return self.default_dataset_config
+        else:
+            return self.dataroot_config.get(dataroot_key, self.default_dataset_config)
+
+    def check_config(self):
+        """Verify all the attributes in the config have been type checked"""
+        if not self.is_completed:
+            raise ConfigurationError("The configuration has not been completed")
+        self.server_config.check_config()
+        self.default_dataset_config.check_config()
+        for dataset_config in self.dataroot_config.values():
+            dataset_config.check_config()
+
+    def update_server_config(self, **kw):
+        self.server_config.update(**kw)
+        self.is_complete = False
+
+    def update_default_dataset_config(self, **kw):
+        self.default_dataset_config.update(**kw)
+        # update all the other dataset configs, if any
+        for value in self.dataroot_config.values():
+            value.update(**kw)
+        self.is_complete = False
+
+    def update_from_config_file(self, config_file):
+        try:
+            with open(config_file) as yml_file:
+                config = yaml.safe_load(yml_file)
+        except yaml.YAMLError as e:
+            raise ConfigurationError(f"The specified config file contained an error: {e}")
+        except OSError as e:
+            raise ConfigurationError(f"Issue retrieving the specified config file: {e}")
+
+        if config.get("server"):
+            self.server_config.update_from_config(config["server"], "server")
+        if config.get("dataset"):
+            self.default_dataset_config.update_from_config(config["dataset"], "dataset")
+
+        per_dataset_config = config.get("per_dataset_config", {})
+        for key, dataroot_config in per_dataset_config.items():
+            # first create and initialize the dataroot with the default config
+            self.add_dataroot_config(key, **config["dataset"])
+            # then apply the per dataset configuration
+            self.dataroot_config[key].update_from_config(dataroot_config, f"per_dataset_config__{key}")
+
+        self.is_complete = False
+
+    def write_config(self, config_file):
+        """output the config to a yaml file"""
+        server = self.server_config.create_mapping(self.server_config.default_config)
+        dataset = self.default_dataset_config.create_mapping(self.default_dataset_config.default_config)
+        config = dict(server={}, dataset={})
+        for attrname in server.keys():
+            config["server__" + attrname] = getattr(self.server_config, attrname)
+        for attrname in dataset.keys():
+            config["dataset__" + attrname] = getattr(self.default_dataset_config, attrname)
+        if self.dataroot_config:
+            config["per_dataset_config"] = {}
+        for dataroot_tag, dataroot_config in self.dataroot_config.items():
+            dataset = dataroot_config.create_mapping(dataroot_config.default_config)
+            for attrname in dataset.keys():
+                config[f"per_dataset_config__{dataroot_tag}__" + attrname] = getattr(dataroot_config, attrname)
+
+        config = unflatten(config, splitter=lambda key: key.split("__"))
+        yaml.dump(config, open(config_file, "w"))
+
+    def changes_from_default(self):
+        """Return all the attribute that are different from the default"""
+        diff_server = self.server_config.changes_from_default()
+        diff_dataset = self.default_dataset_config.changes_from_default()
+        diff = dict(server=diff_server, dataset=diff_dataset)
+        return diff
+
+    def add_dataroot_config(self, dataroot_tag, **kw):
+        """Create a new dataset config object based on the default dataset config, and kw parameters"""
+        if dataroot_tag in self.dataroot_config:
+            raise ConfigurationError(f"dataroot config already exists: {dataroot_tag}")
+        if type(self.server_config.multi_dataset__dataroot) != dict:
+            raise ConfigurationError("The server__multi_dataset__dataroot must be a dictionary")
+        if dataroot_tag not in self.server_config.multi_dataset__dataroot:
+            raise ConfigurationError(f"The dataroot_tag ({dataroot_tag}) not found in server__multi_dataset__dataroot")
+
+        self.is_completed = False
+        self.dataroot_config[dataroot_tag] = DatasetConfig(dataroot_tag, self, self.default_config["dataset"])
+        flat_config = self.default_dataset_config.create_mapping(self.default_dataset_config.default_config)
+        config = {key: value[1] for key, value in flat_config.items()}
+        self.dataroot_config[dataroot_tag].update(**config)
+        self.dataroot_config[dataroot_tag].update_from_config(kw, dataroot_tag)
+
+    def complete_config(self, messagefn=None):
+        """The configure options are checked, and any additional setup based on the config
+        parameters is done"""
+
+        if messagefn is None:
+
+            def noop(message):
+                pass
+
+            messagefn = noop
+
+        # TODO: to give better error messages we can add a mapping between where each config
+        # attribute originated (e.g. command line argument or config file), then in the error
+        # messages we can give correct context for attributes with bad value.
+        context = dict(messagefn=messagefn)
+
+        self.server_config.complete_config(context)
+        self.default_dataset_config.complete_config(context)
+        for dataroot_config in self.dataroot_config.values():
+            dataroot_config.complete_config(context)
+
+        self.is_completed = True
+        self.check_config()
+
+    def get_matrix_data_cache_manager(self):
+        return self.server_config.matrix_data_cache_manager
+
+    def is_multi_dataset(self):
+        return self.server_config.multi_dataset__dataroot is not None
+
+    def get_title(self, data_adaptor):
+        return (
+            self.server_config.single_dataset__title
+            if self.server_config.single_dataset__title
+            else data_adaptor.get_title()
+        )
+
+    def get_about(self, data_adaptor):
+        return (
+            self.server_config.single_dataset__about
+            if self.server_config.single_dataset__about
+            else data_adaptor.get_about()
+        )

server/common/config/base_config.py

@@ -0,0 +1,113 @@
+import copy
+
+from flatten_dict import flatten
+from server.common.errors import ConfigurationError
+
+
+class BaseConfig(object):
+    """
+    This class handles the mechanics of updating and checking attributes.
+    Derived classes are expected to store the actual attributes
+    Currently DatasetConfig and ServerConfig both inherit from BaseConfig.
+    """
+
+    def __init__(self, app_config, default_config, dictval_cases={}):
+        # reference back to the app_config
+        self.app_config = app_config
+        # the complete set of attributes and their default values (unflattened)
+        self.default_config = default_config
+        # attributes where the value may be a dict (and therefore are not flattened)
+        self.dictval_cases = dictval_cases
+        # used to make sure every attribute value is checked
+        self.attr_checked = {key_name: False for key_name in self.create_mapping(default_config).keys()}
+
+    def create_mapping(self, config):
+        """
+        Create a dictionary where the keys are the name of attributes (using double underscore convention)
+        For example: authentication__type
+
+        The values are a tuple,
+        - the first item of the tuple is a tuple of path elements (location in config 'tree')
+        - the second item is the value of the config parameter
+
+        For example: (('authentication', 'type'), 'session'))
+        """
+        config_copy = copy.deepcopy(config)
+        mapping = {}
+
+        # special cases where the value could be a dict.
+        # If its value is not None, the entry is added to the mapping, and not included
+        # in the flattening below.
+        for dictval_case in self.dictval_cases:
+            cur = config_copy
+            for part in dictval_case[:-1]:
+                cur = cur.get(part, {})
+            val = cur.get(dictval_case[-1])
+            if val is not None:
+                key = "__".join(dictval_case)
+                mapping[key] = (dictval_case, val)
+                del cur[dictval_case[-1]]
+
+        flat_config = flatten(config_copy)
+        for key, value in flat_config.items():
+            # name of the attribute
+            attr = "__".join(key)
+            mapping[attr] = (key, value)
+
+        return mapping
+
+    def validate_correct_type_of_configuration_attribute(self, attrname, vtype):
+        val = getattr(self, attrname)
+        if type(vtype) in (list, tuple):
+            if type(val) not in vtype:
+                tnames = ",".join([x.__name__ for x in vtype])
+                raise ConfigurationError(
+                    f"Invalid type for attribute: {attrname}, expected types ({tnames}), got {type(val).__name__}"
+                )
+        else:
+            if type(val) != vtype:
+                raise ConfigurationError(
+                    f"Invalid type for attribute: {attrname}, "
+                    f"expected type {vtype.__name__}, got {type(val).__name__}"
+                )
+
+        self.attr_checked[attrname] = True
+
+    def check_config(self):
+        mapping = self.create_mapping(self.default_config)
+        for key in mapping.keys():
+            if not self.attr_checked[key]:
+                raise ConfigurationError(f"The attr '{key}' has not been checked")
+
+    def update(self, **kw):
+        for key, value in kw.items():
+            if not hasattr(self, key):
+                raise ConfigurationError(f"unknown config parameter {key}.")
+            try:
+                if type(value) == tuple:
+                    # convert tuple values to list values
+                    value = list(value)
+                setattr(self, key, value)
+            except KeyError:
+                raise ConfigurationError(f"Unable to set config parameter {key}.")
+
+            self.attr_checked[key] = False
+
+    def update_from_config(self, config, prefix):
+        mapping = self.create_mapping(config)
+        for attr, (key, value) in mapping.items():
+            if not hasattr(self, attr):
+                raise ConfigurationError(f"Unknown key from config file: {prefix}__{attr}")
+            setattr(self, attr, value)
+
+            self.attr_checked[attr] = False
+
+    def changes_from_default(self):
+        """Return all the attribute that are different from the default"""
+        mapping = self.create_mapping(self.default_config)
+        diff = []
+        for attrname, (key, defval) in mapping.items():
+            curval = getattr(self, attrname)
+            if curval != defval:
+                diff.append((attrname, curval, defval))
+        return diff

server/common/config/client_config.py

@@ -0,0 +1,121 @@
+from server import display_version as cellxgene_display_version
+
+
+def get_client_config(app_config, data_adaptor):
+    """
+    Return the configuration as required by the /config REST route
+    """
+
+    server_config = app_config.server_config
+    dataset_config = data_adaptor.dataset_config
+    annotation = dataset_config.user_annotations
+    auth = server_config.auth
+
+    # FIXME The current set of config is not consistently presented:
+    # we have camalCase, hyphen-text, and underscore_text
+
+    # make sure the configuration has been checked.
+    app_config.check_config()
+
+    # display_names
+    title = app_config.get_title(data_adaptor)
+    about = app_config.get_about(data_adaptor)
+
+    display_names = dict(engine=data_adaptor.get_name(), dataset=title)
+
+    # library_versions
+    library_versions = {}
+    library_versions.update(data_adaptor.get_library_versions())
+    library_versions["cellxgene"] = cellxgene_display_version
+
+    # links
+    links = {"about-dataset": about}
+
+    # parameters
+    parameters = {
+        "layout": dataset_config.embeddings__names,
+        "max-category-items": dataset_config.presentation__max_categories,
+        "obs_names": server_config.single_dataset__obs_names,
+        "var_names": server_config.single_dataset__var_names,
+        "diffexp_lfc_cutoff": dataset_config.diffexp__lfc_cutoff,
+        "backed": server_config.adaptor__anndata_adaptor__backed,
+        "disable-diffexp": not dataset_config.diffexp__enable,
+        "enable-reembedding": dataset_config.embeddings__enable_reembedding,
+        "annotations": False,
+        "annotations_file": None,
+        "annotations_dir": None,
+        "annotations_cell_ontology_enabled": False,
+        "annotations_cell_ontology_obopath": None,
+        "annotations_cell_ontology_terms": None,
+        "custom_colors": dataset_config.presentation__custom_colors,
+        "diffexp-may-be-slow": False,
+        "about_legal_tos": dataset_config.app__about_legal_tos,
+        "about_legal_privacy": dataset_config.app__about_legal_privacy,
+    }
+
+    # corpora dataset_props
+    # TODO/Note: putting info from the dataset into the /config is not ideal.
+    # However, it is definitely not part of /schema, and we do not have a top-level
+    # route for data properties.  Consider creating one at some point.
+    corpora_props = data_adaptor.get_corpora_props()
+    if corpora_props and "default_embedding" in corpora_props:
+        default_embedding = corpora_props["default_embedding"]
+        if isinstance(default_embedding, str) and default_embedding.startswith("X_"):
+            default_embedding = default_embedding[2:]  # drop X_ prefix
+        if default_embedding in data_adaptor.get_embedding_names():
+            parameters["default_embedding"] = default_embedding
+
+    data_adaptor.update_parameters(parameters)
+    if annotation:
+        annotation.update_parameters(parameters, data_adaptor)
+
+    # gather it all together
+    client_config = {}
+    config = client_config["config"] = {}
+    config["displayNames"] = display_names
+    config["library_versions"] = library_versions
+    config["links"] = links
+    config["parameters"] = parameters
+    config["corpora_props"] = corpora_props
+    config["limits"] = {
+        "column_request_max": server_config.limits__column_request_max,
+        "diffexp_cellcount_max": server_config.limits__diffexp_cellcount_max,
+    }
+
+    if dataset_config.app__authentication_enable and auth.is_valid_authentication_type():
+        config["authentication"] = {
+            "requires_client_login": auth.requires_client_login(),
+        }
+        if auth.requires_client_login():
+            config["authentication"].update(
+                {
+                    # Todo why are these stored on the data_adaptor?
+                    "login": auth.get_login_url(data_adaptor),
+                    "logout": auth.get_logout_url(data_adaptor),
+                }
+            )
+
+    return client_config
+
+
+def get_client_userinfo(app_config, data_adaptor):
+    """
+    Return the userinfo as required by the /userinfo REST route
+    """
+
+    server_config = app_config.server_config
+    dataset_config = data_adaptor.dataset_config
+    auth = server_config.auth
+
+    # make sure the configuration has been checked.
+    app_config.check_config()
+
+    if dataset_config.app__authentication_enable and auth.is_valid_authentication_type():
+        userinfo = {}
+        userinfo["userinfo"] = {
+            "is_authenticated": auth.is_user_authenticated(),
+            "username": auth.get_user_name(),
+            "user_id": auth.get_user_id(),
+            "email": auth.get_user_email(),
+        }
+        return userinfo

server/common/config/dataset_config.py

@@ -0,0 +1,234 @@
+import os
+from os.path import splitext, isdir
+
+from server.common.annotations.hosted_tiledb import AnnotationsHostedTileDB
+from server.common.annotations.local_file_csv import AnnotationsLocalFile
+from server.common.config.base_config import BaseConfig
+from server.common.errors import ConfigurationError, OntologyLoadFailure
+from server.compute.scanpy import get_scanpy_module
+from server.data_common.matrix_loader import MatrixDataLoader, MatrixDataType
+from server.db.db_utils import DbUtils
+
+
+class DatasetConfig(BaseConfig):
+    """Manages the config attribute associated with a dataset."""
+
+    def __init__(self, tag, app_config, default_config):
+        super().__init__(app_config, default_config)
+        self.tag = tag
+        try:
+            self.app__scripts = default_config["app"]["scripts"]
+            self.app__inline_scripts = default_config["app"]["inline_scripts"]
+            self.app__about_legal_tos = default_config["app"]["about_legal_tos"]
+            self.app__about_legal_privacy = default_config["app"]["about_legal_privacy"]
+            self.app__authentication_enable = default_config["app"]["authentication_enable"]
+
+            self.presentation__max_categories = default_config["presentation"]["max_categories"]
+            self.presentation__custom_colors = default_config["presentation"]["custom_colors"]
+
+            self.user_annotations__enable = default_config["user_annotations"]["enable"]
+            self.user_annotations__type = default_config["user_annotations"]["type"]
+            self.user_annotations__local_file_csv__directory = default_config["user_annotations"]["local_file_csv"][
+                "directory"
+            ]  # noqa E501
+            self.user_annotations__local_file_csv__file = default_config["user_annotations"]["local_file_csv"]["file"]
+            self.user_annotations__ontology__enable = default_config["user_annotations"]["ontology"]["enable"]
+            self.user_annotations__ontology__obo_location = default_config["user_annotations"]["ontology"][
+                "obo_location"
+            ]  # noqa E501
+            self.user_annotations__hosted_tiledb_array__db_uri = default_config["user_annotations"][
+                "hosted_tiledb_array"
+            ][
+                "db_uri"
+            ]  # noqa E501
+            self.user_annotations__hosted_tiledb_array__hosted_file_directory = default_config["user_annotations"][
+                "hosted_tiledb_array"
+            ][
+                "hosted_file_directory"
+            ]  # noqa E501
+
+            self.embeddings__names = default_config["embeddings"]["names"]
+            self.embeddings__enable_reembedding = default_config["embeddings"]["enable_reembedding"]
+
+            self.diffexp__enable = default_config["diffexp"]["enable"]
+            self.diffexp__lfc_cutoff = default_config["diffexp"]["lfc_cutoff"]
+            self.diffexp__top_n = default_config["diffexp"]["top_n"]
+
+        except KeyError as e:
+            raise ConfigurationError(f"Unexpected config: {str(e)}")
+
+        # The annotation object is created during complete_config and stored here.
+        self.user_annotations = None
+
+    def complete_config(self, context):
+        self.handle_app()
+        self.handle_presentation()
+        self.handle_user_annotations(context)
+        self.handle_embeddings()
+        self.handle_diffexp(context)
+
+    def handle_app(self):
+        self.validate_correct_type_of_configuration_attribute("app__scripts", list)
+        self.validate_correct_type_of_configuration_attribute("app__inline_scripts", list)
+        self.validate_correct_type_of_configuration_attribute("app__about_legal_tos", (type(None), str))
+        self.validate_correct_type_of_configuration_attribute("app__about_legal_privacy", (type(None), str))
+        self.validate_correct_type_of_configuration_attribute("app__authentication_enable", bool)
+
+        # scripts can be string (filename) or dict (attributes). Convert string to dict.
+        scripts = []
+        for script in self.app__scripts:
+            try:
+                if isinstance(script, str):
+                    scripts.append({"src": script})
+                elif isinstance(script, dict) and isinstance(script["src"], str):
+                    scripts.append(script)
+                else:
+                    raise Exception
+            except Exception as e:
+                raise ConfigurationError(f"Scripts must be string or a dict containing an src key: {e}")
+
+        self.app__scripts = scripts
+
+    def handle_presentation(self):
+        self.validate_correct_type_of_configuration_attribute("presentation__max_categories", int)
+        self.validate_correct_type_of_configuration_attribute("presentation__custom_colors", bool)
+
+    def handle_user_annotations(self, context):
+        self.validate_correct_type_of_configuration_attribute("user_annotations__enable", bool)
+        self.validate_correct_type_of_configuration_attribute("user_annotations__type", str)
+        self.validate_correct_type_of_configuration_attribute(
+            "user_annotations__local_file_csv__directory", (type(None), str)
+        )  # noqa E501
+        self.validate_correct_type_of_configuration_attribute(
+            "user_annotations__local_file_csv__file", (type(None), str)
+        )  # noqa E501
+        self.validate_correct_type_of_configuration_attribute("user_annotations__ontology__enable", bool)
+        self.validate_correct_type_of_configuration_attribute(
+            "user_annotations__ontology__obo_location", (type(None), str)
+        )  # noqa E501
+        self.validate_correct_type_of_configuration_attribute(
+            "user_annotations__hosted_tiledb_array__db_uri", (type(None), str)
+        )  # noqa E501
+        self.validate_correct_type_of_configuration_attribute(
+            "user_annotations__hosted_tiledb_array__hosted_file_directory", (type(None), str)
+        )  # noqa E501
+        if self.user_annotations__enable:
+            server_config = self.app_config.server_config
+            if not self.app__authentication_enable:
+                raise ConfigurationError("user annotations requires authentication to be enabled")
+            if not server_config.auth.is_valid_authentication_type():
+                auth_type = server_config.authentication__type
+                raise ConfigurationError(f"authentication method {auth_type} is not compatible with user annotations")
+
+            if self.user_annotations__type == "local_file_csv":
+                self.handle_local_file_csv_annotations()
+            elif self.user_annotations__type == "hosted_tiledb_array":
+                self.handle_hosted_tiledb_annotations()
+            else:
+                raise ConfigurationError('The only annotation type support is "local_file_csv" or "hosted_tiledb_array')
+            if self.user_annotations__ontology__enable or self.user_annotations__ontology__obo_location:
+                try:
+                    self.user_annotations.load_ontology(self.user_annotations__ontology__obo_location)
+                except OntologyLoadFailure as e:
+                    raise ConfigurationError("Unable to load ontology terms\n" + str(e))
+        else:
+            self.check_annotation_config_vars_not_set(context)
+
+    def handle_local_file_csv_annotations(self):
+        dirname = self.user_annotations__local_file_csv__directory
+        filename = self.user_annotations__local_file_csv__file
+        if filename is not None and dirname is not None:
+            raise ConfigurationError("'annotations-file' and 'annotations-dir' may not be used together.")
+
+        if filename is not None:
+            lf_name, lf_ext = splitext(filename)
+            if lf_ext and lf_ext != ".csv":
+                raise ConfigurationError(f"annotation file type must be .csv: {filename}")
+
+        if dirname is not None and not isdir(dirname):
+            try:
+                os.mkdir(dirname)
+            except OSError:
+                raise ConfigurationError("Unable to create directory specified by --annotations-dir")
+
+        self.user_annotations = AnnotationsLocalFile(dirname, filename)
+
+        # if the user has specified a fixed label file, go ahead and validate it
+        # so that we can remove errors early in the process.
+        server_config = self.app_config.server_config
+        if server_config.single_dataset__datapath and self.user_annotations__local_file_csv__file:
+            with server_config.matrix_data_cache_manager.data_adaptor(
+                self.tag, server_config.single_dataset__datapath, self.app_config
+            ) as data_adaptor:
+                data_adaptor.check_new_labels(self.user_annotations.read_labels(data_adaptor))
+
+    def handle_hosted_tiledb_annotations(self):
+        self.validate_correct_type_of_configuration_attribute("user_annotations__hosted_tiledb_array__db_uri", str)
+        self.validate_correct_type_of_configuration_attribute(
+            "user_annotations__hosted_tiledb_array__hosted_file_directory", str
+        )  # noqa E501
+        self.user_annotations = AnnotationsHostedTileDB(
+            directory_path=self.user_annotations__hosted_tiledb_array__hosted_file_directory,
+            db=DbUtils(self.user_annotations__hosted_tiledb_array__db_uri),
+        )
+
+    def check_annotation_config_vars_not_set(self, context):
+        if self.user_annotations__type is not None:
+            dirname = self.user_annotations__local_file_csv__directory
+            filename = self.user_annotations__local_file_csv__file
+            db_uri = self.user_annotations__hosted_tiledb_array__db_uri
+            hosted_file_dirname = self.user_annotations__hosted_tiledb_array__hosted_file_directory
+            if filename is not None:
+                context["messagefn"]("Warning: --annotations-file ignored as annotations are disabled.")
+            if dirname is not None:
+                context["messagefn"]("Warning: --annotations-dir ignored as annotations are disabled.")
+            if db_uri is not None:
+                context["messagefn"]("Warning: db_uri ignored as annotations are disabled.")
+            if hosted_file_dirname is not None:
+                context["messagefn"](
+                    "Warning: hosted_file_directory for hosted_tiledb_array ignored as annotations are disabled."
+                )
+
+        if self.user_annotations__ontology__enable:
+            context["messagefn"]("Warning: --experimental-annotations-ontology ignored as annotations are disabled.")
+        if self.user_annotations__ontology__obo_location is not None:
+            context["messagefn"](
+                "Warning: --experimental-annotations-ontology-obo ignored as annotations are disabled."
+            )
+
+    def handle_embeddings(self):
+        self.validate_correct_type_of_configuration_attribute("embeddings__names", list)
+        self.validate_correct_type_of_configuration_attribute("embeddings__enable_reembedding", bool)
+
+        server_config = self.app_config.server_config
+        if self.embeddings__enable_reembedding:
+            if server_config.single_dataset__datapath:
+                matrix_data_loader = MatrixDataLoader(
+                    server_config.single_dataset__datapath, app_config=self.app_config
+                )
+                if matrix_data_loader.matrix_data_type != MatrixDataType.H5AD:
+                    raise ConfigurationError("enable-reembedding is only supported with H5AD files.")
+                if server_config.adaptor__anndata_adaptor__backed:
+                    raise ConfigurationError("enable-reembedding is not supported when run in --backed mode.")
+
+            try:
+                get_scanpy_module()
+            except NotImplementedError:
+                # Todo add scanpy to requirements.txt and remove this check once re-embeddings is fully supported
+                raise ConfigurationError("Please install scanpy to enable UMAP re-embedding")
+
+    def handle_diffexp(self, context):
+        self.validate_correct_type_of_configuration_attribute("diffexp__enable", bool)
+        self.validate_correct_type_of_configuration_attribute("diffexp__lfc_cutoff", float)
+        self.validate_correct_type_of_configuration_attribute("diffexp__top_n", int)
+
+        server_config = self.app_config.server_config
+        if server_config.single_dataset__datapath:
+            with server_config.matrix_data_cache_manager.data_adaptor(
+                self.tag, server_config.single_dataset__datapath, self.app_config
+            ) as data_adaptor:
+                if self.diffexp__enable and data_adaptor.parameters.get("diffexp_may_be_slow", False):
+                    context["messagefn"](
+                        "CAUTION: due to the size of your dataset, "
+                        "running differential expression may take longer or fail."
+                    )

server/common/config/server_config.py

@@ -0,0 +1,390 @@
+import os
+import sys
+import warnings
+from os.path import basename
+from urllib.parse import urlparse, quote_plus
+
+from server.auth.auth import AuthTypeFactory
+from server.common.config.base_config import BaseConfig
+from server.common.config import DEFAULT_SERVER_PORT, BIG_FILE_SIZE_THRESHOLD
+from server.common.errors import ConfigurationError, DatasetAccessError
+from server.common.data_locator import discover_s3_region_name
+from server.common.utils.utils import is_port_available, find_available_port, custom_format_warning
+from server.compute import diffexp_cxg as diffexp_tiledb
+from server.data_common.matrix_loader import MatrixDataCacheManager, MatrixDataLoader, MatrixDataType
+
+
+class ServerConfig(BaseConfig):
+    """Manages the config attribute associated with the server."""
+
+    def __init__(self, app_config, default_config):
+        dictval_cases = [
+            ("app", "csp_directives"),
+            ("authentication", "params_oauth", "cookie"),
+            ("authentication", "params_oauth", "jwt_decode_options"),
+            ("adaptor", "cxg_adaptor", "tiledb_ctx"),
+            ("multi_dataset", "dataroot"),
+        ]
+        super().__init__(app_config, default_config, dictval_cases)
+
+        try:
+            self.app__verbose = default_config["app"]["verbose"]
+            self.app__debug = default_config["app"]["debug"]
+            self.app__host = default_config["app"]["host"]
+            self.app__port = default_config["app"]["port"]
+            self.app__open_browser = default_config["app"]["open_browser"]
+            self.app__force_https = default_config["app"]["force_https"]
+            self.app__flask_secret_key = default_config["app"]["flask_secret_key"]
+            self.app__generate_cache_control_headers = default_config["app"]["generate_cache_control_headers"]
+            self.app__server_timing_headers = default_config["app"]["server_timing_headers"]
+            self.app__csp_directives = default_config["app"]["csp_directives"]
+            self.app__api_base_url = default_config["app"]["api_base_url"]
+            self.app__web_base_url = default_config["app"]["web_base_url"]
+
+            self.authentication__type = default_config["authentication"]["type"]
+            self.authentication__params_oauth__oauth_api_base_url = default_config["authentication"]["params_oauth"][
+                "oauth_api_base_url"
+            ]  # noqa E501
+            self.authentication__params_oauth__client_id = default_config["authentication"]["params_oauth"]["client_id"]
+            self.authentication__params_oauth__client_secret = default_config["authentication"]["params_oauth"][
+                "client_secret"
+            ]  # noqa E501
+            self.authentication__params_oauth__jwt_decode_options = default_config["authentication"]["params_oauth"][
+                "jwt_decode_options"
+            ]  # noqa E501
+            self.authentication__params_oauth__session_cookie = default_config["authentication"]["params_oauth"][
+                "session_cookie"
+            ]  # noqa E501
+            self.authentication__params_oauth__cookie = default_config["authentication"]["params_oauth"]["cookie"]
+
+            self.multi_dataset__dataroot = default_config["multi_dataset"]["dataroot"]
+            self.multi_dataset__index = default_config["multi_dataset"]["index"]
+            self.multi_dataset__allowed_matrix_types = default_config["multi_dataset"]["allowed_matrix_types"]
+            self.multi_dataset__matrix_cache__max_datasets = default_config["multi_dataset"]["matrix_cache"][
+                "max_datasets"
+            ]  # noqa E501
+            self.multi_dataset__matrix_cache__timelimit_s = default_config["multi_dataset"]["matrix_cache"][
+                "timelimit_s"
+            ]  # noqa E501
+
+            self.single_dataset__datapath = default_config["single_dataset"]["datapath"]
+            self.single_dataset__obs_names = default_config["single_dataset"]["obs_names"]
+            self.single_dataset__var_names = default_config["single_dataset"]["var_names"]
+            self.single_dataset__about = default_config["single_dataset"]["about"]
+            self.single_dataset__title = default_config["single_dataset"]["title"]
+
+            self.diffexp__alg_cxg__max_workers = default_config["diffexp"]["alg_cxg"]["max_workers"]
+            self.diffexp__alg_cxg__cpu_multiplier = default_config["diffexp"]["alg_cxg"]["cpu_multiplier"]
+            self.diffexp__alg_cxg__target_workunit = default_config["diffexp"]["alg_cxg"]["target_workunit"]
+
+            self.data_locator__s3__region_name = default_config["data_locator"]["s3"]["region_name"]
+
+            self.adaptor__cxg_adaptor__tiledb_ctx = default_config["adaptor"]["cxg_adaptor"]["tiledb_ctx"]
+            self.adaptor__anndata_adaptor__backed = default_config["adaptor"]["anndata_adaptor"]["backed"]
+
+            self.limits__diffexp_cellcount_max = default_config["limits"]["diffexp_cellcount_max"]
+            self.limits__column_request_max = default_config["limits"]["column_request_max"]
+
+        except KeyError as e:
+            raise ConfigurationError(f"Unexpected config: {str(e)}")
+
+        # The matrix data cache manager is created during the complete_config and stored here.
+        self.matrix_data_cache_manager = None
+
+        # The authentication object
+        self.auth = None
+
+    def complete_config(self, context):
+        self.handle_app(context)
+        self.handle_data_source()
+        self.handle_authentication()
+        self.handle_data_locator()
+        self.handle_adaptor()  # may depend on data_locator
+        self.handle_single_dataset(context)  # may depend on adaptor
+        self.handle_multi_dataset()  # may depend on adaptor
+        self.handle_diffexp()
+        self.handle_limits()
+
+        self.check_config()
+
+    def handle_app(self, context):
+        self.validate_correct_type_of_configuration_attribute("app__verbose", bool)
+        self.validate_correct_type_of_configuration_attribute("app__debug", bool)
+        self.validate_correct_type_of_configuration_attribute("app__host", str)
+        self.validate_correct_type_of_configuration_attribute("app__port", (type(None), int))
+        self.validate_correct_type_of_configuration_attribute("app__open_browser", bool)
+        self.validate_correct_type_of_configuration_attribute("app__force_https", bool)
+        self.validate_correct_type_of_configuration_attribute("app__flask_secret_key", (type(None), str))
+        self.validate_correct_type_of_configuration_attribute("app__generate_cache_control_headers", bool)
+        self.validate_correct_type_of_configuration_attribute("app__server_timing_headers", bool)
+        self.validate_correct_type_of_configuration_attribute("app__csp_directives", (type(None), dict))
+        self.validate_correct_type_of_configuration_attribute("app__api_base_url", (type(None), str))
+        self.validate_correct_type_of_configuration_attribute("app__web_base_url", (type(None), str))
+
+        if self.app__port:
+            try:
+                if not is_port_available(self.app__host, self.app__port):
+                    raise ConfigurationError(
+                        f"The port selected {self.app__port} is in use, please configure an open port."
+                    )
+            except OverflowError:
+                raise ConfigurationError(f"Invalid port: {self.app__port}")
+        else:
+            try:
+                default_server_port = int(os.environ.get("CXG_SERVER_PORT", DEFAULT_SERVER_PORT))
+            except ValueError:
+                raise ConfigurationError(
+                    "Invalid port from environment variable CXG_SERVER_PORT: " + os.environ.get("CXG_SERVER_PORT")
+                )
+            try:
+                self.app__port = find_available_port(self.app__host, default_server_port)
+            except OverflowError:
+                raise ConfigurationError(f"Invalid port: {default_server_port}")
+
+        if self.app__debug:
+            context["messagefn"]("in debug mode, setting verbose=True and open_browser=False")
+            self.app__verbose = True
+            self.app__open_browser = False
+        else:
+            warnings.formatwarning = custom_format_warning
+
+        if not self.app__verbose:
+            sys.tracebacklimit = 0
+
+        # secret key:
+        #   first, from CXG_SECRET_KEY environment variable
+        #   second, from config file
+        self.app__flask_secret_key = os.environ.get("CXG_SECRET_KEY", self.app__flask_secret_key)
+
+        # CSP Directives are a dict of string: list(string) or string: string
+        if self.app__csp_directives is not None:
+            for k, v in self.app__csp_directives.items():
+                if not isinstance(k, str):
+                    raise ConfigurationError("CSP directive names must be a string.")
+                if isinstance(v, list):
+                    for policy in v:
+                        if not isinstance(policy, str):
+                            raise ConfigurationError("CSP directive value must be a string or list of strings.")
+                elif not isinstance(v, str):
+                    raise ConfigurationError("CSP directive value must be a string or list of strings.")
+
+        if self.app__web_base_url is None:
+            self.app__web_base_url = self.app__api_base_url
+
+    def handle_authentication(self):
+        self.validate_correct_type_of_configuration_attribute("authentication__type", (type(None), str))
+
+        # oauth
+        ptypes = str if self.authentication__type == "oauth" else (type(None), str)
+        self.validate_correct_type_of_configuration_attribute(
+            "authentication__params_oauth__oauth_api_base_url", ptypes
+        )  # noqa E501
+        self.validate_correct_type_of_configuration_attribute("authentication__params_oauth__client_id", ptypes)
+        self.validate_correct_type_of_configuration_attribute("authentication__params_oauth__client_secret", ptypes)
+        self.validate_correct_type_of_configuration_attribute(
+            "authentication__params_oauth__jwt_decode_options", (type(None), dict)
+        )  # noqa E501
+        self.validate_correct_type_of_configuration_attribute("authentication__params_oauth__session_cookie", bool)
+
+        if self.authentication__params_oauth__session_cookie:
+            self.validate_correct_type_of_configuration_attribute(
+                "authentication__params_oauth__cookie", (type(None), dict)
+            )  # noqa E501
+        else:
+            self.validate_correct_type_of_configuration_attribute("authentication__params_oauth__cookie", dict)
+        #   secret key: first, from CXG_OAUTH_CLIENT_SECRET environment variable
+        #   second, from config file
+        self.authentication__params_oauth__client_secret = os.environ.get(
+            "CXG_OAUTH_CLIENT_SECRET", self.authentication__params_oauth__client_secret
+        )
+
+        self.auth = AuthTypeFactory.create(self.authentication__type, self)
+        if self.auth is None:
+            raise ConfigurationError(f"Unknown authentication type: {self.authentication__type}")
+
+    def handle_data_locator(self):
+        self.validate_correct_type_of_configuration_attribute("data_locator__s3__region_name", (type(None), bool, str))
+        if self.data_locator__s3__region_name is True:
+            path = self.single_dataset__datapath or self.multi_dataset__dataroot
+
+            if type(path) == dict:
+                # if multi_dataset__dataroot is a dict, then use the first key
+                # that is in s3.   NOTE:  it is not supported to have dataroots
+                # in different regions.
+                paths = [val.get("dataroot") for val in path.values()]
+                for path in paths:
+                    if path.startswith("s3://"):
+                        break
+            if path.startswith("s3://"):
+                region_name = discover_s3_region_name(path)
+                if region_name is None:
+                    raise ConfigurationError(f"Unable to discover s3 region name from {path}")
+            else:
+                region_name = None
+            self.data_locator__s3__region_name = region_name
+
+    def handle_data_source(self):
+        self.validate_correct_type_of_configuration_attribute("single_dataset__datapath", (str, type(None)))
+        self.validate_correct_type_of_configuration_attribute("multi_dataset__dataroot", (type(None), dict, str))
+
+        if self.single_dataset__datapath and self.multi_dataset__dataroot:
+            raise ConfigurationError(
+                "You must supply either a datapath (for single datasets) or a dataroot (for multidatasets). Not both"
+            )
+        if self.single_dataset__datapath is None and self.multi_dataset__dataroot is None:
+            raise ConfigurationError("You must specify a datapath for a single dataset or a dataroot for multidatasets")
+
+    def handle_single_dataset(self, context):
+        self.validate_correct_type_of_configuration_attribute("single_dataset__datapath", (str, type(None)))
+        self.validate_correct_type_of_configuration_attribute("single_dataset__title", (str, type(None)))
+        self.validate_correct_type_of_configuration_attribute("single_dataset__about", (str, type(None)))
+        self.validate_correct_type_of_configuration_attribute("single_dataset__obs_names", (str, type(None)))
+        self.validate_correct_type_of_configuration_attribute("single_dataset__var_names", (str, type(None)))
+
+        if self.single_dataset__datapath is None:
+            return
+
+        # create the matrix data cache manager:
+        if self.matrix_data_cache_manager is None:
+            self.matrix_data_cache_manager = MatrixDataCacheManager(max_cached=1, timelimit_s=None)
+
+        # preload this data set
+        matrix_data_loader = MatrixDataLoader(self.single_dataset__datapath, app_config=self.app_config)
+        try:
+            matrix_data_loader.pre_load_validation()
+        except DatasetAccessError as e:
+            raise ConfigurationError(str(e))
+
+        file_size = matrix_data_loader.file_size()
+        file_basename = basename(self.single_dataset__datapath)
+        if file_size > BIG_FILE_SIZE_THRESHOLD:
+            context["messagefn"](f"Loading data from {file_basename}, this may take a while...")
+        else:
+            context["messagefn"](f"Loading data from {file_basename}.")
+
+        if self.single_dataset__about:
+
+            def url_check(url):
+                try:
+                    result = urlparse(url)
+                    if all([result.scheme, result.netloc]):
+                        return True
+                    else:
+                        return False
+                except ValueError:
+                    return False
+
+            if not url_check(self.single_dataset__about):
+                raise ConfigurationError(
+                    "Must provide an absolute URL for --about. (Example format: http://example.com)"
+                )
+
+    def handle_multi_dataset(self):
+        self.validate_correct_type_of_configuration_attribute("multi_dataset__dataroot", (type(None), dict, str))
+        self.validate_correct_type_of_configuration_attribute("multi_dataset__index", (type(None), bool, str))
+        self.validate_correct_type_of_configuration_attribute("multi_dataset__allowed_matrix_types", list)
+        self.validate_correct_type_of_configuration_attribute("multi_dataset__matrix_cache__max_datasets", int)
+        self.validate_correct_type_of_configuration_attribute(
+            "multi_dataset__matrix_cache__timelimit_s", (type(None), int, float)
+        )  # noqa E501
+
+        if self.multi_dataset__dataroot is None:
+            return
+
+        if type(self.multi_dataset__dataroot) == str:
+            default_dict = dict(base_url="d", dataroot=self.multi_dataset__dataroot)
+            self.multi_dataset__dataroot = dict(d=default_dict)
+
+        for tag, dataroot_dict in self.multi_dataset__dataroot.items():
+            if "base_url" not in dataroot_dict:
+                raise ConfigurationError(f"error in multi_dataset__dataroot: missing base_url for tag {tag}")
+            if "dataroot" not in dataroot_dict:
+                raise ConfigurationError(f"error in multi_dataset__dataroot: missing dataroot, for tag {tag}")
+
+            base_url = dataroot_dict["base_url"]
+
+            # sanity check for well formed base urls
+            bad = False
+            if type(base_url) != str:
+                bad = True
+            elif os.path.normpath(base_url) != base_url:
+                bad = True
+            else:
+                base_url_parts = base_url.split("/")
+                if [quote_plus(part) for part in base_url_parts] != base_url_parts:
+                    bad = True
+                if ".." in base_url_parts:
+                    bad = True
+            if bad:
+                raise ConfigurationError(f"error in multi_dataset__dataroot base_url {base_url} for tag {tag}")
+
+        # verify all the base_urls are unique
+        base_urls = [d["base_url"] for d in self.multi_dataset__dataroot.values()]
+        if len(base_urls) > len(set(base_urls)):
+            raise ConfigurationError("error in multi_dataset__dataroot:  base_urls must be unique")
+
+        # error checking
+        for mtype in self.multi_dataset__allowed_matrix_types:
+            try:
+                MatrixDataType(mtype)
+            except ValueError:
+                raise ConfigurationError(f'Invalid matrix type in "allowed_matrix_types": {mtype}')
+
+        # create the matrix data cache manager:
+        if self.matrix_data_cache_manager is None:
+            self.matrix_data_cache_manager = MatrixDataCacheManager(
+                max_cached=self.multi_dataset__matrix_cache__max_datasets,
+                timelimit_s=self.multi_dataset__matrix_cache__timelimit_s,
+            )
+
+    def handle_diffexp(self):
+        self.validate_correct_type_of_configuration_attribute("diffexp__alg_cxg__max_workers", (str, int))
+        self.validate_correct_type_of_configuration_attribute("diffexp__alg_cxg__cpu_multiplier", int)
+        self.validate_correct_type_of_configuration_attribute("diffexp__alg_cxg__target_workunit", int)
+
+        max_workers = self.diffexp__alg_cxg__max_workers
+        cpu_multiplier = self.diffexp__alg_cxg__cpu_multiplier
+        cpu_count = os.cpu_count()
+        max_workers = min(max_workers, cpu_multiplier * cpu_count)
+        diffexp_tiledb.set_config(max_workers, self.diffexp__alg_cxg__target_workunit)
+
+    def handle_adaptor(self):
+        # cxg
+        self.validate_correct_type_of_configuration_attribute("adaptor__cxg_adaptor__tiledb_ctx", dict)
+        regionkey = "vfs.s3.region"
+        if regionkey not in self.adaptor__cxg_adaptor__tiledb_ctx:
+            if type(self.data_locator__s3__region_name) == str:
+                self.adaptor__cxg_adaptor__tiledb_ctx[regionkey] = self.data_locator__s3__region_name
+
+        from server.data_cxg.cxg_adaptor import CxgAdaptor
+
+        CxgAdaptor.set_tiledb_context(self.adaptor__cxg_adaptor__tiledb_ctx)
+
+        # anndata
+        self.validate_correct_type_of_configuration_attribute("adaptor__anndata_adaptor__backed", bool)
+
+    def handle_limits(self):
+        self.validate_correct_type_of_configuration_attribute("limits__diffexp_cellcount_max", (type(None), int))
+        self.validate_correct_type_of_configuration_attribute("limits__column_request_max", (type(None), int))
+
+    def exceeds_limit(self, limit_name, value):
+        limit_value = getattr(self, "limits__" + limit_name, None)
+        if limit_value is None:  # disabled
+            return False
+        return value > limit_value
+
+    def get_api_base_url(self):
+        if self.app__api_base_url == "local":
+            return f"http://{self.app__host}:{self.app__port}"
+        if self.app__api_base_url and self.app__api_base_url.endswith("/"):
+            return self.app__api_base_url[:-1]
+        return self.app__api_base_url
+
+    def get_web_base_url(self):
+        if self.app__web_base_url == "local":
+            return f"http://{self.app__host}:{self.app__port}"
+        if self.app__web_base_url is None:
+            return self.get_api_base_url()
+        if self.app__web_base_url.endswith("/"):
+            return self.app__web_base_url[:-1]
+        return self.app__web_base_url

server/common/errors.py

@@ -42,14 +42,14 @@ define_request_exception(
 define_request_exception("ExceedsLimitError", "Raised when an HTTP request exceeds a limit/quota")
 define_request_exception("ColorFormatException", "Raised when color helper functions encounter an unknown color format")
 define_request_exception(
-    "AuthenticationError",
-    "Raised when there is an authentication error",
-    default_status_code=HTTPStatus.UNAUTHORIZED)
+    "AuthenticationError", "Raised when there is an authentication error", default_status_code=HTTPStatus.UNAUTHORIZED
+)
 
 define_request_exception(
     "AnnotationCategoryNameError",
     "Raised when an annotation category name cant be saved",
-    default_status_code=HTTPStatus.UNPROCESSABLE_ENTITY)
+    default_status_code=HTTPStatus.UNPROCESSABLE_ENTITY,
+)
 
 define_exception("OntologyLoadFailure", "Raised when reading the ontology file fails")
 define_exception("ConfigurationError", "Raised when checking configuration errors")

server/common/rest.py

@@ -6,6 +6,7 @@ from http import HTTPStatus
 from flask import make_response, jsonify, current_app, abort
 from werkzeug.urls import url_unquote
 
+from server.common.config.client_config import get_client_config, get_client_userinfo
 from server.common.constants import Axis, DiffExpMode, JSON_NaN_to_num_warning_msg
 from server.common.errors import (
     FilterError,
@@ -117,12 +118,12 @@ def schema_get(data_adaptor):
 
 
 def config_get(app_config, data_adaptor):
-    config = app_config.get_client_config(data_adaptor)
+    config = get_client_config(app_config, data_adaptor)
     return make_response(jsonify(config), HTTPStatus.OK)
 
 
 def userinfo_get(app_config, data_adaptor):
-    config = app_config.get_client_userinfo(data_adaptor)
+    config = get_client_userinfo(app_config, data_adaptor)
     return make_response(jsonify(config), HTTPStatus.OK)
 
 

server/common/utils/cxg_generation_utils.py

@@ -111,7 +111,7 @@ def convert_ndarray_to_cxg_dense_array(ndarray_name, ndarray, ctx):
 
 
 def convert_matrix_to_cxg_array(
-        matrix_name, matrix, encode_as_sparse_array, ctx, column_shift_for_sparse_encoding=None
+    matrix_name, matrix, encode_as_sparse_array, ctx, column_shift_for_sparse_encoding=None
 ):
     """
     Converts a numpy array matrix into a TileDB SparseArray of DenseArray based on whether `encode_as_sparse_array`

server/common/utils/matrix_utils.py

@@ -41,16 +41,19 @@ def is_matrix_sparse(matrix: np.ndarray, sparse_threshold):
         number_of_non_zero_elements += np.count_nonzero(matrix_subset)
         if number_of_non_zero_elements > maximum_number_of_non_zero_elements_in_matrix:
             if end_row_index != total_number_of_rows:
-                percentage_of_non_zero_elements = 100 * number_of_non_zero_elements / (
-                    end_row_index * total_number_of_columns)
+                percentage_of_non_zero_elements = (
+                    100 * number_of_non_zero_elements / (end_row_index * total_number_of_columns)
+                )
                 logging.info(
                     f"Matrix is not sparse. Percentage of non-zero elements (estimate): "
-                    f"{percentage_of_non_zero_elements:6.2f}")
+                    f"{percentage_of_non_zero_elements:6.2f}"
+                )
             else:
                 percentage_of_non_zero_elements = 100 * number_of_non_zero_elements / total_number_of_matrix_elements
                 logging.info(
                     f"Matrix is not sparse. Percentage of non-zero elements (exact): "
-                    f"{percentage_of_non_zero_elements:6.2f}")
+                    f"{percentage_of_non_zero_elements:6.2f}"
+                )
             return False
 
     is_sparse = (100.0 * number_of_non_zero_elements / total_number_of_matrix_elements) < sparse_threshold

server/common/utils/type_conversion_utils.py

@@ -9,8 +9,10 @@ def get_dtypes_and_schemas_of_dataframe(dataframe: pd.DataFrame):
     schema_type_hints_by_column_name = {}
 
     for column_name, column_values in dataframe.items():
-        dtypes_by_column_name[column_name], schema_type_hints_by_column_name[column_name] = \
-            get_dtype_and_schema_of_array(column_values)
+        (
+            dtypes_by_column_name[column_name],
+            schema_type_hints_by_column_name[column_name],
+        ) = get_dtype_and_schema_of_array(column_values)
 
     return dtypes_by_column_name, schema_type_hints_by_column_name
 
@@ -24,8 +26,10 @@ def get_schema_type_hint_of_array(array: pd.Series):
 
 
 def get_dtype_and_schema_of_array(array: pd.Series):
-    return (get_dtype_from_dtype(array.dtype, array_values=array),
-            get_schema_type_hint_from_dtype(array.dtype, array_values=array))
+    return (
+        get_dtype_from_dtype(array.dtype, array_values=array),
+        get_schema_type_hint_from_dtype(array.dtype, array_values=array),
+    )
 
 
 def get_dtype_from_dtype(dtype, array_values=None):
@@ -133,9 +137,11 @@ def can_cast_to_int32(dtype, array_values=None):
         if np.can_cast(dtype, np.int32):
             return True
         ii32 = np.iinfo(np.int32)
-        if not ordered_array_values.empty and (
-                ordered_array_values.min() >= ii32.min and ordered_array_values.max() <= ii32.max) or \
-                ordered_array_values.empty:
+        if (
+            not ordered_array_values.empty
+            and (ordered_array_values.min() >= ii32.min and ordered_array_values.max() <= ii32.max)
+            or ordered_array_values.empty
+        ):
             return True
     return False
 

server/converters/h5ad_data_file.py

@@ -24,14 +24,14 @@ class H5ADDataFile:
     another format (currently just CXG is supported). """
 
     def __init__(
-            self,
-            input_filename,
-            backed=False,
-            dataset_title=None,
-            dataset_about=None,
-            obs_index_column_name=None,
-            vars_index_column_name=None,
-            use_corpora_schema=True,
+        self,
+        input_filename,
+        backed=False,
+        dataset_title=None,
+        dataset_about=None,
+        obs_index_column_name=None,
+        vars_index_column_name=None,
+        use_corpora_schema=True,
     ):
         self.input_filename = input_filename
         self.backed = backed

server/data_common/data_adaptor.py

@@ -5,7 +5,7 @@ import numpy as np
 import pandas as pd
 from server_timing import Timing as ServerTiming
 
-from server.common.app_config import AppFeature, AppConfig
+from server.common.config.app_config import AppConfig
 from server.common.constants import Axis
 from server.common.errors import FilterError, JSONEncodingValueError, ExceedsLimitError
 from server.common.utils.utils import jsonify_numpy
@@ -155,17 +155,6 @@ class DataAdaptor(metaclass=ABCMeta):
         """
         pass
 
-    def get_features(self, annotations=None):
-        """Return list of features, to return as part of the config route"""
-        features = [
-            AppFeature("/cluster/", method="POST", available=False),
-            AppFeature("/layout/obs", method="GET", available=self.get_embedding_names() is not None),
-            AppFeature("/layout/obs", method="PUT", available=self.dataset_config.embeddings__enable_reembedding),
-            AppFeature("/diffexp/", method="POST", available=self.dataset_config.diffexp__enable),
-            AppFeature("/annotations/obs", method="PUT", available=annotations is not None),
-        ]
-        return features
-
     def update_parameters(self, parameters):
         parameters.update(self.parameters)
 
@@ -173,7 +162,7 @@ class DataAdaptor(metaclass=ABCMeta):
         mask = np.zeros((count,), dtype=np.bool)
         for i in filter:
             if type(i) == list:
-                mask[i[0]: i[1]] = True
+                mask[i[0] : i[1]] = True
             else:
                 mask[i] = True
         return mask
@@ -314,7 +303,7 @@ class DataAdaptor(metaclass=ABCMeta):
             top_n = self.dataset_config.diffexp__top_n
 
         if self.server_config.exceeds_limit(
-                "diffexp_cellcount_max", np.count_nonzero(obs_mask_A) + np.count_nonzero(obs_mask_B)
+            "diffexp_cellcount_max", np.count_nonzero(obs_mask_A) + np.count_nonzero(obs_mask_B)
         ):
             raise ExceedsLimitError("Diffexp request exceeds max cell count limit")
 

server/db/cellxgene_orm.py

@@ -1,11 +1,6 @@
 import uuid
 
-from sqlalchemy import (
-    Column,
-    DateTime,
-    ForeignKey,
-    String,
-    func, JSON)
+from sqlalchemy import Column, DateTime, ForeignKey, String, func, JSON
 from sqlalchemy.dialects.postgresql import UUID
 from sqlalchemy.ext.declarative import declarative_base
 from sqlalchemy.orm import relationship

server/db/db_utils.py

@@ -42,9 +42,9 @@ class DbUtils:
 
     def get_or_create_dataset(self, dataset_name):
         try:
-            dataset_id = self.query(
-                table_args=[CellxGeneDataset], filter_args=[CellxGeneDataset.name == dataset_name]
-            )[0].id
+            dataset_id = self.query(table_args=[CellxGeneDataset], filter_args=[CellxGeneDataset.name == dataset_name])[
+                0
+            ].id
         except IndexError:
             dataset_id = uuid.uuid4()
             dataset = CellxGeneDataset(id=dataset_id, name=dataset_name)
@@ -54,9 +54,7 @@ class DbUtils:
 
     def get_or_create_user(self, user_id):
         try:
-            user_id = self.query(
-                table_args=[CellxGeneUser], filter_args=[CellxGeneUser.id == user_id]
-            )[0].id
+            user_id = self.query(table_args=[CellxGeneUser], filter_args=[CellxGeneUser.id == user_id])[0].id
         except IndexError:
             user = CellxGeneUser(id=user_id)
             self.session.add(user)

server/default_config.py

@@ -204,7 +204,6 @@ dataset:
     enable: true
     lfc_cutoff: 0.01
     top_n: 10
-
 """
 
 

server/eb/app.py

@@ -9,7 +9,7 @@ from flask import json
 import logging
 from flask_talisman import Talisman
 from flask_cors import CORS
-from server.common.aws_secret_utils import handle_config_from_secret
+from server.common.config import handle_config_from_secret
 from server.common.errors import SecretKeyRetrievalError
 
 
@@ -26,7 +26,7 @@ SERVERDIR = os.path.dirname(os.path.realpath(__file__))
 sys.path.append(SERVERDIR)
 
 try:
-    from server.common.app_config import AppConfig
+    from server.common.config.app_config import AppConfig
     from server.app.app import Server
     from server.common.data_locator import DataLocator, discover_s3_region_name
 except Exception:
@@ -61,8 +61,7 @@ class WSGIServer(Server):
         csp = {
             "default-src": ["'self'"],
             "connect-src": ["'self'"] + extra_connect_src,
-            "script-src": ["'self'", "'unsafe-eval'"]
-            + obsolete_browser_script_hash + script_hashes,
+            "script-src": ["'self'", "'unsafe-eval'"] + obsolete_browser_script_hash + script_hashes,
             "style-src": ["'self'", "'unsafe-inline'"],
             "img-src": ["'self'", "https://cellxgene.cziscience.com", "data:"],
             "object-src": ["'none'"],
@@ -104,7 +103,7 @@ class WSGIServer(Server):
         if len(script_hashes) == 0:
             logging.error("Content security policy hashes are missing, falling back to unsafe-inline policy")
 
-        return (script_hashes)
+        return script_hashes
 
     @staticmethod
     def compute_inline_csp_hashes(app, app_config):
@@ -173,9 +172,7 @@ try:
         sys.exit(1)
 
     # features are unsupported in the current hosted server
-    app_config.update_default_dataset_config(
-        embeddings__enable_reembedding=False,
-    )
+    app_config.update_default_dataset_config(embeddings__enable_reembedding=False,)
     app_config.update_server_config(multi_dataset__allowed_matrix_types=["cxg"],)
     app_config.complete_config(logging.info)
 

server/test/__init__.py

@@ -13,7 +13,8 @@ import requests
 
 from server.common.annotations.hosted_tiledb import AnnotationsHostedTileDB
 from server.common.annotations.local_file_csv import AnnotationsLocalFile
-from server.common.app_config import AppConfig, DEFAULT_SERVER_PORT
+from server.common.config.app_config import AppConfig
+from server.common.config import DEFAULT_SERVER_PORT
 from server.common.data_locator import DataLocator
 from server.common.utils.utils import find_available_port
 from server.data_common.fbs.matrix import encode_matrix_fbs
@@ -33,8 +34,7 @@ def data_with_tmp_tiledb_annotations(ext: MatrixDataType):
     data_locator = DataLocator(fname)
     config = AppConfig()
     config.update_server_config(
-        multi_dataset__dataroot=data_locator.path,
-        authentication__type="test",
+        multi_dataset__dataroot=data_locator.path, authentication__type="test",
     )
     config.update_default_dataset_config(
         embeddings__names=["umap"],
@@ -42,16 +42,13 @@ def data_with_tmp_tiledb_annotations(ext: MatrixDataType):
         diffexp__lfc_cutoff=0.01,
         user_annotations__type="hosted_tiledb_array",
         user_annotations__hosted_tiledb_array__db_uri="postgresql://postgres:test_pw@localhost:5432",
-        user_annotations__hosted_tiledb_array__hosted_file_directory=tmp_dir
+        user_annotations__hosted_tiledb_array__hosted_file_directory=tmp_dir,
     )
 
     config.complete_config()
 
     data = MatrixDataLoader(data_locator.abspath()).open(config)
-    annotations = AnnotationsHostedTileDB(
-        tmp_dir,
-        DbUtils("postgresql://postgres:test_pw@localhost:5432"),
-    )
+    annotations = AnnotationsHostedTileDB(tmp_dir, DbUtils("postgresql://postgres:test_pw@localhost:5432"),)
     return data, tmp_dir, annotations
 
 

server/test/fixtures/database/__init__.py

@@ -29,34 +29,26 @@ class TestDatabase:
 
     def _create_test_user(self):
         user = CellxGeneUser(id="test_user_id")
-        user2 = CellxGeneUser(id='1234')
+        user2 = CellxGeneUser(id="1234")
         self.db.session.add(user)
         self.db.session.add(user2)
         self.db.session.commit()
 
     def _create_test_dataset(self):
-        dataset = CellxGeneDataset(
-            name="test_dataset",
-        )
+        dataset = CellxGeneDataset(name="test_dataset",)
         self.db.session.add(dataset)
         self.db.session.commit()
 
     def _create_test_annotation(self):
-        dataset = self.db.query([CellxGeneDataset],
-                                [CellxGeneDataset.name == "test_dataset"],
-                                )[0]
-        annotation = Annotation(
-            tiledb_uri="tiledb_uri",
-            user_id="test_user_id",
-            dataset_id=str(dataset.id)
-        )
+        dataset = self.db.query([CellxGeneDataset], [CellxGeneDataset.name == "test_dataset"],)[0]
+        annotation = Annotation(tiledb_uri="tiledb_uri", user_id="test_user_id", dataset_id=str(dataset.id))
         self.db.session.add(annotation)
         self.db.session.commit()
 
     @staticmethod
     def get_random_string():
         letters = string.ascii_lowercase
-        return ''.join(random.choice(letters) for i in range(12))
+        return "".join(random.choice(letters) for i in range(12))
 
     def _create_test_users(self, user_count: int = 10):
         users = []
@@ -80,10 +72,8 @@ class TestDatabase:
         for i in range(annotation_count):
             dataset = self.order_by_random(CellxGeneDataset)
             user = self.order_by_random(CellxGeneUser)
-            annotations.append(Annotation(
-                tiledb_uri=self.get_random_string(),
-                user_id=user.id,
-                dataset_id=str(dataset.id)
-            ))
+            annotations.append(
+                Annotation(tiledb_uri=self.get_random_string(), user_id=user.id, dataset_id=str(dataset.id))
+            )
         self.db.session.add_all(annotations)
         self.db.session.commit()

server/test/fixtures/dataset_config_outline.py

@@ -0,0 +1,37 @@
+f"""
+dataset:
+  app:
+    scripts: {scripts} #list of strs (filenames) or dicts containing keys
+    inline_scripts: {inline_scripts} #list of strs (filenames)
+
+    about_legal_tos: {about_legal_tos}
+    about_legal_privacy: {about_legal_privacy}
+
+    authentication_enable: {authentication_enable}
+
+  presentation:
+    max_categories: {max_categories}
+    custom_colors: {custom_colors}
+
+  user_annotations:
+    enable: {enable_users_annotations}
+    type: {annotation_type}
+    hosted_tiledb_array:
+      db_uri: {db_uri}
+      hosted_file_directory: {hosted_file_directory}
+    local_file_csv:
+      directory: {local_file_csv_directory}
+      file: {local_file_csv_file}
+    ontology:
+      enable: {ontology_enabled}
+      obo_location: {obo_location}
+
+  embeddings:
+    names: {embedding_names}
+    enable_reembedding: {enable_reembedding}
+
+  diffexp:
+    enable: {enable_difexp}
+    lfc_cutoff: {lfc_cutoff}
+    top_n: {top_n}
+"""

server/test/fixtures/server_config_outline.py

@@ -0,0 +1,62 @@
+f"""server:
+  app:
+    verbose: {verbose}
+    debug: {debug}
+    host: {host}
+    port: {port}
+    open_browser: {open_browser}
+    force_https: {force_https}
+    flask_secret_key: {flask_secret_key}
+    generate_cache_control_headers: {generate_cache_control_headers}
+    server_timing_headers: {server_timing_headers}
+    csp_directives: {csp_directives}
+    api_base_url: {api_base_url}
+    web_base_url: {web_base_url}
+  authentication:
+    type: {auth_type}
+    params_oauth:
+      oauth_api_base_url: {oauth_api_base_url}
+      client_id: {client_id}
+      client_secret: {client_secret}
+      jwt_decode_options: {jwt_decode_options}
+      session_cookie:  {session_cookie}
+      cookie:  {cookie}
+
+  multi_dataset:
+    dataroot: {dataroot}
+    index: {index}
+    allowed_matrix_types: {allowed_matrix_types}
+    matrix_cache:
+      max_datasets: {max_cached_datasets}
+      timelimit_s: {timelimit_s}
+
+  single_dataset:
+    datapath: {dataset_datapath}
+    obs_names: {obs_names}
+    var_names: {var_names}
+    about: {about}
+    title: {title}
+
+  diffexp:
+    alg_cxg: # number of threads to use is computed from: min(max_workers, cpu_multipler * cpu_count)
+      max_workers: {diffexp_max_workers}
+      cpu_multiplier: {cpu_multiplier}
+      target_workunit: {target_workunit}  # The target number of matrix elements that are evaluated in one thread.
+
+  data_locator:
+    s3:
+      region_name: {data_locater_region_name}
+
+  adaptor:
+    cxg_adaptor:
+      tiledb_ctx:
+        sm.tile_cache_size:  {cxg_tile_cache_size}
+        sm.num_reader_threads:  {cxg_num_reader_threads}
+
+    anndata_adaptor:
+      backed: {anndata_backed}
+
+  limits:
+    column_request_max: {column_request_max}
+    diffexp_cellcount_max: {diffexp_cellcount_max}
+"""

server/test/performance/run_diffexp.py

@@ -7,7 +7,7 @@ import numpy as np
 import server.compute.diffexp_cxg as diffexp_cxg
 import server.compute.diffexp_generic as diffexp_generic
 
-from server.common.app_config import AppConfig
+from server.common.config.app_config import AppConfig
 from server.data_common.matrix_loader import MatrixDataLoader
 from server.data_cxg.cxg_adaptor import CxgAdaptor
 

server/test/test_database/test_database.py

@@ -16,45 +16,48 @@ class DatabaseTest(unittest.TestCase):
         del cls.db
 
     def test_user_creation(self):
-        one_user = self.db.get(table=CellxGeneUser, entity_id='test_user_id')
-        self.assertEqual(one_user.id, 'test_user_id')
+        one_user = self.db.get(table=CellxGeneUser, entity_id="test_user_id")
+        self.assertEqual(one_user.id, "test_user_id")
         user_count = self.db.session.query(CellxGeneUser).count()
         self.assertGreater(user_count, 10)
 
     def test_dataset_creation(self):
-        one_dataset = self.db.query(table_args=[CellxGeneDataset],
-                                    filter_args=[CellxGeneDataset.name == 'test_dataset'])
-        self.assertEqual(one_dataset[0].name, 'test_dataset')
+        one_dataset = self.db.query(
+            table_args=[CellxGeneDataset], filter_args=[CellxGeneDataset.name == "test_dataset"]
+        )
+        self.assertEqual(one_dataset[0].name, "test_dataset")
         dataset_count = self.db.session.query(CellxGeneDataset).count()
         self.assertGreater(dataset_count, 10)
 
     def test_annotation_creation(self):
-        one_annotation = self.db.query(table_args=[Annotation], filter_args=[Annotation.tiledb_uri == 'tiledb_uri'])[0]
-        self.assertEqual(one_annotation.tiledb_uri, 'tiledb_uri')
+        one_annotation = self.db.query(table_args=[Annotation], filter_args=[Annotation.tiledb_uri == "tiledb_uri"])[0]
+        self.assertEqual(one_annotation.tiledb_uri, "tiledb_uri")
         annotation_count = self.db.session.query(Annotation).count()
         self.assertGreater(annotation_count, 10)
 
     def test_get_most_recent_annotation_for_user_dataset(self):
-        dataset_id = str(self.db.query(table_args=[CellxGeneDataset],
-                                       filter_args=[CellxGeneDataset.name == 'test_dataset'])[0].id)
+        dataset_id = str(
+            self.db.query(table_args=[CellxGeneDataset], filter_args=[CellxGeneDataset.name == "test_dataset"])[0].id
+        )
 
         # have to commit separately because created_at time written on the db server
-        self.db.session.add(Annotation(dataset_id=dataset_id, user_id='test_user_id', tiledb_uri='tiledb_uri_0'))
+        self.db.session.add(Annotation(dataset_id=dataset_id, user_id="test_user_id", tiledb_uri="tiledb_uri_0"))
         self.db.session.commit()
 
-        self.db.session.add(Annotation(dataset_id=dataset_id, user_id='test_user_id', tiledb_uri='tiledb_uri_1'))
+        self.db.session.add(Annotation(dataset_id=dataset_id, user_id="test_user_id", tiledb_uri="tiledb_uri_1"))
         self.db.session.commit()
 
-        self.db.session.add(Annotation(dataset_id=dataset_id, user_id='test_user_id', tiledb_uri='tiledb_uri_2'))
+        self.db.session.add(Annotation(dataset_id=dataset_id, user_id="test_user_id", tiledb_uri="tiledb_uri_2"))
         self.db.session.commit()
 
-        self.db.session.add(Annotation(dataset_id=dataset_id, user_id='test_user_id', tiledb_uri='tiledb_uri_3'))
+        self.db.session.add(Annotation(dataset_id=dataset_id, user_id="test_user_id", tiledb_uri="tiledb_uri_3"))
         self.db.session.commit()
 
-        self.db.session.add(Annotation(dataset_id=dataset_id, user_id='test_user_id', tiledb_uri='tiledb_uri_4'))
+        self.db.session.add(Annotation(dataset_id=dataset_id, user_id="test_user_id", tiledb_uri="tiledb_uri_4"))
         self.db.session.commit()
 
-        most_recent_annotation = self.db.query_for_most_recent(Annotation, [Annotation.dataset_id == dataset_id,
-                                                                            Annotation.user_id == 'test_user_id'])
+        most_recent_annotation = self.db.query_for_most_recent(
+            Annotation, [Annotation.dataset_id == dataset_id, Annotation.user_id == "test_user_id"]
+        )
 
-        self.assertEqual(most_recent_annotation.tiledb_uri, 'tiledb_uri_4')
+        self.assertEqual(most_recent_annotation.tiledb_uri, "tiledb_uri_4")

server/test/unit/auth/test_auth.py

@@ -2,7 +2,7 @@ import unittest
 
 import requests
 
-from server.common.app_config import AppConfig
+from server.common.config.app_config import AppConfig
 from server.test import FIXTURES_ROOT, test_server
 
 
@@ -12,9 +12,7 @@ class AuthTest(unittest.TestCase):
 
     def test_auth_none(self):
         c = AppConfig()
-        c.update_server_config(
-            authentication__type=None, multi_dataset__dataroot=self.dataset_dataroot
-        )
+        c.update_server_config(authentication__type=None, multi_dataset__dataroot=self.dataset_dataroot)
         c.update_default_dataset_config(user_annotations__enable=False)
 
         c.complete_config()
@@ -28,9 +26,7 @@ class AuthTest(unittest.TestCase):
 
     def test_auth_session(self):
         c = AppConfig()
-        c.update_server_config(
-            authentication__type="session", multi_dataset__dataroot=self.dataset_dataroot
-        )
+        c.update_server_config(authentication__type="session", multi_dataset__dataroot=self.dataset_dataroot)
         c.update_default_dataset_config(user_annotations__enable=True)
         c.complete_config()
 
@@ -107,8 +103,8 @@ class AuthTest(unittest.TestCase):
     def test_auth_test_single(self):
         c = AppConfig()
         c.update_server_config(
-            authentication__type="test",
-            single_dataset__datapath=f"{self.dataset_dataroot}/pbmc3k.cxg")
+            authentication__type="test", single_dataset__datapath=f"{self.dataset_dataroot}/pbmc3k.cxg"
+        )
 
         c.complete_config()
 

server/test/unit/auth/test_oauth.py

@@ -9,7 +9,7 @@ from flask import Flask, jsonify, make_response, request, redirect
 from multiprocessing import Process
 
 import jose
-from server.common.app_config import AppConfig
+from server.common.config.app_config import AppConfig
 from server.test import FIXTURES_ROOT, test_server
 
 # This tests the oauth authentication type.
@@ -46,7 +46,7 @@ def token():
         "scope": "openid profile email",
         "expires_in": TOKEN_EXPIRES,
         "token_type": "Bearer",
-        "expires_at": expires_at
+        "expires_at": expires_at,
     }
     return make_response(jsonify(r))
 
@@ -89,9 +89,8 @@ class AuthTest(unittest.TestCase):
             authentication__params_oauth__oauth_api_base_url=f"http://localhost:{PORT}",
             authentication__params_oauth__client_id="mock_client_id",
             authentication__params_oauth__client_secret="mock_client_secret",
-            authentication__params_oauth__jwt_decode_options={
-                "verify_signature": False, "verify_iss": False
-            })
+            authentication__params_oauth__jwt_decode_options={"verify_signature": False, "verify_iss": False},
+        )
 
         app_config.update_server_config(multi_dataset__dataroot=self.dataset_dataroot)
         app_config.complete_config()
@@ -112,7 +111,7 @@ class AuthTest(unittest.TestCase):
             logout_uri = config["config"]["authentication"]["logout"]
 
             self.assertEqual(login_uri, f"{server}/login?dataset=d/pbmc3k.cxg/")
-            self.assertEqual(logout_uri, f"{server}/logout")
+            self.assertEqual(logout_uri, f"{server}/logout?dataset=d/pbmc3k.cxg/")
 
             r = session.get(login_uri)
             # check that the login redirect worked
@@ -122,6 +121,7 @@ class AuthTest(unittest.TestCase):
             userinfo = session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/userinfo").json()
             self.assertTrue(userinfo["userinfo"]["is_authenticated"])
             self.assertEqual(userinfo["userinfo"]["username"], "fake_user")
+            self.assertEqual(userinfo["userinfo"]["email"], "fake_user@email.com")
             self.assertTrue(config["config"]["parameters"]["annotations"])
 
             if cookie_key:
@@ -150,7 +150,7 @@ class AuthTest(unittest.TestCase):
             r = session.get(logout_uri)
             # check that the logout redirect worked
             self.assertEqual(r.history[0].status_code, 302)
-            self.assertEqual(r.url, f"{server}")
+            self.assertEqual(r.url, f"{server}/d/pbmc3k.cxg/")
             config = session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/config").json()
             userinfo = session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/userinfo").json()
             self.assertFalse(userinfo["userinfo"]["is_authenticated"])
@@ -160,9 +160,7 @@ class AuthTest(unittest.TestCase):
     def test_auth_oauth_session(self):
         # test with session cookies
         app_config = AppConfig()
-        app_config.update_server_config(
-            authentication__params_oauth__session_cookie=True,
-        )
+        app_config.update_server_config(authentication__params_oauth__session_cookie=True,)
         self.auth_flow(app_config)
 
     def test_auth_oauth_cookie(self):

server/test/unit/cli/test_launch.py

@@ -0,0 +1,28 @@
+import filecmp
+import os
+import shutil
+import unittest
+
+import yaml
+
+from server.default_config import default_config
+from server.test import FIXTURES_ROOT
+
+
+class CLIPLaunchTests(unittest.TestCase):
+    tmp_dir = os.path.join(FIXTURES_ROOT, "dump_configs")
+
+    @classmethod
+    def setUpClass(cls) -> None:
+        os.mkdir(cls.tmp_dir)
+
+    @classmethod
+    def tearDownClass(cls) -> None:
+        shutil.rmtree(cls.tmp_dir)
+
+
+def test_dump_default_config(self):
+    os.system(f"cellxgene launch --dump-default-config > {self.tmp_dir}/test_config_dump.txt")
+    with open(f"{self.tmp_dir}/expected_config_dump.txt", "w") as expected_config:
+        expected_config.write(yaml.dump(default_config))
+    filecmp.cmp(f"{self.tmp_dir}/expected_config_dump.txt", f"{self.tmp_dir}/test_config_dump.txt")

server/test/unit/common/config/__init__.py

@@ -0,0 +1,246 @@
+import os
+import shutil
+import unittest
+import random
+from unittest import mock
+
+from server.test import FIXTURES_ROOT
+
+
+def mockenv(**envvars):
+    return mock.patch.dict(os.environ, envvars)
+
+
+class ConfigTests(unittest.TestCase):
+    tmp_fixtures_directory = os.path.join(FIXTURES_ROOT, "tmp_dir")
+
+    @classmethod
+    def tearDownClass(cls) -> None:
+        shutil.rmtree(cls.tmp_fixtures_directory)
+
+    @classmethod
+    def setUpClass(cls) -> None:
+        os.makedirs(cls.tmp_fixtures_directory)
+
+    def custom_server_config(
+        self,
+        verbose="false",
+        debug="false",
+        host="localhost",
+        port="null",
+        open_browser="false",
+        force_https="false",
+        flask_secret_key="null",
+        generate_cache_control_headers="false",
+        server_timing_headers="false",
+        csp_directives="null",
+        api_base_url="null",
+        web_base_url="null",
+        auth_type="session",
+        oauth_api_base_url="null",
+        client_id="null",
+        client_secret="null",
+        jwt_decode_options="null",
+        session_cookie="true",
+        cookie="null",
+        dataroot="null",
+        index="false",
+        allowed_matrix_types=[],
+        max_cached_datasets=5,
+        timelimit_s=5,
+        dataset_datapath="null",
+        obs_names="null",
+        var_names="null",
+        about="null",
+        title="null",
+        diffexp_max_workers=64,
+        cpu_multiplier=4,
+        target_workunit="16_000_000",
+        data_locater_region_name="us-east-1",
+        cxg_tile_cache_size=8589934592,
+        cxg_num_reader_threads=32,
+        anndata_backed="false",
+        column_request_max=32,
+        diffexp_cellcount_max="null",
+        config_file_name="server_config.yaml",
+    ):
+        configfile = os.path.join(self.tmp_fixtures_directory, config_file_name)
+        server_config_outline_path = os.path.join(FIXTURES_ROOT, "server_config_outline.py")
+        with open(server_config_outline_path, "r") as config_skeleton:
+            config = config_skeleton.read()
+            server_config = eval(config)
+        with open(configfile, "w") as server_config_file:
+            server_config_file.write(server_config)
+        return configfile
+
+    def custom_app_config(
+        self,
+        verbose="false",
+        debug="false",
+        host="localhost",
+        port="null",
+        open_browser="false",
+        force_https="false",
+        flask_secret_key="null",
+        generate_cache_control_headers="false",
+        server_timing_headers="false",
+        csp_directives="null",
+        api_base_url="null",
+        web_base_url="null",
+        auth_type="session",
+        oauth_api_base_url="null",
+        client_id="null",
+        client_secret="null",
+        jwt_decode_options="null",
+        session_cookie="true",
+        cookie="null",
+        dataroot="null",
+        index="false",
+        allowed_matrix_types=[],
+        max_cached_datasets=5,
+        timelimit_s=5,
+        dataset_datapath="null",
+        obs_names="null",
+        var_names="null",
+        about="null",
+        title="null",
+        diffexp_max_workers=64,
+        cpu_multiplier=4,
+        target_workunit="16_000_000",
+        data_locater_region_name="us-east-1",
+        cxg_tile_cache_size=8589934592,
+        cxg_num_reader_threads=32,
+        anndata_backed="false",
+        column_request_max=32,
+        diffexp_cellcount_max="null",
+        scripts=[],
+        inline_scripts=[],
+        about_legal_tos="null",
+        about_legal_privacy="null",
+        authentication_enable="true",
+        max_categories=1000,
+        custom_colors="true",
+        enable_users_annotations="true",
+        annotation_type="local_file_csv",
+        db_uri="null",
+        hosted_file_directory="null",
+        local_file_csv_directory="null",
+        local_file_csv_file="null",
+        ontology_enabled="false",
+        obo_location="null",
+        embedding_names=[],
+        enable_reembedding="false",
+        enable_difexp="true",
+        lfc_cutoff=0.01,
+        top_n=10,
+        config_file_name="app_config.yml",
+    ):
+        random_num = random.randrange(999999)
+        configfile = os.path.join(self.tmp_fixtures_directory, config_file_name)
+        server_config = self.custom_server_config(
+            verbose=verbose,
+            debug=debug,
+            host=host,
+            port=port,
+            open_browser=open_browser,
+            force_https=force_https,
+            flask_secret_key=flask_secret_key,
+            generate_cache_control_headers=generate_cache_control_headers,
+            server_timing_headers=server_timing_headers,
+            csp_directives=csp_directives,
+            api_base_url=api_base_url,
+            web_base_url=web_base_url,
+            auth_type=auth_type,
+            oauth_api_base_url=oauth_api_base_url,
+            client_id=client_id,
+            client_secret=client_secret,
+            jwt_decode_options=jwt_decode_options,
+            session_cookie=session_cookie,
+            cookie=cookie,
+            dataroot=dataroot,
+            index=index,
+            allowed_matrix_types=allowed_matrix_types,
+            max_cached_datasets=max_cached_datasets,
+            timelimit_s=timelimit_s,
+            dataset_datapath=dataset_datapath,
+            obs_names=obs_names,
+            var_names=var_names,
+            about=about,
+            title=title,
+            diffexp_max_workers=diffexp_max_workers,
+            cpu_multiplier=cpu_multiplier,
+            target_workunit=target_workunit,
+            data_locater_region_name=data_locater_region_name,
+            cxg_tile_cache_size=cxg_tile_cache_size,
+            cxg_num_reader_threads=cxg_num_reader_threads,
+            anndata_backed=anndata_backed,
+            column_request_max=column_request_max,
+            diffexp_cellcount_max=diffexp_cellcount_max,
+            config_file_name=f"temp_server_config_{random_num}.yml",
+        )
+        dataset_config = self.custom_dataset_config(
+            scripts=scripts,
+            inline_scripts=inline_scripts,
+            about_legal_tos=about_legal_tos,
+            about_legal_privacy=about_legal_privacy,
+            authentication_enable=authentication_enable,
+            max_categories=max_categories,
+            custom_colors=custom_colors,
+            enable_users_annotations=enable_users_annotations,
+            annotation_type=annotation_type,
+            db_uri=db_uri,
+            hosted_file_directory=hosted_file_directory,
+            local_file_csv_directory=local_file_csv_directory,
+            local_file_csv_file=local_file_csv_file,
+            ontology_enabled=ontology_enabled,
+            obo_location=obo_location,
+            embedding_names=embedding_names,
+            enable_reembedding=enable_reembedding,
+            enable_difexp=enable_difexp,
+            lfc_cutoff=lfc_cutoff,
+            top_n=top_n,
+            config_file_name=f"temp_dataset_config_{random_num}.yml",
+        )
+        with open(server_config) as server_config:
+            with open(dataset_config) as dataset_config:
+                with open(configfile, "w") as app_config_file:
+                    for line in server_config:
+                        app_config_file.write(line)
+                    for line in dataset_config:
+                        app_config_file.write(line)
+
+        return configfile
+
+    def custom_dataset_config(
+        self,
+        scripts=[],
+        inline_scripts=[],
+        about_legal_tos="null",
+        about_legal_privacy="null",
+        authentication_enable="true",
+        max_categories=1000,
+        custom_colors="true",
+        enable_users_annotations="true",
+        annotation_type="local_file_csv",
+        db_uri="null",
+        hosted_file_directory="null",
+        local_file_csv_directory="null",
+        local_file_csv_file="null",
+        ontology_enabled="false",
+        obo_location="null",
+        embedding_names=[],
+        enable_reembedding="false",
+        enable_difexp="true",
+        lfc_cutoff=0.01,
+        top_n=10,
+        config_file_name="dataset_config.yml",
+    ):
+        configfile = os.path.join(self.tmp_fixtures_directory, config_file_name)
+        dataset_config_outline_path = os.path.join(FIXTURES_ROOT, "dataset_config_outline.py")
+        with open(dataset_config_outline_path, "r") as config_skeleton:
+            config = config_skeleton.read()
+            dataset_config = eval(config)
+        with open(configfile, "w") as dataset_config_file:
+            dataset_config_file.write(dataset_config)
+
+        return configfile

server/test/unit/common/config/test_app_config.py

@@ -0,0 +1,140 @@
+import os
+import tempfile
+import unittest
+
+import yaml
+
+from server.default_config import default_config
+from server.common.config.app_config import AppConfig
+from server.test.unit.common.config import ConfigTests
+from server.common.errors import ConfigurationError
+from server.test import FIXTURES_ROOT
+
+
+class AppConfigTest(ConfigTests):
+    def setUp(self):
+        self.config_file_name = f"{unittest.TestCase.id(self).split('.')[-1]}.yml"
+        self.config = AppConfig()
+        self.config.update_server_config(multi_dataset__dataroot=FIXTURES_ROOT)
+        self.server_config = self.config.server_config
+        self.config.complete_config()
+
+        message_list = []
+
+        def noop(message):
+            message_list.append(message)
+
+        messagefn = noop
+        self.context = dict(messagefn=messagefn, messages=message_list)
+
+    def get_config(self, **kwargs):
+        file_name = self.custom_app_config(
+            dataroot=f"{FIXTURES_ROOT}", config_file_name=self.config_file_name, **kwargs
+        )
+        config = AppConfig()
+        config.update_from_config_file(file_name)
+        return config
+
+    def test_get_default_config_correctly_reads_default_config_file(self):
+        app_default_config = AppConfig().default_config
+
+        expected_config = yaml.load(default_config, Loader=yaml.Loader)
+
+        server_config = app_default_config["server"]
+        dataset_config = app_default_config["dataset"]
+
+        expected_server_config = expected_config["server"]
+        expected_dataset_config = expected_config["dataset"]
+
+        self.assertDictEqual(app_default_config, expected_config)
+        self.assertDictEqual(server_config, expected_server_config)
+        self.assertDictEqual(dataset_config, expected_dataset_config)
+
+    def test_get_dataset_config_returns_default_dataset_config_for_single_datasets(self):
+        datapath = f"{FIXTURES_ROOT}/1e4dfec4-c0b2-46ad-a04e-ff3ffb3c0a8f.h5ad"
+        file_name = self.custom_app_config(dataset_datapath=datapath, config_file_name=self.config_file_name)
+        config = AppConfig()
+        config.update_from_config_file(file_name)
+
+        self.assertEqual(config.get_dataset_config(""), config.default_dataset_config)
+
+    def test_update_server_config_updates_server_config_and_config_status(self):
+        config = self.get_config()
+        config.complete_config()
+        config.check_config()
+        config.update_server_config(multi_dataset__dataroot=FIXTURES_ROOT)
+        with self.assertRaises(ConfigurationError):
+            config.server_config.check_config()
+
+    def test_write_config_outputs_yaml_with_all_config_vars(self):
+        config = self.get_config()
+        config.write_config(f"{FIXTURES_ROOT}/tmp_dir/write_config.yml")
+        with open(f"{FIXTURES_ROOT}/tmp_dir/{self.config_file_name}", "r") as default_config:
+            default_config_yml = yaml.safe_load(default_config)
+
+        with open(f"{FIXTURES_ROOT}/tmp_dir/write_config.yml", "r") as output_config:
+            output_config_yml = yaml.safe_load(output_config)
+        self.maxDiff = None
+        self.assertEqual(default_config_yml, output_config_yml)
+
+    def test_update_app_config(self):
+        config = AppConfig()
+        config.update_server_config(app__verbose=True, multi_dataset__dataroot="datadir")
+        vars = config.server_config.changes_from_default()
+        self.assertCountEqual(vars, [("app__verbose", True, False), ("multi_dataset__dataroot", "datadir", None)])
+
+        config = AppConfig()
+        config.update_default_dataset_config(app__scripts=(), app__inline_scripts=())
+        vars = config.server_config.changes_from_default()
+        self.assertCountEqual(vars, [])
+
+        config = AppConfig()
+        config.update_default_dataset_config(app__scripts=[], app__inline_scripts=[])
+        vars = config.default_dataset_config.changes_from_default()
+        self.assertCountEqual(vars, [])
+
+        config = AppConfig()
+        config.update_default_dataset_config(app__scripts=("a", "b"), app__inline_scripts=["c", "d"])
+        vars = config.default_dataset_config.changes_from_default()
+        self.assertCountEqual(vars, [("app__scripts", ["a", "b"], []), ("app__inline_scripts", ["c", "d"], [])])
+
+    def test_configfile_no_dataset_section(self):
+        # test a config file without a dataset section
+
+        with tempfile.TemporaryDirectory() as tempdir:
+            configfile = os.path.join(tempdir, "config.yaml")
+            with open(configfile, "w") as fconfig:
+                config = """
+                server:
+                    multi_dataset:
+                        dataroot: test_dataroot
+
+                """
+                fconfig.write(config)
+
+            app_config = AppConfig()
+            app_config.update_from_config_file(configfile)
+            server_changes = app_config.server_config.changes_from_default()
+            dataset_changes = app_config.default_dataset_config.changes_from_default()
+            self.assertEqual(server_changes, [("multi_dataset__dataroot", "test_dataroot", None)])
+            self.assertEqual(dataset_changes, [])
+
+    def test_configfile_no_server_section(self):
+        # test a config file without a dataset section
+
+        with tempfile.TemporaryDirectory() as tempdir:
+            configfile = os.path.join(tempdir, "config.yaml")
+            with open(configfile, "w") as fconfig:
+                config = """
+                dataset:
+                    user_annotations:
+                        enable: false
+                """
+                fconfig.write(config)
+
+            app_config = AppConfig()
+            app_config.update_from_config_file(configfile)
+            server_changes = app_config.server_config.changes_from_default()
+            dataset_changes = app_config.default_dataset_config.changes_from_default()
+            self.assertEqual(server_changes, [])
+            self.assertEqual(dataset_changes, [("user_annotations__enable", False, True)])

server/test/unit/common/config/test_base_config.py

@@ -0,0 +1,63 @@
+import unittest
+
+from server.common.config.app_config import AppConfig
+from server.test import FIXTURES_ROOT
+from server.test.unit.common.config import ConfigTests
+from server.common.errors import ConfigurationError
+
+
+class BaseConfigTest(ConfigTests):
+    def setUp(self):
+        self.config_file_name = f"{unittest.TestCase.id(self).split('.')[-1]}.yml"
+        self.config = AppConfig()
+        self.config.update_server_config(multi_dataset__dataroot=FIXTURES_ROOT)
+        self.server_config = self.config.server_config
+        self.config.complete_config()
+
+        message_list = []
+
+        def noop(message):
+            message_list.append(message)
+
+        messagefn = noop
+        self.context = dict(messagefn=messagefn, messages=message_list)
+
+    def get_config(self, **kwargs):
+        file_name = self.custom_app_config(
+            dataroot=f"{FIXTURES_ROOT}", config_file_name=self.config_file_name, **kwargs
+        )
+        config = AppConfig()
+        config.update_from_config_file(file_name)
+        return config
+
+    def test_mapping_creation_returns_map_of_server_and_dataset_config(self):
+        config = AppConfig()
+        mapping = config.default_dataset_config.create_mapping(config.default_config)
+        self.assertIsNotNone(mapping["server__app__verbose"])
+        self.assertIsNotNone(mapping["dataset__presentation__max_categories"])
+        self.assertIsNotNone(mapping["dataset__user_annotations__ontology__obo_location"])
+        self.assertIsNotNone(mapping["server__multi_dataset__allowed_matrix_types"])
+
+    def test_changes_from_default_returns_list_of_nondefault_config_values(self):
+        config = self.get_config(verbose="true", lfc_cutoff=0.05)
+        server_changes = config.server_config.changes_from_default()
+        dataset_changes = config.default_dataset_config.changes_from_default()
+
+        self.assertEqual(
+            server_changes,
+            [
+                ("app__verbose", True, False),
+                ("multi_dataset__dataroot", FIXTURES_ROOT, None),
+                ("multi_dataset__matrix_cache__timelimit_s", 5, 30),
+                ("data_locator__s3__region_name", "us-east-1", True),
+            ],
+        )
+        self.assertEqual(dataset_changes, [("diffexp__lfc_cutoff", 0.05, 0.01)])
+
+    def test_check_config_throws_error_if_attr_has_not_been_checked(self):
+        config = self.get_config(verbose="true")
+        config.complete_config()
+        config.check_config()
+        config.update_server_config(app__verbose=False)
+        with self.assertRaises(ConfigurationError):
+            config.check_config()

server/test/unit/common/config/test_dataset_config.py

@@ -0,0 +1,260 @@
+import os
+import tempfile
+
+import requests
+import unittest
+from unittest.mock import patch
+
+from server.common.annotations.hosted_tiledb import AnnotationsHostedTileDB
+from server.common.annotations.local_file_csv import AnnotationsLocalFile
+from server.common.config.app_config import AppConfig
+from server.common.config.base_config import BaseConfig
+from server.test import test_server, PROJECT_ROOT, FIXTURES_ROOT
+
+from server.common.errors import ConfigurationError
+from server.test.unit.common.config import ConfigTests
+
+
+class TestDatasetConfig(ConfigTests):
+    def setUp(self):
+        self.config_file_name = f"{unittest.TestCase.id(self).split('.')[-1]}.yml"
+        self.config = AppConfig()
+        self.config.update_server_config(multi_dataset__dataroot=FIXTURES_ROOT)
+        self.dataset_config = self.config.default_dataset_config
+        self.config.complete_config()
+        message_list = []
+
+        def noop(message):
+            message_list.append(message)
+
+        messagefn = noop
+        self.context = dict(messagefn=messagefn, messages=message_list)
+
+    def get_config(self, **kwargs):
+        file_name = self.custom_app_config(
+            dataroot=f"{FIXTURES_ROOT}", config_file_name=self.config_file_name, **kwargs
+        )
+        config = AppConfig()
+        config.update_from_config_file(file_name)
+        return config
+
+    def test_init_datatset_config_sets_vars_from_default_config(self):
+        config = AppConfig()
+        self.assertEqual(config.default_dataset_config.presentation__max_categories, 1000)
+        self.assertEqual(config.default_dataset_config.user_annotations__type, "local_file_csv")
+        self.assertEqual(config.default_dataset_config.diffexp__lfc_cutoff, 0.01)
+        self.assertIsNone(config.default_dataset_config.user_annotations__ontology__obo_location)
+
+    @patch("server.common.config.dataset_config.BaseConfig.validate_correct_type_of_configuration_attribute")
+    def test_complete_config_checks_all_attr(self, mock_check_attrs):
+        mock_check_attrs.side_effect = BaseConfig.validate_correct_type_of_configuration_attribute()
+        self.dataset_config.complete_config(self.context)
+        self.assertEqual(mock_check_attrs.call_count, 21)
+
+    def test_app_sets_script_vars(self):
+        config = self.get_config(scripts=["path/to/script"])
+        config.default_dataset_config.handle_app()
+
+        self.assertEqual(config.default_dataset_config.app__scripts, [{"src": "path/to/script"}])
+
+        config = self.get_config(scripts=[{"src": "path/to/script", "more": "different/script/path"}])
+        config.default_dataset_config.handle_app()
+        self.assertEqual(
+            config.default_dataset_config.app__scripts, [{"src": "path/to/script", "more": "different/script/path"}]
+        )
+
+        config = self.get_config(scripts=["path/to/script", "different/script/path"])
+        config.default_dataset_config.handle_app()
+        # TODO @madison -- is this the desired functionality?
+        self.assertEqual(
+            config.default_dataset_config.app__scripts, [{"src": "path/to/script"}, {"src": "different/script/path"}]
+        )
+
+        config = self.get_config(scripts=[{"more": "different/script/path"}])
+        with self.assertRaises(ConfigurationError):
+            config.default_dataset_config.handle_app()
+
+    def test_handle_user_annotations_ensures_auth_is_enabled_with_valid_auth_type(self):
+        config = self.get_config(enable_users_annotations="true", authentication_enable="false")
+        config.server_config.complete_config(self.context)
+        with self.assertRaises(ConfigurationError):
+            config.default_dataset_config.handle_user_annotations(self.context)
+
+        config = self.get_config(enable_users_annotations="true", authentication_enable="true", auth_type="pretend")
+        with self.assertRaises(ConfigurationError):
+            config.server_config.complete_config(self.context)
+
+    def test_handle_user_annotations__adds_warning_message_if_annotation_vars_set_when_annotations_disabled(self):
+        config = self.get_config(
+            enable_users_annotations="false", authentication_enable="false", db_uri="shouldnt/be/set"
+        )
+        config.default_dataset_config.handle_user_annotations(self.context)
+
+        self.assertEqual(self.context["messages"], ["Warning: db_uri ignored as annotations are disabled."])
+
+    @patch("server.common.config.dataset_config.DbUtils")
+    def test_handle_user_annotations__instantiates_user_annotations_class_correctly(self, mock_db_utils):
+        mock_db_utils.return_value = "123"
+        config = self.get_config(
+            enable_users_annotations="true", authentication_enable="true", annotation_type="local_file_csv"
+        )
+        config.server_config.complete_config(self.context)
+        config.default_dataset_config.handle_user_annotations(self.context)
+        self.assertIsInstance(config.default_dataset_config.user_annotations, AnnotationsLocalFile)
+
+        config = self.get_config(
+            enable_users_annotations="true",
+            authentication_enable="true",
+            annotation_type="hosted_tiledb_array",
+            db_uri="gotta/set/this",
+            hosted_file_directory="and/this",
+        )
+        config.server_config.complete_config(self.context)
+        config.default_dataset_config.handle_user_annotations(self.context)
+        self.assertIsInstance(config.default_dataset_config.user_annotations, AnnotationsHostedTileDB)
+
+        config = self.get_config(
+            enable_users_annotations="true", authentication_enable="true", annotation_type="NOT_REAL"
+        )
+        config.server_config.complete_config(self.context)
+        with self.assertRaises(ConfigurationError):
+            config.default_dataset_config.handle_user_annotations(self.context)
+
+    def test_handle_local_file_csv_annotations__sets_dir_if_not_passed_in(self):
+        config = self.get_config(
+            enable_users_annotations="true", authentication_enable="true", annotation_type="local_file_csv"
+        )
+        config.server_config.complete_config(self.context)
+        config.default_dataset_config.handle_local_file_csv_annotations()
+        self.assertIsInstance(config.default_dataset_config.user_annotations, AnnotationsLocalFile)
+        cwd = os.getcwd()
+        self.assertEqual(config.default_dataset_config.user_annotations._get_output_dir(), cwd)
+
+    def test_handle_embeddings__checks_data_file_types(self):
+        file_name = self.custom_app_config(
+            embedding_names=["name1", "name2"],
+            enable_reembedding="true",
+            dataset_datapath=f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad",
+            anndata_backed="true",
+            config_file_name=self.config_file_name,
+        )
+        config = AppConfig()
+        config.update_from_config_file(file_name)
+        config.server_config.complete_config(self.context)
+        with self.assertRaises(ConfigurationError):
+            config.default_dataset_config.handle_embeddings()
+
+    def test_handle_diffexp__raises_warning_for_large_datasets(self):
+        config = self.get_config(lfc_cutoff=0.02, enable_difexp="true", top_n=15)
+        config.server_config.complete_config(self.context)
+        config.default_dataset_config.handle_diffexp(self.context)
+        self.assertEqual(len(self.context["messages"]), 0)
+
+    def test_multi_dataset(self):
+        config = AppConfig()
+        # test for illegal url_dataroots
+        for illegal in ("../b", "!$*", "\\n", "", "(bad)"):
+            config.update_server_config(
+                multi_dataset__dataroot={"tag": {"base_url": illegal, "dataroot": "{PROJECT_ROOT}/example-dataset"}}
+            )
+            with self.assertRaises(ConfigurationError):
+                config.complete_config()
+
+        # test for legal url_dataroots
+        for legal in ("d", "this.is-okay_", "a/b"):
+            config.update_server_config(
+                multi_dataset__dataroot={"tag": {"base_url": legal, "dataroot": "{PROJECT_ROOT}/example-dataset"}}
+            )
+            config.complete_config()
+
+        # test that multi dataroots work end to end
+        config.update_server_config(
+            multi_dataset__dataroot=dict(
+                s1=dict(dataroot=f"{PROJECT_ROOT}/example-dataset", base_url="set1/1/2"),
+                s2=dict(dataroot=f"{FIXTURES_ROOT}", base_url="set2"),
+                s3=dict(dataroot=f"{FIXTURES_ROOT}", base_url="set3"),
+            )
+        )
+
+        # Change this default to test if the dataroot overrides below work.
+        config.update_default_dataset_config(app__about_legal_tos="tos_default.html")
+
+        # specialize the configs for set1
+        config.add_dataroot_config(
+            "s1", user_annotations__enable=False, diffexp__enable=True, app__about_legal_tos="tos_set1.html"
+        )
+
+        # specialize the configs for set2
+        config.add_dataroot_config(
+            "s2", user_annotations__enable=True, diffexp__enable=False, app__about_legal_tos="tos_set2.html"
+        )
+
+        # no specializations for set3 (they get the default dataset config)
+        config.complete_config()
+
+        with test_server(app_config=config) as server:
+            session = requests.Session()
+
+            response = session.get(f"{server}/set1/1/2/pbmc3k.h5ad/api/v0.2/config")
+            data_config = response.json()
+            assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
+            assert data_config["config"]["parameters"]["annotations"] is False
+            assert data_config["config"]["parameters"]["disable-diffexp"] is False
+            assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_set1.html"
+
+            response = session.get(f"{server}/set2/pbmc3k.cxg/api/v0.2/config")
+            data_config = response.json()
+            assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
+            assert data_config["config"]["parameters"]["annotations"] is True
+            assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_set2.html"
+
+            response = session.get(f"{server}/set3/pbmc3k.cxg/api/v0.2/config")
+            data_config = response.json()
+            assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
+            assert data_config["config"]["parameters"]["annotations"] is True
+            assert data_config["config"]["parameters"]["disable-diffexp"] is False
+            assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_default.html"
+
+            response = session.get(f"{server}/health")
+            assert response.json()["status"] == "pass"
+
+    def test_configfile_with_specialization(self):
+        # test that per_dataset_config config load the default config, then the specialized config
+
+        with tempfile.TemporaryDirectory() as tempdir:
+            configfile = os.path.join(tempdir, "config.yaml")
+            with open(configfile, "w") as fconfig:
+                config = """
+                server:
+                    multi_dataset:
+                        dataroot:
+                            test:
+                                base_url: test
+                                dataroot: fake_dataroot
+
+                dataset:
+                    user_annotations:
+                        enable: false
+                        type: hosted_tiledb_array
+                        hosted_tiledb_array:
+                            db_uri: fake_db_uri
+                            hosted_file_directory: fake_dir
+
+                per_dataset_config:
+                    test:
+                        user_annotations:
+                            enable: true
+                """
+                fconfig.write(config)
+
+            app_config = AppConfig()
+            app_config.update_from_config_file(configfile)
+
+            test_config = app_config.dataroot_config["test"]
+
+            # test config from default
+            self.assertEqual(test_config.user_annotations__type, "hosted_tiledb_array")
+            self.assertEqual(test_config.user_annotations__hosted_tiledb_array__db_uri, "fake_db_uri")
+
+            # test config from specialization
+            self.assertTrue(test_config.user_annotations__enable)

server/test/unit/common/config/test_server_config.py

@@ -0,0 +1,335 @@
+import os
+import unittest
+from unittest import mock
+from unittest.mock import patch
+
+from server.common.config.base_config import BaseConfig
+from server.common.utils.utils import find_available_port
+from server.test import PROJECT_ROOT, FIXTURES_ROOT
+
+import requests
+
+from server.common.config.app_config import AppConfig
+from server.common.errors import ConfigurationError
+from server.test import test_server
+from server.test.unit.common.config import ConfigTests
+
+
+def mockenv(**envvars):
+    return mock.patch.dict(os.environ, envvars)
+
+
+class TestServerConfig(ConfigTests):
+    def setUp(self):
+        self.config_file_name = f"{unittest.TestCase.id(self).split('.')[-1]}.yml"
+        self.config = AppConfig()
+        self.config.update_server_config(multi_dataset__dataroot=FIXTURES_ROOT)
+        self.server_config = self.config.server_config
+        self.config.complete_config()
+
+        message_list = []
+
+        def noop(message):
+            message_list.append(message)
+
+        messagefn = noop
+        self.context = dict(messagefn=messagefn, messages=message_list)
+
+    def get_config(self, **kwargs):
+        file_name = self.custom_app_config(
+            dataroot=f"{FIXTURES_ROOT}", config_file_name=self.config_file_name, **kwargs
+        )
+        config = AppConfig()
+        config.update_from_config_file(file_name)
+        return config
+
+    def test_init_raises_error_if_default_config_is_invalid(self):
+        invalid_config = self.get_config(port="not_valid")
+        with self.assertRaises(ConfigurationError):
+            invalid_config.complete_config()
+
+    @patch("server.common.config.server_config.BaseConfig.validate_correct_type_of_configuration_attribute")
+    def test_complete_config_checks_all_attr(self, mock_check_attrs):
+        mock_check_attrs.side_effect = BaseConfig.validate_correct_type_of_configuration_attribute()
+        self.server_config.complete_config(self.context)
+        self.assertEqual(mock_check_attrs.call_count, 40)
+
+    def test_handle_app__throws_error_if_port_doesnt_exist(self):
+        config = self.get_config(port=99999999)
+        with self.assertRaises(ConfigurationError):
+            config.server_config.handle_app(self.context)
+
+    @patch("server.common.config.server_config.discover_s3_region_name")
+    def test_handle_data_locator_works_for_default_types(self, mock_discover_region_name):
+        mock_discover_region_name.return_value = None
+        # Default config
+        self.assertEqual(self.config.server_config.data_locator__s3__region_name, None)
+        # hard coded
+        config = self.get_config()
+        self.assertEqual(config.server_config.data_locator__s3__region_name, "us-east-1")
+        # incorrectly formatted
+        dataroot = {
+            "d1": {"base_url": "set1", "dataroot": "/path/to/set1_datasets/"},
+            "d2": {"base_url": "set2/subdir", "dataroot": "s3://shouldnt/work"},
+        }
+        file_name = self.custom_app_config(
+            dataroot=dataroot, config_file_name=self.config_file_name, data_locater_region_name="true"
+        )
+        config = AppConfig()
+        config.update_from_config_file(file_name)
+        with self.assertRaises(ConfigurationError):
+            config.server_config.handle_data_locator()
+
+    @patch("server.common.config.server_config.discover_s3_region_name")
+    def test_handle_data_locator_can_read_from_dataroot(self, mock_discover_region_name):
+        mock_discover_region_name.return_value = "us-west-2"
+        dataroot = {
+            "d1": {"base_url": "set1", "dataroot": "/path/to/set1_datasets/"},
+            "d2": {"base_url": "set2/subdir", "dataroot": "s3://hosted-cellxgene-dev"},
+        }
+        file_name = self.custom_app_config(
+            dataroot=dataroot, config_file_name=self.config_file_name, data_locater_region_name="true"
+        )
+        config = AppConfig()
+        config.update_from_config_file(file_name)
+        config.server_config.handle_data_locator()
+        self.assertEqual(config.server_config.data_locator__s3__region_name, "us-west-2")
+        mock_discover_region_name.assert_called_once_with("s3://hosted-cellxgene-dev")
+
+    def test_handle_app___can_use_envar_port(self):
+        config = self.get_config(port=24)
+        self.assertEqual(config.server_config.app__port, 24)
+
+        # Note if the port is set in the config file it will NOT be overwritten by a different envvar
+        os.environ["CXG_SERVER_PORT"] = "4008"
+        self.config = AppConfig()
+        self.config.server_config.handle_app(self.context)
+        self.assertEqual(self.config.server_config.app__port, 4008)
+
+    def test_handle_app__can_get_secret_key_from_envvar_or_config_file_with_envvar_given_preference(self):
+        config = self.get_config(flask_secret_key="KEY_FROM_FILE")
+        self.assertEqual(config.server_config.app__flask_secret_key, "KEY_FROM_FILE")
+
+        os.environ["CXG_SECRET_KEY"] = "KEY_FROM_ENV"
+        config.server_config.handle_app(self.context)
+        self.assertEqual(config.server_config.app__flask_secret_key, "KEY_FROM_ENV")
+
+    def test_handle_app__sets_web_base_url(self):
+        config = self.get_config(web_base_url="anything.com")
+        self.assertEqual(config.server_config.app__web_base_url, "anything.com")
+
+    def test_handle_auth__gets_client_secret_from_envvars_or_config_with_envvars_given_preference(self):
+        config = self.get_config(client_secret="KEY_FROM_FILE")
+        config.server_config.handle_authentication()
+        self.assertEqual(config.server_config.authentication__params_oauth__client_secret, "KEY_FROM_FILE")
+
+        os.environ["CXG_OAUTH_CLIENT_SECRET"] = "KEY_FROM_ENV"
+        config.server_config.handle_authentication()
+
+        self.assertEqual(config.server_config.authentication__params_oauth__client_secret, "KEY_FROM_ENV")
+
+    def test_handle_data_source__errors_when_passed_zero_or_two_dataroots(self):
+        file_name = self.custom_app_config(
+            dataroot=f"{FIXTURES_ROOT}",
+            config_file_name="two_data_roots.yml",
+            dataset_datapath=f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad",
+        )
+        config = AppConfig()
+        config.update_from_config_file(file_name)
+        with self.assertRaises(ConfigurationError):
+            config.server_config.handle_data_source()
+
+        file_name = self.custom_app_config(config_file_name="zero_roots.yml")
+        config = AppConfig()
+        config.update_from_config_file(file_name)
+        with self.assertRaises(ConfigurationError):
+            config.server_config.handle_data_source()
+
+    def test_get_api_base_url_works(self):
+
+        # test the api_base_url feature, and that it can contain a path
+        config = AppConfig()
+        backend_port = find_available_port("localhost", 10000)
+        config.update_server_config(
+            app__api_base_url=f"http://localhost:{backend_port}/additional/path",
+            multi_dataset__dataroot=f"{PROJECT_ROOT}/example-dataset",
+        )
+
+        config.complete_config()
+
+        with test_server(["-p", str(backend_port)], app_config=config) as server:
+            session = requests.Session()
+            self.assertEqual(server, f"http://localhost:{backend_port}")
+            response = session.get(f"{server}/additional/path/d/pbmc3k.h5ad/api/v0.2/config")
+            self.assertEqual(response.status_code, 200)
+            data_config = response.json()
+            self.assertEqual(data_config["config"]["displayNames"]["dataset"], "pbmc3k")
+
+            # test the health check at the correct url
+            response = session.get(f"{server}/additional/path/health")
+            assert response.json()["status"] == "pass"
+
+            # also check that the old URL still works.
+            # NOTE:  this old URL location will soon be deprecated, and when that happens
+            # this check can be removed.
+            response = session.get(f"{server}/health")
+            assert response.json()["status"] == "pass"
+
+    def test_get_web_base_url_works(self):
+        config = self.get_config(web_base_url="www.thisisawebsite.com")
+        web_base_url = config.server_config.get_web_base_url()
+        self.assertEqual(web_base_url, "www.thisisawebsite.com")
+
+        config = self.get_config(web_base_url="local", port=12)
+        web_base_url = config.server_config.get_web_base_url()
+        self.assertEqual(web_base_url, "http://localhost:12")
+
+        config = self.get_config(web_base_url="www.thisisawebsite.com/")
+        web_base_url = config.server_config.get_web_base_url()
+        self.assertEqual(web_base_url, "www.thisisawebsite.com")
+
+        config = self.get_config(api_base_url="www.api_base.com/")
+        web_base_url = config.server_config.get_web_base_url()
+        self.assertEqual(web_base_url, "www.api_base.com")
+
+    def test_config_for_single_dataset(self):
+        file_name = self.custom_app_config(
+            config_file_name="single_dataset.yml", dataset_datapath=f"{FIXTURES_ROOT}/pbmc3k.cxg"
+        )
+        config = AppConfig()
+        config.update_from_config_file(file_name)
+        config.server_config.handle_single_dataset(self.context)
+        self.assertIsNotNone(config.server_config.matrix_data_cache_manager)
+
+        file_name = self.custom_app_config(
+            config_file_name="single_dataset_with_about.yml",
+            about="www.cziscience.com",
+            dataset_datapath=f"{FIXTURES_ROOT}/pbmc3k.cxg",
+        )
+        config = AppConfig()
+        config.update_from_config_file(file_name)
+        with self.assertRaises(ConfigurationError):
+            config.server_config.handle_single_dataset(self.context)
+
+    def test_multi_dataset_raises_error_for_illegal_routes(self):
+        # test for illegal url_dataroots
+        for illegal in ("../b", "!$*", "\\n", "", "(bad)"):
+            self.config.update_server_config(
+                multi_dataset__dataroot={"tag": {"base_url": illegal, "dataroot": "{PROJECT_ROOT}/example-dataset"}}
+            )
+            with self.assertRaises(ConfigurationError):
+                self.config.complete_config()
+
+    def test_multidataset_works_for_legal_routes(self):
+        # test for legal url_dataroots
+        for legal in ("d", "this.is-okay_", "a/b"):
+            self.config.update_server_config(
+                multi_dataset__dataroot={"tag": {"base_url": legal, "dataroot": "{PROJECT_ROOT}/example-dataset"}}
+            )
+            self.config.complete_config()
+
+    def test_mulitdatasets_work_e2e(self):
+        # test that multi dataroots work end to end
+        self.config.update_server_config(
+            multi_dataset__dataroot=dict(
+                s1=dict(dataroot=f"{PROJECT_ROOT}/example-dataset", base_url="set1/1/2"),
+                s2=dict(dataroot=f"{FIXTURES_ROOT}", base_url="set2"),
+                s3=dict(dataroot=f"{FIXTURES_ROOT}", base_url="set3"),
+            )
+        )
+
+        # Change this default to test if the dataroot overrides below work.
+        self.config.update_default_dataset_config(app__about_legal_tos="tos_default.html")
+
+        # specialize the configs for set1
+        self.config.add_dataroot_config(
+            "s1", user_annotations__enable=False, diffexp__enable=True, app__about_legal_tos="tos_set1.html"
+        )
+
+        # specialize the configs for set2
+        self.config.add_dataroot_config(
+            "s2", user_annotations__enable=True, diffexp__enable=False, app__about_legal_tos="tos_set2.html"
+        )
+
+        # no specializations for set3 (they get the default dataset config)
+        self.config.complete_config()
+
+        with test_server(app_config=self.config) as server:
+            session = requests.Session()
+
+            response = session.get(f"{server}/set1/1/2/pbmc3k.h5ad/api/v0.2/config")
+            data_config = response.json()
+            assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
+            assert data_config["config"]["parameters"]["annotations"] is False
+            assert data_config["config"]["parameters"]["disable-diffexp"] is False
+            assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_set1.html"
+
+            response = session.get(f"{server}/set2/pbmc3k.cxg/api/v0.2/config")
+            data_config = response.json()
+            assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
+            assert data_config["config"]["parameters"]["annotations"] is True
+            assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_set2.html"
+
+            response = session.get(f"{server}/set3/pbmc3k.cxg/api/v0.2/config")
+            data_config = response.json()
+            assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
+            assert data_config["config"]["parameters"]["annotations"] is True
+            assert data_config["config"]["parameters"]["disable-diffexp"] is False
+            assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_default.html"
+
+            response = session.get(f"{server}/health")
+            assert response.json()["status"] == "pass"
+
+    @patch("server.common.config.server_config.diffexp_tiledb.set_config")
+    def test_handle_diffexp(self, mock_tiledb_config):
+        custom_config_file = self.custom_app_config(
+            dataroot=f"{FIXTURES_ROOT}",
+            cpu_multiplier=3,
+            diffexp_max_workers=1,
+            target_workunit=4,
+            config_file_name=self.config_file_name,
+        )
+        config = AppConfig()
+        config.update_from_config_file(custom_config_file)
+        config.server_config.handle_diffexp()
+        # called with the min of diffexp_max_workers and cpus*cpu_multiplier
+        mock_tiledb_config.assert_called_once_with(1, 4)
+
+    @patch("server.data_cxg.cxg_adaptor.CxgAdaptor.set_tiledb_context")
+    def test_handle_adaptor(self, mock_tiledb_context):
+        custom_config = self.custom_app_config(
+            dataroot=f"{FIXTURES_ROOT}", cxg_tile_cache_size=10, cxg_num_reader_threads=2
+        )
+        config = AppConfig()
+        config.update_from_config_file(custom_config)
+        config.server_config.handle_adaptor()
+        mock_tiledb_context.assert_called_once_with(
+            {"sm.tile_cache_size": 10, "sm.num_reader_threads": 2, "vfs.s3.region": "us-east-1"}
+        )
+
+    @mockenv(CXG_AWS_SECRET_NAME="TESTING", CXG_AWS_SECRET_REGION_NAME="TEST_REGION")
+    @patch("server.common.config.get_secret_key")
+    def test_get_config_vars_from_aws_secrets(self, mock_get_secret_key):
+        mock_get_secret_key.return_value = {
+            "flask_secret_key": "mock_flask_secret",
+            "oauth_client_secret": "mock_oauth_secret",
+            "db_uri": "mock_db_uri",
+        }
+
+        config = AppConfig()
+
+        with self.assertLogs(level="INFO") as logger:
+            from server.common.config import handle_config_from_secret
+
+            # should not throw error
+            # "AttributeError: 'XConfig' object has no attribute 'x'"
+            handle_config_from_secret(config)
+
+            # should log 3 lines (one for each var set from a secret)
+            self.assertEqual(len(logger.output), 3)
+            self.assertIn("INFO:root:set app__flask_secret_key from secret", logger.output[0])
+            self.assertIn("INFO:root:set authentication__params_oauth__client_secret from secret", logger.output[1])
+            self.assertIn("INFO:root:set user_annotations__hosted_tiledb_array__db_uri from secret", logger.output[2])
+            self.assertEqual(config.server_config.app__flask_secret_key, "mock_flask_secret")
+            self.assertEqual(config.server_config.authentication__params_oauth__client_secret, "mock_oauth_secret")
+            self.assertEqual(config.default_dataset_config.user_annotations__hosted_tiledb_array__db_uri, "mock_db_uri")

server/test/unit/common/test_api.py

@@ -8,8 +8,14 @@ import requests
 
 import server.test.unit.decode_fbs as decode_fbs
 from server.data_common.matrix_loader import MatrixDataType
-from server.test import (data_with_tmp_annotations, make_fbs, PROJECT_ROOT, FIXTURES_ROOT, start_test_server,
-                         stop_test_server)
+from server.test import (
+    data_with_tmp_annotations,
+    make_fbs,
+    PROJECT_ROOT,
+    FIXTURES_ROOT,
+    start_test_server,
+    stop_test_server,
+)
 from server.test.fixtures.fixtures import pbmc3k_colors
 
 BAD_FILTER = {"filter": {"obs": {"annotation_value": [{"name": "xyz"}]}}}
@@ -43,7 +49,6 @@ class EndPoints(object):
         result_data = result.json()
         self.assertIn("library_versions", result_data["config"])
         self.assertEqual(result_data["config"]["displayNames"]["dataset"], "pbmc3k")
-        self.assertEqual(len(result_data["config"]["features"]), 5)
 
     def test_get_layout_fbs(self):
         endpoint = "layout/obs"
@@ -381,11 +386,14 @@ class EndPointsAnndata(unittest.TestCase, EndPoints):
 
     @classmethod
     def setUpClass(cls):
-        cls._setupClass(cls, [
-            f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad",
-            "--disable-annotations",
-            "--experimental-enable-reembedding",
-        ])
+        cls._setupClass(
+            cls,
+            [
+                f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad",
+                "--disable-annotations",
+                "--experimental-enable-reembedding",
+            ],
+        )
 
     @classmethod
     def tearDownClass(cls):
@@ -403,10 +411,7 @@ class EndPointsCxg(unittest.TestCase, EndPoints):
 
     @classmethod
     def setUpClass(cls):
-        cls._setupClass(cls, [
-            f"{FIXTURES_ROOT}/pbmc3k.cxg",
-            "--disable-annotations",
-        ])
+        cls._setupClass(cls, [f"{FIXTURES_ROOT}/pbmc3k.cxg", "--disable-annotations"])
 
     @classmethod
     def tearDownClass(cls):
@@ -423,7 +428,7 @@ class EndPointsAnndataAnnotations(unittest.TestCase, EndPointsAnnotations):
         cls.data, cls.tmp_dir, cls.annotations = data_with_tmp_annotations(
             MatrixDataType.H5AD, annotations_fixture=True
         )
-        cls._setupClass(cls, ["--annotations-file", cls.annotations.output_file, cls.data.get_location(), ])
+        cls._setupClass(cls, ["--annotations-file", cls.annotations.output_file, cls.data.get_location()])
 
     @classmethod
     def tearDownClass(cls):
@@ -439,11 +444,7 @@ class EndPointsCxgAnnotations(unittest.TestCase, EndPointsAnnotations):
     @classmethod
     def setUpClass(cls):
         cls.data, cls.tmp_dir, cls.annotations = data_with_tmp_annotations(MatrixDataType.CXG, annotations_fixture=True)
-        cls._setupClass(cls, [
-            "--annotations-file",
-            cls.annotations.output_file,
-            cls.data.get_location(),
-        ])
+        cls._setupClass(cls, ["--annotations-file", cls.annotations.output_file, cls.data.get_location()])
 
     @classmethod
     def tearDownClass(cls):

server/test/unit/common/test_app_config.py

@@ -1,249 +0,0 @@
-import os
-import unittest
-from unittest import mock
-from unittest.mock import patch
-import tempfile
-
-import requests
-
-from server.common.app_config import AppConfig
-from server.common.errors import ConfigurationError
-from server.common.utils.utils import find_available_port
-from server.test import PROJECT_ROOT, test_server, FIXTURES_ROOT
-
-
-# NOTE, there are more tests that should be written for AppConfig.
-# this is just a start.
-
-def mockenv(**envvars):
-    return mock.patch.dict(os.environ, envvars)
-
-
-class AppConfigTest(unittest.TestCase):
-    def test_update(self):
-        config = AppConfig()
-        config.update_server_config(app__verbose=True, multi_dataset__dataroot="datadir")
-        vars = config.server_config.changes_from_default()
-        self.assertCountEqual(vars, [("app__verbose", True, False), ("multi_dataset__dataroot", "datadir", None)])
-
-        config = AppConfig()
-        config.update_default_dataset_config(app__scripts=(), app__inline_scripts=())
-        vars = config.server_config.changes_from_default()
-        self.assertCountEqual(vars, [])
-
-        config = AppConfig()
-        config.update_default_dataset_config(app__scripts=[], app__inline_scripts=[])
-        vars = config.default_dataset_config.changes_from_default()
-        self.assertCountEqual(vars, [])
-
-        config = AppConfig()
-        config.update_default_dataset_config(app__scripts=("a", "b"), app__inline_scripts=["c", "d"])
-        vars = config.default_dataset_config.changes_from_default()
-        self.assertCountEqual(vars, [("app__scripts", ["a", "b"], []), ("app__inline_scripts", ["c", "d"], [])])
-
-    def test_multi_dataset(self):
-
-        config = AppConfig()
-        # test for illegal url_dataroots
-        for illegal in ("../b", "!$*", "\\n", "", "(bad)"):
-            config.update_server_config(
-                multi_dataset__dataroot={"tag": {"base_url": illegal, "dataroot": "{PROJECT_ROOT}/example-dataset"}}
-            )
-            with self.assertRaises(ConfigurationError):
-                config.complete_config()
-
-        # test for legal url_dataroots
-        for legal in ("d", "this.is-okay_", "a/b"):
-            config.update_server_config(
-                multi_dataset__dataroot={"tag": {"base_url": legal, "dataroot": "{PROJECT_ROOT}/example-dataset"}}
-            )
-            config.complete_config()
-
-        # test that multi dataroots work end to end
-        config.update_server_config(
-            multi_dataset__dataroot=dict(
-                s1=dict(dataroot=f"{PROJECT_ROOT}/example-dataset", base_url="set1/1/2"),
-                s2=dict(dataroot=f"{FIXTURES_ROOT}", base_url="set2"),
-                s3=dict(dataroot=f"{FIXTURES_ROOT}", base_url="set3"),
-            )
-        )
-
-        # Change this default to test if the dataroot overrides below work.
-        config.update_default_dataset_config(app__about_legal_tos="tos_default.html")
-
-        # specialize the configs for set1
-        config.add_dataroot_config(
-            "s1", user_annotations__enable=False, diffexp__enable=True, app__about_legal_tos="tos_set1.html"
-        )
-
-        # specialize the configs for set2
-        config.add_dataroot_config(
-            "s2", user_annotations__enable=True, diffexp__enable=False, app__about_legal_tos="tos_set2.html"
-        )
-
-        # no specializations for set3 (they get the default dataset config)
-        config.complete_config()
-
-        with test_server(app_config=config) as server:
-            session = requests.Session()
-
-            response = session.get(f"{server}/set1/1/2/pbmc3k.h5ad/api/v0.2/config")
-            data_config = response.json()
-            assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
-            assert data_config["config"]["parameters"]["annotations"] is False
-            assert data_config["config"]["parameters"]["disable-diffexp"] is False
-            assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_set1.html"
-
-            response = session.get(f"{server}/set2/pbmc3k.cxg/api/v0.2/config")
-            data_config = response.json()
-            assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
-            assert data_config["config"]["parameters"]["annotations"] is True
-            assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_set2.html"
-
-            response = session.get(f"{server}/set3/pbmc3k.cxg/api/v0.2/config")
-            data_config = response.json()
-            assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
-            assert data_config["config"]["parameters"]["annotations"] is True
-            assert data_config["config"]["parameters"]["disable-diffexp"] is False
-            assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_default.html"
-
-            response = session.get(f"{server}/health")
-            assert response.json()["status"] == "pass"
-
-    @mockenv(CXG_AWS_SECRET_NAME="TESTING", CXG_AWS_SECRET_REGION_NAME="TEST_REGION")
-    @patch('server.common.aws_secret_utils.get_secret_key')
-    def test_get_config_vars_from_aws_secrets(self, mock_get_secret_key):
-        mock_get_secret_key.return_value = {
-            "flask_secret_key": "mock_flask_secret",
-            "oauth_client_secret": "mock_oauth_secret",
-            "db_uri": "mock_db_uri"
-        }
-
-        config = AppConfig()
-
-        with self.assertLogs(level="INFO") as logger:
-            from server.common.aws_secret_utils import handle_config_from_secret
-            # should not throw error
-            # "AttributeError: 'XConfig' object has no attribute 'x'"
-            handle_config_from_secret(config)
-
-            # should log 3 lines (one for each var set from a secret)
-            self.assertEqual(len(logger.output), 3)
-            self.assertIn('INFO:root:set app__flask_secret_key from secret', logger.output[0])
-            self.assertIn('INFO:root:set authentication__params_oauth__client_secret from secret', logger.output[1])
-            self.assertIn('INFO:root:set user_annotations__hosted_tiledb_array__db_uri from secret', logger.output[2])
-            self.assertEqual(config.server_config.app__flask_secret_key, "mock_flask_secret")
-            self.assertEqual(config.server_config.authentication__params_oauth__client_secret, "mock_oauth_secret")
-            self.assertEqual(config.default_dataset_config.user_annotations__hosted_tiledb_array__db_uri, "mock_db_uri")
-
-    def test_api_base_url(self):
-
-        # test the api_base_url feature, and that it can contain a path
-        config = AppConfig()
-        backend_port = find_available_port("localhost", 10000)
-        config.update_server_config(
-            app__api_base_url=f"http://localhost:{backend_port}/additional/path",
-            multi_dataset__dataroot=f"{PROJECT_ROOT}/example-dataset"
-        )
-
-        config.complete_config()
-
-        with test_server(["-p", str(backend_port)], app_config=config) as server:
-            session = requests.Session()
-            self.assertEqual(server, f"http://localhost:{backend_port}")
-            response = session.get(f"{server}/additional/path/d/pbmc3k.h5ad/api/v0.2/config")
-            self.assertEqual(response.status_code, 200)
-            data_config = response.json()
-            self.assertEqual(data_config["config"]["displayNames"]["dataset"], "pbmc3k")
-
-            # test the health check at the correct url
-            response = session.get(f"{server}/additional/path/health")
-            assert response.json()["status"] == "pass"
-
-            # also check that the old URL still works.
-            # NOTE:  this old URL location will soon be deprecated, and when that happens
-            # this check can be removed.
-            response = session.get(f"{server}/health")
-            assert response.json()["status"] == "pass"
-
-    def test_configfile_with_specialization(self):
-        # test that per_dataset_config config load the default config, then the specialized config
-
-        with tempfile.TemporaryDirectory() as tempdir:
-            configfile = os.path.join(tempdir, "config.yaml")
-            with open(configfile, "w") as fconfig:
-                config = """
-                server:
-                    multi_dataset:
-                        dataroot:
-                            test:
-                                base_url: test
-                                dataroot: fake_dataroot
-
-                dataset:
-                    user_annotations:
-                        enable: false
-                        type: hosted_tiledb_array
-                        hosted_tiledb_array:
-                            db_uri: fake_db_uri
-                            hosted_file_directory: fake_dir
-
-                per_dataset_config:
-                    test:
-                        user_annotations:
-                            enable: true
-                """
-                fconfig.write(config)
-
-            app_config = AppConfig()
-            app_config.update_from_config_file(configfile)
-
-            test_config = app_config.dataroot_config["test"]
-
-            # test config from default
-            self.assertEqual(test_config.user_annotations__type, "hosted_tiledb_array")
-            self.assertEqual(test_config.user_annotations__hosted_tiledb_array__db_uri, "fake_db_uri")
-
-            # test config from specialization
-            self.assertTrue(test_config.user_annotations__enable)
-
-    def test_configfile_no_dataset_section(self):
-        # test a config file without a dataset section
-
-        with tempfile.TemporaryDirectory() as tempdir:
-            configfile = os.path.join(tempdir, "config.yaml")
-            with open(configfile, "w") as fconfig:
-                config = """
-                server:
-                    multi_dataset:
-                        dataroot: test_dataroot
-
-                """
-                fconfig.write(config)
-
-            app_config = AppConfig()
-            app_config.update_from_config_file(configfile)
-            server_changes = app_config.server_config.changes_from_default()
-            dataset_changes = app_config.default_dataset_config.changes_from_default()
-            self.assertEqual(server_changes, [("multi_dataset__dataroot", "test_dataroot", None)])
-            self.assertEqual(dataset_changes, [])
-
-    def test_configfile_no_server_section(self):
-        # test a config file without a dataset section
-
-        with tempfile.TemporaryDirectory() as tempdir:
-            configfile = os.path.join(tempdir, "config.yaml")
-            with open(configfile, "w") as fconfig:
-                config = """
-                dataset:
-                    user_annotations:
-                        enable: false
-                """
-                fconfig.write(config)
-
-            app_config = AppConfig()
-            app_config.update_from_config_file(configfile)
-            server_changes = app_config.server_config.changes_from_default()
-            dataset_changes = app_config.default_dataset_config.changes_from_default()
-            self.assertEqual(server_changes, [])
-            self.assertEqual(dataset_changes, [("user_annotations__enable", False, True)])

server/test/unit/common/test_corpora.py

@@ -87,24 +87,17 @@ class CorporaRESTAPITest(unittest.TestCase):
     def setCorporaFields(cls, path):
         adata = anndata.read_h5ad(path)
         corpora_props = {
-            "version": {
-                "corpora_schema_version": "1.0.0",
-                "corpora_encoding_version": "0.1.0"
-            },
+            "version": {"corpora_schema_version": "1.0.0", "corpora_encoding_version": "0.1.0"},
             "title": "PBMC3K",
-            "contributors": json.dumps([
-                {"name": "name"}
-            ]),
-            "layer_descriptions": {
-                "X": "raw counts"
-            },
+            "contributors": json.dumps([{"name": "name"}]),
+            "layer_descriptions": {"X": "raw counts"},
             "organism": "human",
             "organism_ontology_term_id": "unknown",
             "project_name": "test project",
             "project_description": "test description",
-            "project_links": json.dumps([
-                {"link_name": "test link", "link_type": "SUMMARY", "link_url": "https://a.u.r.l/"}
-            ]),
+            "project_links": json.dumps(
+                [{"link_name": "test link", "link_type": "SUMMARY", "link_url": "https://a.u.r.l/"}]
+            ),
             "default_embedding": "X_tsne",
         }
         adata.uns.update(corpora_props)

server/test/unit/common/test_writable_annotation.py

@@ -27,7 +27,7 @@ class auth(object):
 
 class WritableTileDBStoredAnnotationTest(unittest.TestCase):
     def setUp(self):
-        self.user_id = '1234'
+        self.user_id = "1234"
         self.data, self.tmp_dir, self.annotations = data_with_tmp_tiledb_annotations(MatrixDataType.H5AD)
         self.data.dataset_config.user_annotations = self.annotations
         self.db = self.annotations.db
@@ -38,7 +38,7 @@ class WritableTileDBStoredAnnotationTest(unittest.TestCase):
         }
         self.fbs = make_fbs(self.test_dict)
         self.df = pd.DataFrame(self.test_dict)
-        self.app = Flask('fake_app')
+        self.app = Flask("fake_app")
         self.app.__setattr__("auth", auth)
 
     def tearDown(self):
@@ -65,8 +65,7 @@ class WritableTileDBStoredAnnotationTest(unittest.TestCase):
             self.annotations.write_labels(self.df, self.data)
             dataset_id = self.db.query([CellxGeneDataset], [CellxGeneDataset.name == self.data.get_location()])[0].id
             annotation = self.db.query_for_most_recent(
-                Annotation,
-                [Annotation.user_id == self.user_id, Annotation.dataset_id == str(dataset_id)]
+                Annotation, [Annotation.user_id == self.user_id, Annotation.dataset_id == str(dataset_id)]
             )
             # retrieve tiledb array
             df = tiledb.open(annotation.tiledb_uri)
@@ -78,7 +77,7 @@ class WritableTileDBStoredAnnotationTest(unittest.TestCase):
 
     def test_write_labels_creates_a_dataset_if_it_doesnt_exist(self):
         with self.app.test_request_context():
-            new_name = 'new_dataset/location'
+            new_name = "new_dataset/location"
             self.data.get_location = MagicMock(return_value=new_name)
             num_datasets = len(self.db.query([CellxGeneDataset]))
             self.annotation_put_fbs(self.fbs)
@@ -130,15 +129,14 @@ class WritableTileDBStoredAnnotationTest(unittest.TestCase):
             with self.assertRaises(KeyError):
                 self.annotation_put_fbs(fbs_bad)
 
-    @patch('server.common.annotations.hosted_tiledb.current_app')
+    @patch("server.common.annotations.hosted_tiledb.current_app")
     def test_write_labels_stores_df_as_tiledb_array(self, mock_user_id):
-        mock_user_id.auth.get_user_id.return_value = '1234'
+        mock_user_id.auth.get_user_id.return_value = "1234"
         self.annotations.write_labels(self.df, self.data)
         # get uri
         dataset_id = self.db.query([CellxGeneDataset], [CellxGeneDataset.name == self.data.get_location()])[0].id
         annotation = self.db.query_for_most_recent(
-            Annotation,
-            [Annotation.user_id == '1234', Annotation.dataset_id == str(dataset_id)]
+            Annotation, [Annotation.user_id == "1234", Annotation.dataset_id == str(dataset_id)]
         )
 
         df = tiledb.open(annotation.tiledb_uri)
@@ -270,20 +268,3 @@ class WritableAnnotationTest(unittest.TestCase):
             all_col_schema["cat_B"],
             {"name": "cat_B", "type": "categorical", "categories": ["label_B"], "writable": True},
         )
-
-    def test_config(self):
-        features = self.data.get_features(self.annotations)
-
-        # test each for singular presence and accuracy of available flag
-        def check_feature(method, path, available):
-            feature = list(
-                filter(lambda f: f.method == method and f.path == path and f.available == available, features)
-            )
-            self.assertIsNotNone(feature)
-            self.assertEqual(len(feature), 1)
-
-        check_feature("POST", "/cluster/", False)
-        check_feature("POST", "/diffexp/", self.data.dataset_config.diffexp__enable)
-        check_feature("GET", "/layout/obs", True)
-        check_feature("PUT", "/layout/obs", self.data.dataset_config.embeddings__enable_reembedding)
-        check_feature("PUT", "/annotations/obs", True)

server/test/unit/common/utils/test_cxg_generation_utils.py

@@ -8,8 +8,12 @@ import numpy as np
 import tiledb
 from pandas import Series, DataFrame
 
-from server.common.utils.cxg_generation_utils import (convert_dictionary_to_cxg_group, convert_dataframe_to_cxg_array,
-                                                      convert_ndarray_to_cxg_dense_array, convert_matrix_to_cxg_array)
+from server.common.utils.cxg_generation_utils import (
+    convert_dictionary_to_cxg_group,
+    convert_dataframe_to_cxg_array,
+    convert_ndarray_to_cxg_dense_array,
+    convert_matrix_to_cxg_array,
+)
 
 PROJECT_ROOT = popen("git rev-parse --show-toplevel").read().strip()
 
@@ -28,8 +32,9 @@ class TestCxgGenerationUtils(unittest.TestCase):
         dictionary_name = "favorite_desserts"
         expected_array_directory = f"{self.testing_cxg_temp_directory}/{dictionary_name}"
 
-        convert_dictionary_to_cxg_group(self.testing_cxg_temp_directory, random_dictionary,
-                                        group_metadata_name=dictionary_name)
+        convert_dictionary_to_cxg_group(
+            self.testing_cxg_temp_directory, random_dictionary, group_metadata_name=dictionary_name
+        )
 
         array = tiledb.open(expected_array_directory)
         actual_stored_metadata = dict(array.meta.items())
@@ -44,13 +49,16 @@ class TestCxgGenerationUtils(unittest.TestCase):
         random_dataframe_name = f"random_dataframe_{uuid4()}"
         random_dataframe = DataFrame(data={"int_category": random_int_category, "bool_category": random_bool_category})
 
-        convert_dataframe_to_cxg_array(self.testing_cxg_temp_directory, random_dataframe_name, random_dataframe,
-                                       "int_category", tiledb.Ctx())
+        convert_dataframe_to_cxg_array(
+            self.testing_cxg_temp_directory, random_dataframe_name, random_dataframe, "int_category", tiledb.Ctx()
+        )
 
         expected_array_directory = f"{self.testing_cxg_temp_directory}/{random_dataframe_name}"
         expected_array_metadata = {
-            "cxg_schema": json.dumps({"int_category": {"type": "int32"}, "bool_category": {"type": "boolean"},
-                                      "index": "int_category"})}
+            "cxg_schema": json.dumps(
+                {"int_category": {"type": "int32"}, "bool_category": {"type": "boolean"}, "index": "int_category"}
+            )
+        }
 
         actual_stored_dataframe_array = tiledb.open(expected_array_directory)
         actual_stored_dataframe_metadata = dict(actual_stored_dataframe_array.meta.items())
@@ -95,7 +103,7 @@ class TestCxgGenerationUtils(unittest.TestCase):
 
         self.assertTrue(path.isdir(matrix_name))
         self.assertTrue(isinstance(actual_stored_array, tiledb.SparseArray))
-        self.assertTrue(actual_stored_array[:, :][''].size == 0)
+        self.assertTrue(actual_stored_array[:, :][""].size == 0)
 
     def test__convert_matrix_to_cxg_array__sparse_array_only_store_nonzeros(self):
         matrix = np.zeros([3, 3])
@@ -110,10 +118,10 @@ class TestCxgGenerationUtils(unittest.TestCase):
 
         self.assertTrue(path.isdir(matrix_name))
         self.assertTrue(isinstance(actual_stored_array, tiledb.SparseArray))
-        self.assertTrue(actual_stored_array[0, 0][''] == 1)
-        self.assertTrue(actual_stored_array[1, 1][''] == 1)
-        self.assertTrue(actual_stored_array[2, 2][''] == 2)
-        self.assertTrue(actual_stored_array[:, :][''].size == 3)
+        self.assertTrue(actual_stored_array[0, 0][""] == 1)
+        self.assertTrue(actual_stored_array[1, 1][""] == 1)
+        self.assertTrue(actual_stored_array[2, 2][""] == 2)
+        self.assertTrue(actual_stored_array[:, :][""].size == 3)
 
     def test__convert_matrix_to_cxg_array__sparse_array_with_column_encoding_empty_array(self):
         matrix_name = f"{self.testing_cxg_temp_directory}/awesome_column_shift_matrix_{uuid4()}"
@@ -122,14 +130,15 @@ class TestCxgGenerationUtils(unittest.TestCase):
         # a matrix of zeros which is sparse.
         column_shift = np.ones((3, 2))
 
-        convert_matrix_to_cxg_array(matrix_name, matrix, True, tiledb.Ctx(),
-                                    column_shift_for_sparse_encoding=column_shift)
+        convert_matrix_to_cxg_array(
+            matrix_name, matrix, True, tiledb.Ctx(), column_shift_for_sparse_encoding=column_shift
+        )
 
         actual_stored_array = tiledb.open(matrix_name)
 
         self.assertTrue(path.isdir(matrix_name))
         self.assertTrue(isinstance(actual_stored_array, tiledb.SparseArray))
-        self.assertTrue(actual_stored_array[:, :][''].size == 0)
+        self.assertTrue(actual_stored_array[:, :][""].size == 0)
 
     def test__convert_matrix_to_cxg_array__sparse_array_with_column_encoding_partial_array(self):
         matrix_name = f"{self.testing_cxg_temp_directory}/awesome_column_shift_matrix_{uuid4()}"
@@ -137,13 +146,14 @@ class TestCxgGenerationUtils(unittest.TestCase):
         # Only column shift the first column of ones.
         column_shift = np.array([[1, 0], [1, 0]])
 
-        convert_matrix_to_cxg_array(matrix_name, matrix, True, tiledb.Ctx(),
-                                    column_shift_for_sparse_encoding=column_shift)
+        convert_matrix_to_cxg_array(
+            matrix_name, matrix, True, tiledb.Ctx(), column_shift_for_sparse_encoding=column_shift
+        )
 
         actual_stored_array = tiledb.open(matrix_name)
 
         self.assertTrue(path.isdir(matrix_name))
         self.assertTrue(isinstance(actual_stored_array, tiledb.SparseArray))
-        self.assertTrue(actual_stored_array[0, 1][''] == 1)
-        self.assertTrue(actual_stored_array[1, 1][''] == 1)
-        self.assertTrue(actual_stored_array[:, :][''].size == 2)
+        self.assertTrue(actual_stored_array[0, 1][""] == 1)
+        self.assertTrue(actual_stored_array[1, 1][""] == 1)
+        self.assertTrue(actual_stored_array[:, :][""].size == 2)

server/test/unit/common/utils/test_matrix_utils.py

@@ -6,7 +6,6 @@ from server.common.utils.matrix_utils import is_matrix_sparse, get_column_shift_
 
 
 class TestMatrixUtils(unittest.TestCase):
-
     def test__is_matrix_sparse__zero_and_one_hundred_percent_threshold(self):
         matrix = np.array([1, 2, 3])
 

server/test/unit/common/utils/test_sanitization_utils.py

@@ -4,7 +4,6 @@ from server.common.utils.sanitization_utils import sanitize_values_in_list, sani
 
 
 class TestSanitizationUtils(unittest.TestCase):
-
     def test__sanitize_values_in_list__not_strings_raises_exception(self):
         keys_to_sanitize = [1, 2, 3]