Merge branch 'main' into colinmegill/geneset-prototype

.bumpversion.cfg

@@ -1,5 +1,19 @@
 [bumpversion]
 current_version = 0.16.0
+commit = True
+# The below regex details an acceptable version number by naming the groups (major, minor, patch, prerel, and
+# prerelversion) and also specifying the valid values for each group (integers, `\d+`,  for major, minor, patch, and
+# prerelversion and only `rc` as the acceptable value for prerel).
+parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(?:-(?P<prerel>rc)\.(?P<prerelversion>\d+))?
+serialize =
+	{major}.{minor}.{patch}-{prerel}.{prerelversion}
+	{major}.{minor}.{patch}
+
+[bumpversion:part:prerel]
+optional_value = release
+values =
+	rc
+	release
 
 [bumpversion:file:setup.py]
 search = version="{current_version}"

.github/workflows/compatibility_tests.yml

@@ -67,11 +67,6 @@ jobs:
         uses: actions/checkout@v2
         with:
           path: cellxgene
-      - name: Checkout tools repo
-        uses: actions/checkout@v2
-        with:
-          repository: theislab/anndata
-          path: anndata
       - name: Install dependencies
         run: |
           cd cellxgene
@@ -82,7 +77,7 @@ jobs:
           # 2. install cellxgene
           pip install --upgrade cellxgene
           # 3. install anndata
-          cd ../anndata && pip install -e .
+          pip install git+https://github.com/theislab/anndata
       - name: Tests
         run: cd cellxgene && make unit-test ${{ matrix.test-suite }}
 
@@ -102,17 +97,11 @@ jobs:
         uses: actions/checkout@v2
         with:
           path: cellxgene
-      - name: Checkout tools repo
-        uses: actions/checkout@v2
-        with:
-          repository: theislab/anndata
-          path: anndata
       - name: Install dependencies
         run: |
           cd cellxgene
           sed -i -E 's/^anndata[>=]=[0-9]+.[0-9]+.[0-9]+$/anndata/g' server/requirements.txt
           make pydist install-dist dev-env
-          cd ../anndata
+          pip install git+https://github.com/theislab/anndata
-          pip install -e .
       - name: Tests
         run: cd cellxgene && make unit-test ${{ matrix.test-suite }}

Makefile

@@ -99,22 +99,32 @@ pydist: build
 
 # RELEASE HELPERS
 
-# create new version to commit to main
+# Create new version to commit to main
-.PHONY: release-stage-1
+.PHONY: create-release-candidate
-release-stage-1: dev-env bump clean-lite gen-package-lock
+create-release-candidate: dev-env bump-version clean-lite gen-package-lock
 	@echo "Version bumped part:$(PART) and client built. Ready to commit and push"
 
-# build dist and release to dev pypi
+# Bump the release candidate version if needed (i.e. the previous release candidate had errors).
-.PHONY: release-stage-2
+.PHONY: recreate-release-candidate
-release-stage-2: dev-env pydist twine
+recreate-release-candidate: dev-env bump-release-candidate clean-lite gen-package-lock
+	@echo "Version bumped part:$(PART) and client built. Ready to commit and push"
+
+# Build dist and release to Test PyPI
+.PHONY: release-candidate-to-test-pypi
+release-candidate-to-test-pypi: dev-env pydist twine
 	@echo "Dist built and uploaded to test.pypi.org"
 	@echo "Test the install:"
 	@echo "    make install-release-test"
-	@echo "Then upload to Pypi prod:"
-	@echo "    make twine-prod"
 
-.PHONY: release-stage-final
+# Build final dist (gets rid of the rc tag) and release final candidate to TestPyPI
-release-stage-final: twine-prod
+.PHONY: release-final-to-test-pypi
+release-final-to-test-pypi: dev-env bump-release clean-lite gen-package-lock pydist twine
+	@echo "Final release dist built and uploaded to test.pypi.org"
+	@echo "Test the install:"
+	@echo "    make install-release-test"
+
+.PHONY: release-final
+release-final: twine-prod
 	@echo "Release uploaded to pypi.org"
 
 # DANGER: releases directly to prod
@@ -136,11 +146,22 @@ dev-env-client:
 dev-env-server:
 	pip install -r server/requirements-dev.txt
 
-# give PART=[major, minor, part] as param to make bump
+# Set PART=[major, minor, patch] as param to make bump.
-.PHONY: bump
+# This will create a release candidate. (i.e. 0.16.1 -> 0.16.2-rc.0 for a patch bump)
-bump:
+.PHONY: bump-version
+bump-version:
 	bumpversion --config-file .bumpversion.cfg $(PART)
 
+# Increments the release candidate version (i.e. 0.16.2-rc.1 -> 0.16.2-rc.2)
+.PHONY: bump-release-candidate
+bump-release-candidate:
+	bumpversion --config-file .bumpversion.cfg prerelversion --allow-dirty
+
+# Finalizes the release candidate by removing the release candidate tag (i.e. 0.16.2-rc.2 -> 0.16.2).
+.PHONY: bump-release
+bump-release:
+	bumpversion --config-file .bumpversion.cfg prerel --allow-dirty
+
 .PHONY: twine
 twine:
 	twine upload --repository-url https://test.pypi.org/legacy/ dist/*

README.md

@@ -79,23 +79,7 @@ This project was started with the sole goal of empowering the scientific communi
 
 If you believe you have found a security issue, we would appreciate notification. Please send email to <security@chanzuckerberg.com>.
 
-# About
+# Inspiration
-
-### Core team
-
-The current core team:
-
-- Colin Megill, frontend & product design
-- Bruce Martin, software engineer
-- Sidney Bell, computational biologist
-- Lia Prins, designer
-- Severiano Badajoz, software engineer
-
-We would also like to gratefully acknowledge contributions from past core team members:
-
-- Charlotte Weaver, software engineer
-
-### Inspiration
 
 We've been heavily inspired by several other related single-cell visualization projects, including the [UCSC Cell Browswer](http://cells.ucsc.edu/), [Cytoscape](http://www.cytoscape.org/), [Xena](https://xena.ucsc.edu/), [ASAP](https://asap.epfl.ch/), [Gene Pattern](http://genepattern-notebook.org/), and many others. We hope to explore collaborations where useful as this community works together on improving interactive visualization for single-cell data.
 

client/__tests__/e2e/__snapshots__/e2e.test.js.snap

@@ -1,5 +1,5 @@
 // Jest Snapshot v1, https://goo.gl/fbAQLP
 
-exports[`did launch page launched 1`] = `"<span style=\\"width: 185px; display: flex; overflow: hidden; justify-content: flex-start;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">pbm</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">c3k</span><span style=\\"position: absolute; right: 0px; color: inherit;\\">c3k</span></span></span>"`;
+exports[`did launch page launched 1`] = `"<span style=\\"max-width: 155px; display: flex; overflow: hidden; justify-content: flex-start;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">pbm</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">c3k</span><span style=\\"position: absolute; right: 0px; color: inherit;\\">c3k</span></span></span>"`;
 
 exports[`metadata loads categories and values from dataset appear 1`] = `"<div style=\\"display: flex; justify-content: space-between; align-items: baseline;\\"><div style=\\"display: flex; justify-content: flex-start; align-items: flex-start;\\"><label class=\\"bp3-control bp3-checkbox\\" for=\\"category-select-louvain\\"><input id=\\"category-select-louvain\\" data-testclass=\\"category-select\\" data-testid=\\"louvain:category-select\\" type=\\"checkbox\\" checked=\\"\\"><span class=\\"bp3-control-indicator\\"></span></label><span role=\\"menuitem\\" tabindex=\\"0\\" data-testclass=\\"category-expand\\" data-testid=\\"louvain:category-expand\\" style=\\"cursor: pointer;\\"><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\"><span data-testid=\\"louvain:category-label\\" aria-label=\\"louvain\\" class=\\"\\" tabindex=\\"0\\" style=\\"max-width: 265px;\\"><span style=\\"max-width: 265px; display: flex; overflow: hidden; justify-content: flex-start;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">lou</span><span style=\\"position: relative; overflow: hidden; white-space: nowrap;\\"><span style=\\"color: transparent;\\">vain</span><span style=\\"position: absolute; right: 0px; color: inherit;\\">vain</span></span></span></span></span></span><svg stroke=\\"currentColor\\" fill=\\"currentColor\\" stroke-width=\\"0\\" viewBox=\\"0 0 320 512\\" data-testclass=\\"category-expand-is-not-expanded\\" height=\\"1em\\" width=\\"1em\\" xmlns=\\"http://www.w3.org/2000/svg\\" style=\\"font-size: 10px; margin-left: 5px;\\"><path d=\\"M285.476 272.971L91.132 467.314c-9.373 9.373-24.569 9.373-33.941 0l-22.667-22.667c-9.357-9.357-9.375-24.522-.04-33.901L188.505 256 34.484 101.255c-9.335-9.379-9.317-24.544.04-33.901l22.667-22.667c9.373-9.373 24.569-9.373 33.941 0L285.475 239.03c9.373 9.372 9.373 24.568.001 33.941z\\"></path></svg></span></div><div><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\"><a role=\\"button\\" data-testclass=\\"colorby\\" data-testid=\\"colorby-louvain\\" class=\\"bp3-button\\" tabindex=\\"0\\"><span icon=\\"tint\\" class=\\"bp3-icon bp3-icon-tint\\"><svg data-icon=\\"tint\\" width=\\"16\\" height=\\"16\\" viewBox=\\"0 0 16 16\\"><desc>tint</desc><path d=\\"M7.88 1s-4.9 6.28-4.9 8.9c.01 2.82 2.34 5.1 4.99 5.1 2.65-.01 5.03-2.3 5.03-5.13C12.99 7.17 7.88 1 7.88 1z\\" fill-rule=\\"evenodd\\"></path></svg></span></a></span></span></div></div><div style=\\"margin-left: 26px;\\"></div><div></div>"`;

client/configuration/eslint/eslint.js

@@ -39,6 +39,7 @@ module.exports = {
     },
   },
   rules: {
+    "react/jsx-no-target-blank": "off",
     "eslint-comments/require-description": ["error"],
     "no-magic-numbers": "off",
     "no-nested-ternary": "off",

client/src/actions/embedding.js

@@ -5,14 +5,14 @@ action creators related to embeddings choice
 import { AnnoMatrixObsCrossfilter } from "../annoMatrix";
 import { _setEmbeddingSubset } from "../util/stateManager/viewStackHelpers";
 
-export async function _switchEmbedding(prevAnnoMatrix, newEmbeddingName) {
+export async function _switchEmbedding(prevAnnoMatrix, prevCrossfilter, newEmbeddingName) {
   /*
   DRY helper used by this and reembedding action creators
   */
   const base = prevAnnoMatrix.base();
   const embeddingDf = await base.fetch("emb", newEmbeddingName);
   const annoMatrix = _setEmbeddingSubset(prevAnnoMatrix, embeddingDf);
-  const obsCrossfilter = await new AnnoMatrixObsCrossfilter(annoMatrix).select(
+  const obsCrossfilter = await new AnnoMatrixObsCrossfilter(annoMatrix, prevCrossfilter.obsCrossfilter).select(
     "emb",
     newEmbeddingName,
     {
@@ -30,9 +30,10 @@ export const layoutChoiceAction = (newLayoutChoice) => async (
   On layout choice, make sure we have selected all on the previous layout, AND the new
   layout.
   */
-  const { annoMatrix: prevAnnoMatrix } = getState();
+  const { annoMatrix: prevAnnoMatrix, obsCrossfilter: prevCrossfilter } = getState();
   const [annoMatrix, obsCrossfilter] = await _switchEmbedding(
     prevAnnoMatrix,
+    prevCrossfilter,
     newLayoutChoice
   );
   dispatch({

client/src/actions/index.js

@@ -43,7 +43,7 @@ async function configFetch(dispatch) {
 
 async function userInfoFetch(dispatch) {
   return fetchJson("userinfo").then((response) => {
-    const userinfo = { ...response.userinfo };
+    const { userinfo } = response || {};
     dispatch({
       type: "userinfo load complete",
       userinfo,

client/src/components/categorical/category/index.js

@@ -491,19 +491,7 @@ const CategoryRender = React.memo(
       /*
       Entire category has a single value, special case.
       */
-      const theOneValue = categorySummary.categoryValues[0];
+      return null;
-      return (
-        <div style={{ marginBottom: 10, marginTop: 4 }}>
-          <Truncate>
-            <span style={{ maxWidth: 150, fontWeight: 700 }}>
-              {metadataField}
-            </span>
-          </Truncate>
-          <Truncate>
-            <span style={{ maxWidth: 150 }}>{`: ${theOneValue}`}</span>
-          </Truncate>
-        </div>
-      );
     }
 
     /*

client/src/components/categorical/value/index.js

@@ -81,7 +81,7 @@ class CategoryValue extends React.Component {
   get shouldRenderStackedBarOrHistogram() {
     const { colorAccessor, isColorBy, annotations } = this.props;
 
-    return colorAccessor && !isColorBy && !annotations.isEditingLabelName;
+    return !!colorAccessor && !isColorBy && !annotations.isEditingLabelName;
   }
 
   handleDeleteValue = () => {
@@ -439,7 +439,9 @@ class CategoryValue extends React.Component {
 
     if (
       !this.shouldRenderStackedBarOrHistogram ||
-      !AnnotationsHelpers.isContinuousAnnotation(schema, colorAccessor)
+      // This function returns true on categorical annotations(when stacked bar should not render),
+      //  in cases where the colorAccessor is a gene this function will return undefined since genes do not live on the schema
+      AnnotationsHelpers.isCategoricalAnnotation(schema, colorAccessor) === true
     ) {
       return null;
     }

client/src/components/infoDrawer/infoDrawer.js

@@ -0,0 +1,118 @@
+import React, { PureComponent } from "react";
+import { connect, shallowEqual } from "react-redux";
+import { Drawer } from "@blueprintjs/core";
+import Async from "react-async";
+
+import InfoFormat from "./infoFormat";
+import {
+  selectableCategoryNames,
+  createCategorySummaryFromDfCol,
+} from "../../util/stateManager/controlsHelpers";
+
+@connect((state) => {
+  return {
+    annoMatrix: state.annoMatrix,
+    schema: state.annoMatrix.schema,
+    datasetTitle: state.config?.displayNames?.dataset ?? "",
+    aboutURL: state.config?.links?.["about-dataset"],
+    isOpen: state.controls.datasetDrawer,
+    dataPortalProps: state.config?.["corpora_props"] ?? {},
+  };
+})
+class InfoDrawer extends PureComponent {
+  static watchAsync(props, prevProps) {
+    return !shallowEqual(props.watchProps, prevProps.watchProps);
+  }
+
+  fetchAsyncProps = async (props) => {
+    const { schema } = props.watchProps;
+    const { annoMatrix } = this.props;
+
+    const allCategoryNames = selectableCategoryNames(schema).sort();
+
+    const nonUserAnnoCategories = allCategoryNames.map((catName) => {
+      const isUserAnno = schema?.annotations?.obsByName[catName]?.writable;
+      if (!isUserAnno) return annoMatrix.fetch("obs", catName);
+      return null;
+    });
+    const singleValueCategories = (
+      await Promise.all(nonUserAnnoCategories)
+    ).reduce((acc, categoryData, i) => {
+      const catName = allCategoryNames[i];
+
+      const column = categoryData.icol(0);
+      const colSchema = schema.annotations.obsByName[catName];
+
+      const categorySummary = createCategorySummaryFromDfCol(column, colSchema);
+
+      const { numCategoryValues } = categorySummary;
+      //  Add to the array if the category has only one value
+      if (numCategoryValues === 1) {
+        acc.set(catName, categorySummary.allCategoryValues[0]);
+      }
+      return acc;
+    }, new Map());
+
+    return { singleValueCategories };
+  };
+
+  handleClose = () => {
+    const { dispatch } = this.props;
+
+    dispatch({ type: "toggle dataset drawer" });
+  };
+
+  render() {
+    const {
+      position,
+      aboutURL,
+      datasetTitle,
+      schema,
+      isOpen,
+      dataPortalProps,
+    } = this.props;
+
+    return (
+      <Drawer
+        title="Dataset Overview"
+        onClose={this.handleClose}
+        {...{ isOpen, position }}
+      >
+        <Async
+          watchFn={InfoDrawer.watchAsync}
+          promiseFn={this.fetchAsyncProps}
+          watchProps={{ schema }}
+        >
+          <Async.Pending>
+            <InfoFormat
+              skeleton
+              {...{ datasetTitle, aboutURL, dataPortalProps }}
+            />
+          </Async.Pending>
+          <Async.Rejected>
+            {(error) => {
+              console.error(error);
+              return <span>Failed to load info</span>;
+            }}
+          </Async.Rejected>
+          <Async.Fulfilled>
+            {(asyncProps) => {
+              const { singleValueCategories } = asyncProps;
+              return (
+                <InfoFormat
+                  {...{
+                    datasetTitle,
+                    aboutURL,
+                    singleValueCategories,
+                    dataPortalProps,
+                  }}
+                />
+              );
+            }}
+          </Async.Fulfilled>
+        </Async>
+      </Drawer>
+    );
+  }
+}
+export default InfoDrawer;

client/src/components/infoDrawer/infoFormat.js

@@ -0,0 +1,194 @@
+import { H3, H1, UL, Classes } from "@blueprintjs/core";
+import React from "react";
+
+const renderContributors = (contributors, affiliations, skeleton) => {
+  // eslint-disable-next-line no-constant-condition --  Temp removed contributor section to avoid publishing PII
+  if (!contributors || contributors.length === 0 || true) return null;
+  return (
+    <>
+      <H3 className={skeleton ? Classes.SKELETON : null}>Contributors</H3>
+      <p className={skeleton ? Classes.SKELETON : null}>
+        {contributors.map((contributor) => {
+          const { email, name, institution } = contributor;
+
+          return (
+            <span key={name}>
+              {name}
+              {email && `(${email})`}
+              <sup>{affiliations.indexOf(institution) + 1}</sup>
+            </span>
+          );
+        })}
+      </p>
+      {renderAffiliations(affiliations, skeleton)}
+    </>
+  );
+};
+
+// generates a list of unique institutions by order of appearance in contributors
+const buildAffiliations = (contributors = []) => {
+  const affiliations = [];
+  contributors.forEach((contributor) => {
+    const { institution } = contributor;
+    if (affiliations.indexOf(institution) === -1) {
+      affiliations.push(institution);
+    }
+  });
+  return affiliations;
+};
+
+const renderAffiliations = (affiliations, skeleton) => {
+  if (affiliations.length === 0) return null;
+  return (
+    <>
+      <H3 className={skeleton ? Classes.SKELETON : null}>Affiliations</H3>
+      <UL>
+        {affiliations.map((item, index) => (
+          <div key={item} className={skeleton ? Classes.SKELETON : null}>
+            <sup>{index + 1}</sup>
+            {"  "}
+            {item}
+          </div>
+        ))}
+      </UL>
+    </>
+  );
+};
+
+const renderDOILink = (type, doi, skeleton) => {
+  if (!doi) return null;
+  return (
+    <>
+      <H3 className={skeleton ? Classes.SKELETON : null}>{type}</H3>
+      <p className={skeleton ? Classes.SKELETON : null}>
+        <a href={doi} target="_blank" rel="noopener">
+          {doi}
+        </a>
+      </p>
+    </>
+  );
+};
+
+const renderOrganism = (organism, skeleton) => {
+  if (!organism) return null;
+  return (
+    <>
+      <H3 className={skeleton ? Classes.SKELETON : null}>Organism</H3>
+      <p className={skeleton ? Classes.SKELETON : null}>{organism}</p>
+    </>
+  );
+};
+
+// Render list of metadata attributes found in categorical field
+// Ignores categories with empty or null values
+const renderSingleValueCategories = (singleValueCategories, skeleton) => {
+  if (singleValueCategories.size === 0) return null;
+  return (
+    <>
+      <H3 className={skeleton ? Classes.SKELETON : null}>Dataset Metadata</H3>
+      <UL>
+        {Array.from(singleValueCategories).map((pair) => {
+          if (!pair[1] || pair[1] === "") return null;
+          return (
+            <li
+              className={skeleton ? Classes.SKELETON : null}
+              key={pair[0]}
+            >{`${pair[0]}: ${pair[1]}`}</li>
+          );
+        })}
+      </UL>
+    </>
+  );
+};
+
+// Renders any links found in the config where link_type is not "SUMMARY"
+// If there are no links in the config, render the aboutURL
+const renderLinks = (projectLinks, aboutURL, skeleton) => {
+  if (!projectLinks && !aboutURL) return null;
+  if (projectLinks)
+    return (
+      <>
+        <H3 className={skeleton ? Classes.SKELETON : null}>Project Links</H3>
+        <UL>
+          {projectLinks.map((link) => {
+            if (link.link_type === "SUMMARY") return null;
+            return (
+              <li
+                key={link.link_name}
+                className={skeleton ? Classes.SKELETON : null}
+              >
+                <a href={link.link_url} target="_blank" rel="noopener">
+                  {link.link_name}
+                </a>
+              </li>
+            );
+          })}
+        </UL>
+      </>
+    );
+
+  return (
+    <>
+      <H3 className={skeleton ? Classes.SKELETON : null}>More Info</H3>
+      <p>
+        <a
+          className={skeleton ? Classes.SKELETON : null}
+          href={aboutURL}
+          target="_blank"
+          rel="noopener"
+        >
+          {aboutURL}
+        </a>
+      </p>
+    </>
+  );
+};
+
+const NUM_CATEGORIES = 8;
+
+// Generates arbitrary placeholder array for singleValueCategories skeleton shape
+const singleValueCategoriesPlaceholder = Array.from(Array(NUM_CATEGORIES)).map(
+  (_, index) => {
+    return [index, index];
+  }
+);
+
+const InfoFormat = React.memo(
+  ({
+    datasetTitle,
+    singleValueCategories = new Map(singleValueCategoriesPlaceholder),
+    aboutURL = "thisisabouthtelengthofaurl",
+    dataPortalProps = {},
+    skeleton = false,
+  }) => {
+    if (dataPortalProps.corpora_schema_version === "1.0.0") {
+      dataPortalProps = {};
+    }
+    const {
+      title,
+      publication_doi: doi,
+      preprint_doi: preprintDOI,
+      organism,
+      contributors,
+      project_links: projectLinks,
+    } = dataPortalProps;
+
+    const affiliations = buildAffiliations(contributors);
+
+    return (
+      <div style={{ margin: 24, overflow: "auto" }}>
+        <H1 className={skeleton ? Classes.SKELETON : null}>
+          {title ?? datasetTitle}
+        </H1>
+        {renderContributors(contributors, affiliations, skeleton)}
+        {renderDOILink("DOI", doi, skeleton)}
+        {renderDOILink("Preprint DOI", preprintDOI, skeleton)}
+        {renderOrganism(organism, skeleton)}
+        {renderSingleValueCategories(singleValueCategories, skeleton)}
+        {renderLinks(projectLinks, aboutURL, skeleton)}
+      </div>
+    );
+  }
+);
+
+export default InfoFormat;

client/src/components/leftSidebar/topLeftLogoAndTitle.js

@@ -1,22 +1,27 @@
 // jshint esversion: 6
 import React from "react";
 import { connect } from "react-redux";
+import { Button } from "@blueprintjs/core";
+import { IconNames } from "@blueprintjs/icons";
+
 import * as globals from "../../globals";
 import Logo from "../framework/logo";
 import Truncate from "../util/truncate";
+import InfoDrawer from "../infoDrawer/infoDrawer";
 
-const DATASET_TITLE_WIDTH = 190;
 const DATASET_TITLE_FONT_SIZE = 14;
 
 @connect((state) => ({
   datasetTitle: state.config?.displayNames?.dataset ?? "",
-  aboutURL: state.config?.links?.["about-dataset"],
-  scatterplotXXaccessor: state.controls.scatterplotXXaccessor,
-  scatterplotYYaccessor: state.controls.scatterplotYYaccessor,
 }))
 class LeftSideBar extends React.Component {
+  handleClick = () => {
+    const { dispatch } = this.props;
+    dispatch({ type: "toggle dataset drawer" });
+  };
+
   render() {
-    const { datasetTitle, aboutURL } = this.props;
+    const { datasetTitle } = this.props;
 
     return (
       <div
@@ -53,39 +58,23 @@ class LeftSideBar extends React.Component {
           </span>
           gene
         </span>
-        <div
+        <Button
+          minimal
+          icon={IconNames.BOOK}
           style={{
             fontSize: DATASET_TITLE_FONT_SIZE,
-            position: "relative",
+            position: "absolute",
-            top: -6,
+            right: 10,
-            display: "inline-block",
-            width: DATASET_TITLE_WIDTH,
-            marginLeft: "7px",
-            height: "1.2em",
-            overflow: "hidden",
-            wordBreak: "break-all",
           }}
+          onClick={this.handleClick}
         >
-          {aboutURL ? (
+          <Truncate>
-            <Truncate>
+            <span style={{ maxWidth: 155 }} data-testid="header">
-              <a
+              {datasetTitle}
-                style={{ width: 185 }}
+            </span>
-                href={aboutURL}
+          </Truncate>
-                data-testid="header"
+        </Button>
-                target="_blank"
+        <InfoDrawer />
-                rel="noopener noreferrer"
-              >
-                {datasetTitle}
-              </a>
-            </Truncate>
-          ) : (
-            <Truncate>
-              <span style={{ width: 185 }} data-testid="header">
-                {datasetTitle}
-              </span>
-            </Truncate>
-          )}
-        </div>
       </div>
     );
   }

client/src/components/menubar/index.js

@@ -250,10 +250,7 @@ class MenuBar extends React.PureComponent {
       >
         <AuthButtons auth={auth} userinfo={userinfo} />
         <InformationMenu
-          libraryVersions={libraryVersions}
+          {...{ libraryVersions, aboutLink, tosURL, privacyURL, dispatch }}
-          aboutLink={aboutLink}
-          tosURL={tosURL}
-          privacyURL={privacyURL}
         />
         <UndoRedoReset
           dispatch={dispatch}

client/src/components/menubar/infoMenu.js

@@ -1,25 +1,25 @@
 // jshint esversion: 6
 import React from "react";
 import { Button, Popover, Menu, MenuItem, Position } from "@blueprintjs/core";
+import { IconNames } from "@blueprintjs/icons";
 import styles from "./menubar.css";
 
+const handleClick = (dispatch) => {
+  dispatch({ type: "toggle dataset drawer" });
+};
+
 const InformationMenu = React.memo((props) => {
-  const { libraryVersions, aboutLink, tosURL, privacyURL } = props;
+  const { libraryVersions, tosURL, privacyURL, dispatch } = props;
   return (
     <div className={`bp3-button-group ${styles.menubarButton}`}>
       <Popover
         content={
           <Menu>
-            {aboutLink ? (
+            <MenuItem
-              <MenuItem
+              onClick={() => handleClick(dispatch)}
-                href={aboutLink}
+              icon={IconNames.BOOK}
-                target="_blank"
+              text="Dataset Overview"
-                icon="document-open"
+            />
-                text="About this dataset"
-              />
-            ) : (
-              ""
-            )}
 
             <MenuItem
               href="https://chanzuckerberg.github.io/cellxgene/"

client/src/components/termsPrompt/index.js

@@ -84,7 +84,7 @@ class TermsPrompt extends React.PureComponent {
           }}
           href={tosURL}
           target="_blank"
-          rel="noopener noreferrer"
+          rel="noopener"
         >
           terms of service
         </a>
@@ -106,7 +106,7 @@ class TermsPrompt extends React.PureComponent {
           }}
           href={privacyURL}
           target="_blank"
-          rel="noopener noreferrer"
+          rel="noopener"
         >
           privacy policy
         </a>

client/src/reducers/controls.js

@@ -20,6 +20,8 @@ const Controls = (
     scatterplotXXaccessor: null, // just easier to read
     scatterplotYYaccessor: null,
     graphRenderCounter: 0 /* integer as <Component key={graphRenderCounter} - a change in key forces a remount */,
+
+    datasetDrawer: false,
   },
   action
 ) => {
@@ -162,6 +164,12 @@ const Controls = (
         scatterplotYYaccessor: null,
       };
 
+    /**************************
+          Dataset Drawer
+     **************************/
+    case "toggle dataset drawer":
+      return { ...state, datasetDrawer: !state.datasetDrawer };
+
     default:
       return state;
   }

dev_docs/release_process.md

@@ -1,4 +1,4 @@
-# cellxgene release process
+# cellxgene Release Process
 
 _This document defines the release process for cellxgene_
 
@@ -16,71 +16,65 @@ The release process should result in the following side-effects:
 
 Note all release tags pushed to GitHub MUST follow semantic versioning.
 
-## Recipe
+## Releasing a Major or Minor Version of cellxgene
 
-Follow these steps to create a release.
+Please scroll down the section below for how to release a patch version. Follow these steps to create a major or minor release.
 
 1.  Preparation:
     -   python3.6 environment, and a cellxgene clone
-    -   Define the release version number, using [semantic versioning](https://semver.org/),
+    -   Define the release version number, using [semantic versioning](https://semver.org/), and specifying all three digits (e.g., 0.3.0)
-        and specifying all three digits (eg, 0.3.0)
+    -   Write the release title and release notes and add to [release notes document](https://docs.google.com/document/d/1KnHwkYfhyWO5H8BDcMu7y3ogjvq5Yi4OwpmZ8DB6w0Y/edit)
-    -   Write the release title and release notes and add to
+2.  Create a release branch, eg, `release-version-0.16.0`
-        [release notes document](https://docs.google.com/document/d/1KnHwkYfhyWO5H8BDcMu7y3ogjvq5Yi4OwpmZ8DB6w0Y/edit)
+3.  In the release branch, run `make create-release-candidate PART=[major | minor | patch]` where you choose major/minor/patch depending on which part of the version is being bumped (e.g., `0.2.9` -> `0.3.0` is minor version bump). This will bump the version and create a release *candidate* version (i.e. `0.3.0-rc.0`).
-2.  Create a release branch, eg, `release-version`
+4.  Commit and push the new branch. This will trigger tests to ensure that your branch isn't broken.
-3.  In the release branch:
+5. Upload the release candidate to Test PyPI by running the command `make release-candidate-to-test-pypi`. (Make sure you are registered for PyPI and Test PyPI and you have write access to the cellxgene PyPI package for both).
-    -   Run `make release-stage-1 PART=[major | minor | patch]` where you choose major/minor/patch depending on which part of the version
+6. Verify the release candidate in a fresh virtual environment by running `make install-release-test` which installs the cellxgene build you just uploaded the Test PyPI.
-        is being bumped (eg, 0.2.9->0.3 is minor).
+7. If you find errors with the release candidate, run `make recreate-release-candidate` to increment the release candidate version (i.e. `0.3.0-rc.0` -> `0.3.0-rc.1`). Then go back to Steps 5 and 6 to re-upload and re-test the new release candidate.
-4.  Commit and push the new branch
+8. If everything looks good, push the release to Test PyPI without the release candidate tag by running the command `make release-final-to-test-pypi` (i.e. `0.3.0-rc.1` -> `0.3.0`).
-5.  Create a PR for the release.
+    - **NOTE:** Once you push the final release version to Test PyPI, you cannot ever re-upload the build again. If you need to make changes to the build, you will have to "burn" the version number and bump the part again and go back to step 1 with a brand new version number. For example, if you upload `0.3.0` to Test PyPI and realize there's a bug, you will have to create a new version `0.4.0` and there will be no `0.3.0` version of cellxgene. This is why testing the release candidate is very important.
-    -   [optional] As needed, conduct PR review.
+9.  Create a PR for the release and conduct a PR review.
-6.  Merge to the `main` branch
+10.  Merge to the `main` branch.
-7.  Publish to pypi by performing the following steps (assumes you that you have registered for pypi,
+11.  Publish to PyPI (prod) (assuming you that you have registered for PyPI, and that you have write access to the cellxgene pypi package) by running `make release-final`.
-    and that you have write access to the cellxgene pypi package):
+12. Test the installation in a fresh virtual environment by running `pip install --no-cache-dir cellxgene`.
-    -   Build the distribution and upload to test pypi `make release-stage-2`
+13.  Create Github release using the version number and release notes ([instructions](https://help.github.com/articles/creating-releases/)):
-    -   Test the test installation in a fresh virtual environment using `make install-release-test`
+     - Draft new release
-    -   Upload the package to real pypi using `make release-stage-final`
+     - Type version name matching release version number from (1)
-    -   Test the installation in a fresh virtual environment using `pip install cellxgene`
+     - Select `main` as release branch (ensure you merged the release PR)
-8.  Create Github release using the version number and release notes
+     - Type title `Release {version num}`
-    ([instructions](https://help.github.com/articles/creating-releases/)).
+     - [optional] Check pre-release if this release is not ready for production
-    -   Draft new release
+     - Publish Release
-    -   Type version name matching release version number from (1)
-    -   Select `main` as release branch (ensure you merged the release PR)
-    -   Type title `Release {version num}`
-    -   [optional] Check pre-release if this release is not ready for production
-    -   Publish Release
 
-The optional steps are for testing purposes, and are recommended
+The optional steps are for testing purposes, and are recommended for publishing any major releases, and any releases that significantly change the packaging (e.g. new bundled files, new dependencies, etc.)
-for publishing any major releases, and any releases that significantly
-change the packaging (e.g. new bundled files, new dependencies, etc.)
 
-### Point release (special case)
+### Releasing a Patch Version of cellxgene (special case)
 
-To make a bugfix release (a point release) when there are already other changes in `main` we need to do a modified version of our release process. The difference is that instead of using `main` we are going make our release branch off of the tag for the release we want to patch. We cherrypick the commits that we want to include in the patch. Then instead of merging to `main`, we create the release directly off of the branch.
+To make a bugfix release (a point release/patch release) when there are already other changes in `main` we need to do a modified version of our release process. The difference is that instead of using `main` we are going make our release branch off of the tag for the release we want to patch. We cherrypick the commits that we want to include in the patch. Then instead of merging to `main`, we create the release directly off of the branch.
 
 1.  (same as above) Preparation:
     -   python3.6 environment, and a cellxgene clone
-    -   Define the release version number, using [semantic versioning](https://semver.org/),
+    -   Define the release version number, using [semantic versioning](https://semver.org/), and specifying all three digits (e.g., 0.3.2) (for this you will update the last digit to represent a bugfix change).
-        and specifying all three digits (eg, 0.3.0) (for this you will update the last digit to represent a bugfix change)
+    -   Write the release title and release notes and add to [release notes document](https://docs.google.com/document/d/1KnHwkYfhyWO5H8BDcMu7y3ogjvq5Yi4OwpmZ8DB6w0Y/edit)
-    -   Write the release title and release notes and add to
-        [release notes document](https://docs.google.com/document/d/1KnHwkYfhyWO5H8BDcMu7y3ogjvq5Yi4OwpmZ8DB6w0Y/edit)
 2.  Create a release branch off of the tag for the release you want to update.
-    -   Checkout the tag for the release you want to fix. ex. if we are fixing 0.9.0: `git checkout 0.9.0`
+    -   Checkout the tag for the release you want to fix. For example, if we are fixing 0.9.0: `git checkout 0.9.0`.
-    -   Create a branch from that tag. `git branch release-0.9.1`
+    -   Create a branch from that tag. `git branch release-version-0.9.1`
 3.  Cherrypick the commits that you want included in this patch.
-    -   Test that the cherrypicked commits landed and fixed the issue
+    -   Test that the cherrypicked commits landed and fixed the issue locally.
-    -   We WILL NOT merge this branch back into `main`, these commits should already exist in `main`.
+    -   We **WILL NOT** merge this branch back into `main` as these commits should already exist in `main`.
-4.  In the release branch:
+4.  In the release branch (i.e. `release-version-0.9.1`), run `make create-release-candidate PART=patch` to bump the patch version and create the first release candidate (i.e. `0.9.1-rc.0`).
-    -   Run `make release-stage-1 PART=patch`.
+5. Run `make release-candidate-to-test-pypi` to upload the release candidate to Test PyPI.
-5.  Commit and push the new branch. DO NOT MAKE A PR OR MERGE TO `main`.
+6. Verify the release candidate in a fresh virtual environment by running `make install-release-test` which installs the cellxgene build you just uploaded the Test PyPI.
-    -   wait for release to pass the tests
+7. If you find errors with the release candidate, run `make recreate-release-candidate` to increment the release candidate version (i.e. `0.9.1-rc.0` -> `0.9.1-rc.1`). Then go back to Steps 5 and 6 to re-upload and re-test the new release candidate.
-6.  Publish to pypi by performing the following steps (assumes you that you have registered for pypi,
+8. If everything looks good, push the final version of the release to Test PyPI without the release candidate tag by running the command `make release-final-to-test-pypi` (i.e. `0.9.1-rc.1` -> `0.9.1`).
-    and that you have write access to the cellxgene pypi package): - Build the distribution and upload to test pypi `make release-stage-2` - Test the test installation in a fresh virtual environment using `make install-release-test` - Upload the package to real pypi using `make release-stage-final` - Test the installation in a fresh virtual environment using
+    - **NOTE:** Once you push the final release version to Test PyPI, you cannot ever re-upload the build again. If you need to make changes to the build, you will have to "burn" the version number and bump the part again and go back to step 1 with a brand new version number. For example, if you upload `0.9.1` to Test PyPI and realize there's a bug, you will have to create a new version `0.9.2` and there will be no `0.9.1` version of cellxgene. This is why testing the release candidate is very important.
-    `pip install --no-cache-dir cellxgene`
+9.  Commit and push the new branch. DO NOT MAKE A PR OR MERGE TO `main`.
-7.  Create Github release using the version number and release notes
+    -   Wait for release to pass the tests.
+10.  Publish to PyPI (prod) (assuming you that you have registered for PyPI, and that you have write access to the cellxgene pypi package) by running `make release-final`.
+11. Test the installation in a fresh virtual environment by running `pip install --no-cache-dir cellxgene`.
+12.  Create Github release using the version number and release notes
     ([instructions](https://help.github.com/articles/creating-releases/)).
     -   Draft new release
     -   Type version name matching release version number from (1)
-    -   _Different than above_ Select the release-branch you pushed at step 5 as release branch
+    -   [**_Different than above_**] Select the release-branch you pushed at step 5 as release branch
     -   Type title `Release {version num}`
     -   [optional] Check pre-release if this release is not ready for production
     -   Publish Release
@@ -93,36 +87,52 @@ _PyPi doesn't allow you to reupload a release with the same version number_
 If you accidentally burned a release number you want to use on prod, you have a few options:
 
 1. OPTION 1: Create distribution `make pydist`; test release locally `pip install dist/<release tarball>`;
-   then upload to prod `make release-stage-final`.
+   then upload to prod `make release-final`.
 2. OPTION 2: (DANGER) release directly to prod: `make release-directly-to-prod`.
-3. OPTION 3: If the release was burned on prod as well run from Step 3 again with option
+3. OPTION 3: If the release was burned on prod as well run from Step 3 again with option PART=patch until you get to an unburned version.
-   PART=patch until you get to an unburned version.
 
 ### The release doesn't install or fails your tests when you install it
 
 Delete it from pypi - Go to pypi.org -> sign in -> go to the cellxgene package -> click manage -> then in the options drop down click delete -> follow the instructions. You will not be able to use that release number again. If it is a minor bug and not a major regression, you can just release a patch.
 
-### If you need to run stage final on a different computer than stage 2
+### If you need to run the final upload to PyPI (prod) on a different computer than where you ran the command to upload to Test PyPI.
 
-If you run stage final without running stage 2 first, the dist will not have been build on the computer running stage final. The solution is to run `make release-directly-to-prod`. This both builds the distribution files and then releases directly to prod pypi.org.
+If you run `make release-final` without running `make release-final-to-test-pypi` first, the dist will not have been build on the computer running the final PyPI push. The solution is to run `make release-directly-to-prod`. This both builds the distribution files and then releases directly to prod pypi.org.
 
-## Stage Details
+## Command Details
 
-### Stage 1 - `make release-stage-1`
+### Initial creation stage - `make create-release-candidate PART=[major | minor | patch]`
 
     1. Pip installs requirements-dev
-    2. Bumps version by [PART]
+    2. Bumps version by [PART] and creates the first release candidate.
     3. Deletes build directory, client/build, dist and cellxgene.egg-info
     4. Creates the package-lock.json
 
-### Stage 2 - `make release-stage-2`
+### Test PyPI upload stage - `make release-candidate-to-test-pypi`
 
     1. Pip installs requirements-dev
     2. Builds client and server
     3. Creates distribution release (sdist)
     4. Uploads to test.pypi.org
     
-### Stage final - `make release-stage-final`
+### Recreating release candidate stage(s) - `make recreate-release-candidate`
+
+    1. Pip installs requirements-dev
+    2. Bumps release candidate version number.
+    3. Deletes build directory, client/build, dist and cellxgene.egg-info
+    4. Creates the package-lock.json
+
+### Penultimate stage, final release to Test PyPI - `make release-final-to-test-pypi`
+    1. Pip installs requirements-dev
+    2. Removes release candidate tag from the version number.
+    3. Deletes build directory, client/build, dist and cellxgene.egg-info
+    4. Creates the package-lock.json
+    5. Pip installs requirements-dev
+    6. Builds client and server
+    7. Creates distribution release (sdist)
+    8. Uploads to test.pypi.org
+
+### Final stage - `make release-final`
 
     ** Does not build distribution **
     1. Uploads to pypi.org

server/app/app.py

@@ -2,6 +2,9 @@ import datetime
 import logging
 from functools import wraps
 from http import HTTPStatus
+from urllib.parse import urlparse
+import hashlib
+import os
 
 from flask import Flask, redirect, current_app, make_response, render_template, abort, Blueprint, request, \
     send_from_directory
@@ -84,10 +87,12 @@ def dataset_index(url_dataroot=None, dataset=None):
         cache_manager = current_app.matrix_data_cache_manager
         with cache_manager.data_adaptor(url_dataroot, location, app_config) as data_adaptor:
             data_adaptor.set_uri_path(f"{url_dataroot}/{dataset}")
-            dataset_title = app_config.get_title(data_adaptor)
+            args = {
-            return render_template(
+                "SCRIPTS" : scripts,
-                "index.html", datasetTitle=dataset_title, SCRIPTS=scripts, INLINE_SCRIPTS=inline_scripts
+                "INLINE_SCRIPTS" : inline_scripts
-            )
+            }
+            return render_template("index.html", **args)
+
     except DatasetAccessError as e:
         return common_rest.abort_and_log(
             e.status_code, f"Invalid dataset {dataset}: {e.message}", loglevel=logging.INFO, include_exc_info=True
@@ -179,9 +184,9 @@ def dataroot_test_index():
             data += f"<p>Logged in as {auth.get_user_id()} / {auth.get_user_name()} / {auth.get_user_email()}</p>"
         if auth.requires_client_login():
             if server_config.auth.is_user_authenticated():
-                data += "<p><a href='/logout'>Logout</a></p>"
+                data += f"<p><a href='{auth.get_logout_url(None)}'>Logout</a></p>"
             else:
-                data += "<p><a href='/login'>Login</a></p>"
+                data += f"<p><a href='{auth.get_login_url(None)}'>Login</a></p>"
 
     datasets = []
     for dataroot_dict in server_config.multi_dataset__dataroot.values():
@@ -228,7 +233,8 @@ class DatasetResource(Resource):
 
 
 class SchemaAPI(DatasetResource):
-    @cache_control(public=True, max_age=ONE_WEEK)
+    # TODO @mdunitz separate dataset schema and user schema
+    @cache_control(no_store=True)
     @rest_get_data_adaptor
     def get(self, data_adaptor):
         return common_rest.schema_get(data_adaptor)
@@ -329,6 +335,25 @@ def get_api_resources(bp_api, url_dataroot=None):
     return api
 
 
+def handle_api_base_url(app, app_config):
+    """If an api_base_url is provided, then an inline script is generated to
+    handle the new API prefix"""
+    api_base_url = app_config.server_config.get_api_base_url()
+    if not api_base_url:
+        return
+
+    sha256 = hashlib.sha256(api_base_url.encode()).hexdigest()
+    script_name = f"api_base_url-{sha256}.js"
+    script_path = os.path.join(app.root_path, "../common/web/templates", script_name)
+    with open(script_path, "w") as fout:
+        fout.write("window.CELLXGENE.API.prefix = `" + api_base_url + "${location.pathname}api/`;\n")
+
+    dataset_configs = [app_config.default_dataset_config] + list(app_config.dataroot_config.values())
+    for dataset_config in dataset_configs:
+        inline_scripts = dataset_config.app__inline_scripts
+        inline_scripts.append(script_name)
+
+
 class Server:
     @staticmethod
     def _before_adding_routes(app, app_config):
@@ -337,6 +362,7 @@ class Server:
 
     def __init__(self, app_config):
         self.app = Flask(__name__, static_folder=None)
+        handle_api_base_url(self.app, app_config)
         self._before_adding_routes(self.app, app_config)
         self.app.json_encoder = Float32JSONEncoder
         server_config = app_config.server_config
@@ -353,6 +379,12 @@ class Server:
         self.app.register_blueprint(webbp)
 
         api_version = "/api/v0.2"
+        api_base_url = server_config.get_api_base_url()
+        api_path = "/"
+        if api_base_url:
+            parse = urlparse(api_base_url)
+            api_path = parse.path
+
         if app_config.is_multi_dataset():
             # NOTE:  These routes only allow the dataset to be in the directory
             # of the dataroot, and not a subdirectory.  We may want to change
@@ -360,7 +392,8 @@ class Server:
             for dataroot_dict in server_config.multi_dataset__dataroot.values():
                 url_dataroot = dataroot_dict["base_url"]
                 bp_api = Blueprint(
-                    f"api_dataset_{url_dataroot}", __name__, url_prefix=f"/{url_dataroot}/<dataset>" + api_version
+                    f"api_dataset_{url_dataroot}", __name__,
+                    url_prefix=f"{api_path}/{url_dataroot}/<dataset>" + api_version
                 )
                 resources = get_api_resources(bp_api, url_dataroot)
                 self.app.register_blueprint(resources.blueprint)
@@ -378,7 +411,7 @@ class Server:
                 )
 
         else:
-            bp_api = Blueprint("api", __name__, url_prefix=api_version)
+            bp_api = Blueprint("api", __name__, url_prefix=f"{api_path}{api_version}")
             resources = get_api_resources(bp_api)
             self.app.register_blueprint(resources.blueprint)
             self.app.add_url_rule(

server/auth/auth_oauth.py

@@ -1,7 +1,7 @@
 from flask import session, request, redirect, current_app, after_this_request, has_request_context, g
 from server.auth.auth import AuthTypeClientBase, AuthTypeFactory
 from server.common.errors import AuthenticationError, ConfigurationError
-from urllib.parse import urlencode
+from urllib.parse import urlencode, urlparse
 import json
 import requests
 import base64
@@ -29,7 +29,9 @@ class Tokens:
         self.id_token = id_token
         self.refresh_token = refresh_token
         self.expires_at = expires_at
-        if not (access_token and id_token and refresh_token and expires_at):
+
+        # expires_at may be None after a token refresh, and so it is not checked here
+        if not (access_token and id_token and refresh_token):
             raise KeyError(str(self.__dict__))
 
 
@@ -43,13 +45,20 @@ class AuthTypeOAuth(AuthTypeClientBase):
         if missingimport:
             raise ConfigurationError(f"oauth requires these modules: {', '.join(missingimport)}")
         self.algorithms = ["RS256"]
-        self.api_base_url = server_config.authentication__params_oauth__api_base_url
+        self.oauth_api_base_url = server_config.authentication__params_oauth__oauth_api_base_url
         self.client_id = server_config.authentication__params_oauth__client_id
         self.client_secret = server_config.authentication__params_oauth__client_secret
-        self.callback_base_url = server_config.authentication__params_oauth__callback_base_url
         self.session_cookie = server_config.authentication__params_oauth__session_cookie
         self.cookie_params = server_config.authentication__params_oauth__cookie
+        self.jwt_decode_options = server_config.authentication__params_oauth__jwt_decode_options
+
         self._validate_cookie_params()
+        self._validate_jwt_decode_options()
+
+        self.api_base_url = server_config.get_api_base_url()
+        self.web_base_url = server_config.get_web_base_url()
+        if self.api_base_url is None:
+            raise ConfigurationError("oauth requires the app__api_base_url to be set")
 
         # set the audience
         self.audience = self.client_id
@@ -58,11 +67,13 @@ class AuthTypeOAuth(AuthTypeClientBase):
         # The JSON Web Key Set (JWKS) is a set of keys which contains the public keys used to verify
         # any JSON Web Token (JWT) issued by the authorization server and signed using the RS256
         try:
-            jwksloc = f"{self.api_base_url}/.well-known/jwks.json"
+            jwksloc = f"{self.oauth_api_base_url}/.well-known/jwks.json"
             jwksurl = requests.get(jwksloc)
             self.jwks = jwksurl.json()
         except Exception:
-            raise ConfigurationError(f"error in oauth, api_url_base: {self.api_base_url}, cannot access {jwksloc}")
+            raise ConfigurationError(
+                f"error in oauth, api_url_base: {self.oauth_api_base_url}, cannot access {jwksloc}"
+            )
 
     def _validate_cookie_params(self):
         """check the cookie_params, and raise a ConfigurationError if there is something wrong"""
@@ -79,6 +90,20 @@ class AuthTypeOAuth(AuthTypeClientBase):
         if "key" not in keys:
             raise ConfigurationError("must have a key (name) in the cookie params")
 
+    def _validate_jwt_decode_options(self):
+        """check the jwt_decode_options, and raise a ConfigurationError if there is something wrong"""
+        if self.jwt_decode_options is None:
+            self.jwt_decode_options = {}
+            return
+
+        valid_keys = {
+            "verify_signature", "verify_aud", "verify_iat", "verify_exp", "verify_nbf", "verify_iss",
+            "verify_sub", "verify_jti", "verify_at_hash", "leeway"}
+        keys = set(self.jwt_decode_options.keys())
+        unknown = keys - valid_keys
+        if unknown:
+            raise ConfigurationError(f"unexpected key in jwt_decode_options: {', '.join(unknown)}")
+
     def is_valid_authentication_type(self):
         return True
 
@@ -86,27 +111,22 @@ class AuthTypeOAuth(AuthTypeClientBase):
         return True
 
     def add_url_rules(self, app):
-        app.add_url_rule("/login", "login", self.login, methods=["GET"])
+        parse = urlparse(self.api_base_url)
-        app.add_url_rule("/logout", "logout", self.logout, methods=["GET"])
+        app.add_url_rule(f"{parse.path}/login", "login", self.login, methods=["GET"])
-        app.add_url_rule("/oauth2/callback", "callback", self.callback, methods=["GET"])
+        app.add_url_rule(f"{parse.path}/logout", "logout", self.logout, methods=["GET"])
+        app.add_url_rule(f"{parse.path}/oauth2/callback", "callback", self.callback, methods=["GET"])
 
     def complete_setup(self, flask_app):
         self.oauth = OAuth(flask_app)
-        if self.callback_base_url is None:
-            # In this case, assume the server is running on the same host as the client,
-            # and the oauth provider has been configured
-            # with a callback that understands a localhost callback (e.g. A http://localhost:5005).
-            server_config = flask_app.app_config.server_config
-            self.callback_base_url = f"http://{server_config.app__host}:{server_config.app__port}"
 
         self.client = self.oauth.register(
             "auth0",
             client_id=self.client_id,
             client_secret=self.client_secret,
-            api_base_url=self.api_base_url,
+            api_base_url=self.oauth_api_base_url,
-            refresh_token_url=f"{self.api_base_url}/oauth/token",
+            refresh_token_url=f"{self.oauth_api_base_url}/oauth/token",
-            access_token_url=f"{self.api_base_url}/oauth/token",
+            access_token_url=f"{self.oauth_api_base_url}/oauth/token",
-            authorize_url=f"{self.api_base_url}/authorize",
+            authorize_url=f"{self.oauth_api_base_url}/authorize",
             client_kwargs={"scope": "openid profile email offline_access"},
         )
 
@@ -136,9 +156,9 @@ class AuthTypeOAuth(AuthTypeClientBase):
         response.cache_control.update(dict(public=True, max_age=0, no_store=True, no_cache=True, must_revalidate=True))
 
     def login(self):
-        callbackurl = f"{self.callback_base_url}/oauth2/callback"
+        callbackurl = f"{self.api_base_url}/oauth2/callback"
         return_path = request.args.get("dataset", "")
-        return_to = f"{self.callback_base_url}/{return_path}"
+        return_to = f"{self.web_base_url}/{return_path}/"
         # save the return path in the session cookie, accessed in the callback function
         session["oauth_callback_redirect"] = return_to
         response = self.client.authorize_redirect(redirect_uri=callbackurl)
@@ -147,7 +167,7 @@ class AuthTypeOAuth(AuthTypeClientBase):
 
     def logout(self):
         self.remove_tokens()
-        params = {"returnTo": self.callback_base_url, "client_id": self.client_id}
+        params = {"returnTo": self.web_base_url, "client_id": self.client_id}
         response = redirect(self.client.api_base_url + "/v2/logout?" + urlencode(params))
         self.update_response(response)
         return response
@@ -226,14 +246,14 @@ class AuthTypeOAuth(AuthTypeClientBase):
 
     def get_login_url(self, data_adaptor):
         """Return the url for the login route"""
-        if current_app.app_config.is_multi_dataset():
+        if data_adaptor and current_app.app_config.is_multi_dataset():
-            return f"/login?dataset={data_adaptor.uri_path}/"
+            return f"{self.api_base_url}/login?dataset={data_adaptor.uri_path}/"
         else:
-            return "/login"
+            return f"{self.api_base_url}/login"
 
     def get_logout_url(self, data_adaptor):
         """Return the url for the logout route"""
-        return "/logout"
+        return f"{self.api_base_url}/logout"
 
     def check_jwt_payload(self, id_token):
         try:
@@ -252,18 +272,14 @@ class AuthTypeOAuth(AuthTypeClientBase):
                     "e": key.get("e"),
                 }
         if rsa_key:
-            options = {}
-            if not rsa_key["n"] or not rsa_key["e"]:
-                # this is a mock auth server, do not validate
-                options = {"verify_signature": False, "verify_iss": False}
             try:
                 payload = jwt.decode(
                     id_token,
                     rsa_key,
                     algorithms=self.algorithms,
                     audience=self.audience,
-                    issuer=self.api_base_url + "/",
+                    issuer=self.oauth_api_base_url + "/",
-                    options=options,
+                    options=self.jwt_decode_options,
                 )
                 return payload
 
@@ -319,7 +335,7 @@ class AuthTypeOAuth(AuthTypeClientBase):
             "client_secret": self.client_secret,
         }
         headers = {"content-type": "application/x-www-form-urlencoded"}
-        request = requests.post(f"{self.api_base_url}/oauth/token", urlencode(params), headers=headers)
+        request = requests.post(f"{self.oauth_api_base_url}/oauth/token", urlencode(params), headers=headers)
         if request.status_code != 200:
             # unable to refresh the token, log the user out
             self.remove_tokens()

server/cli/upgrade.py

@@ -1,17 +1,16 @@
-import click
 import re
-import requests
 
+import click
+import requests
 from requests.exceptions import ConnectionError
+
 from .. import __version__
 
 # Official SemVer regex: https://semver.org/
 SEMVER_FORMAT = re.compile(
-    r"^(?P<major>0|[1-9]\d*)\.(?P<minor>0|[1-9]\d*)\.(?P<patch>0|[1-9]\d*)"
+    r"^(?P<major>0|[1-9]\d*)\.(?P<minor>0|[1-9]\d*)\.(?P<patch>0|[1-9]\d*)(?:-(?P<prerelease>(?:0|[1-9]\d*|\d*["
-    + r"(?:-(?P<prerelease>(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)"
+    r"a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?P<buildmetadata>[0-9a-zA-Z-]+("
-    + r"(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?"
+    r"?:\.[0-9a-zA-Z-]+)*))?$")
-    + r"(?:\+(?P<buildmetadata>[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$"
-)
 
 
 def log_upgrade_check():
@@ -23,8 +22,8 @@ def log_upgrade_check():
         release_tag_generator = (r["tag_name"] for r in _request_cellxgene_releases())
         latest_release = next(release_tag_generator, lambda tag_name: validate_version_str(tag_name))
         if version_gt(latest_release, __version__):
-            click.echo(f"There's a new version of cellxgene available ({latest_release})!")
+            click.echo(f"There's a new version of cellxgene available ({latest_release})!", err=True)
-            click.echo("To upgrade, run the following: pip install --upgrade cellxgene\n")
+            click.echo("To upgrade, run the following: pip install --upgrade cellxgene\n", err=True)
     except (ConnectionError, RateLimitException):
         click.echo("Upgrade check failed.\n")
 

server/common/app_config.py

@@ -9,6 +9,7 @@ import yaml
 from flatten_dict import flatten, unflatten
 
 import server.compute.diffexp_cxg as diffexp_tiledb
+import server.compute.scanpy
 from server import display_version as cellxgene_display_version
 from server.auth.auth import AuthTypeFactory
 from server.common.annotations.hosted_tiledb import AnnotationsHostedTileDB
@@ -99,7 +100,10 @@ class AppConfig(object):
 
         per_dataset_config = config.get("per_dataset_config", {})
         for key, dataroot_config in per_dataset_config.items():
-            self.add_dataroot_config(key, **dataroot_config)
+            # first create and initialize the dataroot with the default config
+            self.add_dataroot_config(key, **config["dataset"])
+            # then apply the per dataset configuration
+            self.dataroot_config[key].update_from_config(dataroot_config, f"per_dataset_config__{key}")
 
         self.is_complete = False
 
@@ -416,6 +420,7 @@ class ServerConfig(BaseConfig):
         dictval_cases = [
             ("app", "csp_directives"),
             ("authentication", "params_oauth", "cookie"),
+            ("authentication", "params_oauth", "jwt_decode_options"),
             ("adaptor", "cxg_adaptor", "tiledb_ctx"),
             ("multi_dataset", "dataroot"),
         ]
@@ -433,13 +438,17 @@ class ServerConfig(BaseConfig):
             self.app__generate_cache_control_headers = dc["app"]["generate_cache_control_headers"]
             self.app__server_timing_headers = dc["app"]["server_timing_headers"]
             self.app__csp_directives = dc["app"]["csp_directives"]
+            self.app__api_base_url = dc["app"]["api_base_url"]
+            self.app__web_base_url = dc["app"]["web_base_url"]
 
             self.authentication__type = dc["authentication"]["type"]
-            self.authentication__params_oauth__api_base_url = dc["authentication"]["params_oauth"]["api_base_url"]
+            self.authentication__params_oauth__oauth_api_base_url = dc["authentication"]["params_oauth"][
+                "oauth_api_base_url"
+            ]
             self.authentication__params_oauth__client_id = dc["authentication"]["params_oauth"]["client_id"]
             self.authentication__params_oauth__client_secret = dc["authentication"]["params_oauth"]["client_secret"]
-            self.authentication__params_oauth__callback_base_url = \
+            self.authentication__params_oauth__jwt_decode_options = dc["authentication"]["params_oauth"][
-                dc["authentication"]["params_oauth"]["callback_base_url"]
+                "jwt_decode_options"]
             self.authentication__params_oauth__session_cookie = dc["authentication"]["params_oauth"]["session_cookie"]
             self.authentication__params_oauth__cookie = dc["authentication"]["params_oauth"]["cookie"]
 
@@ -500,6 +509,8 @@ class ServerConfig(BaseConfig):
         self.check_attr("app__generate_cache_control_headers", bool)
         self.check_attr("app__server_timing_headers", bool)
         self.check_attr("app__csp_directives", (type(None), dict))
+        self.check_attr("app__api_base_url", (type(None), str))
+        self.check_attr("app__web_base_url", (type(None), str))
 
         if self.app__port:
             try:
@@ -548,15 +559,18 @@ class ServerConfig(BaseConfig):
                 elif not isinstance(v, str):
                     raise ConfigurationError("CSP directive value must be a string or list of strings.")
 
+        if self.app__web_base_url is None:
+            self.app__web_base_url = self.app__api_base_url
+
     def handle_authentication(self, context):
         self.check_attr("authentication__type", (type(None), str))
 
         # oauth
         ptypes = str if self.authentication__type == "oauth" else (type(None), str)
-        self.check_attr("authentication__params_oauth__api_base_url", ptypes)
+        self.check_attr("authentication__params_oauth__oauth_api_base_url", ptypes)
         self.check_attr("authentication__params_oauth__client_id", ptypes)
         self.check_attr("authentication__params_oauth__client_secret", ptypes)
-        self.check_attr("authentication__params_oauth__callback_base_url", (type(None), str))
+        self.check_attr("authentication__params_oauth__jwt_decode_options", (type(None), dict))
         self.check_attr("authentication__params_oauth__session_cookie", bool)
 
         if self.authentication__params_oauth__session_cookie:
@@ -742,6 +756,22 @@ class ServerConfig(BaseConfig):
             return False
         return value > limit_value
 
+    def get_api_base_url(self):
+        if self.app__api_base_url == "local":
+            return f"http://{self.app__host}:{self.app__port}"
+        if self.app__api_base_url and self.app__api_base_url.endswith("/"):
+            return self.app__api_base_url[:-1]
+        return self.app__api_base_url
+
+    def get_web_base_url(self):
+        if self.app__web_base_url == "local":
+            return f"http://{self.app__host}:{self.app__port}"
+        if self.app__web_base_url is None:
+            return self.get_api_base_url()
+        if self.app__web_base_url.endswith("/"):
+            return self.app__web_base_url[:-1]
+        return self.api__web_base_url
+
 
 class DatasetConfig(BaseConfig):
     """Manages the config attribute associated with a dataset."""
@@ -768,7 +798,7 @@ class DatasetConfig(BaseConfig):
             self.user_annotations__ontology__obo_location = dc["user_annotations"]["ontology"]["obo_location"]
             self.user_annotations__hosted_tiledb_array__db_uri = dc["user_annotations"]["hosted_tiledb_array"]["db_uri"]
             self.user_annotations__hosted_tiledb_array__hosted_file_directory = \
-                dc["user_annotations"]["hosted_tiledb_array"]["hosted_file_directory"]  # noqa E501
+                dc["user_annotations"][ "hosted_tiledb_array" ][ "hosted_file_directory" ]  # noqa E501
 
             self.embeddings__names = dc["embeddings"]["names"]
             self.embeddings__enable_reembedding = dc["embeddings"]["enable_reembedding"]
@@ -898,16 +928,21 @@ class DatasetConfig(BaseConfig):
         self.check_attr("embeddings__enable_reembedding", bool)
 
         server_config = self.app_config.server_config
-        if server_config.single_dataset__datapath:
+        if self.embeddings__enable_reembedding:
-            if self.embeddings__enable_reembedding:
+            if server_config.single_dataset__datapath:
                 matrix_data_loader = MatrixDataLoader(
                     server_config.single_dataset__datapath, app_config=self.app_config
                 )
                 if matrix_data_loader.matrix_data_type != MatrixDataType.H5AD:
-                    raise ConfigurationError("'enable-reembedding is only supported with H5AD files.")
+                    raise ConfigurationError("enable-reembedding is only supported with H5AD files.")
                 if server_config.adaptor__anndata_adaptor__backed:
                     raise ConfigurationError("enable-reembedding is not supported when run in --backed mode.")
 
+            try:
+                server.compute.scanpy.get_scanpy_module()
+            except NotImplementedError:
+                raise ConfigurationError("Please install scanpy to enable UMAP re-embedding")
+
     def handle_diffexp(self, context):
         self.check_attr("diffexp__enable", bool)
         self.check_attr("diffexp__lfc_cutoff", float)

server/common/default_config.py

@@ -14,6 +14,20 @@ server:
     server_timing_headers: false
     csp_directives: null
 
+    # By default, cellxgene will serve api requests from the same base url as the webpage.
+    # In general api_base_url and web_base_url will not need to be set.
+    # There are two reasons to set these parameters:
+    #  1. Oauth authentication is used; the oauth server will redirect back to the api_base_url after login,
+    #     which then redirects back to the web_base_url.  If the web_base_url is not set, it will default to
+    #     the api_base_url.  If oauth authentication is used, the api_base_url must be set.
+    #     For a local test (where the server runs on "http://localhost:<port>"), then the api_base_url may be
+    #     set to the string "local".
+    #  2. The cellxgene deploymnent is in an environment where the webpage and api have
+    #     different base urls.  In this case both api_base_url and web_base_url must be set.
+    #     It is up to the server admin to ensure that the networking is setup correctly for this environment.
+    api_base_url: null
+    web_base_url: null
+
   authentication:
     # The authentication types may be "none", "session", "oauth"
     # none:  No authentication support, features like user_annotations must not be enabled.
@@ -22,16 +36,17 @@ server:
     type: session
 
     params_oauth:
-       # url to the auth server
+       # url to the oauth server
-       api_base_url: null
+       oauth_api_base_url: null
        # client_id of this app
        client_id: null
        # the client_secret known to the auth server and this app
        client_secret: null
-       # cellxgene server location;
+       # jwt_decode_options, to specify non default decode options define
-       # the browser will be redirected to locations relative to this location during login and logout.
+       # jwt_decode_options to be a dictionary with key/values described by
-       # A value of None, indicates the client and server are on the localhost.  http://localhost:<port> will be used.
+       # the options parameter of the jose.jwt.decode function:
-       callback_base_url: null
+       # (https://python-jose.readthedocs.io/en/latest/jwt/api.html)
+       jwt_decode_options: null
 
        # if true, the jwt containing the id_token is stored in a session cookie
        session_cookie:  true

server/compute/scanpy.py

@@ -43,7 +43,7 @@ def scanpy_umap(adata, obs_mask=None, pca_options={}, neighbors_options={}, umap
     for k in list(adata.uns.keys()):
         del adata.uns[k]
 
-    sc.pp.pca(adata, zero_center=None, n_comps=min(adata.n_obs - 1, 50), **pca_options)
+    sc.pp.pca(adata, zero_center=None, n_comps=min(adata.n_vars - 1, 50), **pca_options)
     sc.pp.neighbors(adata, **neighbors_options)
     sc.tl.umap(adata, **umap_options)
 

server/eb/Makefile

@@ -34,7 +34,8 @@ build: clean
 	     cp customize/requirements.txt artifact.dir;  \
 	  fi ; \
 	  if [ -d customize/deploy ] ; then \
-		  cp -r customize/deploy artifact.dir/server/common/web/static;  \
+		 mkdir -p artifact.dir/server/common/web/static/cellxgene; \
+		 cp -r customize/deploy artifact.dir/server/common/web/static/cellxgene;  \
 	  fi; \
 	  if [ -d customize/inline_scripts ] ; then \
 	      cp -r customize/inline_scripts/* artifact.dir/server/common/web/templates;  \

server/eb/README.md

@@ -95,7 +95,7 @@ To use this feature, do the following:
 
 * In this directory, create a sub directory called "customize/deploy/".
 * Copy the files you want to serve into this directory
-* modify your configuration file to set the location to these file:  /static/deploy/<filename>
+* modify your configuration file to set the location to these file:  /static/cellxgene/deploy/<filename>
 
 Example:  you want to include an "about_legal_tos" and "about_legal_privacy" page to cellxgene.
 Assume files called "tos.html" and "privacy.html" exist.
@@ -106,9 +106,9 @@ $ cp <source_dir>/tos.html customize/deploy/tos.html
 $ cp <source_dir>/privacy.html customize/deploy/privacy.html
 
 # edit config.yaml
-$ grep "/static/deploy" config.yaml
+$ grep "/static/cellxgene/deploy" config.yaml
-about_legal_tos: /static/deploy/tos.html
+about_legal_tos: /static/cellxgene/deploy/tos.html
-about_legal_privacy: /static/deploy/privacy.html
+about_legal_privacy: /static/cellxgene/deploy/privacy.html
 ```
 
 #### Inline javascript scripts

server/eb/app.py

@@ -4,10 +4,11 @@ import sys
 import os
 import hashlib
 import base64
+from urllib.parse import urlparse
 from flask import json
 import logging
 from flask_talisman import Talisman
-
+from flask_cors import CORS
 from server.common.aws_secret_utils import handle_config_from_secret
 from server.common.errors import SecretKeyRetrievalError
 
@@ -41,6 +42,14 @@ class WSGIServer(Server):
     def _before_adding_routes(app, app_config):
         script_hashes = WSGIServer.get_csp_hashes(app, app_config)
         server_config = app_config.server_config
+
+        # add the api_base_url to the connect_src csp header.
+        extra_connect_src = []
+        api_base_url = server_config.get_api_base_url()
+        if api_base_url:
+            parse_api_base_url = urlparse(api_base_url)
+            extra_connect_src = [f"{parse_api_base_url.scheme}://{parse_api_base_url.netloc}"]
+
         # This hash should be in sync with the script within
         # `client/configuration/webpack/obsoleteHTMLTemplate.html`
 
@@ -51,7 +60,7 @@ class WSGIServer(Server):
         obsolete_browser_script_hash = ["'sha256-/rmgOi/skq9MpiZxPv6lPb1PNSN+Uf4NaUHO/IjyfwM='"]
         csp = {
             "default-src": ["'self'"],
-            "connect-src": ["'self'"],
+            "connect-src": ["'self'"] + extra_connect_src,
             "script-src": ["'self'", "'unsafe-eval'"]
             + obsolete_browser_script_hash + script_hashes,
             "style-src": ["'self'", "'unsafe-inline'"],
@@ -70,6 +79,13 @@ class WSGIServer(Server):
                     v = [v]
                 csp[k] = csp.get(k, []) + v
 
+        # Add the web_base_url to the CORS header
+        web_base_url = server_config.get_web_base_url()
+        if web_base_url:
+            web_base_url_parse = urlparse(web_base_url)
+            allowed_origin = f"{web_base_url_parse.scheme}://{web_base_url_parse.netloc}"
+            CORS(app, supports_credentials=True, origins=allowed_origin)
+
         Talisman(
             app, force_https=server_config.app__force_https, frame_options="DENY", content_security_policy=csp,
         )

server/test/__init__.py

@@ -131,16 +131,24 @@ def start_test_server(command_line_args=[], app_config=None):
 
     where the server can be accessed within the context, and is terminated when
     the context is exited.
-    The port is automatically set using find_available_port.
+    The port is automatically set using find_available_port, unless passed in as a command line arg.
     The verbose flag is automatically set to True.
     If an app_config is provided, then this function writes a temporary
     yaml config file, which this server will read and parse.
     """
 
-    start = random.randint(DEFAULT_SERVER_PORT, 2 ** 16 - 1)
+    command = ["cellxgene", "--no-upgrade-check", "launch", "--verbose"]
-    port = int(os.environ.get("CXG_SERVER_PORT", start))
+    if "-p" in command_line_args:
-    port = find_available_port("localhost", port)
+        port = int(command_line_args[command_line_args.index("-p") + 1])
-    command = ["cellxgene", "--no-upgrade-check", "launch", "--verbose", "--port=%d" % port] + command_line_args
+    elif "--port" in command_line_args:
+        port = int(command_line_args[command_line_args.index("--port") + 1])
+    else:
+        start = random.randint(DEFAULT_SERVER_PORT, 2 ** 16 - 1)
+        port = int(os.environ.get("CXG_SERVER_PORT", start))
+        port = find_available_port("localhost", port)
+        command += ["--port=%d" % port]
+
+    command += command_line_args
 
     tempdir = None
     if app_config:

server/test/unit/auth/test_oauth.py

@@ -19,7 +19,7 @@ from server.test import FIXTURES_ROOT, test_server
 # oauth server.
 
 # number of seconds that the oauth token is valid
-TOKEN_EXPIRES = 5
+TOKEN_EXPIRES = 2
 
 # Create a mocked out oauth token, which servers all the endpoints needed by the oauth type.
 mock_oauth_app = Flask("mock_oauth_app")
@@ -34,17 +34,19 @@ def authorize():
 
 @mock_oauth_app.route("/oauth/token", methods=["POST"])
 def token():
+    now = time.time()
+    expires_at = now + TOKEN_EXPIRES
     headers = dict(alg="RS256", kid="fake_kid")
-    payload = dict(name="fake_user", sub="fake_id", email="fake_user@email.com", email_verified=True)
+    payload = dict(name="fake_user", sub="fake_id", email="fake_user@email.com", email_verified=True, exp=expires_at)
     jwt = jose.jwt.encode(claims=payload, key="mysecret", algorithm="HS256", headers=headers)
     r = {
-        "access_token": f"access-{time.time()}",
+        "access_token": f"access-{now}",
         "id_token": jwt,
-        "refresh_token": f"random-{time.time()}",
+        "refresh_token": f"random-{now}",
         "scope": "openid profile email",
         "expires_in": TOKEN_EXPIRES,
         "token_type": "Bearer",
-        "expires_at": time.time() + TOKEN_EXPIRES,
+        "expires_at": expires_at
     }
     return make_response(jsonify(r))
 
@@ -81,6 +83,19 @@ class AuthTest(unittest.TestCase):
 
     def auth_flow(self, app_config, cookie_key=None):
 
+        app_config.update_server_config(
+            app__api_base_url="local",
+            authentication__type="oauth",
+            authentication__params_oauth__oauth_api_base_url=f"http://localhost:{PORT}",
+            authentication__params_oauth__client_id="mock_client_id",
+            authentication__params_oauth__client_secret="mock_client_secret",
+            authentication__params_oauth__jwt_decode_options={
+                "verify_signature": False, "verify_iss": False
+            })
+
+        app_config.update_server_config(multi_dataset__dataroot=self.dataset_dataroot)
+        app_config.complete_config()
+
         with test_server(app_config=app_config) as server:
             session = requests.Session()
 
@@ -96,10 +111,10 @@ class AuthTest(unittest.TestCase):
             login_uri = config["config"]["authentication"]["login"]
             logout_uri = config["config"]["authentication"]["logout"]
 
-            self.assertEqual(login_uri, "/login?dataset=d/pbmc3k.cxg/")
+            self.assertEqual(login_uri, f"{server}/login?dataset=d/pbmc3k.cxg/")
-            self.assertEqual(logout_uri, "/logout")
+            self.assertEqual(logout_uri, f"{server}/logout")
 
-            r = session.get(f"{server}/{login_uri}")
+            r = session.get(login_uri)
             # check that the login redirect worked
             self.assertEqual(r.history[0].status_code, 302)
             self.assertEqual(r.url, f"{server}/d/pbmc3k.cxg/")
@@ -113,13 +128,13 @@ class AuthTest(unittest.TestCase):
                 cookie = session.cookies.get(cookie_key)
                 token = json.loads(base64.b64decode(cookie))
                 access_token_before = token.get("access_token")
-                expires_at_before = token.get("expires_at")
+                id_token_before = token.get("id_token")
 
                 # let the token expire
                 time.sleep(TOKEN_EXPIRES + 1)
 
                 # check that refresh works
-                session.get(f"{server}/{login_uri}")
+                session.get(login_uri)
                 userinfo = session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/userinfo").json()
                 self.assertTrue(userinfo["userinfo"]["is_authenticated"])
                 self.assertEqual(userinfo["userinfo"]["username"], "fake_user")
@@ -127,12 +142,12 @@ class AuthTest(unittest.TestCase):
                 cookie = session.cookies.get(cookie_key)
                 token = json.loads(base64.b64decode(cookie))
                 access_token_after = token.get("access_token")
-                expires_at_after = token.get("expires_at")
+                id_token_after = token.get("id_token")
 
                 self.assertNotEqual(access_token_before, access_token_after)
-                self.assertTrue(expires_at_after - expires_at_before > TOKEN_EXPIRES)
+                self.assertNotEqual(id_token_before, id_token_after)
 
-            r = session.get(f"{server}/{logout_uri}")
+            r = session.get(logout_uri)
             # check that the logout redirect worked
             self.assertEqual(r.history[0].status_code, 302)
             self.assertEqual(r.url, f"{server}")
@@ -146,31 +161,16 @@ class AuthTest(unittest.TestCase):
         # test with session cookies
         app_config = AppConfig()
         app_config.update_server_config(
-            authentication__type="oauth",
-            authentication__params_oauth__api_base_url=f"http://localhost:{PORT}",
-            authentication__params_oauth__client_id="mock_client_id",
-            authentication__params_oauth__client_secret="mock_client_secret",
             authentication__params_oauth__session_cookie=True,
         )
-
-        app_config.update_server_config(multi_dataset__dataroot=self.dataset_dataroot)
-        app_config.complete_config()
-
         self.auth_flow(app_config)
 
     def test_auth_oauth_cookie(self):
         # test with specified cookie
         app_config = AppConfig()
         app_config.update_server_config(
-            authentication__type="oauth",
-            authentication__params_oauth__api_base_url=f"http://localhost:{PORT}",
-            authentication__params_oauth__client_id="mock_client_id",
-            authentication__params_oauth__client_secret="mock_client_secret",
             authentication__params_oauth__session_cookie=False,
             authentication__params_oauth__cookie=dict(key="test_cxguser", httponly=True, max_age=60),
         )
 
-        app_config.update_server_config(multi_dataset__dataroot=self.dataset_dataroot)
-        app_config.complete_config()
-
         self.auth_flow(app_config, "test_cxguser")

server/test/unit/common/test_app_config.py

@@ -2,11 +2,13 @@ import os
 import unittest
 from unittest import mock
 from unittest.mock import patch
+import tempfile
 
 import requests
 
 from server.common.app_config import AppConfig
 from server.common.errors import ConfigurationError
+from server.common.utils.utils import find_available_port
 from server.test import PROJECT_ROOT, test_server, FIXTURES_ROOT
 
 
@@ -19,46 +21,46 @@ def mockenv(**envvars):
 
 class AppConfigTest(unittest.TestCase):
     def test_update(self):
-        c = AppConfig()
+        config = AppConfig()
-        c.update_server_config(app__verbose=True, multi_dataset__dataroot="datadir")
+        config.update_server_config(app__verbose=True, multi_dataset__dataroot="datadir")
-        v = c.server_config.changes_from_default()
+        vars = config.server_config.changes_from_default()
-        self.assertCountEqual(v, [("app__verbose", True, False), ("multi_dataset__dataroot", "datadir", None)])
+        self.assertCountEqual(vars, [("app__verbose", True, False), ("multi_dataset__dataroot", "datadir", None)])
 
-        c = AppConfig()
+        config = AppConfig()
-        c.update_default_dataset_config(app__scripts=(), app__inline_scripts=())
+        config.update_default_dataset_config(app__scripts=(), app__inline_scripts=())
-        v = c.server_config.changes_from_default()
+        vars = config.server_config.changes_from_default()
-        self.assertCountEqual(v, [])
+        self.assertCountEqual(vars, [])
 
-        c = AppConfig()
+        config = AppConfig()
-        c.update_default_dataset_config(app__scripts=[], app__inline_scripts=[])
+        config.update_default_dataset_config(app__scripts=[], app__inline_scripts=[])
-        v = c.default_dataset_config.changes_from_default()
+        vars = config.default_dataset_config.changes_from_default()
-        self.assertCountEqual(v, [])
+        self.assertCountEqual(vars, [])
 
-        c = AppConfig()
+        config = AppConfig()
-        c.update_default_dataset_config(app__scripts=("a", "b"), app__inline_scripts=["c", "d"])
+        config.update_default_dataset_config(app__scripts=("a", "b"), app__inline_scripts=["c", "d"])
-        v = c.default_dataset_config.changes_from_default()
+        vars = config.default_dataset_config.changes_from_default()
-        self.assertCountEqual(v, [("app__scripts", ["a", "b"], []), ("app__inline_scripts", ["c", "d"], [])])
+        self.assertCountEqual(vars, [("app__scripts", ["a", "b"], []), ("app__inline_scripts", ["c", "d"], [])])
 
     def test_multi_dataset(self):
 
-        c = AppConfig()
+        config = AppConfig()
         # test for illegal url_dataroots
         for illegal in ("../b", "!$*", "\\n", "", "(bad)"):
-            c.update_server_config(
+            config.update_server_config(
                 multi_dataset__dataroot={"tag": {"base_url": illegal, "dataroot": "{PROJECT_ROOT}/example-dataset"}}
             )
             with self.assertRaises(ConfigurationError):
-                c.complete_config()
+                config.complete_config()
 
         # test for legal url_dataroots
         for legal in ("d", "this.is-okay_", "a/b"):
-            c.update_server_config(
+            config.update_server_config(
                 multi_dataset__dataroot={"tag": {"base_url": legal, "dataroot": "{PROJECT_ROOT}/example-dataset"}}
             )
-            c.complete_config()
+            config.complete_config()
 
         # test that multi dataroots work end to end
-        c.update_server_config(
+        config.update_server_config(
             multi_dataset__dataroot=dict(
                 s1=dict(dataroot=f"{PROJECT_ROOT}/example-dataset", base_url="set1/1/2"),
                 s2=dict(dataroot=f"{FIXTURES_ROOT}", base_url="set2"),
@@ -67,46 +69,46 @@ class AppConfigTest(unittest.TestCase):
         )
 
         # Change this default to test if the dataroot overrides below work.
-        c.update_default_dataset_config(app__about_legal_tos="tos_default.html")
+        config.update_default_dataset_config(app__about_legal_tos="tos_default.html")
 
         # specialize the configs for set1
-        c.add_dataroot_config(
+        config.add_dataroot_config(
             "s1", user_annotations__enable=False, diffexp__enable=True, app__about_legal_tos="tos_set1.html"
         )
 
         # specialize the configs for set2
-        c.add_dataroot_config(
+        config.add_dataroot_config(
             "s2", user_annotations__enable=True, diffexp__enable=False, app__about_legal_tos="tos_set2.html"
         )
 
         # no specializations for set3 (they get the default dataset config)
-        c.complete_config()
+        config.complete_config()
 
-        with test_server(app_config=c) as server:
+        with test_server(app_config=config) as server:
             session = requests.Session()
 
-            r = session.get(f"{server}/set1/1/2/pbmc3k.h5ad/api/v0.2/config")
+            response = session.get(f"{server}/set1/1/2/pbmc3k.h5ad/api/v0.2/config")
-            data_config = r.json()
+            data_config = response.json()
             assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
             assert data_config["config"]["parameters"]["annotations"] is False
             assert data_config["config"]["parameters"]["disable-diffexp"] is False
             assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_set1.html"
 
-            r = session.get(f"{server}/set2/pbmc3k.cxg/api/v0.2/config")
+            response = session.get(f"{server}/set2/pbmc3k.cxg/api/v0.2/config")
-            data_config = r.json()
+            data_config = response.json()
             assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
             assert data_config["config"]["parameters"]["annotations"] is True
             assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_set2.html"
 
-            r = session.get(f"{server}/set3/pbmc3k.cxg/api/v0.2/config")
+            response = session.get(f"{server}/set3/pbmc3k.cxg/api/v0.2/config")
-            data_config = r.json()
+            data_config = response.json()
             assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
             assert data_config["config"]["parameters"]["annotations"] is True
             assert data_config["config"]["parameters"]["disable-diffexp"] is False
             assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_default.html"
 
-            r = session.get(f"{server}/health")
+            response = session.get(f"{server}/health")
-            assert r.json()["status"] == "pass"
+            assert response.json()["status"] == "pass"
 
     @mockenv(CXG_AWS_SECRET_NAME="TESTING", CXG_AWS_SECRET_REGION_NAME="TEST_REGION")
     @patch('server.common.aws_secret_utils.get_secret_key')
@@ -133,3 +135,64 @@ class AppConfigTest(unittest.TestCase):
             self.assertEqual(config.server_config.app__flask_secret_key, "mock_flask_secret")
             self.assertEqual(config.server_config.authentication__params_oauth__client_secret, "mock_oauth_secret")
             self.assertEqual(config.default_dataset_config.user_annotations__hosted_tiledb_array__db_uri, "mock_db_uri")
+
+    def test_api_base_url(self):
+
+        # test the api_base_url feature, and that it can contain a path
+        config = AppConfig()
+        backend_port = find_available_port("localhost", 10000)
+        config.update_server_config(
+            app__api_base_url=f"http://localhost:{backend_port}/additional/path/before/dataroot",
+            multi_dataset__dataroot=f"{PROJECT_ROOT}/example-dataset"
+        )
+
+        config.complete_config()
+
+        with test_server(["-p", str(backend_port)], app_config=config) as server:
+            session = requests.Session()
+            self.assertEqual(server, f"http://localhost:{backend_port}")
+            response = session.get(f"{server}/additional/path/before/dataroot/d/pbmc3k.h5ad/api/v0.2/config")
+            self.assertEqual(response.status_code, 200)
+            data_config = response.json()
+            self.assertEqual(data_config["config"]["displayNames"]["dataset"], "pbmc3k")
+
+    def test_configfile_with_specialization(self):
+        # test that per_dataset_config config load the default config, then the specialized config
+
+        with tempfile.TemporaryDirectory() as tempdir:
+            configfile = os.path.join(tempdir, "config.yaml")
+            with open(configfile, "w") as fconfig:
+                config = """
+                server:
+                    multi_dataset:
+                        dataroot:
+                            test:
+                                base_url: test
+                                dataroot: fake_dataroot
+
+                dataset:
+                    user_annotations:
+                        enable: false
+                        type: hosted_tiledb_array
+                        hosted_tiledb_array:
+                            db_uri: fake_db_uri
+                            hosted_file_directory: fake_dir
+
+                per_dataset_config:
+                    test:
+                        user_annotations:
+                            enable: true
+                """
+                fconfig.write(config)
+
+            app_config = AppConfig()
+            app_config.update_from_config_file(configfile)
+
+            test_config = app_config.dataroot_config["test"]
+
+            # test config from default
+            self.assertEqual(test_config.user_annotations__type, "hosted_tiledb_array")
+            self.assertEqual(test_config.user_annotations__hosted_tiledb_array__db_uri, "fake_db_uri")
+
+            # test config from specialization
+            self.assertTrue(test_config.user_annotations__enable)