Merge branch 'master' into colinmegill/geneset-prototype

.github/workflows/push_tests.yml

@@ -4,6 +4,7 @@ on:
   push:
     branches: master
   pull_request:
+    branches: '*'
 
 env:
   JEST_ENV: prod
@@ -31,20 +32,15 @@ jobs:
         run: |
           pip install flake8
           cd client
-          npm i "eslint" "eslint-config-airbnb" "eslint-config-prettier" "eslint-loader" "eslint-plugin-filenames" "eslint-plugin-import" "eslint-plugin-jest" "eslint-plugin-jsx-a11y" "eslint-plugin-react" "eslint-plugin-react-hooks"
+          npm i "eslint" "eslint-config-airbnb" "eslint-config-prettier" "eslint-loader" "eslint-plugin-filenames" "eslint-plugin-import" "eslint-plugin-jest" "eslint-plugin-jsx-a11y" "eslint-plugin-react" "eslint-plugin-react-hooks" "eslint-plugin-prettier"
       - name: Lint with flake8
         run: |
           make lint-server
-      # - name: Lint all with eslint
+      - name: Lint src with eslint
-      #   working-directory: ./client
+        working-directory: ./client
-      #   if: github.event_name	!= 'pull_request'
+        run: |
-      #   run: |
+          make lint
-      #     make lint
+
-      # - name: Lint diff with eslint
-      #   working-directory: ./client
-      #   if: github.event_name	== 'pull_request'
-      #   run: |
-      #     make lint-diff
 
   unit-test:
     runs-on: ubuntu-latest

.gitignore

@@ -51,3 +51,5 @@ data
 # Jekyll
 docs/_site/
 docs/Gemfile.lock
+
+client/.eslintcache

Makefile

@@ -84,11 +84,6 @@ lint-server:
 lint-client:
 	cd client && $(MAKE) lint
 
-.PHONY: lint-diff-client
-lint-diff-client:
-	cd client && $(MAKE) lint-diff
-
-
 # CREATING DISTRIBUTION RELEASE
 
 .PHONY: pydist

client/Makefile

@@ -25,16 +25,7 @@ build:
 
 .PHONY: lint
 lint:
-	npx eslint .
+	npx eslint ./src/
-
-.PHONY: lint-diff
-lint-diff:
-# Get client diff against master, find all js/jsx files, remove the client prefex, run eslinst against result
-	git diff --name-only --diff-filter=d origin/master...HEAD -- ../client/ \
-		| grep -E "(.*)\.(jsx|js)$$" \
-		| sed "s/client\///" \
-		| $(if $(IS_DARWIN),xargs ./node_modules/.bin/eslint,xargs -r ./node_modules/.bin/eslint)
-
 
 # Development convenience methods
 .PHONY: start-frontend

client/__tests__/e2e/__snapshots__/e2e.test.js.snap

@@ -0,0 +1,5 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`did launch page launched 1`] = `"<span style=\\"width: 185px; display: flex; overflow: hidden; justify-content: flex-start;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">pbm</span><span style=\\"color: transparent; position: relative; overflow: hidden; white-space: nowrap;\\">c3k<span style=\\"position: absolute; right: 0px; color: initial;\\">c3k</span></span></span>"`;
+
+exports[`metadata loads categories and values from dataset appear 1`] = `"<div style=\\"display: flex; justify-content: space-between; align-items: baseline;\\"><div style=\\"display: flex; justify-content: flex-start; align-items: flex-start;\\"><label class=\\"bp3-control bp3-checkbox\\" for=\\"category-select-louvain\\"><input id=\\"category-select-louvain\\" data-testclass=\\"category-select\\" data-testid=\\"louvain:category-select\\" type=\\"checkbox\\" checked=\\"\\"><span class=\\"bp3-control-indicator\\"></span></label><span role=\\"menuitem\\" tabindex=\\"0\\" data-testid=\\"louvain:category-expand\\" style=\\"cursor: pointer;\\"><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\"><span data-testid=\\"louvain:category-label\\" aria-label=\\"louvain\\" class=\\"\\" tabindex=\\"0\\" style=\\"max-width: 265px;\\"><span style=\\"max-width: 265px; display: flex; overflow: hidden; justify-content: flex-start;\\"><span style=\\"overflow: hidden; text-overflow: ellipsis; white-space: nowrap; flex-shrink: 1; min-width: 5px;\\">lou</span><span style=\\"color: transparent; position: relative; overflow: hidden; white-space: nowrap;\\">vain<span style=\\"position: absolute; right: 0px; color: initial;\\">vain</span></span></span></span></span></span><svg stroke=\\"currentColor\\" fill=\\"currentColor\\" stroke-width=\\"0\\" viewBox=\\"0 0 320 512\\" data-testclass=\\"category-expand-is-not-expanded\\" height=\\"1em\\" width=\\"1em\\" xmlns=\\"http://www.w3.org/2000/svg\\" style=\\"font-size: 10px; margin-left: 5px;\\"><path d=\\"M285.476 272.971L91.132 467.314c-9.373 9.373-24.569 9.373-33.941 0l-22.667-22.667c-9.357-9.357-9.375-24.522-.04-33.901L188.505 256 34.484 101.255c-9.335-9.379-9.317-24.544.04-33.901l22.667-22.667c9.373-9.373 24.569-9.373 33.941 0L285.475 239.03c9.373 9.372 9.373 24.568.001 33.941z\\"></path></svg></span></div><div><span class=\\"bp3-popover-wrapper\\"><span class=\\"bp3-popover-target\\"><a role=\\"button\\" data-testclass=\\"colorby\\" data-testid=\\"colorby-louvain\\" class=\\"bp3-button\\" tabindex=\\"0\\"><span icon=\\"tint\\" class=\\"bp3-icon bp3-icon-tint\\"><svg data-icon=\\"tint\\" width=\\"16\\" height=\\"16\\" viewBox=\\"0 0 16 16\\"><desc>tint</desc><path d=\\"M7.88 1s-4.9 6.28-4.9 8.9c.01 2.82 2.34 5.1 4.99 5.1 2.65-.01 5.03-2.3 5.03-5.13C12.99 7.17 7.88 1 7.88 1z\\" fill-rule=\\"evenodd\\"></path></svg></span></a></span></span></div></div><div style=\\"margin-left: 26px;\\"><div></div></div><div></div>"`;

client/__tests__/e2e/cellxgeneActions.js

@@ -44,9 +44,10 @@ export const cellxgeneActions = (page, utils) => ({
       (rows) =>
         Object.fromEntries(
           rows.map((row) => {
-            const cat = row.querySelector(
+            const cat = row
-              "[data-testclass='categorical-value']"
+              .querySelector("[data-testclass='categorical-value']")
-            ).innerText;
+              .getAttribute("aria-label");
+
             const count = row.querySelector(
               "[data-testclass='categorical-value-count']"
             ).innerText;

client/__tests__/e2e/e2e.test.js

@@ -30,12 +30,12 @@ describe("did launch", () => {
     const element = await utils.getOneElementInnerHTML(
       "[data-testid='header']"
     );
-    expect(element).toBe(data.title);
+    expect(element).toMatchSnapshot();
   });
 
   test("terms of service, if they are there", async () => {
     try {
-      await utils.clickOn("tos-cookies-accept", { timeout: 500 });
+      await utils.clickOn("tos-cookies-accept", { timeout: 3000 });
     } catch {
       console.warn("No terms of service footer detected.");
     }
@@ -48,10 +48,10 @@ describe("did launch", () => {
 describe("metadata loads", () => {
   test("categories and values from dataset appear", async () => {
     for (const label in data.categorical) {
-      const categoryName = await utils.getOneElementInnerText(
+      const elem = await utils.getOneElementInnerHTML(
         `[data-testid="category-${label}"]`
       );
-      expect(categoryName).toMatch(label);
+      expect(elem).toMatchSnapshot();
       await utils.clickOn(`${label}:category-expand`);
       const categories = await cxgActions.getAllCategoriesAndCounts(label);
       expect(Object.keys(categories)).toMatchObject(
@@ -326,13 +326,16 @@ describe("ui elements don't error", () => {
 
 describe("centroid labels", () => {
   test("labels are created", async () => {
-    await utils.clickOn("centroid-label-toggle");
     const labels = Object.keys(data.categorical);
+
+    await utils.clickOn(`colorby-${labels[0]}`);
+    await utils.clickOn("centroid-label-toggle");
     /* eslint-disable no-await-in-loop */
     // Toggle colorby for each category and check to see if labels are generated
     for (let i = 0, { length } = labels; i < length; i += 1) {
       const label = labels[i];
-      await utils.clickOn(`colorby-${label}`);
+      // first label is already enabled
+      if (i !== 0) await utils.clickOn(`colorby-${label}`);
       const generatedLabels = await utils.getAllByClass("centroid-label");
       // Number of labels generated should be equal to size of the object
       expect(generatedLabels).toHaveLength(
@@ -346,8 +349,8 @@ describe("centroid labels", () => {
 describe("graph overlay", () => {
   test("transform centroids correctly", async () => {
     const category = Object.keys(data.categorical)[0];
-    await utils.clickOn("centroid-label-toggle");
     await utils.clickOn(`colorby-${category}`);
+    await utils.clickOn("centroid-label-toggle");
     await utils.clickOn("mode-pan-zoom");
     const panCoords = await cxgActions.calcDragCoordinates(
       "layout-graph",

client/__tests__/e2e/e2eAnnotations.test.js

@@ -201,16 +201,18 @@ describe.each([
   });
 
   async function assertCategoryExists(categoryName) {
-    const handle = await utils.waitByID(`${categoryName}:category-expand`);
+    const handle = await utils.waitByID(`${categoryName}:category-label`);
-    const result = await handle.evaluate((node) => node.innerText);
+
-    // slice beginning and end of category name result to account for truncation of long names
+    const result = await handle.evaluate((node) =>
-    expect(result.slice(0, 10)).toBe(categoryName.slice(0, 10));
+      node.getAttribute("aria-label")
-    expect(result.slice(-10)).toBe(categoryName.slice(-10));
+    );
+
+    expect(result).toBe(categoryName);
   }
 
   async function assertCategoryDoesNotExist(categoryName) {
     const result = await page.$(
-      `[data-testid='${categoryName}:category-expand']`
+      `[data-testid='${categoryName}:category-label']`
     );
     expect(result).toBeNull();
   }
@@ -222,7 +224,9 @@ describe.each([
     const previous = await utils.waitByID(
       `categorical-value-${categoryName}-${labelName}`
     );
-    expect(await previous.evaluate((node) => node.innerText)).toBe(labelName);
+    expect(
+      await previous.evaluate((node) => node.getAttribute("aria-label"))
+    ).toBe(labelName);
   }
 
   async function assertLabelDoesNotExist(categoryName, labelName) {

client/__tests__/e2e/puppeteerUtils.js

@@ -50,8 +50,8 @@ export const puppeteerUtils = (page) => ({
     return click;
   },
 
-  async getOneElementInnerHTML(selector) {
+  async getOneElementInnerHTML(selector, options = {}) {
-    await page.waitForSelector(selector);
+    await page.waitForSelector(selector, options);
     return page.$eval(selector, (el) => el.innerHTML);
   },
 

client/__tests__/util/centroid.test.js

@@ -4,6 +4,7 @@ import calcCentroid from "../../src/util/centroid";
 
 import quantile from "../../src/util/quantile";
 import * as Universe from "../../src/util/stateManager/universe";
+import { matrixFBSToDataframe } from "../../src/util/stateManager/matrix";
 import * as World from "../../src/util/stateManager/world";
 import * as REST from "./stateManager/sampleResponses";
 import { ControlsHelpers as CH } from "../../src/util/stateManager";
@@ -22,16 +23,13 @@ describe("centroid", () => {
       ...universe,
       ...Universe.addObsAnnotations(
         universe,
-        Universe.matrixFBSToDataframe(REST.annotationsObs)
+        matrixFBSToDataframe(REST.annotationsObs)
       ),
       ...Universe.addVarAnnotations(
         universe,
-        Universe.matrixFBSToDataframe(REST.annotationsVar)
+        matrixFBSToDataframe(REST.annotationsVar)
-      ),
-      ...Universe.addObsLayout(
-        universe,
-        Universe.matrixFBSToDataframe(REST.layoutObs)
       ),
+      ...Universe.addObsLayout(universe, matrixFBSToDataframe(REST.layoutObs)),
     };
     world = World.createWorldFromEntireUniverse(universe);
 

client/__tests__/util/dataframe/dataframe.test.js

@@ -38,8 +38,8 @@ describe("dataframe constructor", () => {
 
     expect(df.rowIndex).toBeInstanceOf(Dataframe.DenseInt32Index);
     expect(df.colIndex).toBeInstanceOf(Dataframe.KeyIndex);
-    expect(df.rowIndex.keys()).toEqual(new Int32Array([2, 1, 0]));
+    expect(df.rowIndex.labels()).toEqual(new Int32Array([2, 1, 0]));
-    expect(df.colIndex.keys()).toEqual(["A", "B"]);
+    expect(df.colIndex.labels()).toEqual(["A", "B"]);
 
     expect(df.at(0, "A")).toEqual(2);
     expect(df.at(2, "B")).toEqual(3);
@@ -138,7 +138,7 @@ describe("dataframe subsetting", () => {
         new Float32Array([4.4, 5.5, 6.6]),
         ["red", "green", "blue"],
       ],
-      null,
+      null, // identity index
       new Dataframe.KeyIndex(["int32", "string", "float32", "colors"])
     );
 
@@ -153,12 +153,12 @@ describe("dataframe subsetting", () => {
       expect(dfA.col("colors").asArray()).toEqual(
         sourceDf.col("colors").asArray()
       );
-      expect(dfA.rowIndex.keys()).toEqual(sourceDf.rowIndex.keys());
+      expect(dfA.rowIndex.labels()).toEqual(sourceDf.rowIndex.labels());
-      expect(dfA.colIndex.keys()).toEqual(["colors"]);
+      expect(dfA.colIndex.labels()).toEqual(["colors"]);
     });
 
     test("all rows, two columns", () => {
-      const dfB = sourceDf.subset(null, ["colors", "float32"]);
+      const dfB = sourceDf.subset(null, ["float32", "colors"]);
       expect(dfB).toBeDefined();
       expect(dfB.dims).toEqual([3, 2]);
       expect(dfB.iat(0, 0)).toBeCloseTo(4.4);
@@ -177,8 +177,8 @@ describe("dataframe subsetting", () => {
       expect(dfB.col("float32").asArray()).toEqual(
         sourceDf.col("float32").asArray()
       );
-      expect(dfB.rowIndex.keys()).toEqual(sourceDf.rowIndex.keys());
+      expect(dfB.rowIndex.labels()).toEqual(sourceDf.rowIndex.labels());
-      expect(dfB.colIndex.keys()).toEqual(["float32", "colors"]);
+      expect(dfB.colIndex.labels()).toEqual(["float32", "colors"]);
     });
 
     test("one row, all columns", () => {
@@ -189,8 +189,8 @@ describe("dataframe subsetting", () => {
       expect(dfC.iat(0, 1)).toEqual("B");
       expect(dfC.iat(0, 2)).toBeCloseTo(5.5);
       expect(dfC.iat(0, 3)).toEqual("green");
-      expect(dfC.rowIndex.keys()).toEqual(new Int32Array([1]));
+      expect(dfC.rowIndex.labels()).toEqual(new Int32Array([1]));
-      expect(dfC.colIndex.keys()).toEqual(sourceDf.colIndex.keys());
+      expect(dfC.colIndex.labels()).toEqual(sourceDf.colIndex.labels());
     });
 
     test("two rows, all columns", () => {
@@ -201,8 +201,17 @@ describe("dataframe subsetting", () => {
       expect(dfD.icol(1).asArray()).toEqual(["A", "C"]);
       expect(dfD.icol(2).asArray()).toEqual(new Float32Array([4.4, 6.6]));
       expect(dfD.icol(3).asArray()).toEqual(["red", "blue"]);
-      expect(dfD.rowIndex.keys()).toEqual(new Int32Array([0, 2]));
+      expect(dfD.rowIndex.labels()).toEqual(new Int32Array([0, 2]));
-      expect(dfD.colIndex.keys()).toEqual(sourceDf.colIndex.keys());
+      expect(dfD.colIndex.labels()).toEqual(sourceDf.colIndex.labels());
+
+      // reverse the row order
+      const dfDr = sourceDf.subset([2, 0], null);
+      expect(dfDr.icol(0).asArray()).toEqual(new Int32Array([2, 0]));
+      expect(dfDr.icol(1).asArray()).toEqual(["C", "A"]);
+      expect(dfDr.icol(2).asArray()).toEqual(new Float32Array([6.6, 4.4]));
+      expect(dfDr.icol(3).asArray()).toEqual(["blue", "red"]);
+      expect(dfDr.rowIndex.labels()).toEqual(new Int32Array([2, 0]));
+      expect(dfDr.colIndex.labels()).toEqual(sourceDf.colIndex.labels());
     });
 
     test("all rows, all columns", () => {
@@ -213,8 +222,8 @@ describe("dataframe subsetting", () => {
       expect(dfE.icol(1).asArray()).toEqual(sourceDf.icol(1).asArray());
       expect(dfE.icol(2).asArray()).toEqual(sourceDf.icol(2).asArray());
       expect(dfE.icol(3).asArray()).toEqual(sourceDf.icol(3).asArray());
-      expect(dfE.rowIndex.keys()).toEqual(sourceDf.rowIndex.keys());
+      expect(dfE.rowIndex.labels()).toEqual(sourceDf.rowIndex.labels());
-      expect(dfE.colIndex.keys()).toEqual(sourceDf.colIndex.keys());
+      expect(dfE.colIndex.labels()).toEqual(sourceDf.colIndex.labels());
     });
 
     test("two rows, two colums", () => {
@@ -223,8 +232,17 @@ describe("dataframe subsetting", () => {
       expect(dfF.dims).toEqual([2, 2]);
       expect(dfF.icol(0).asArray()).toEqual(new Int32Array([0, 2]));
       expect(dfF.icol(1).asArray()).toEqual(new Float32Array([4.4, 6.6]));
-      expect(dfF.rowIndex.keys()).toEqual(new Int32Array([0, 2]));
+      expect(dfF.rowIndex.labels()).toEqual(new Int32Array([0, 2]));
-      expect(dfF.colIndex.keys()).toEqual(["int32", "float32"]);
+      expect(dfF.colIndex.labels()).toEqual(["int32", "float32"]);
+
+      // reverse the row and column order
+      const dfFr = sourceDf.subset([2, 0], ["float32", "int32"]);
+      expect(dfFr).toBeDefined();
+      expect(dfFr.dims).toEqual([2, 2]);
+      expect(dfFr.icol(0).asArray()).toEqual(new Float32Array([6.6, 4.4]));
+      expect(dfFr.icol(1).asArray()).toEqual(new Int32Array([2, 0]));
+      expect(dfFr.rowIndex.labels()).toEqual(new Int32Array([2, 0]));
+      expect(dfFr.colIndex.labels()).toEqual(["float32", "int32"]);
     });
 
     test("withRowIndex", () => {
@@ -271,8 +289,49 @@ describe("dataframe subsetting", () => {
     expect(dfA.dims).toEqual([2, 2]);
     expect(dfA.icol(0).asArray()).toEqual(new Int32Array([1, 2]));
     expect(dfA.icol(1).asArray()).toEqual(["green", "blue"]);
-    expect(dfA.rowIndex.keys()).toEqual(new Int32Array([4, 6]));
+    expect(dfA.rowIndex.labels()).toEqual(new Int32Array([4, 6]));
-    expect(dfA.colIndex.keys()).toEqual(["int32", "colors"]);
+    expect(dfA.colIndex.labels()).toEqual(["int32", "colors"]);
+  });
+
+  describe("isubset", () => {
+    const sourceDf = new Dataframe.Dataframe(
+      [3, 4],
+      [
+        new Int32Array([0, 1, 2]),
+        ["A", "B", "C"],
+        new Float32Array([4.4, 5.5, 6.6]),
+        ["red", "green", "blue"],
+      ],
+      null, // identity index
+      new Dataframe.KeyIndex(["int32", "string", "float32", "colors"])
+    );
+
+    test("one row, all cols", () => {
+      const dfA = sourceDf.isubset([1], null);
+      expect(dfA.dims).toEqual([1, 4]);
+      expect(dfA.icol(0).asArray()).toEqual(new Int32Array([1]));
+      expect(dfA.icol(1).asArray()).toEqual(["B"]);
+      expect(dfA.icol(2).asArray()).toEqual(new Float32Array([5.5]));
+      expect(dfA.icol(3).asArray()).toEqual(["green"]);
+    });
+
+    test("all rows, two cols", () => {
+      const dfA = sourceDf.isubset(null, [1, 2]);
+      expect(dfA.dims).toEqual([3, 2]);
+      expect(dfA.icol(0).asArray()).toEqual(["A", "B", "C"]);
+      expect(dfA.icol(1).asArray()).toEqual(new Float32Array([4.4, 5.5, 6.6]));
+      expect(dfA.col("string")).toBe(dfA.icol(0));
+      expect(dfA.col("float32")).toBe(dfA.icol(1));
+    });
+
+    test("out of order rows", () => {
+      const dfA = sourceDf.isubset([2, 0], null);
+      expect(dfA.dims).toEqual([2, 4]);
+      expect(dfA.icol(0).asArray()).toEqual(new Int32Array([2, 0]));
+      expect(dfA.icol(1).asArray()).toEqual(["C", "A"]);
+      expect(dfA.icol(2).asArray()).toEqual(new Float32Array([6.6, 4.4]));
+      expect(dfA.icol(3).asArray()).toEqual(["blue", "red"]);
+    });
   });
 });
 
@@ -310,8 +369,8 @@ describe("dataframe factories", () => {
     expect(dfB).not.toBe(dfA);
     expect(dfB.dims).toEqual(dfA.dims);
     expect(dfB).toHaveLength(dfA.length);
-    expect(dfB.rowIndex.keys()).toEqual(dfA.rowIndex.keys());
+    expect(dfB.rowIndex.labels()).toEqual(dfA.rowIndex.labels());
-    expect(dfB.colIndex.keys()).toEqual(dfA.colIndex.keys());
+    expect(dfB.colIndex.labels()).toEqual(dfA.colIndex.labels());
     for (let i = 0, l = dfB.dims[1]; i < l; i += 1) {
       expect(dfB.icol(i).asArray()).toEqual(dfA.icol(i).asArray());
     }
@@ -336,9 +395,9 @@ describe("dataframe factories", () => {
       expect(dfA.icol(1).asArray()).toEqual([true, false]);
       expect(dfA.icol(2).asArray()).toEqual([1, 0]);
       expect(dfA.col("numbers").asArray()).toEqual([1, 0]);
-      expect(dfA.colIndex.keys()).toEqual(["colors", "bools", "numbers"]);
+      expect(dfA.colIndex.labels()).toEqual(["colors", "bools", "numbers"]);
-      expect(df.colIndex.keys()).toEqual(["colors", "bools"]);
+      expect(df.colIndex.labels()).toEqual(["colors", "bools"]);
-      expect(df.rowIndex.keys()).toEqual(dfA.rowIndex.keys());
+      expect(df.rowIndex.labels()).toEqual(dfA.rowIndex.labels());
     });
 
     test("DenseInt32Index", () => {
@@ -361,9 +420,9 @@ describe("dataframe factories", () => {
       expect(dfA.col(74).asArray()).toEqual(["red", "blue"]);
       expect(dfA.col(75).asArray()).toEqual([true, false]);
       expect(dfA.col(72).asArray()).toEqual([1, 0]);
-      expect(dfA.colIndex.keys()).toEqual(new Int32Array([74, 75, 72]));
+      expect(dfA.colIndex.labels()).toEqual(new Int32Array([74, 75, 72]));
-      expect(df.colIndex.keys()).toEqual(new Int32Array([74, 75]));
+      expect(df.colIndex.labels()).toEqual(new Int32Array([74, 75]));
-      expect(df.rowIndex.keys()).toEqual(dfA.rowIndex.keys());
+      expect(df.rowIndex.labels()).toEqual(dfA.rowIndex.labels());
     });
 
     test("DenseInt32Index promote", () => {
@@ -386,9 +445,9 @@ describe("dataframe factories", () => {
       expect(dfA.col(74).asArray()).toEqual(["red", "blue"]);
       expect(dfA.col(75).asArray()).toEqual([true, false]);
       expect(dfA.col(999).asArray()).toEqual([1, 0]);
-      expect(dfA.colIndex.keys()).toEqual(new Int32Array([74, 75, 999]));
+      expect(dfA.colIndex.labels()).toEqual(new Int32Array([74, 75, 999]));
-      expect(df.colIndex.keys()).toEqual(new Int32Array([74, 75]));
+      expect(df.colIndex.labels()).toEqual(new Int32Array([74, 75]));
-      expect(df.rowIndex.keys()).toEqual(dfA.rowIndex.keys());
+      expect(df.rowIndex.labels()).toEqual(dfA.rowIndex.labels());
     });
 
     test("IdentityInt32Index with last", () => {
@@ -411,9 +470,9 @@ describe("dataframe factories", () => {
       expect(dfA.col(0).asArray()).toEqual(["red", "blue"]);
       expect(dfA.col(1).asArray()).toEqual([true, false]);
       expect(dfA.col(2).asArray()).toEqual([1, 0]);
-      expect(dfA.colIndex.keys()).toEqual(new Int32Array([0, 1, 2]));
+      expect(dfA.colIndex.labels()).toEqual(new Int32Array([0, 1, 2]));
-      expect(df.colIndex.keys()).toEqual(new Int32Array([0, 1]));
+      expect(df.colIndex.labels()).toEqual(new Int32Array([0, 1]));
-      expect(df.rowIndex.keys()).toEqual(dfA.rowIndex.keys());
+      expect(df.rowIndex.labels()).toEqual(dfA.rowIndex.labels());
     });
 
     test("IdentityInt32Index promote", () => {
@@ -436,9 +495,9 @@ describe("dataframe factories", () => {
       expect(dfA.col(0).asArray()).toEqual(["red", "blue"]);
       expect(dfA.col(1).asArray()).toEqual([true, false]);
       expect(dfA.col(99).asArray()).toEqual([1, 0]);
-      expect(dfA.colIndex.keys()).toEqual(new Int32Array([0, 1, 99]));
+      expect(dfA.colIndex.labels()).toEqual(new Int32Array([0, 1, 99]));
-      expect(df.colIndex.keys()).toEqual(new Int32Array([0, 1]));
+      expect(df.colIndex.labels()).toEqual(new Int32Array([0, 1]));
-      expect(df.rowIndex.keys()).toEqual(dfA.rowIndex.keys());
+      expect(df.rowIndex.labels()).toEqual(dfA.rowIndex.labels());
     });
 
     describe("handle column dimensions correctly", () => {
@@ -517,25 +576,25 @@ describe("dataframe factories", () => {
       const dfLikeA = dfEmpty.withColsFrom(dfA);
       expect(dfLikeA).toBeDefined();
       expect(dfLikeA.dims).toEqual(dfA.dims);
-      expect(dfLikeA.colIndex.keys()).toEqual(dfA.colIndex.keys());
+      expect(dfLikeA.colIndex.labels()).toEqual(dfA.colIndex.labels());
       expect(dfLikeA.rowIndex).toEqual(dfA.rowIndex);
-      expect(dfLikeA.rowIndex.keys()).toEqual(dfA.rowIndex.keys());
+      expect(dfLikeA.rowIndex.labels()).toEqual(dfA.rowIndex.labels());
       expect(dfLikeA.icol(0).asArray()).toEqual(dfA.icol(0).asArray());
 
       const dfAlsoLikeA = dfA.withColsFrom(dfEmpty);
       expect(dfAlsoLikeA).toBeDefined();
       expect(dfAlsoLikeA.dims).toEqual(dfA.dims);
-      expect(dfAlsoLikeA.colIndex.keys()).toEqual(dfA.colIndex.keys());
+      expect(dfAlsoLikeA.colIndex.labels()).toEqual(dfA.colIndex.labels());
       expect(dfAlsoLikeA.rowIndex).toEqual(dfA.rowIndex);
-      expect(dfAlsoLikeA.rowIndex.keys()).toEqual(dfA.rowIndex.keys());
+      expect(dfAlsoLikeA.rowIndex.labels()).toEqual(dfA.rowIndex.labels());
       expect(dfAlsoLikeA.icol(0).asArray()).toEqual(dfA.icol(0).asArray());
 
       const dfC = dfA.withColsFrom(dfB);
       expect(dfC).toBeDefined();
       expect(dfC.dims).toEqual([2, 2]);
-      expect(dfC.colIndex.keys()).toEqual(["colors", "bools"]);
+      expect(dfC.colIndex.labels()).toEqual(["colors", "bools"]);
       expect(dfC.rowIndex).toEqual(dfA.rowIndex);
-      expect(dfC.rowIndex.keys()).toEqual(dfA.rowIndex.keys());
+      expect(dfC.rowIndex.labels()).toEqual(dfA.rowIndex.labels());
       expect(dfC.col("colors").asArray()).toEqual(["red", "blue"]);
       expect(dfC.col("bools").asArray()).toEqual([true, false]);
     });
@@ -562,21 +621,21 @@ describe("dataframe factories", () => {
       const dfX = dfEmpty.withColsFrom(dfB, ["colors", "bools"]);
       expect(dfX).toBeDefined();
       expect(dfX.dims).toEqual([2, 2]);
-      expect(dfX.colIndex.keys()).toEqual(["colors", "bools"]);
+      expect(dfX.colIndex.labels()).toEqual(["colors", "bools"]);
       expect(dfX.rowIndex).toEqual(dfB.rowIndex);
       expect(dfX.icol(0).asArray()).toEqual(dfB.icol(0).asArray());
 
       const dfY = dfA.withColsFrom(dfB, ["numbers"]);
       expect(dfY).toBeDefined();
       expect(dfY.dims).toEqual([2, 2]);
-      expect(dfY.colIndex.keys()).toEqual(["colors", "numbers"]);
+      expect(dfY.colIndex.labels()).toEqual(["colors", "numbers"]);
       expect(dfY.rowIndex).toEqual(dfA.rowIndex);
       expect(dfY.icol(0).asArray()).toEqual(dfA.icol(0).asArray());
 
       const dfZ = dfA.withColsFrom(dfEmpty, []);
       expect(dfZ).toBeDefined();
       expect(dfZ.dims).toEqual(dfA.dims);
-      expect(dfZ.colIndex.keys()).toEqual(dfA.colIndex.keys());
+      expect(dfZ.colIndex.labels()).toEqual(dfA.colIndex.labels());
       expect(dfZ.rowIndex).toEqual(dfA.rowIndex);
       expect(dfZ.icol(0).asArray()).toEqual(dfA.icol(0).asArray());
 
@@ -604,7 +663,7 @@ describe("dataframe factories", () => {
       const dfX = dfA.withColsFrom(dfB, { colors: "_colors", bools: "_bools" });
       expect(dfX).toBeDefined();
       expect(dfX.dims).toEqual([2, 3]);
-      expect(dfX.colIndex.keys()).toEqual(["colors", "_colors", "_bools"]);
+      expect(dfX.colIndex.labels()).toEqual(["colors", "_colors", "_bools"]);
       expect(dfX.rowIndex).toEqual(dfA.rowIndex);
       expect(dfX.icol(0).asArray()).toEqual(dfA.icol(0).asArray());
       expect(dfX.col("_colors").asArray()).toBe(dfB.col("colors").asArray());
@@ -630,9 +689,9 @@ describe("dataframe factories", () => {
       expect(dfA.icol(0).asArray()).toEqual([true, false]);
       expect(dfA.icol(1).asArray()).toEqual([1, 0]);
       expect(dfA.col("numbers").asArray()).toEqual([1, 0]);
-      expect(dfA.colIndex.keys()).toEqual(["bools", "numbers"]);
+      expect(dfA.colIndex.labels()).toEqual(["bools", "numbers"]);
-      expect(df.colIndex.keys()).toEqual(["colors", "bools", "numbers"]);
+      expect(df.colIndex.labels()).toEqual(["colors", "bools", "numbers"]);
-      expect(df.rowIndex.keys()).toEqual(dfA.rowIndex.keys());
+      expect(df.rowIndex.labels()).toEqual(dfA.rowIndex.labels());
     });
 
     test("IdentityInt32Index drop first", () => {
@@ -654,9 +713,9 @@ describe("dataframe factories", () => {
       expect(dfA.icol(1).asArray()).toEqual([1, 0]);
       expect(df.col(1).asArray()).toEqual(dfA.col(1).asArray());
       expect(df.col(2).asArray()).toEqual(dfA.col(2).asArray());
-      expect(dfA.colIndex.keys()).toEqual(new Int32Array([1, 2]));
+      expect(dfA.colIndex.labels()).toEqual(new Int32Array([1, 2]));
-      expect(df.colIndex.keys()).toEqual(new Int32Array([0, 1, 2]));
+      expect(df.colIndex.labels()).toEqual(new Int32Array([0, 1, 2]));
-      expect(df.rowIndex.keys()).toEqual(dfA.rowIndex.keys());
+      expect(df.rowIndex.labels()).toEqual(dfA.rowIndex.labels());
     });
 
     test("IdentityInt32Index drop last", () => {
@@ -678,9 +737,9 @@ describe("dataframe factories", () => {
       expect(dfA.icol(1).asArray()).toEqual([true, false]);
       expect(df.col(0).asArray()).toEqual(dfA.col(0).asArray());
       expect(df.col(1).asArray()).toEqual(dfA.col(1).asArray());
-      expect(dfA.colIndex.keys()).toEqual(new Int32Array([0, 1]));
+      expect(dfA.colIndex.labels()).toEqual(new Int32Array([0, 1]));
-      expect(df.colIndex.keys()).toEqual(new Int32Array([0, 1, 2]));
+      expect(df.colIndex.labels()).toEqual(new Int32Array([0, 1, 2]));
-      expect(df.rowIndex.keys()).toEqual(dfA.rowIndex.keys());
+      expect(df.rowIndex.labels()).toEqual(dfA.rowIndex.labels());
     });
 
     test("DenseInt32Index", () => {
@@ -702,9 +761,9 @@ describe("dataframe factories", () => {
       expect(dfA.icol(1).asArray()).toEqual([1, 0]);
       expect(dfA.col(100).asArray()).toEqual([1, 0]);
       expect(dfA.col(102).asArray()).toEqual(["red", "blue"]);
-      expect(dfA.colIndex.keys()).toEqual(new Int32Array([102, 100]));
+      expect(dfA.colIndex.labels()).toEqual(new Int32Array([102, 100]));
-      expect(df.colIndex.keys()).toEqual(new Int32Array([102, 101, 100]));
+      expect(df.colIndex.labels()).toEqual(new Int32Array([102, 101, 100]));
-      expect(df.rowIndex.keys()).toEqual(dfA.rowIndex.keys());
+      expect(df.rowIndex.labels()).toEqual(dfA.rowIndex.labels());
     });
   });
 
@@ -766,8 +825,8 @@ describe("dataframe factories", () => {
         new Dataframe.KeyIndex(["A", "B"])
       );
       const dfB = dfA.renameCol("B", "C");
-      expect(dfA.colIndex.keys()).toEqual(["A", "B"]);
+      expect(dfA.colIndex.labels()).toEqual(["A", "B"]);
-      expect(dfB.colIndex.keys()).toEqual(["A", "C"]);
+      expect(dfB.colIndex.labels()).toEqual(["A", "C"]);
       expect(dfA.dims).toMatchObject(dfB.dims);
       expect(dfA.columns()).toMatchObject(dfB.columns());
     });
@@ -857,3 +916,84 @@ describe("dataframe col", () => {
     expect(df.col("B").indexOf(true)).toBeUndefined();
   });
 });
+
+describe("label indexing", () => {
+
+  test("IdentityInt32Index", () => {
+    const idx = new Dataframe.IdentityInt32Index(12); // [0, 12)
+
+    expect(Dataframe.isLabelIndex(idx)).toBeTruthy();
+
+    expect(idx.labels()).toEqual(new Int32Array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]));
+    expect(idx.getLabel(1)).toEqual(1);
+    expect(idx.getOffset(1)).toEqual(1);
+    expect(idx.getOffsets([1,3])).toEqual([1,3])
+    expect(idx.getLabels([1, 3])).toEqual([1,3])
+    expect(idx.size()).toEqual(12);
+
+    expect(idx.subset([2]).labels()).toEqual([2]);
+    expect(idx.subset([2, 3, 4]).labels()).toEqual(new Int32Array([2, 3, 4]));
+    expect(idx.subset([0, 1, 2, 3]).labels()).toEqual(new Int32Array([0, 1, 2, 3]));
+
+    expect(idx.isubset([2]).labels()).toEqual([2]);
+    expect(idx.isubset([2, 3, 4]).labels()).toEqual(new Int32Array([2, 3, 4]));
+    expect(idx.isubset([0, 1, 2, 3]).labels()).toEqual(new Int32Array([0, 1, 2, 3]));
+
+    expect(idx.subset([0, 1, 2, 3, 4])).toBeInstanceOf(Dataframe.IdentityInt32Index);
+    expect(idx.subset([2, 1, 0])).toBeInstanceOf(Dataframe.IdentityInt32Index);
+    expect(idx.subset([1, 2, 3, 4])).toBeInstanceOf(Dataframe.DenseInt32Index);
+    expect(idx.subset([0, 1, 3, 4])).toBeInstanceOf(Dataframe.DenseInt32Index);
+    expect(idx.subset([0, 1, 2, 3, 10])).toBeInstanceOf(Dataframe.DenseInt32Index);
+    expect(idx.subset([4, 3, 2, 1])).toBeInstanceOf(Dataframe.DenseInt32Index);
+    expect(idx.subset([4])).toBeInstanceOf(Dataframe.KeyIndex);
+
+    expect(idx.withLabel(99).labels()).toEqual(new Int32Array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 99]));
+    expect(idx.dropLabel(0).labels()).toEqual(new Int32Array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]));
+    expect(idx.dropLabel(11).labels()).toEqual(new Int32Array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]));
+    expect(idx.dropLabel(5).labels()).toEqual(new Int32Array([0, 1, 2, 3, 4, 6, 7, 8, 9, 10, 11]));
+  });
+
+  test("DenseInt32Index", () => {
+
+    const idx = new Dataframe.DenseInt32Index([99, 1002, 48, 0, 22]);
+
+    expect(Dataframe.isLabelIndex(idx)).toBeTruthy();
+
+    expect(idx.labels()).toEqual(new Int32Array([99, 1002, 48, 0, 22]));
+    expect(idx.size()).toEqual(5);
+    expect(idx.getOffset(1002)).toEqual(1);
+    expect(idx.getOffset(0)).toEqual(3);
+    expect(idx.getLabel(0)).toEqual(99);
+    expect(idx.getLabels(new Int32Array([2, 4]))).toEqual(new Int32Array([48, 22]));
+    expect(idx.getLabels([2, 4])).toEqual([48, 22]);
+    expect(idx.getOffsets([0, 48])).toEqual([3, 2]);
+
+    expect(idx.subset([1002, 0, 99]).labels()).toEqual(new Int32Array([1002, 0, 99]))
+    expect(idx.getOffsets(idx.subset([1002, 0, 99]).labels())).toEqual(new Int32Array([1, 3, 0]));
+    expect(idx.isubset([4, 1, 2]).labels()).toEqual(new Int32Array([22, 1002, 48]));
+
+    expect(idx.withLabel(88).labels()).toEqual(new Int32Array([99, 1002, 48, 0, 22, 88]));
+    expect(idx.withLabel(88).getOffset(88)).toEqual(5);
+    expect(idx.dropLabel(48).labels()).toEqual(new Int32Array([99, 1002, 0, 22]));
+  });
+
+  test("KeyIndex", () => {
+    const idx = new Dataframe.KeyIndex(["red", "green", "blue"]);
+
+    expect(Dataframe.isLabelIndex(idx)).toBeTruthy();
+
+    expect(idx.labels()).toEqual(["red", "green", "blue"]);
+    expect(idx.size()).toEqual(3);
+    expect(idx.getOffset("blue")).toEqual(2);
+    expect(idx.getLabel(1)).toEqual("green");
+
+    expect(idx.subset(["green"]).labels()).toEqual(["green"]);
+    expect(idx.subset(["green", "red"]).labels()).toEqual(["green", "red"]);
+    expect(idx.isubset([2, 1, 0]).labels()).toEqual(["blue", "green", "red"]);
+
+    expect(idx.withLabel("yo").labels()).toEqual(["red", "green", "blue", "yo"]);
+    expect(idx.withLabel("yo").getOffset("yo")).toEqual(3);
+    expect(idx.dropLabel("blue").labels()).toEqual(["red", "green"]);
+});
+
+})

client/__tests__/util/stateManager/controlsHelpers.test.js

@@ -4,11 +4,21 @@ test controls helpers
 import { subsetAndResetGeneLists } from "../../../src/util/stateManager/controlsHelpers";
 import * as globals from "../../../src/globals";
 
+
 describe("controls helpers", () => {
   test("subsetAndResetGeneLists", () => {
-    const geneList = [...Array(150).keys()].map(
+    const geneList = [];
-      () => Math.random().toString(36).substring(2, 6) // random string of 4 characters
+    const genRandGene = () => Math.random().toString(36).substring(2, 6);
-    );
+
+    // build a unique set of genes
+    for (let i = 0; i < 150; i += 1) {
+      let randGene = genRandGene();
+      while (geneList.includes(randGene)) randGene = genRandGene();
+      geneList.push(randGene);
+    }
+    // insert duplicates
+    geneList[0] = "dupl";
+    geneList[20] = "dupl";
     const state = {
       userDefinedGenes: geneList.slice(0, 20),
       diffexpGenes: geneList.slice(20),
@@ -16,12 +26,14 @@ describe("controls helpers", () => {
     const [newUserDefinedGenes, newDiffExpGenes] = subsetAndResetGeneLists(
       state
     );
+    const expectedNewUserDefinedGenes = [
+      ...geneList.slice(0, 20),
+      ...geneList.slice(21)
+    ].slice(0, globals.maxGenes);
     expect(globals.maxUserDefinedGenes).toBeLessThan(globals.maxGenes);
     expect(geneList.length).toBeGreaterThan(globals.maxGenes);
     expect(newUserDefinedGenes).toHaveLength(globals.maxGenes);
-    expect(newUserDefinedGenes).toStrictEqual(
+    expect(newUserDefinedGenes).toStrictEqual(expectedNewUserDefinedGenes);
-      geneList.slice(0, globals.maxGenes)
-    );
     expect(newDiffExpGenes).toStrictEqual([]);
   });
 });

client/__tests__/util/stateManager/fbs.test.js

@@ -27,7 +27,7 @@ describe("encode/decode", () => {
     const dfWithColIdx = new Dataframe([3, 4], columns, null, colIndex);
     const dfB = decodeMatrixFBS(encodeMatrixFBS(dfWithColIdx));
     expect([dfB.nRows, dfB.nCols]).toEqual(dfWithColIdx.dims);
-    expect(dfB.colIdx).toEqual(colIndex.keys());
+    expect(dfB.colIdx).toEqual(colIndex.labels());
     expect(dfB.rowIdx).toBeNull();
     expect(dfB.columns).toEqual(columns);
   });

client/__tests__/util/stateManager/universe.test.js

@@ -1,4 +1,5 @@
 import * as Universe from "../../../src/util/stateManager/universe";
+import { matrixFBSToDataframe } from "../../../src/util/stateManager/matrix";
 import * as Dataframe from "../../../src/util/dataframe";
 import * as REST from "./sampleResponses";
 
@@ -51,16 +52,13 @@ describe("createUniverseFromResponse", () => {
       ...universe,
       ...Universe.addObsAnnotations(
         universe,
-        Universe.matrixFBSToDataframe(REST.annotationsObs)
+        matrixFBSToDataframe(REST.annotationsObs)
       ),
       ...Universe.addVarAnnotations(
         universe,
-        Universe.matrixFBSToDataframe(REST.annotationsVar)
+        matrixFBSToDataframe(REST.annotationsVar)
-      ),
-      ...Universe.addObsLayout(
-        universe,
-        Universe.matrixFBSToDataframe(REST.layoutObs)
       ),
+      ...Universe.addObsLayout(universe, matrixFBSToDataframe(REST.layoutObs)),
     };
 
     expect(universe).toMatchObject(
@@ -80,7 +78,7 @@ describe("createUniverseFromResponse", () => {
       REST.schema.schema.annotations.obs.columns.length,
     ]);
     expect(universe.obsLayout.dims).toEqual([nObs, 2]);
-    expect(universe.obsLayout.colIndex.keys()).toEqual(
+    expect(universe.obsLayout.colIndex.labels()).toEqual(
       universe.schema.layout.obs[0].dims
     );
     expect(universe.varAnnotations.dims).toEqual([

client/__tests__/util/stateManager/world.test.js

@@ -1,5 +1,6 @@
 import _ from "lodash";
 import * as Universe from "../../../src/util/stateManager/universe";
+import { matrixFBSToDataframe } from "../../../src/util/stateManager/matrix";
 import * as World from "../../../src/util/stateManager/world";
 import * as Dataframe from "../../../src/util/dataframe";
 import Crossfilter from "../../../src/util/typedCrossfilter";
@@ -26,16 +27,13 @@ const defaultBigBang = () => {
     ...universe,
     ...Universe.addObsAnnotations(
       universe,
-      Universe.matrixFBSToDataframe(REST.annotationsObs)
+      matrixFBSToDataframe(REST.annotationsObs)
     ),
     ...Universe.addVarAnnotations(
       universe,
-      Universe.matrixFBSToDataframe(REST.annotationsVar)
+      matrixFBSToDataframe(REST.annotationsVar)
-    ),
-    ...Universe.addObsLayout(
-      universe,
-      Universe.matrixFBSToDataframe(REST.layoutObs)
     ),
+    ...Universe.addObsLayout(universe, matrixFBSToDataframe(REST.layoutObs)),
   };
 
   /* create world */
@@ -59,9 +57,9 @@ describe("createWorldFromEntireUniverse", () => {
     const universe = Universe.createUniverseFromResponse(
       _.cloneDeep(REST.config),
       _.cloneDeep(REST.schema),
-      Universe.matrixFBSToDataframe(_.cloneDeep(REST.annotationsObs)),
+      matrixFBSToDataframe(_.cloneDeep(REST.annotationsObs)),
-      Universe.matrixFBSToDataframe(_.cloneDeep(REST.annotationsVar)),
+      matrixFBSToDataframe(_.cloneDeep(REST.annotationsVar)),
-      Universe.matrixFBSToDataframe(_.cloneDeep(REST.layoutObs))
+      matrixFBSToDataframe(_.cloneDeep(REST.layoutObs))
     );
     expect(universe).toBeDefined();
 
@@ -144,16 +142,16 @@ describe("createWorldFromCurrentSelection", () => {
       })
     );
 
-    expect(world.obsAnnotations.rowIndex.keys()).toEqual(
+    expect(world.obsAnnotations.rowIndex.labels()).toEqual(
       new Int32Array(matchingIndices)
     );
-    expect(world.obsAnnotations.colIndex.keys()).toEqual(
+    expect(world.obsAnnotations.colIndex.labels()).toEqual(
-      universe.obsAnnotations.colIndex.keys()
+      universe.obsAnnotations.colIndex.labels()
     );
-    expect(world.obsLayout.rowIndex.keys()).toEqual(
+    expect(world.obsLayout.rowIndex.labels()).toEqual(
       new Int32Array(matchingIndices)
     );
-    expect(world.obsLayout.colIndex.keys()).toEqual(
+    expect(world.obsLayout.colIndex.labels()).toEqual(
       world.schema.layout.obs[0].dims
     );
   });

client/configuration/eslint/eslint.js

@@ -1,7 +1,7 @@
 module.exports = {
   root: true,
   parser: "babel-eslint",
-  extends: ["airbnb", "prettier", "prettier/react"],
+  extends: ["airbnb", "plugin:prettier/recommended", "prettier/react"],
   env: { browser: true, commonjs: true, es6: true },
   globals: { expect: true },
   parserOptions: {
@@ -21,13 +21,7 @@ module.exports = {
     "react/jsx-filename-extension": "off",
     "comma-dangle": "off",
     "no-underscore-dangle": "off",
-    quotes: ["error", "double"],
     "implicit-arrow-linebreak": "off",
-    "operator-linebreak": [
-      "error",
-      "after",
-      { overrides: { "?": "before", ":": "before" } },
-    ],
     "no-console": "off",
     "spaced-comment": ["error", "always", { exceptions: ["*"] }],
     "no-param-reassign": "off",

client/configuration/lint-staged/lint-staged.config.js

@@ -0,0 +1,3 @@
+module.exports = {
+  "./src/**/*.js": "eslint --fix",
+};

client/package-lock.json

@@ -2547,6 +2547,15 @@
         }
       }
     },
+    "@samverschueren/stream-to-observable": {
+      "version": "0.3.0",
+      "resolved": "https://registry.npmjs.org/@samverschueren/stream-to-observable/-/stream-to-observable-0.3.0.tgz",
+      "integrity": "sha512-MI4Xx6LHs4Webyvi6EbspgyAb4D2Q2VtnCQ1blOJcoLS6mVa8lNN2rkIy1CVxfTUpoyIbCTkXES1rLXztFD1lg==",
+      "dev": true,
+      "requires": {
+        "any-observable": "^0.3.0"
+      }
+    },
     "@sentry/cli": {
       "version": "1.52.3",
       "resolved": "https://registry.npmjs.org/@sentry/cli/-/cli-1.52.3.tgz",
@@ -2744,6 +2753,12 @@
       "integrity": "sha512-uM4mnmsIIPK/yeO+42F2RQhGUIs39K2RFmugcJANppXe6J1nvH87PvzPZYpza7Xhhs8Yn9yIAVdLZ84z61+0xQ==",
       "dev": true
     },
+    "@types/parse-json": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/@types/parse-json/-/parse-json-4.0.0.tgz",
+      "integrity": "sha512-//oorEZjL6sbPcKUaCdIGlIUeH26mgzimjBB77G6XRgnDl/L5wOnpyBGRe/Mmf5CVW3PwEBE1NjiMZ/ssFh4wA==",
+      "dev": true
+    },
     "@types/prettier": {
       "version": "1.19.1",
       "resolved": "https://registry.npmjs.org/@types/prettier/-/prettier-1.19.1.tgz",
@@ -3263,6 +3278,12 @@
       "integrity": "sha512-uMgjozySS8adZZYePpaWs8cxB9/kdzmpX6SgJZ+wbz1K5eYk5QMYDVJaZKhxyIHUdnnJkfR7SVgStgH7LkGUyg==",
       "dev": true
     },
+    "any-observable": {
+      "version": "0.3.0",
+      "resolved": "https://registry.npmjs.org/any-observable/-/any-observable-0.3.0.tgz",
+      "integrity": "sha512-/FQM1EDkTsf63Ub2C6O7GuYFDsSXUwsaZDurV0np41ocwq0jthUAYCmhBX9f+KwlaCgIuWyr/4WlUQUBfKfZog==",
+      "dev": true
+    },
     "anymatch": {
       "version": "3.1.1",
       "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.1.tgz",
@@ -5378,6 +5399,60 @@
         "restore-cursor": "^3.1.0"
       }
     },
+    "cli-truncate": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/cli-truncate/-/cli-truncate-2.1.0.tgz",
+      "integrity": "sha512-n8fOixwDD6b/ObinzTrp1ZKFzbgvKZvuz/TvejnLn1aQfC6r52XEx85FmuC+3HI+JM7coBRXUvNqEU2PHVrHpg==",
+      "dev": true,
+      "requires": {
+        "slice-ansi": "^3.0.0",
+        "string-width": "^4.2.0"
+      },
+      "dependencies": {
+        "ansi-styles": {
+          "version": "4.2.1",
+          "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.2.1.tgz",
+          "integrity": "sha512-9VGjrMsG1vePxcSweQsN20KY/c4zN0h9fLjqAbwbPfahM3t+NL+M9HC8xeXG2I8pX5NoamTGNuomEUFI7fcUjA==",
+          "dev": true,
+          "requires": {
+            "@types/color-name": "^1.1.1",
+            "color-convert": "^2.0.1"
+          }
+        },
+        "astral-regex": {
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/astral-regex/-/astral-regex-2.0.0.tgz",
+          "integrity": "sha512-Z7tMw1ytTXt5jqMcOP+OQteU1VuNK9Y02uuJtKQ1Sv69jXQKKg5cibLwGJow8yzZP+eAc18EmLGPal0bp36rvQ==",
+          "dev": true
+        },
+        "color-convert": {
+          "version": "2.0.1",
+          "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+          "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
+          "dev": true,
+          "requires": {
+            "color-name": "~1.1.4"
+          }
+        },
+        "color-name": {
+          "version": "1.1.4",
+          "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+          "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
+          "dev": true
+        },
+        "slice-ansi": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/slice-ansi/-/slice-ansi-3.0.0.tgz",
+          "integrity": "sha512-pSyv7bSTC7ig9Dcgbw9AuRNUb5k5V6oDudjZoMBSr13qpLBG7tB+zgCkARjq7xIUgdz5P1Qe8u+rSGdouOOIyQ==",
+          "dev": true,
+          "requires": {
+            "ansi-styles": "^4.0.0",
+            "astral-regex": "^2.0.0",
+            "is-fullwidth-code-point": "^3.0.0"
+          }
+        }
+      }
+    },
     "cli-width": {
       "version": "2.2.0",
       "resolved": "https://registry.npmjs.org/cli-width/-/cli-width-2.2.0.tgz",
@@ -5585,6 +5660,12 @@
       "integrity": "sha1-3dgA2gxmEnOTzKWVDqloo6rxJTs=",
       "dev": true
     },
+    "compare-versions": {
+      "version": "3.6.0",
+      "resolved": "https://registry.npmjs.org/compare-versions/-/compare-versions-3.6.0.tgz",
+      "integrity": "sha512-W6Af2Iw1z4CB7q4uU4hv646dW9GQuBM+YpC0UvUCWSD8w90SJjp+ujJuXaEMtAXBtSqGfMPuFOVn4/+FlaqfBA==",
+      "dev": true
+    },
     "component-emitter": {
       "version": "1.3.0",
       "resolved": "https://registry.npmjs.org/component-emitter/-/component-emitter-1.3.0.tgz",
@@ -6517,6 +6598,12 @@
         "mimic-response": "^2.0.0"
       }
     },
+    "dedent": {
+      "version": "0.7.0",
+      "resolved": "https://registry.npmjs.org/dedent/-/dedent-0.7.0.tgz",
+      "integrity": "sha1-JJXduvbrh0q7Dhvp3yLS5aVEMmw=",
+      "dev": true
+    },
     "deep-equal": {
       "version": "1.1.1",
       "resolved": "https://registry.npmjs.org/deep-equal/-/deep-equal-1.1.1.tgz",
@@ -6548,6 +6635,23 @@
       "integrity": "sha512-FJ3UgI4gIl+PHZm53knsuSFpE+nESMr7M4v9QcgB7S63Kj/6WqMiFQJpBBYz1Pt+66bZpP3Q7Lye0Oo9MPKEdg==",
       "dev": true
     },
+    "defaults": {
+      "version": "1.0.3",
+      "resolved": "https://registry.npmjs.org/defaults/-/defaults-1.0.3.tgz",
+      "integrity": "sha1-xlYFHpgX2f8I7YgUd/P+QBnz730=",
+      "dev": true,
+      "requires": {
+        "clone": "^1.0.2"
+      },
+      "dependencies": {
+        "clone": {
+          "version": "1.0.4",
+          "resolved": "https://registry.npmjs.org/clone/-/clone-1.0.4.tgz",
+          "integrity": "sha1-2jCcwmPfFZlMaIypAheco8fNfH4=",
+          "dev": true
+        }
+      }
+    },
     "define-properties": {
       "version": "1.1.3",
       "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.1.3.tgz",
@@ -6911,6 +7015,12 @@
       "integrity": "sha512-zcUd1p/7yzTSdWkCTrqGvbnEOASy96d0RJL/lc5BDJoO23Z3G/VHd0yIPbguDU9n8QNUTCigLO7oEdtOb7fp2A==",
       "dev": true
     },
+    "elegant-spinner": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/elegant-spinner/-/elegant-spinner-2.0.0.tgz",
+      "integrity": "sha512-5YRYHhvhYzV/FC4AiMdeSIg3jAYGq9xFvbhZMpPlJoBsfYgrw2DSCYeXfat6tYBu45PWiyRr3+flaCPPmviPaA==",
+      "dev": true
+    },
     "elliptic": {
       "version": "6.5.2",
       "resolved": "https://registry.npmjs.org/elliptic/-/elliptic-6.5.2.tgz",
@@ -6976,6 +7086,15 @@
         }
       }
     },
+    "enquirer": {
+      "version": "2.3.5",
+      "resolved": "https://registry.npmjs.org/enquirer/-/enquirer-2.3.5.tgz",
+      "integrity": "sha512-BNT1C08P9XD0vNg3J475yIUG+mVdp9T6towYFHUv897X0KoHBjB1shyrNmhmtHWKP17iSWgo7Gqh7BBuzLZMSA==",
+      "dev": true,
+      "requires": {
+        "ansi-colors": "^3.2.1"
+      }
+    },
     "entities": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/entities/-/entities-2.0.0.tgz",
@@ -7411,6 +7530,15 @@
         }
       }
     },
+    "eslint-plugin-prettier": {
+      "version": "3.1.3",
+      "resolved": "https://registry.npmjs.org/eslint-plugin-prettier/-/eslint-plugin-prettier-3.1.3.tgz",
+      "integrity": "sha512-+HG5jmu/dN3ZV3T6eCD7a4BlAySdN7mLIbJYo0z1cFQuI+r2DiTJEFeF68ots93PsnrMxbzIZ2S/ieX+mkrBeQ==",
+      "dev": true,
+      "requires": {
+        "prettier-linter-helpers": "^1.0.0"
+      }
+    },
     "eslint-plugin-react": {
       "version": "7.19.0",
       "resolved": "https://registry.npmjs.org/eslint-plugin-react/-/eslint-plugin-react-7.19.0.tgz",
@@ -7907,6 +8035,12 @@
       "integrity": "sha512-8UEa58QDLauDNfpbrX55Q9jrGHThw2ZMdOky5Gl1CDtVeJDPVrG4Jxx1N8jw2gkWaff5UUuX1KJd+9zGe2B+ZA==",
       "dev": true
     },
+    "fast-diff": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/fast-diff/-/fast-diff-1.2.0.tgz",
+      "integrity": "sha512-xJuoT5+L99XlZ8twedaRf6Ax2TgQVxvgZOYoPKqZufmJib0tL2tegPBOZb1pVNgIhlqDlA0eO0c3wBvQcmzx4w==",
+      "dev": true
+    },
     "fast-json-stable-stringify": {
       "version": "2.1.0",
       "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz",
@@ -8290,6 +8424,15 @@
         "locate-path": "^3.0.0"
       }
     },
+    "find-versions": {
+      "version": "3.2.0",
+      "resolved": "https://registry.npmjs.org/find-versions/-/find-versions-3.2.0.tgz",
+      "integrity": "sha512-P8WRou2S+oe222TOCHitLy8zj+SIsVJh52VP4lvXkaFVnOFFdoWv1H1Jjvel1aI6NCFOAaeAVm8qrI0odiLcww==",
+      "dev": true,
+      "requires": {
+        "semver-regex": "^2.0.0"
+      }
+    },
     "findup-sync": {
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/findup-sync/-/findup-sync-3.0.0.tgz",
@@ -8694,6 +8837,12 @@
       "integrity": "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==",
       "dev": true
     },
+    "get-own-enumerable-property-symbols": {
+      "version": "3.0.2",
+      "resolved": "https://registry.npmjs.org/get-own-enumerable-property-symbols/-/get-own-enumerable-property-symbols-3.0.2.tgz",
+      "integrity": "sha512-I0UBV/XOz1XkIJHEUDMZAbzCThU/H8DxmSfmdGcKPnVhu2VfFqr34jr9777IyaTYvxjedWhqVIilEDsCdP5G6g==",
+      "dev": true
+    },
     "get-stdin": {
       "version": "6.0.0",
       "resolved": "https://registry.npmjs.org/get-stdin/-/get-stdin-6.0.0.tgz",
@@ -9301,6 +9450,100 @@
       "integrity": "sha512-SEQu7vl8KjNL2eoGBLF3+wAjpsNfA9XMlXAYj/3EdaNfAlxKthD1xjEQfGOUhllCGGJVNY34bRr6lPINhNjyZw==",
       "dev": true
     },
+    "husky": {
+      "version": "4.2.5",
+      "resolved": "https://registry.npmjs.org/husky/-/husky-4.2.5.tgz",
+      "integrity": "sha512-SYZ95AjKcX7goYVZtVZF2i6XiZcHknw50iXvY7b0MiGoj5RwdgRQNEHdb+gPDPCXKlzwrybjFjkL6FOj8uRhZQ==",
+      "dev": true,
+      "requires": {
+        "chalk": "^4.0.0",
+        "ci-info": "^2.0.0",
+        "compare-versions": "^3.6.0",
+        "cosmiconfig": "^6.0.0",
+        "find-versions": "^3.2.0",
+        "opencollective-postinstall": "^2.0.2",
+        "pkg-dir": "^4.2.0",
+        "please-upgrade-node": "^3.2.0",
+        "slash": "^3.0.0",
+        "which-pm-runs": "^1.0.0"
+      },
+      "dependencies": {
+        "cosmiconfig": {
+          "version": "6.0.0",
+          "resolved": "https://registry.npmjs.org/cosmiconfig/-/cosmiconfig-6.0.0.tgz",
+          "integrity": "sha512-xb3ZL6+L8b9JLLCx3ZdoZy4+2ECphCMo2PwqgP1tlfVq6M6YReyzBJtvWWtbDSpNr9hn96pkCiZqUcFEc+54Qg==",
+          "dev": true,
+          "requires": {
+            "@types/parse-json": "^4.0.0",
+            "import-fresh": "^3.1.0",
+            "parse-json": "^5.0.0",
+            "path-type": "^4.0.0",
+            "yaml": "^1.7.2"
+          }
+        },
+        "find-up": {
+          "version": "4.1.0",
+          "resolved": "https://registry.npmjs.org/find-up/-/find-up-4.1.0.tgz",
+          "integrity": "sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==",
+          "dev": true,
+          "requires": {
+            "locate-path": "^5.0.0",
+            "path-exists": "^4.0.0"
+          }
+        },
+        "locate-path": {
+          "version": "5.0.0",
+          "resolved": "https://registry.npmjs.org/locate-path/-/locate-path-5.0.0.tgz",
+          "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==",
+          "dev": true,
+          "requires": {
+            "p-locate": "^4.1.0"
+          }
+        },
+        "p-locate": {
+          "version": "4.1.0",
+          "resolved": "https://registry.npmjs.org/p-locate/-/p-locate-4.1.0.tgz",
+          "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==",
+          "dev": true,
+          "requires": {
+            "p-limit": "^2.2.0"
+          }
+        },
+        "parse-json": {
+          "version": "5.0.0",
+          "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-5.0.0.tgz",
+          "integrity": "sha512-OOY5b7PAEFV0E2Fir1KOkxchnZNCdowAJgQ5NuxjpBKTRP3pQhwkrkxqQjeoKJ+fO7bCpmIZaogI4eZGDMEGOw==",
+          "dev": true,
+          "requires": {
+            "@babel/code-frame": "^7.0.0",
+            "error-ex": "^1.3.1",
+            "json-parse-better-errors": "^1.0.1",
+            "lines-and-columns": "^1.1.6"
+          }
+        },
+        "path-exists": {
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-4.0.0.tgz",
+          "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==",
+          "dev": true
+        },
+        "path-type": {
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/path-type/-/path-type-4.0.0.tgz",
+          "integrity": "sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==",
+          "dev": true
+        },
+        "pkg-dir": {
+          "version": "4.2.0",
+          "resolved": "https://registry.npmjs.org/pkg-dir/-/pkg-dir-4.2.0.tgz",
+          "integrity": "sha512-HRDzbaKjC+AOWVXxAU/x54COGeIv9eb+6CkDSQoNTt4XyWoIJvuPsXizxu/Fr23EiekbtZwmh1IcIG/l/a10GQ==",
+          "dev": true,
+          "requires": {
+            "find-up": "^4.0.0"
+          }
+        }
+      }
+    },
     "iconv-lite": {
       "version": "0.4.24",
       "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz",
@@ -9869,6 +10112,12 @@
         "has": "^1.0.3"
       }
     },
+    "is-regexp": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/is-regexp/-/is-regexp-1.0.0.tgz",
+      "integrity": "sha1-/S2INUXEa6xaYz57mgnof6LLUGk=",
+      "dev": true
+    },
     "is-resolvable": {
       "version": "1.1.0",
       "resolved": "https://registry.npmjs.org/is-resolvable/-/is-resolvable-1.1.0.tgz",
@@ -11739,6 +11988,208 @@
         "type-check": "~0.3.2"
       }
     },
+    "lines-and-columns": {
+      "version": "1.1.6",
+      "resolved": "https://registry.npmjs.org/lines-and-columns/-/lines-and-columns-1.1.6.tgz",
+      "integrity": "sha1-HADHQ7QzzQpOgHWPe2SldEDZ/wA=",
+      "dev": true
+    },
+    "lint-staged": {
+      "version": "10.2.4",
+      "resolved": "https://registry.npmjs.org/lint-staged/-/lint-staged-10.2.4.tgz",
+      "integrity": "sha512-doTMGKXQAT34c3S3gwDrTnXmCZp/z1/92D8suPqqh755sKPT18ew1NoPNHxJdrvv1D4WrJ7CEnx79Ns3EdEFbg==",
+      "dev": true,
+      "requires": {
+        "chalk": "^4.0.0",
+        "commander": "^5.1.0",
+        "cosmiconfig": "^6.0.0",
+        "debug": "^4.1.1",
+        "dedent": "^0.7.0",
+        "execa": "^4.0.1",
+        "listr2": "^2.0.2",
+        "log-symbols": "^4.0.0",
+        "micromatch": "^4.0.2",
+        "normalize-path": "^3.0.0",
+        "please-upgrade-node": "^3.2.0",
+        "string-argv": "0.3.1",
+        "stringify-object": "^3.3.0"
+      },
+      "dependencies": {
+        "commander": {
+          "version": "5.1.0",
+          "resolved": "https://registry.npmjs.org/commander/-/commander-5.1.0.tgz",
+          "integrity": "sha512-P0CysNDQ7rtVw4QIQtm+MRxV66vKFSvlsQvGYXZWR3qFU0jlMKHZZZgw8e+8DSah4UDKMqnknRDQz+xuQXQ/Zg==",
+          "dev": true
+        },
+        "cosmiconfig": {
+          "version": "6.0.0",
+          "resolved": "https://registry.npmjs.org/cosmiconfig/-/cosmiconfig-6.0.0.tgz",
+          "integrity": "sha512-xb3ZL6+L8b9JLLCx3ZdoZy4+2ECphCMo2PwqgP1tlfVq6M6YReyzBJtvWWtbDSpNr9hn96pkCiZqUcFEc+54Qg==",
+          "dev": true,
+          "requires": {
+            "@types/parse-json": "^4.0.0",
+            "import-fresh": "^3.1.0",
+            "parse-json": "^5.0.0",
+            "path-type": "^4.0.0",
+            "yaml": "^1.7.2"
+          }
+        },
+        "cross-spawn": {
+          "version": "7.0.2",
+          "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.2.tgz",
+          "integrity": "sha512-PD6G8QG3S4FK/XCGFbEQrDqO2AnMMsy0meR7lerlIOHAAbkuavGU/pOqprrlvfTNjvowivTeBsjebAL0NSoMxw==",
+          "dev": true,
+          "requires": {
+            "path-key": "^3.1.0",
+            "shebang-command": "^2.0.0",
+            "which": "^2.0.1"
+          }
+        },
+        "execa": {
+          "version": "4.0.1",
+          "resolved": "https://registry.npmjs.org/execa/-/execa-4.0.1.tgz",
+          "integrity": "sha512-SCjM/zlBdOK8Q5TIjOn6iEHZaPHFsMoTxXQ2nvUvtPnuohz3H2dIozSg+etNR98dGoYUp2ENSKLL/XaMmbxVgw==",
+          "dev": true,
+          "requires": {
+            "cross-spawn": "^7.0.0",
+            "get-stream": "^5.0.0",
+            "human-signals": "^1.1.1",
+            "is-stream": "^2.0.0",
+            "merge-stream": "^2.0.0",
+            "npm-run-path": "^4.0.0",
+            "onetime": "^5.1.0",
+            "signal-exit": "^3.0.2",
+            "strip-final-newline": "^2.0.0"
+          }
+        },
+        "get-stream": {
+          "version": "5.1.0",
+          "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-5.1.0.tgz",
+          "integrity": "sha512-EXr1FOzrzTfGeL0gQdeFEvOMm2mzMOglyiOXSTpPC+iAjAKftbr3jpCMWynogwYnM+eSj9sHGc6wjIcDvYiygw==",
+          "dev": true,
+          "requires": {
+            "pump": "^3.0.0"
+          }
+        },
+        "is-stream": {
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.0.tgz",
+          "integrity": "sha512-XCoy+WlUr7d1+Z8GgSuXmpuUFC9fOhRXglJMx+dwLKTkL44Cjd4W1Z5P+BQZpr+cR93aGP4S/s7Ftw6Nd/kiEw==",
+          "dev": true
+        },
+        "npm-run-path": {
+          "version": "4.0.1",
+          "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-4.0.1.tgz",
+          "integrity": "sha512-S48WzZW777zhNIrn7gxOlISNAqi9ZC/uQFnRdbeIHhZhCA6UqpkOT8T1G7BvfdgP4Er8gF4sUbaS0i7QvIfCWw==",
+          "dev": true,
+          "requires": {
+            "path-key": "^3.0.0"
+          }
+        },
+        "parse-json": {
+          "version": "5.0.0",
+          "resolved": "https://registry.npmjs.org/parse-json/-/parse-json-5.0.0.tgz",
+          "integrity": "sha512-OOY5b7PAEFV0E2Fir1KOkxchnZNCdowAJgQ5NuxjpBKTRP3pQhwkrkxqQjeoKJ+fO7bCpmIZaogI4eZGDMEGOw==",
+          "dev": true,
+          "requires": {
+            "@babel/code-frame": "^7.0.0",
+            "error-ex": "^1.3.1",
+            "json-parse-better-errors": "^1.0.1",
+            "lines-and-columns": "^1.1.6"
+          }
+        },
+        "path-key": {
+          "version": "3.1.1",
+          "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz",
+          "integrity": "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q==",
+          "dev": true
+        },
+        "path-type": {
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/path-type/-/path-type-4.0.0.tgz",
+          "integrity": "sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==",
+          "dev": true
+        },
+        "shebang-command": {
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz",
+          "integrity": "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA==",
+          "dev": true,
+          "requires": {
+            "shebang-regex": "^3.0.0"
+          }
+        },
+        "shebang-regex": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/shebang-regex/-/shebang-regex-3.0.0.tgz",
+          "integrity": "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A==",
+          "dev": true
+        },
+        "which": {
+          "version": "2.0.2",
+          "resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz",
+          "integrity": "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA==",
+          "dev": true,
+          "requires": {
+            "isexe": "^2.0.0"
+          }
+        }
+      }
+    },
+    "listr2": {
+      "version": "2.0.4",
+      "resolved": "https://registry.npmjs.org/listr2/-/listr2-2.0.4.tgz",
+      "integrity": "sha512-oJaAcplPsa72rKW0eg4P4LbEJjhH+UO2I8uqR/I2wzHrVg16ohSfUy0SlcHS21zfYXxtsUpL8YXGHjyfWMR0cg==",
+      "dev": true,
+      "requires": {
+        "@samverschueren/stream-to-observable": "^0.3.0",
+        "chalk": "^4.0.0",
+        "cli-cursor": "^3.1.0",
+        "cli-truncate": "^2.1.0",
+        "elegant-spinner": "^2.0.0",
+        "enquirer": "^2.3.5",
+        "figures": "^3.2.0",
+        "indent-string": "^4.0.0",
+        "log-update": "^4.0.0",
+        "p-map": "^4.0.0",
+        "pad": "^3.2.0",
+        "rxjs": "^6.5.5",
+        "through": "^2.3.8",
+        "uuid": "^7.0.2"
+      },
+      "dependencies": {
+        "indent-string": {
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/indent-string/-/indent-string-4.0.0.tgz",
+          "integrity": "sha512-EdDDZu4A2OyIK7Lr/2zG+w5jmbuk1DVBnEwREQvBzspBJkCEbRa8GxU1lghYcaGJCnRWibjDXlq779X1/y5xwg==",
+          "dev": true
+        },
+        "p-map": {
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/p-map/-/p-map-4.0.0.tgz",
+          "integrity": "sha512-/bjOqmgETBYB5BoEeGVea8dmvHb2m9GLy1E9W43yeyfP6QQCZGFNa+XRceJEuDB6zqr+gKpIAmlLebMpykw/MQ==",
+          "dev": true,
+          "requires": {
+            "aggregate-error": "^3.0.0"
+          }
+        },
+        "rxjs": {
+          "version": "6.5.5",
+          "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-6.5.5.tgz",
+          "integrity": "sha512-WfQI+1gohdf0Dai/Bbmk5L5ItH5tYqm3ki2c5GdWhKjalzjg93N3avFjVStyZZz+A2Em+ZxKH5bNghw9UeylGQ==",
+          "dev": true,
+          "requires": {
+            "tslib": "^1.9.0"
+          }
+        },
+        "uuid": {
+          "version": "7.0.3",
+          "resolved": "https://registry.npmjs.org/uuid/-/uuid-7.0.3.tgz",
+          "integrity": "sha512-DPSke0pXhTZgoF/d+WSt2QaKMCFSfx7QegxEWT+JOuHF5aWrKEn0G+ztjuJg/gG8/ItK+rbPCD/yNv8yyih6Cg==",
+          "dev": true
+        }
+      }
+    },
     "load-bmfont": {
       "version": "1.4.0",
       "resolved": "https://registry.npmjs.org/load-bmfont/-/load-bmfont-1.4.0.tgz",
@@ -11948,6 +12399,71 @@
       "integrity": "sha1-E2Xt9DFIBIHvDRxolXpe2Z1J984=",
       "dev": true
     },
+    "log-symbols": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-4.0.0.tgz",
+      "integrity": "sha512-FN8JBzLx6CzeMrB0tg6pqlGU1wCrXW+ZXGH481kfsBqer0hToTIiHdjH4Mq8xJUbvATujKCvaREGWpGUionraA==",
+      "dev": true,
+      "requires": {
+        "chalk": "^4.0.0"
+      }
+    },
+    "log-update": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/log-update/-/log-update-4.0.0.tgz",
+      "integrity": "sha512-9fkkDevMefjg0mmzWFBW8YkFP91OrizzkW3diF7CpG+S2EYdy4+TVfGwz1zeF8x7hCx1ovSPTOE9Ngib74qqUg==",
+      "dev": true,
+      "requires": {
+        "ansi-escapes": "^4.3.0",
+        "cli-cursor": "^3.1.0",
+        "slice-ansi": "^4.0.0",
+        "wrap-ansi": "^6.2.0"
+      },
+      "dependencies": {
+        "ansi-styles": {
+          "version": "4.2.1",
+          "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.2.1.tgz",
+          "integrity": "sha512-9VGjrMsG1vePxcSweQsN20KY/c4zN0h9fLjqAbwbPfahM3t+NL+M9HC8xeXG2I8pX5NoamTGNuomEUFI7fcUjA==",
+          "dev": true,
+          "requires": {
+            "@types/color-name": "^1.1.1",
+            "color-convert": "^2.0.1"
+          }
+        },
+        "astral-regex": {
+          "version": "2.0.0",
+          "resolved": "https://registry.npmjs.org/astral-regex/-/astral-regex-2.0.0.tgz",
+          "integrity": "sha512-Z7tMw1ytTXt5jqMcOP+OQteU1VuNK9Y02uuJtKQ1Sv69jXQKKg5cibLwGJow8yzZP+eAc18EmLGPal0bp36rvQ==",
+          "dev": true
+        },
+        "color-convert": {
+          "version": "2.0.1",
+          "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
+          "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==",
+          "dev": true,
+          "requires": {
+            "color-name": "~1.1.4"
+          }
+        },
+        "color-name": {
+          "version": "1.1.4",
+          "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz",
+          "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==",
+          "dev": true
+        },
+        "slice-ansi": {
+          "version": "4.0.0",
+          "resolved": "https://registry.npmjs.org/slice-ansi/-/slice-ansi-4.0.0.tgz",
+          "integrity": "sha512-qMCMfhY040cVHT43K9BFygqYbUPFZKHOg7K73mtTWJRb8pyP3fzf4Ixd5SzdEJQ6MRUg/WBnOLxghZtKKurENQ==",
+          "dev": true,
+          "requires": {
+            "ansi-styles": "^4.0.0",
+            "astral-regex": "^2.0.0",
+            "is-fullwidth-code-point": "^3.0.0"
+          }
+        }
+      }
+    },
     "lolex": {
       "version": "5.1.2",
       "resolved": "https://registry.npmjs.org/lolex/-/lolex-5.1.2.tgz",
@@ -12903,6 +13419,12 @@
         "mimic-fn": "^2.1.0"
       }
     },
+    "opencollective-postinstall": {
+      "version": "2.0.2",
+      "resolved": "https://registry.npmjs.org/opencollective-postinstall/-/opencollective-postinstall-2.0.2.tgz",
+      "integrity": "sha512-pVOEP16TrAO2/fjej1IdOyupJY8KDUM1CvsaScRbw6oddvpQoOfGk4ywha0HKKVAD6RkW4x6Q+tNBwhf3Bgpuw==",
+      "dev": true
+    },
     "optimize-css-assets-webpack-plugin": {
       "version": "5.0.3",
       "resolved": "https://registry.npmjs.org/optimize-css-assets-webpack-plugin/-/optimize-css-assets-webpack-plugin-5.0.3.tgz",
@@ -13022,6 +13544,15 @@
         "semver": "^5.1.0"
       }
     },
+    "pad": {
+      "version": "3.2.0",
+      "resolved": "https://registry.npmjs.org/pad/-/pad-3.2.0.tgz",
+      "integrity": "sha512-2u0TrjcGbOjBTJpyewEl4hBO3OeX5wWue7eIFPzQTg6wFSvoaHcBTTUY5m+n0hd04gmTCPuY0kCpVIVuw5etwg==",
+      "dev": true,
+      "requires": {
+        "wcwidth": "^1.0.1"
+      }
+    },
     "pako": {
       "version": "1.0.11",
       "resolved": "https://registry.npmjs.org/pako/-/pako-1.0.11.tgz",
@@ -13376,6 +13907,15 @@
         }
       }
     },
+    "please-upgrade-node": {
+      "version": "3.2.0",
+      "resolved": "https://registry.npmjs.org/please-upgrade-node/-/please-upgrade-node-3.2.0.tgz",
+      "integrity": "sha512-gQR3WpIgNIKwBMVLkpMUeR3e1/E1y42bqDQZfql+kDeXd8COYfM8PQA4X6y7a8u9Ua9FHmsrrmirW2vHs45hWg==",
+      "dev": true,
+      "requires": {
+        "semver-compare": "^1.0.0"
+      }
+    },
     "pn": {
       "version": "1.1.0",
       "resolved": "https://registry.npmjs.org/pn/-/pn-1.1.0.tgz",
@@ -14041,6 +14581,21 @@
       "integrity": "sha1-1PRWKwzjaW5BrFLQ4ALlemNdxtw=",
       "dev": true
     },
+    "prettier": {
+      "version": "2.0.5",
+      "resolved": "https://registry.npmjs.org/prettier/-/prettier-2.0.5.tgz",
+      "integrity": "sha512-7PtVymN48hGcO4fGjybyBSIWDsLU4H4XlvOHfq91pz9kkGlonzwTfYkaIEwiRg/dAJF9YlbsduBAgtYLi+8cFg==",
+      "dev": true
+    },
+    "prettier-linter-helpers": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/prettier-linter-helpers/-/prettier-linter-helpers-1.0.0.tgz",
+      "integrity": "sha512-GbK2cP9nraSSUF9N2XwUwqfzlAFlMNYYl+ShE/V+H8a9uNl/oUqB1w2EL54Jh0OlyRSd8RfWYJ3coVS4TROP2w==",
+      "dev": true,
+      "requires": {
+        "fast-diff": "^1.1.2"
+      }
+    },
     "pretty-bytes": {
       "version": "4.0.2",
       "resolved": "https://registry.npmjs.org/pretty-bytes/-/pretty-bytes-4.0.2.tgz",
@@ -15456,6 +16011,12 @@
       "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==",
       "dev": true
     },
+    "semver-compare": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/semver-compare/-/semver-compare-1.0.0.tgz",
+      "integrity": "sha1-De4hahyUGrN+nvsXiPavxf9VN/w=",
+      "dev": true
+    },
     "semver-diff": {
       "version": "2.1.0",
       "resolved": "https://registry.npmjs.org/semver-diff/-/semver-diff-2.1.0.tgz",
@@ -15465,6 +16026,12 @@
         "semver": "^5.0.3"
       }
     },
+    "semver-regex": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/semver-regex/-/semver-regex-2.0.0.tgz",
+      "integrity": "sha512-mUdIBBvdn0PLOeP3TEkMH7HHeUP3GjsXCwKarjv/kGmUFOYg1VqEemKhoQpWMu6X2I8kHeuVdGibLGkVK+/5Qw==",
+      "dev": true
+    },
     "send": {
       "version": "0.17.1",
       "resolved": "https://registry.npmjs.org/send/-/send-0.17.1.tgz",
@@ -16154,6 +16721,12 @@
       "integrity": "sha1-J5siXfHVgrH1TmWt3UNS4Y+qBxM=",
       "dev": true
     },
+    "string-argv": {
+      "version": "0.3.1",
+      "resolved": "https://registry.npmjs.org/string-argv/-/string-argv-0.3.1.tgz",
+      "integrity": "sha512-a1uQGz7IyVy9YwhqjZIZu1c8JO8dNIe20xBmSS6qu9kv++k3JGzCVmprbNN5Kn+BgzD5E7YYwg1CcjuJMRNsvg==",
+      "dev": true
+    },
     "string-length": {
       "version": "3.1.0",
       "resolved": "https://registry.npmjs.org/string-length/-/string-length-3.1.0.tgz",
@@ -16227,6 +16800,17 @@
         "safe-buffer": "~5.1.0"
       }
     },
+    "stringify-object": {
+      "version": "3.3.0",
+      "resolved": "https://registry.npmjs.org/stringify-object/-/stringify-object-3.3.0.tgz",
+      "integrity": "sha512-rHqiFh1elqCQ9WPLIC8I0Q/g/wj5J1eMkyoiD6eoQApWHP0FtlK7rqnhmabL5VUY9JQCcqwwvlOaSuutekgyrw==",
+      "dev": true,
+      "requires": {
+        "get-own-enumerable-property-symbols": "^3.0.0",
+        "is-obj": "^1.0.1",
+        "is-regexp": "^1.0.0"
+      }
+    },
     "strip-ansi": {
       "version": "5.2.0",
       "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-5.2.0.tgz",
@@ -17775,6 +18359,15 @@
         "neo-async": "^2.5.0"
       }
     },
+    "wcwidth": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/wcwidth/-/wcwidth-1.0.1.tgz",
+      "integrity": "sha1-8LDc+RW8X/FSivrbLA4XtTLaL+g=",
+      "dev": true,
+      "requires": {
+        "defaults": "^1.0.3"
+      }
+    },
     "webidl-conversions": {
       "version": "4.0.2",
       "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-4.0.2.tgz",
@@ -18535,6 +19128,12 @@
       "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==",
       "dev": true
     },
+    "yaml": {
+      "version": "1.10.0",
+      "resolved": "https://registry.npmjs.org/yaml/-/yaml-1.10.0.tgz",
+      "integrity": "sha512-yr2icI4glYaNG+KWONODapy2/jDdMSDnrONSjblABjD9B4Z5LgiircSt8m8sRZFNi08kG9Sm0uSHtEmP3zaEGg==",
+      "dev": true
+    },
     "yargs": {
       "version": "15.3.1",
       "resolved": "https://registry.npmjs.org/yargs/-/yargs-15.3.1.tgz",

client/package.json

@@ -84,6 +84,7 @@
     "eslint-plugin-import": "^2.20.2",
     "eslint-plugin-jest": "^23.8.2",
     "eslint-plugin-jsx-a11y": "^6.2.3",
+    "eslint-plugin-prettier": "^3.1.3",
     "eslint-plugin-react": "^7.19.0",
     "eslint-plugin-react-hooks": "^2.5.1",
     "express": "^4.17.1",
@@ -91,11 +92,14 @@
     "file-loader": "^6.0.0",
     "html-webpack-inline-source-plugin": "^1.0.0-beta.2",
     "html-webpack-plugin": "^4.0.0",
+    "husky": "^4.2.5",
     "jest": "^25.2.7",
     "jest-puppeteer": "^4.4.0",
     "json-loader": "^0.5.7",
+    "lint-staged": "^10.2.4",
     "mini-css-extract-plugin": "^0.9.0",
     "optimize-css-assets-webpack-plugin": "^5.0.3",
+    "prettier": "^2.0.5",
     "puppeteer": "^2.1.1",
     "rimraf": "^3.0.2",
     "serve-favicon": "^2.5.0",
@@ -146,5 +150,10 @@
         ]
       }
     }
+  },
+  "husky": {
+    "hooks": {
+      "pre-commit": "lint-staged --config \"./configuration/lint-staged/lint-staged.config.js\""
+    }
   }
 }

client/src/actions/index.js

@@ -29,7 +29,7 @@ async function obsAnnotationFetchAndLoad(dispatch, schema) {
         fetchBinary(
           `annotations/obs?annotation-name=${encodeURIComponent(col.name)}`
         )
-          .then((buffer) => Universe.matrixFBSToDataframe(buffer))
+          .then((buffer) => MatrixFBS.matrixFBSToDataframe(buffer))
           .then((df) =>
             dispatch({
               type: "universe: column load success",
@@ -52,7 +52,7 @@ async function varAnnotationFetchAndLoad(dispatch, schema) {
   return Promise.all(
     names.map((name) =>
       fetchBinary(`annotations/var?annotation-name=${encodeURIComponent(name)}`)
-        .then((buffer) => Universe.matrixFBSToDataframe(buffer))
+        .then((buffer) => MatrixFBS.matrixFBSToDataframe(buffer))
         .then((df) =>
           dispatch({
             type: "universe: column load success",
@@ -77,7 +77,7 @@ function layoutFetchAndLoad(dispatch, schema) {
       plimit.add(() =>
         fetchBinary(
           `layout/obs?layout-name=${encodeURIComponent(e)}`
-        ).then((buffer) => Universe.matrixFBSToDataframe(buffer))
+        ).then((buffer) => MatrixFBS.matrixFBSToDataframe(buffer))
       )
     )
   ).then((dfs) =>

client/src/actions/reembed.js

@@ -1,5 +1,5 @@
 import { API } from "../globals";
-import { Universe } from "../util/stateManager";
+import { MatrixFBS } from "../util/stateManager";
 import {
   postNetworkErrorToast,
   postAsyncSuccessToast,
@@ -24,7 +24,7 @@ function abortableFetch(request, opts, timeout = 0) {
 
 async function doReembedFetch(dispatch, getState) {
   const state = getState();
-  let cells = state.world.obsAnnotations.rowIndex.keys();
+  let cells = state.world.obsAnnotations.rowIndex.labels();
 
   // These lines ensure that we convert any TypedArray to an Array.
   // This is necessary because JSON.stringify() does some very strange
@@ -80,7 +80,7 @@ export function requestReembed() {
       const res = await doReembedFetch(dispatch, getState);
       const schema = JSON.parse(res.headers.get("CxG-Schema"));
       const buffer = await res.arrayBuffer();
-      const df = Universe.matrixFBSToDataframe(buffer);
+      const df = MatrixFBS.matrixFBSToDataframe(buffer);
       dispatch({
         type: "reembed: request completed",
       });

client/src/components/app.js

@@ -21,7 +21,6 @@ import actions from "../actions";
   graphRenderCounter: state.controls.graphRenderCounter,
 }))
 class App extends React.Component {
-
   componentDidMount() {
     const { dispatch } = this.props;
 
@@ -68,22 +67,21 @@ class App extends React.Component {
             error loading
           </div>
         ) : null}
-        {loading ? null : <Layout>
+        {loading ? null : (
-          <LeftSideBar/>
+          <Layout>
-          {viewportRef =>
+            <LeftSideBar />
-            <>
+            {(viewportRef) => (
-              <MenuBar/>
+              <>
-              <Autosave/>
+                <MenuBar />
-              <TermsOfServicePrompt/>
+                <Autosave />
-              <Legend viewportRef={viewportRef}/>
+                <TermsOfServicePrompt />
-              <Graph
+                <Legend viewportRef={viewportRef} />
-                key={graphRenderCounter}
+                <Graph key={graphRenderCounter} viewportRef={viewportRef} />
-                viewportRef={viewportRef}
+              </>
-              />
+            )}
-            </>
+            <RightSideBar />
-          }
+          </Layout>
-          <RightSideBar/>
+        )}
-        </Layout>}
       </Container>
     );
   }

client/src/components/categorical/category/annoMenuCategory.js

@@ -6,9 +6,13 @@ import {
   MenuItem,
   Popover,
   Position,
+  Tooltip,
+  Icon,
   PopoverInteractionKind,
 } from "@blueprintjs/core";
 
+import * as globals from "../../../globals";
+
 @connect((state) => ({
   annotations: state.annotations,
 }))
@@ -56,46 +60,57 @@ class AnnoMenuCategory extends React.PureComponent {
     return (
       <>
         {isUserAnno ? (
-          <Popover
+          <>
-            interactionKind={PopoverInteractionKind.HOVER}
+            <Tooltip
-            boundary="window"
+              content={createText}
-            position={Position.RIGHT_TOP}
+              position="bottom"
-            content={
+              hoverOpenDelay={globals.tooltipHoverOpenDelay}
-              <Menu>
+            >
-                <MenuItem
+              <Button
-                  icon="tag"
+                style={{ marginLeft: 0, marginRight: 2 }}
-                  data-testclass="handleAddNewLabelToCategory"
+                data-testclass="handleAddNewLabelToCategory"
-                  data-testid={`${metadataField}:add-new-label-to-category`}
+                data-testid={`${metadataField}:add-new-label-to-category`}
-                  onClick={this.activateAddNewLabelMode}
+                icon={<Icon icon="plus" iconSize={10} />}
-                  text={createText}
+                onClick={this.activateAddNewLabelMode}
-                />
+                small
-                <MenuItem
+                minimal
-                  icon="edit"
+              />
-                  disabled={annotations.isEditingCategoryName}
+            </Tooltip>
-                  data-testclass="activateEditCategoryMode"
+            <Popover
-                  data-testid={`${metadataField}:edit-category-mode`}
+              interactionKind={PopoverInteractionKind.HOVER}
-                  onClick={this.activateEditCategoryMode}
+              boundary="window"
-                  text={editText}
+              position={Position.RIGHT_TOP}
-                />
+              content={
-                <MenuItem
+                <Menu>
-                  icon="delete"
+                  <MenuItem
-                  intent="danger"
+                    icon="edit"
-                  data-testclass="handleDeleteCategory"
+                    disabled={annotations.isEditingCategoryName}
-                  data-testid={`${metadataField}:delete-category`}
+                    data-testclass="activateEditCategoryMode"
-                  onClick={this.handleDeleteCategory}
+                    data-testid={`${metadataField}:edit-category-mode`}
-                  text={deleteText}
+                    onClick={this.activateEditCategoryMode}
-                />
+                    text={editText}
-              </Menu>
+                  />
-            }
+                  <MenuItem
-          >
+                    icon="delete"
-            <Button
+                    intent="danger"
-              style={{ marginLeft: 0 }}
+                    data-testclass="handleDeleteCategory"
-              data-testclass="seeActions"
+                    data-testid={`${metadataField}:delete-category`}
-              data-testid={`${metadataField}:see-actions`}
+                    onClick={this.handleDeleteCategory}
-              icon="more"
+                    text={deleteText}
-              minimal
+                  />
-            />
+                </Menu>
-          </Popover>
+              }
+            >
+              <Button
+                style={{ marginLeft: 0, marginRight: 5 }}
+                data-testclass="seeActions"
+                data-testid={`${metadataField}:see-actions`}
+                icon={<Icon icon="more" iconSize={10} />}
+                small
+                minimal
+              />
+            </Popover>
+          </>
         ) : null}
       </>
     );

client/src/components/categorical/category/index.js

@@ -2,20 +2,18 @@ import React from "react";
 import _ from "lodash";
 import { connect } from "react-redux";
 import { FaChevronRight, FaChevronDown } from "react-icons/fa";
-import {
+import { AnchorButton, Button, Tooltip } from "@blueprintjs/core";
-  AnchorButton,
-  Button,
-  Tooltip,
-  Icon,
-  Position,
-} from "@blueprintjs/core";
 import CategoryFlipperLayout from "./categoryFlipperLayout";
 import AnnoMenu from "./annoMenuCategory";
 import AnnoDialogEditCategoryName from "./annoDialogEditCategoryName";
 import AnnoDialogAddLabel from "./annoDialogAddLabel";
+import Truncate from "../../util/truncate";
 
 import * as globals from "../../../globals";
-import maybeTruncateString from "../../../util/maybeTruncateString";
+
+const LABEL_WIDTH = globals.leftSidebarWidth - 100;
+const ANNO_BUTTON_WIDTH = 50;
+const LABEL_WIDTH_ANNO = LABEL_WIDTH - ANNO_BUTTON_WIDTH;
 
 @connect((state, ownProps) => {
   const { metadataField } = ownProps;
@@ -120,10 +118,6 @@ class Category extends React.Component {
     We are still loading this category, so render a "busy" signal.
     */
     const { metadataField } = this.props;
-    const truncatedString = maybeTruncateString(
-      metadataField,
-      globals.categoryDisplayStringMaxLength
-    );
 
     const checkboxID = `category-select-${metadataField}`;
 
@@ -151,26 +145,17 @@ class Category extends React.Component {
               <input disabled id={checkboxID} checked type="checkbox" />
               <span className="bp3-control-indicator" />
             </label>
-            <Tooltip
+            <Truncate>
-              content={metadataField}
-              disabled={truncatedString === null}
-              hoverOpenDelay={globals.tooltipHoverOpenDelayQuick}
-              position={Position.LEFT}
-              usePortal
-              modifiers={{
-                preventOverflow: { enabled: false },
-                hide: { enabled: false },
-              }}
-            >
               <span
                 style={{
                   cursor: "pointer",
                   display: "inline-block",
+                  width: LABEL_WIDTH,
                 }}
               >
-                {truncatedString || metadataField}
+                {metadataField}
               </span>
-            </Tooltip>
+            </Truncate>
           </div>
           <div>
             <Button minimal loading intent="primary" />
@@ -205,21 +190,22 @@ class Category extends React.Component {
       false
     );
 
-    const truncatedString = maybeTruncateString(
-      metadataField,
-      globals.categoryDisplayStringMaxLength
-    );
-
     if (
       !isUserAnno &&
       schema?.annotations?.obsByName[metadataField]?.categories?.length === 1
     ) {
       return (
         <div style={{ marginBottom: 10, marginTop: 4 }}>
-          <span style={{ fontWeight: 700 }}>
+          <Truncate>
-            {truncatedString || metadataField}
+            <span style={{ maxWidth: 150, fontWeight: 700 }}>
-          </span>
+              {metadataField}
-          : {schema.annotations.obsByName[metadataField].categories[0]}
+            </span>
+          </Truncate>
+          <Truncate>
+            <span style={{ maxWidth: 150 }}>
+              {`: ${schema.annotations.obsByName[metadataField].categories[0]}`}
+            </span>
+          </Truncate>
         </div>
       );
     }
@@ -252,49 +238,42 @@ class Category extends React.Component {
             />
             <span className="bp3-control-indicator" />
           </label>
-          <Tooltip
+          <span
-            content={metadataField}
+            role="menuitem"
-            disabled={truncatedString === null}
+            tabIndex="0"
-            hoverOpenDelay={globals.tooltipHoverOpenDelayQuick}
+            data-testid={`${metadataField}:category-expand`}
-            position={Position.LEFT}
+            onKeyPress={(e) => {
-            usePortal
+              if (e.key === "Enter") {
-            modifiers={{
+                this.handleCategoryClick();
-              preventOverflow: { enabled: false },
+              }
-              hide: { enabled: false },
             }}
+            style={{
+              cursor: "pointer",
+            }}
+            onClick={this.handleCategoryClick}
           >
-            <span
+            <Truncate>
-              role="menuitem"
+              <span
-              tabIndex="0"
+                style={{
-              data-testid={`${metadataField}:category-expand`}
+                  maxWidth: isUserAnno ? LABEL_WIDTH_ANNO : LABEL_WIDTH,
-              onKeyPress={(e) => {
+                }}
-                if (e.key === "Enter") {
+                data-testid={`${metadataField}:category-label`}
-                  this.handleCategoryClick();
+              >
-                }
+                {metadataField}
-              }}
+              </span>
-              style={{
+            </Truncate>
-                cursor: "pointer",
+            {isExpanded ? (
-                display: "inline-block",
+              <FaChevronDown
-              }}
+                data-testclass="category-expand-is-expanded"
-              onClick={this.handleCategoryClick}
+                style={{ fontSize: 10, marginLeft: 5 }}
-            >
+              />
-              {isUserAnno ? (
+            ) : (
-                <Icon style={{ marginRight: 5 }} icon="tag" iconSize={16} />
+              <FaChevronRight
-              ) : null}
+                data-testclass="category-expand-is-not-expanded"
-              {truncatedString || metadataField}
+                style={{ fontSize: 10, marginLeft: 5 }}
-              {isExpanded ? (
+              />
-                <FaChevronDown
+            )}
-                  data-testclass="category-expand-is-expanded"
+          </span>
-                  style={{ fontSize: 10, marginLeft: 5 }}
-                />
-              ) : (
-                <FaChevronRight
-                  data-testclass="category-expand-is-not-expanded"
-                  style={{ fontSize: 10, marginLeft: 5 }}
-                />
-              )}
-            </span>
-          </Tooltip>
         </div>
         {<AnnoDialogEditCategoryName metadataField={metadataField} />}
         {<AnnoDialogAddLabel metadataField={metadataField} />}

client/src/components/categorical/value/index.js

@@ -7,17 +7,17 @@ import {
   MenuItem,
   Popover,
   Position,
+  Icon,
   PopoverInteractionKind,
-  Tooltip,
 } from "@blueprintjs/core";
 import Occupancy from "./occupancy";
 import * as globals from "../../../globals";
 import styles from "../categorical.css";
 import AnnoDialog from "../annoDialog";
 import LabelInput from "../labelInput";
+import Truncate from "../../util/truncate";
 
 import { AnnotationsHelpers } from "../../../util/stateManager";
-import maybeTruncateString from "../../../util/maybeTruncateString";
 import { labelPrompt, isLabelErroneous } from "../labelUtil";
 
 /* this is defined outside of the class so we can use it in connect() */
@@ -316,13 +316,6 @@ class CategoryValue extends React.Component {
       categories = schema.annotations.obsByName[colorAccessor]?.categories;
     }
 
-    const truncatedString = maybeTruncateString(
-      displayString,
-      colorAccessor && !isColorBy
-        ? globals.categoryLabelDisplayStringShortLength
-        : globals.categoryLabelDisplayStringLongLength
-    );
-
     const editModeActive =
       isUserAnno &&
       annotations.labelEditable.category === metadataField &&
@@ -331,6 +324,26 @@ class CategoryValue extends React.Component {
 
     const valueToggleLabel = `value-toggle-checkbox-${displayString}`;
 
+    const LEFT_MARGIN = 33;
+    const CHECKBOX = 26;
+    const CELL_NUMBER = 61;
+    const ANNO_MENU = 26;
+    const LABEL_MARGIN = 24;
+
+    const otherElementsWidth =
+      LEFT_MARGIN +
+      CHECKBOX +
+      CELL_NUMBER +
+      LABEL_MARGIN +
+      (isUserAnno ? ANNO_MENU : 0);
+
+    const OCCUPANCY_WIDTH = 100;
+
+    const labelWidth =
+      colorAccessor && !isColorBy
+        ? globals.leftSidebarWidth - otherElementsWidth - OCCUPANCY_WIDTH
+        : globals.leftSidebarWidth - otherElementsWidth;
+
     return (
       <div
         key={i}
@@ -343,7 +356,7 @@ class CategoryValue extends React.Component {
         }
         data-testclass="categorical-row"
         style={{
-          padding: "4px 7px",
+          padding: "4px 0px 4px 7px",
           display: "flex",
           alignItems: "baseline",
           justifyContent: "space-between",
@@ -383,21 +396,12 @@ class CategoryValue extends React.Component {
                 onMouseLeave={this.handleMouseEnter}
               />
             </label>
-            <Tooltip
+            <Truncate>
-              content={displayString}
-              disabled={truncatedString === null}
-              hoverOpenDelay={globals.tooltipHoverOpenDelayQuick}
-              position={Position.LEFT}
-              usePortal
-              modifiers={{
-                preventOverflow: { enabled: false },
-                hide: { enabled: false },
-              }}
-            >
               <span
                 data-testid={`categorical-value-${metadataField}-${displayString}`}
                 data-testclass="categorical-value"
                 style={{
+                  width: labelWidth,
                   color:
                     displayString === globals.unassignedCategoryLabel
                       ? "#ababab"
@@ -410,13 +414,13 @@ class CategoryValue extends React.Component {
                   overflow: "hidden",
                   lineHeight: "1.1em",
                   height: "1.1em",
-                  wordBreak: "break-all",
                   verticalAlign: "middle",
+                  marginRight: LABEL_MARGIN,
                 }}
               >
-                {truncatedString || displayString}
+                {displayString}
               </span>
-            </Tooltip>
+            </Truncate>
             {editModeActive ? (
               <div>
                 <AnnoDialog
@@ -561,14 +565,14 @@ class CategoryValue extends React.Component {
                 >
                   <Button
                     style={{
-                      marginLeft: 0,
+                      marginLeft: 2,
                       position: "relative",
                       top: -1,
                       minHeight: 16,
                     }}
                     data-testclass="seeActions"
                     data-testid={`${metadataField}:${displayString}:see-actions`}
-                    icon="more"
+                    icon={<Icon icon="more" iconSize={10} />}
                     small
                     minimal
                   />

client/src/components/categorical/value/occupancy.js

@@ -168,7 +168,7 @@ class Occupancy extends React.PureComponent {
             else this.createHistogram();
           }}
         />
-        <div key="text" style={{ fontFamily: "Roboto", fontSize: "14px" }}>
+        <div key="text" style={{ fontSize: "14px" }}>
           <p style={{ margin: "0" }}>
             This histograms shows the distribution of{" "}
             <strong>{colorAccessor}</strong> within{" "}

client/src/components/continuousLegend/index.js

@@ -105,10 +105,12 @@ const continuous = (selectorId, colorscale, colorAccessor) => {
   colorScale: state.colors.scale,
 }))
 class ContinuousLegend extends React.Component {
-
   componentDidUpdate(prevProps) {
     const { colorAccessor, colorScale } = this.props;
-    if (prevProps.colorAccessor !== colorAccessor || prevProps.colorScale !== colorScale) {
+    if (
+      prevProps.colorAccessor !== colorAccessor ||
+      prevProps.colorScale !== colorScale
+    ) {
       /* always remove it, if it's not continuous we don't put it back. */
       d3.select("#continuous_legend").selectAll("*").remove();
     }
@@ -130,7 +132,9 @@ class ContinuousLegend extends React.Component {
     return (
       <div
         id="continuous_legend"
-        ref={ref => {this.ref = ref}}
+        ref={(ref) => {
+          this.ref = ref;
+        }}
         style={{
           display: colorAccessor ? "inherit" : "none",
           position: "absolute",

client/src/components/framework/container.js

@@ -2,19 +2,21 @@
 import React from "react";
 
 function Container(props) {
-  const {children} = props;
+  const { children } = props;
-  return <div
+  return (
-    className="container"
+    <div
-    style={{
+      className="container"
-      height: "calc(100vh - (100vh - 100%))",
+      style={{
-      width: "calc(100vw - (100vw - 100%))",
+        height: "calc(100vh - (100vh - 100%))",
-      position: "absolute",
+        width: "calc(100vw - (100vw - 100%))",
-      top: 0,
+        position: "absolute",
-      left: 0,
+        top: 0,
-    }}
+        left: 0,
-  >
+      }}
-    {children}
+    >
-  </div>
+      {children}
+    </div>
+  );
 }
 
 export default Container;

client/src/components/framework/layout.js

@@ -2,9 +2,7 @@
 import React from "react";
 import * as globals from "../../globals";
 
-
 class Layout extends React.Component {
-
   /*
     Layout - this react component contains all the layout style and logic for the application once it has loaded.
 
@@ -26,60 +24,66 @@ class Layout extends React.Component {
 
   render() {
     const { children } = this.props;
-    const [ leftSidebar, renderGraph, rightSidebar ] = children;
+    const [leftSidebar, renderGraph, rightSidebar] = children;
-    return <div
+    return (
-      style={{
+      <div
-        display: "grid",
+        style={{
-        gridTemplateColumns: `
+          display: "grid",
-          [left-sidebar-start] ${globals.leftSidebarWidth+1}px
+          gridTemplateColumns: `
+          [left-sidebar-start] ${globals.leftSidebarWidth + 1}px
           [left-sidebar-end graph-start] auto
-          [graph-end right-sidebar-start] ${globals.rightSidebarWidth+1}px [right-sidebar-end]
+          [graph-end right-sidebar-start] ${
+            globals.rightSidebarWidth + 1
+          }px [right-sidebar-end]
         `,
-        gridTemplateRows: "[top] auto [bottom]",
+          gridTemplateRows: "[top] auto [bottom]",
-        gridTemplateAreas: "left-sidebar | graph | right-sidebar",
+          gridTemplateAreas: "left-sidebar | graph | right-sidebar",
-        columnGap: "0px",
+          columnGap: "0px",
-        justifyItems: "stretch",
+          justifyItems: "stretch",
-        alignItems: "stretch",
+          alignItems: "stretch",
-        height: "inherit",
-        width: "inherit",
-        position: "relative",
-        top: 0,
-        left: 0,
-        minWidth: "1240px",
-      }}
-    >
-      <div
-        style={{
-          gridArea: "top / left-sidebar-start / bottom / left-sidebar-end",
-          position: "relative",
           height: "inherit",
-          overflowY: "auto"
+          width: "inherit",
+          position: "relative",
+          top: 0,
+          left: 0,
+          minWidth: "1240px",
         }}
       >
-        {leftSidebar}
+        <div
+          style={{
+            gridArea: "top / left-sidebar-start / bottom / left-sidebar-end",
+            position: "relative",
+            height: "inherit",
+            overflowY: "auto",
+          }}
+        >
+          {leftSidebar}
+        </div>
+        <div
+          style={{
+            zIndex: 0,
+            gridArea: "top / graph-start / bottom / graph-end",
+            position: "relative",
+            height: "inherit",
+          }}
+          ref={(ref) => {
+            this.viewportRef = ref;
+          }}
+        >
+          {this.viewportRef ? renderGraph(this.viewportRef) : null}
+        </div>
+        <div
+          style={{
+            gridArea: "top / right-sidebar-start / bottom / right-sidebar-end",
+            position: "relative",
+            height: "inherit",
+            overflowY: "auto",
+          }}
+        >
+          {rightSidebar}
+        </div>
       </div>
-      <div
+    );
-        style={{
-          zIndex: 0,
-          gridArea: "top / graph-start / bottom / graph-end",
-          position: "relative",
-          height: "inherit",
-        }}
-        ref={ref => { this.viewportRef = ref; }}
-      >
-        {this.viewportRef ? renderGraph(this.viewportRef) : null}
-      </div>
-      <div
-        style={{
-          gridArea: "top / right-sidebar-start / bottom / right-sidebar-end",
-          position: "relative",
-          height: "inherit",
-          overflowY: "auto"
-        }}
-      >
-        {rightSidebar}
-      </div>
-    </div>;
   }
 }
 

client/src/components/graph/overlays/centroidLabels.js

@@ -8,6 +8,7 @@ export default
   colorAccessor: state.colors.colorAccessor,
   dilatedValue: state.pointDilation.categoryField,
   labels: state.centroidLabels.labels,
+  categoricalSelection: state.categoricalSelection,
 }))
 class CentroidLabels extends PureComponent {
   // Check to see if centroids have either just been displayed or removed from the overlay
@@ -33,11 +34,21 @@ class CentroidLabels extends PureComponent {
       dilatedValue,
       dispatch,
       colorAccessor,
+      categoricalSelection,
     } = this.props;
 
+    if (!colorAccessor || labels.size === undefined || labels.size === 0)
+      return null;
+
+    const {
+      categoryValueIndices,
+      categoryValueSelected,
+    } = categoricalSelection?.[colorAccessor];
+
     const labelSVGS = [];
     let fontSize = "15px";
     let fontWeight = null;
+    const deselectOpacity = 0.375;
     labels.forEach((coords, label) => {
       fontSize = "15px";
       fontWeight = null;
@@ -46,6 +57,8 @@ class CentroidLabels extends PureComponent {
         fontWeight = "800";
       }
 
+      const selected = categoryValueSelected[categoryValueIndices.get(label)];
+
       // Mirror LSB middle truncation
       let displayLabel = label;
       if (displayLabel.length > categoryLabelDisplayStringLongLength) {
@@ -72,11 +85,11 @@ class CentroidLabels extends PureComponent {
             textAnchor="middle"
             data-label={label}
             style={{
-              fontFamily: "Roboto Condensed",
               fontSize,
               fontWeight,
               fill: "black",
               userSelect: "none",
+              opacity: selected ? 1 : deselectOpacity,
             }}
             onMouseEnter={(e) =>
               dispatch({

client/src/components/graph/overlays/graphOverlayLayer.js

@@ -81,7 +81,7 @@ export default class GraphOverlayLayer extends PureComponent {
           position: "absolute",
           top: 0,
           left: 0,
-          zIndex: 1,
+          zIndex: 2,
           backgroundColor: displaying ? "rgba(255, 255, 255, 0.55)" : "",
         }}
         onMouseMove={handleCanvasEvent}

client/src/components/graph/setupSVGandBrush.js

@@ -14,7 +14,7 @@ export default (
   handleDragAction,
   handleEndAction,
   handleCancelAction,
-  viewport,
+  viewport
 ) => {
   const svg = d3.select("#graph-wrapper").select("#lasso-layer");
 
@@ -23,7 +23,7 @@ export default (
       .brush()
       .extent([
         [0, 0],
-        [viewport.width, viewport.height]
+        [viewport.width, viewport.height],
       ])
       .on("start", handleStartAction)
       .on("brush", handleDragAction)

client/src/components/leftSidebar/index.js

@@ -11,7 +11,6 @@ import TopLeftLogoAndTitle from "./topLeftLogoAndTitle";
   scatterplotYYaccessor: state.controls.scatterplotYYaccessor,
 }))
 class LeftSideBar extends React.Component {
-
   render() {
     const { scatterplotXXaccessor, scatterplotYYaccessor } = this.props;
     return (

client/src/components/leftSidebar/topLeftLogoAndTitle.js

@@ -3,6 +3,10 @@ import React from "react";
 import { connect } from "react-redux";
 import * as globals from "../../globals";
 import Logo from "../framework/logo";
+import Truncate from "../util/truncate";
+
+const DATASET_TITLE_WIDTH = 190;
+const DATASET_TITLE_FONT_SIZE = 14;
 
 @connect((state) => ({
   datasetTitle: state.config?.displayNames?.dataset ?? "",
@@ -14,16 +18,6 @@ class LeftSideBar extends React.Component {
   render() {
     const { datasetTitle, aboutURL } = this.props;
 
-    const displayTitle =
-      datasetTitle.length > globals.datasetTitleMaxCharacterCount
-        ? `${datasetTitle.substring(
-            0,
-            Math.floor(globals.datasetTitleMaxCharacterCount / 2)
-          )}…${datasetTitle.slice(
-            -Math.floor(globals.datasetTitleMaxCharacterCount / 2)
-          )}`
-        : datasetTitle;
-
     return (
       <div
         style={{
@@ -60,26 +54,36 @@ class LeftSideBar extends React.Component {
           gene
         </span>
         <div
-          data-testid="header"
           style={{
-            fontSize: 14,
+            fontSize: DATASET_TITLE_FONT_SIZE,
             position: "relative",
             top: -6,
             display: "inline-block",
-            width: "190px",
+            width: DATASET_TITLE_WIDTH,
             marginLeft: "7px",
             height: "1.2em",
             overflow: "hidden",
             wordBreak: "break-all",
           }}
-          title={datasetTitle}
         >
           {aboutURL ? (
-            <a href={aboutURL} target="_blank" rel="noopener noreferrer">
+            <Truncate>
-              {displayTitle}
+              <a
-            </a>
+                style={{ width: 185 }}
+                href={aboutURL}
+                data-testid="header"
+                target="_blank"
+                rel="noopener noreferrer"
+              >
+                {datasetTitle}
+              </a>
+            </Truncate>
           ) : (
-            displayTitle
+            <Truncate>
+              <span style={{ width: 185 }} data-testid="header">
+                {datasetTitle}
+              </span>
+            </Truncate>
           )}
         </div>
       </div>

client/src/components/menubar/clip.js

@@ -35,9 +35,7 @@ function Clip(props) {
       : "";
 
   return (
-    <div
+    <div className={`bp3-button-group ${styles.menubarButton}`}>
-      className={`bp3-button-group ${styles.menubarButton}`}
-    >
       <Popover
         target={
           <Tooltip

client/src/components/menubar/index.js

@@ -37,6 +37,7 @@ import DiffexpButtons from "./diffexpButtons";
   showCentroidLabels: state.centroidLabels.showLabels,
   tosURL: state.config?.parameters?.["about_legal_tos"],
   privacyURL: state.config?.parameters?.["about_legal_privacy"],
+  categoricalSelection: state.categoricalSelection,
 }))
 class MenuBar extends React.Component {
   static isValidDigitKeyEvent(e) {
@@ -203,9 +204,13 @@ class MenuBar extends React.Component {
       showCentroidLabels,
       privacyURL,
       tosURL,
+      categoricalSelection,
+      colorAccessor,
     } = this.props;
     const { pendingClipPercentiles } = this.state;
 
+    const isColoredByCategorical = !!categoricalSelection?.[colorAccessor];
+
     // constants used to create selection tool button
     const [selectionTooltip, selectionButtonIcon] =
       selectionTool === "brush"
@@ -267,6 +272,7 @@ class MenuBar extends React.Component {
             onClick={this.handleCentroidChange}
             active={showCentroidLabels}
             intent={showCentroidLabels ? "primary" : "none"}
+            disabled={!isColoredByCategorical}
           />
         </Tooltip>
         <ButtonGroup className={styles.menubarButton}>

client/src/components/menubar/infoMenu.js

@@ -6,9 +6,7 @@ import styles from "./menubar.css";
 function InformationMenu(props) {
   const { libraryVersions, aboutLink, tosURL, privacyURL } = props;
   return (
-    <div
+    <div className={`bp3-button-group ${styles.menubarButton}`}>
-      className={`bp3-button-group ${styles.menubarButton}`}
-    >
       <Popover
         content={
           <Menu>

client/src/components/menubar/menubar.css

@@ -1,4 +1,4 @@
 :local(.menubarButton) {
-    margin-top: 8px;
+  margin-top: 8px;
-    margin-left: 8px;
+  margin-left: 8px;
 }

client/src/components/menubar/subset.js

@@ -12,9 +12,7 @@ function Subset(props) {
   } = props;
 
   return (
-    <ButtonGroup
+    <ButtonGroup className={styles.menubarButton}>
-     className={styles.menubarButton}
-    >
       <Tooltip
         content="Subset to currently selected cells and associated metadata"
         position="bottom"
@@ -26,7 +24,7 @@ function Subset(props) {
           disabled={!subsetPossible}
           icon="pie-chart"
           onClick={handleSubset}
-  />
+        />
       </Tooltip>
       <Tooltip
         content="Undo subset and show all cells and associated metadata"

client/src/components/menubar/undoRedo.js

@@ -7,9 +7,7 @@ import styles from "./menubar.css";
 function InformationMenu(props) {
   const { undoDisabled, redoDisabled, dispatch } = props;
   return (
-    <div
+    <div className={`bp3-button-group ${styles.menubarButton}`}>
-      className={`bp3-button-group ${styles.menubarButton}`}
-    >
       <Tooltip
         content="Undo"
         position="bottom"

client/src/components/rightSidebar/index.js

@@ -5,12 +5,11 @@ import Continuous from "../continuous/continuous";
 import GeneExpression from "../geneExpression";
 import * as globals from "../../globals";
 
-@connect(state => ({
+@connect((state) => ({
   scatterplotXXaccessor: state.controls.scatterplotXXaccessor,
   scatterplotYYaccessor: state.controls.scatterplotYYaccessor,
 }))
 class RightSidebar extends React.Component {
-
   render() {
     return (
       <div
@@ -22,7 +21,7 @@ class RightSidebar extends React.Component {
           position: "relative",
           overflowY: "inherit",
           height: "inherit",
-          width: "inherit"
+          width: "inherit",
         }}
       >
         <GeneExpression />

client/src/components/util/truncate.js

@@ -0,0 +1,85 @@
+import React, { cloneElement } from "react";
+import { Tooltip, Position } from "@blueprintjs/core";
+
+import { tooltipHoverOpenDelayQuick } from "../../globals";
+
+const SPLIT_STYLE = {
+  display: "flex",
+  overflow: "hidden",
+  justifyContent: "flex-start",
+};
+
+const FIRST_HALF_STYLE = {
+  overflow: "hidden",
+  textOverflow: "ellipsis",
+  whiteSpace: "nowrap",
+  flexShrink: 1,
+  minWidth: "5px",
+};
+
+const SECOND_HALF_INNER_STYLE = {
+  position: "absolute",
+  right: 0,
+};
+const SECOND_HALF_STYLE = {
+  color: "transparent",
+  position: "relative",
+  overflow: "hidden",
+  whiteSpace: "nowrap",
+};
+
+export default (props) => {
+  const { children } = props;
+  // Truncate only support a single child with a text child
+
+  if (
+    React.Children.count(children) !== 1 ||
+    React.Children.count(children.props?.children) !== 1
+  ) {
+    throw Error("Only pass a single child with text to Truncate");
+  }
+  const originalString = children.props.children;
+
+  const firstString = originalString.substr(0, originalString.length / 2);
+  const secondString = originalString.substr(originalString.length / 2);
+
+  const inheritedColor = children.props.style.color;
+
+  const splitStyle = { ...children.props.style, ...SPLIT_STYLE };
+  const secondHalfInnerStyle = {
+    ...SECOND_HALF_INNER_STYLE,
+    color: inheritedColor || "initial",
+  };
+
+  const truncatedJSX = (
+    <span style={splitStyle}>
+      <span style={FIRST_HALF_STYLE}>{firstString}</span>
+      <span style={SECOND_HALF_STYLE}>
+        {secondString}
+        <span style={secondHalfInnerStyle}>{secondString}</span>
+      </span>
+    </span>
+  );
+
+  // clone children, changing the children(text) to the truncated string
+  const newChildren = React.Children.map(children, (child) =>
+    cloneElement(child, {
+      children: truncatedJSX,
+      "aria-label": originalString,
+    })
+  );
+  return (
+    <Tooltip
+      content={originalString}
+      hoverOpenDelay={tooltipHoverOpenDelayQuick}
+      position={Position.LEFT}
+      usePortal
+      modifiers={{
+        preventOverflow: { enabled: false },
+        hide: { enabled: false },
+      }}
+    >
+      {newChildren}
+    </Tooltip>
+  );
+};

client/src/reducers/categoricalSelection.js

@@ -31,7 +31,7 @@ const CategoricalSelection = (
       const names = CH.selectableCategoryNames(
         world.schema,
         CH.maxCategoryItems(prevSharedState.config),
-        dataframe.colIndex.keys()
+        dataframe.colIndex.labels()
       );
       if (names.length === 0) return state;
       return {

client/src/reducers/centroidLabels.js

@@ -63,6 +63,7 @@ const centroidLabels = (state = initialState, action, sharedNextState) => {
       };
 
     case "color by continuous metadata":
+    case "color by expression":
       return { ...state, labels: [] };
 
     case "reset centroid labels":

client/src/reducers/world.js

@@ -93,7 +93,7 @@ const WorldReducer = (
           let worldValSlice = val;
           if (!World.worldEqUniverse(state, universe)) {
             worldValSlice = universeVarData
-              .subset(state.obsAnnotations.rowIndex.keys(), [key], null)
+              .subset(state.obsAnnotations.rowIndex.labels(), [key], null)
               .icol(0)
               .asArray();
           }
@@ -129,10 +129,10 @@ const WorldReducer = (
       //
       let clippedVarData = state.varData;
       const keysToDrop = clippedVarData.colIndex
-        .keys()
+        .labels()
         .filter((k) => !unclippedVarData.hasCol(k));
       const keysToAdd = unclippedVarData.colIndex
-        .keys()
+        .labels()
         .filter((k) => !clippedVarData.hasCol(k));
       keysToDrop.forEach((k) => {
         clippedVarData = clippedVarData.dropCol(k);
@@ -171,7 +171,7 @@ const WorldReducer = (
       let newAnnotation = null;
       if (!World.worldEqUniverse(state, universe)) {
         newAnnotation = universe.obsAnnotations
-          .subset(state.obsAnnotations.rowIndex.keys(), [name], null)
+          .subset(state.obsAnnotations.rowIndex.labels(), [name], null)
           .icol(0)
           .asArray();
       } else {
@@ -303,7 +303,7 @@ const WorldReducer = (
       let schema = origSchema;
 
       // alias the names the server sent us, in case they were not the same as the schema
-      const embedingLabels = embedding.colIndex.keys();
+      const embedingLabels = embedding.colIndex.labels();
       const labels = {
         [embedingLabels[0]]: dims[0],
         [embedingLabels[1]]: dims[1],

client/src/util/dataframe/dataframe.js

@@ -1,6 +1,5 @@
 import { IdentityInt32Index, isLabelIndex } from "./labelIndex";
 // weird cross-dependency that we should clean up someday...
-import { sortArray } from "../typedCrossfilter/sort";
 import {
   isTypedArray,
   isArrayOrTypedArray,
@@ -389,7 +388,7 @@ class Dataframe {
     let dstLabels;
     if (!labels) {
       // combine all columns
-      dstLabels = dataframe.colIndex.keys();
+      dstLabels = dataframe.colIndex.labels();
       srcLabels = dstLabels;
     } else if (Array.isArray(labels)) {
       // combine subset of keys with no aliasing
@@ -537,7 +536,12 @@ class Dataframe {
   }
 
   static empty(rowIndex = null, colIndex = null) {
-    return new Dataframe([0, 0], [], rowIndex, colIndex);
+    const dims = [
+      rowIndex ? rowIndex.size() : 0,
+      colIndex ? colIndex.size() : 0,
+    ];
+    if (dims[0] && dims[1]) throw new Error("not an empty dataframe");
+    return new Dataframe(dims, new Array(dims[1]), rowIndex, colIndex);
   }
 
   static create(dims, columnarData) {
@@ -551,97 +555,59 @@ class Dataframe {
     return new Dataframe(dims, columnarData, null, null);
   }
 
-  __subset(rowOffsets, colOffsets, withRowIndex) {
+  __subset(newRowIndex, newColIndex) {
     const dims = [...this.dims];
 
-    const getSortedLabelAndOffsets = (offsets, index) => {
+    /* subset columns */
-      /*
+    let { __columns, colIndex } = this;
-      Given offsets, return both offsets and associated lables,
+    if (newColIndex) {
-      sorted by offset.
+      const colOffsets = this.colIndex.getOffsets(newColIndex.labels());
-      */
+      __columns = new Array(colOffsets.length);
-      if (!offsets) {
+      for (let i = 0, l = colOffsets.length; i < l; i += 1) {
-        return [null, null];
+        __columns[i] = this.__columns[colOffsets[i]];
       }
-      const sortedOffsets = sortArray(offsets);
+      colIndex = newColIndex;
-      const sortedLabels = new Array(sortedOffsets.length);
-      for (let i = 0, l = sortedOffsets.length; i < l; i += 1) {
-        sortedLabels[i] = index.getLabel(sortedOffsets[i]);
-      }
-      return [sortedLabels, sortedOffsets];
-    };
-
-    let { colIndex } = this;
-    if (colOffsets) {
-      let colLabels;
-      [colLabels, colOffsets] = getSortedLabelAndOffsets(
-        colOffsets,
-        this.colIndex
-      );
       dims[1] = colOffsets.length;
-      colIndex = this.colIndex.subsetLabels(colLabels);
     }
 
     let { rowIndex } = this;
-    if (withRowIndex) rowIndex = withRowIndex;
+    if (newRowIndex) {
-    if (rowOffsets) {
+      const rowOffsets = this.rowIndex.getOffsets(newRowIndex.labels());
-      let rowLabels;
+      __columns = __columns.map((col) => {
-      [rowLabels, rowOffsets] = getSortedLabelAndOffsets(
-        rowOffsets,
-        this.rowIndex
-      );
-      dims[0] = rowLabels.length;
-      if (!withRowIndex) rowIndex = this.rowIndex.subsetLabels(rowLabels);
-    }
-
-    /* subset columns */
-    let columns = this.__columns;
-    if (colOffsets) {
-      columns = new Array(colOffsets.length);
-      for (let i = 0, l = colOffsets.length; i < l; i += 1) {
-        columns[i] = this.__columns[colOffsets[i]];
-      }
-    }
-
-    /* subset rows */
-    if (rowOffsets) {
-      columns = columns.map((col) => {
         const newCol = new col.constructor(rowOffsets.length);
         for (let i = 0, l = rowOffsets.length; i < l; i += 1) {
           newCol[i] = col[rowOffsets[i]];
         }
         return newCol;
       });
+      rowIndex = newRowIndex;
+      dims[0] = rowOffsets.length;
     }
 
     if (dims[0] === 0 || dims[1] === 0) return Dataframe.empty();
-    return new Dataframe(dims, columns, rowIndex, colIndex);
+    return new Dataframe(dims, __columns, rowIndex, colIndex);
   }
 
   subset(rowLabels, colLabels = null, withRowIndex = null) {
     /*
     Subset by row/col labels.
 
-    withRowIndex allows assignment of new row index during subset operation.
+    withRowIndex allows subset with an index, rather than rowLabels.
-    If withRowIndex === null, it will reset the index to identity (offset)
+    If withRowIndex is specified, rowLabels is ignored.
-    indexing.  if withRowIndex is a label index object, it will be used
-    for the new dataframe.
     */
-    const toOffsets = (labels, index) => {
+    let rowIndex = null;
-      if (!labels) {
+    if (withRowIndex) {
-        return null;
+      rowIndex = withRowIndex;
-      }
+    } else if (rowLabels) {
-      return labels.map((label) => {
+      rowIndex = this.rowIndex.subset(rowLabels);
-        const off = index.getOffset(label);
+    }
-        if (off === undefined) {
-          throw new RangeError(`unknown label: ${label}`);
-        }
-        return off;
-      });
-    };
 
-    const rowOffsets = toOffsets(rowLabels, this.rowIndex);
+    let colIndex = null;
-    const colOffsets = toOffsets(colLabels, this.colIndex);
+    if (colLabels) {
-    return this.__subset(rowOffsets, colOffsets, withRowIndex);
+      colIndex = this.colIndex.subset(colLabels);
+    }
+
+    return this.__subset(rowIndex, colIndex);
   }
 
   isubset(rowOffsets, colOffsets = null, withRowIndex = null) {
@@ -653,7 +619,19 @@ class Dataframe {
     indexing. If withRowIndex is a label index object, it will be used
     for the new dataframe.
     */
-    return this.__subset(rowOffsets, colOffsets, withRowIndex);
+    let rowIndex = null;
+    if (withRowIndex) {
+      rowIndex = withRowIndex;
+    } else if (rowOffsets) {
+      rowIndex = this.rowIndex.isubset(rowOffsets);
+    }
+
+    let colIndex = null;
+    if (colOffsets) {
+      colIndex = this.colIndex.isubset(colOffsets);
+    }
+
+    return this.__subset(rowIndex, colIndex);
   }
 
   isubsetMask(rowMask, colMask = null, withRowIndex = null) {
@@ -690,7 +668,7 @@ class Dataframe {
     };
     const rowOffsets = toList(rowMask, nRows);
     const colOffsets = toList(colMask, nCols);
-    return this.__subset(rowOffsets, colOffsets, withRowIndex);
+    return this.isubset(rowOffsets, colOffsets, withRowIndex);
   }
 
   /**
@@ -790,7 +768,7 @@ class Dataframe {
     Return true if this is an empty dataframe, ie, has dimensions [0,0]
     */
     const [rows, cols] = this.dims;
-    return rows === 0 && cols === 0;
+    return rows === 0 || cols === 0;
   }
 
   /****

client/src/util/dataframe/index.js

@@ -1,2 +1,7 @@
 export { default as Dataframe } from "./dataframe";
-export { DenseInt32Index, IdentityInt32Index, KeyIndex } from "./labelIndex";
+export {
+  DenseInt32Index,
+  IdentityInt32Index,
+  KeyIndex,
+  isLabelIndex,
+} from "./labelIndex";

client/src/util/dataframe/labelIndex.js

@@ -32,10 +32,10 @@ class IdentityInt32Index {
     this.maxOffset = maxOffset;
   }
 
-  keys() {
+  labels() {
     // memoize
     const k = fillRange(new Int32Array(this.maxOffset));
-    this.keys = function keys() {
+    this.labels = function labels() {
       return k;
     };
     return k;
@@ -47,12 +47,24 @@ class IdentityInt32Index {
     return i;
   }
 
+  // eslint-disable-next-line class-methods-use-this
+  getOffsets(arr) {
+    // labels to offsets
+    return arr;
+  }
+
   // eslint-disable-next-line class-methods-use-this
   getLabel(i) {
     // offset to label
     return i;
   }
 
+  // eslint-disable-next-line class-methods-use-this
+  getLabels(arr) {
+    // offsets to labels
+    return arr;
+  }
+
   size() {
     return this.maxOffset;
   }
@@ -62,6 +74,9 @@ class IdentityInt32Index {
     time/space decision - based on the resulting density
     */
     const [minLabel, maxLabel] = extent(labelArray);
+    if (minLabel === 0 && maxLabel === labelArray.length - 1)
+      return new IdentityInt32Index(labelArray.length);
+
     const labelSpaceSize = maxLabel - minLabel + 1;
     const density = labelSpaceSize / this.maxOffset;
     /* 0.1 is a magic number, that needs testing to optimize */
@@ -71,30 +86,43 @@ class IdentityInt32Index {
     return new DenseInt32Index(labelArray, [minLabel, maxLabel]);
   }
 
-  subsetLabels(labelArray) {
+  subset(labels) {
-    return this.__promote(labelArray);
+    /* validate subset */
+    const { maxOffset } = this;
+    for (let i = 0, l = labels.length; i < l; i += 1) {
+      const label = labels[i];
+      if (label < 0 || label >= maxOffset)
+        throw new RangeError(`offset or label: ${label}`);
+    }
+    return this.__promote(labels);
+  }
+
+  /* identity index - labels are offsets */
+  isubset(offsets) {
+    return this.subset(offsets);
   }
 
   withLabel(label) {
     if (label === this.maxOffset) {
       return new IdentityInt32Index(label + 1);
     }
-    return this.__promote([...this.keys(), label]);
+    return this.__promote([...this.labels(), label]);
   }
 
   withLabels(labels) {
-    return this.__promote([...this.keys(), ...labels]);
+    return this.__promote([...this.labels(), ...labels]);
   }
 
   dropLabel(label) {
     if (label === this.maxOffset - 1) {
       return new IdentityInt32Index(label);
     }
-    const labelArray = [...this.keys()];
+    const labelArray = [...this.labels()];
     labelArray.splice(labelArray.indexOf(label), 1);
     return this.__promote(labelArray);
   }
 }
+
 class DenseInt32Index {
   /*
   DenseInt32Index indexes integer labels, and uses Int32Array typed arrays
@@ -129,12 +157,29 @@ class DenseInt32Index {
     this.getOffset = function getOffset(l) {
       return index[l - minLabel];
     };
+
+    this.getOffsets = function getOffsets(arr) {
+      const res = new arr.constructor(arr.length);
+      for (let i = 0, len = arr.length; i < len; i += 1) {
+        res[i] = index[arr[i] - minLabel];
+      }
+      return res;
+    };
+
     this.getLabel = function getLabel(i) {
       return rindex[i];
     };
+
+    this.getLabels = function getLabels(arr) {
+      const res = new arr.constructor(arr.length);
+      for (let i = 0, len = arr.length; i < len; i += 1) {
+        res[i] = rindex[arr[i]];
+      }
+      return res;
+    };
   }
 
-  keys() {
+  labels() {
     return this.rindex;
   }
 
@@ -158,20 +203,44 @@ class DenseInt32Index {
     return new DenseInt32Index(labelArray, [minLabel, maxLabel]);
   }
 
-  subsetLabels(labelArray) {
+  subset(labels) {
-    return this.__promote(labelArray);
+    /* validate subset */
+    for (let i = 0, l = labels.length; i < l; i += 1) {
+      const label = labels[i];
+      const offset = this.getOffset(label);
+      if (offset === undefined || offset === -1)
+        throw new RangeError(`unknown label: ${label}`);
+    }
+
+    return this.__promote(labels);
+  }
+
+  // eslint-disable-next-line class-methods-use-this
+  isubset(offsets) {
+    /* validate subset */
+    const { rindex } = this;
+    const maxOffset = rindex.length;
+    const labels = new Int32Array(offsets.length);
+    for (let i = 0, l = offsets.length; i < l; i += 1) {
+      const offset = offsets[i];
+      if (offset < 0 || offset >= maxOffset)
+        throw new RangeError(`out of bounds offset: ${offset}`);
+      labels[i] = rindex[offset];
+    }
+
+    return this.__promote(labels);
   }
 
   withLabel(label) {
-    return this.__promote([...this.keys(), label]);
+    return this.__promote([...this.labels(), label]);
   }
 
   withLabels(labels) {
-    return this.__promote([...this.keys(), ...labels]);
+    return this.__promote([...this.labels(), ...labels]);
   }
 
   dropLabel(label) {
-    const labelArray = [...this.keys()];
+    const labelArray = [...this.labels()];
     labelArray.splice(labelArray.indexOf(label), 1);
     return this.__promote(labelArray);
   }
@@ -207,12 +276,29 @@ class KeyIndex {
     this.getOffset = function getOffset(k) {
       return index.get(k);
     };
+
+    this.getOffsets = function getOffsets(arr) {
+      const res = new arr.constructor(arr.length);
+      for (let i = 0, len = arr.length; i < len; i += 1) {
+        res[i] = index.get(arr[i]);
+      }
+      return res;
+    };
+
     this.getLabel = function getLabel(i) {
       return rindex[i];
     };
+
+    this.getLabels = function getLabels(arr) {
+      const res = new arr.constructor(arr.length);
+      for (let i = 0, len = arr.length; i < len; i += 1) {
+        res[i] = rindex[arr[i]];
+      }
+      return res;
+    };
   }
 
-  keys() {
+  labels() {
     return this.rindex;
   }
 
@@ -220,9 +306,30 @@ class KeyIndex {
     return this.rindex.length;
   }
 
+  subset(labels) {
+    /* validate subset */
+    for (let i = 0, l = labels.length; i < l; i += 1) {
+      const label = labels[i];
+      const offset = this.getOffset(label);
+      if (offset === undefined) throw new RangeError(`unknown label: ${label}`);
+    }
+
+    return new KeyIndex(labels);
+  }
+
   // eslint-disable-next-line class-methods-use-this
-  subsetLabels(labelArray) {
+  isubset(offsets) {
-    return new KeyIndex(labelArray);
+    const { rindex } = this;
+    const maxOffset = rindex.length;
+    const labels = new Array(offsets.length);
+    for (let i = 0, l = offsets.length; i < l; i += 1) {
+      const offset = offsets[i];
+      if (offset < 0 || offset >= maxOffset)
+        throw new RangeError(`out of bounds offset: ${offset}`);
+      labels[i] = rindex[offset];
+    }
+
+    return new KeyIndex(labels);
   }
 
   withLabel(label) {

client/src/util/maybeTruncateString.js

@@ -1,12 +0,0 @@
-const maybeTruncateString = (str, maxLength) => {
-  let truncatedString = null;
-  if (str.length > maxLength) {
-    truncatedString = `${str.slice(0, maxLength / 2)}…${str.slice(
-      -maxLength / 2
-    )}`;
-  }
-
-  return truncatedString;
-};
-
-export default maybeTruncateString;

client/src/util/stateManager/annotationsHelpers.js

@@ -100,7 +100,7 @@ export function setLabelByValue(df, colName, fromLabel, toLabel) {
   /* 
 	in the dataframe column `colName`, set any value of `fromLabel` to `toLabel`
 	*/
-  const keys = df.colIndex.keys();
+  const keys = df.colIndex.labels();
   const ndf = df.mapColumns((col, colIdx) => {
     if (colName !== keys[colIdx]) return col;
 
@@ -118,7 +118,7 @@ export function setLabelByMask(df, colName, mask, label) {
   /* 
 	in the dataframe column `colName`, set the masked rows to 'label'
 	*/
-  const keys = df.colIndex.keys();
+  const keys = df.colIndex.labels();
   const ndf = df.mapColumns((col, colIdx) => {
     if (colName !== keys[colIdx]) return col;
 

client/src/util/stateManager/controlsHelpers.js

@@ -187,7 +187,7 @@ export function pruneVarDataCache(varData, needed) {
   if (numOverWatermark <= 0) return varData;
 
   const { colIndex } = varData;
-  const all = colIndex.keys();
+  const all = colIndex.labels();
   const unused = _.difference(all, needed);
   if (unused.length > 0) {
     // sort by offset in the dataframe - ie, psuedo-LRU
@@ -203,9 +203,9 @@ export function pruneVarDataCache(varData, needed) {
 
 export function subsetAndResetGeneLists(state) {
   const { userDefinedGenes, diffexpGenes } = state;
-  const newUserDefinedGenes = []
+  const newUserDefinedGenes = _.uniq(
-    .concat(userDefinedGenes, diffexpGenes)
+    [].concat(userDefinedGenes, diffexpGenes)
-    .slice(0, globals.maxGenes);
+  ).slice(0, globals.maxGenes);
   const newDiffExpGenes = [];
   return [newUserDefinedGenes, newDiffExpGenes];
 }

client/src/util/stateManager/matrix.js

@@ -1,7 +1,12 @@
 import { flatbuffers } from "flatbuffers";
 import { NetEncoding } from "./matrix_generated";
-import { isTypedArray } from "../typeHelpers";
+import { isTypedArray, isFpTypedArray } from "../typeHelpers";
-import { IdentityInt32Index, DenseInt32Index, KeyIndex } from "../dataframe";
+import {
+  Dataframe,
+  IdentityInt32Index,
+  DenseInt32Index,
+  KeyIndex,
+} from "../dataframe";
 
 const utf8Decoder = new TextDecoder("utf-8");
 
@@ -133,14 +138,14 @@ export function encodeMatrixFBS(df) {
         encColIndex = encodeTypedArray(
           builder,
           encColIndexUType,
-          df.colIndex.keys()
+          df.colIndex.labels()
         );
       } else if (colIndexType === KeyIndex) {
         encColIndexUType = NetEncoding.TypedArray.JSONEncodedArray;
         encColIndex = encodeTypedArray(
           builder,
           encColIndexUType,
-          utf8Encoder.encode(JSON.stringify(df.colIndex.keys()))
+          utf8Encoder.encode(JSON.stringify(df.colIndex.labels()))
         );
       } else {
         throw new Error("Index type FBS encoding unsupported");
@@ -162,3 +167,79 @@ export function encodeMatrixFBS(df) {
   builder.finish(root);
   return builder.asUint8Array();
 }
+
+function promoteTypedArray(o) {
+  /*
+  Decide what internal data type to use for the data returned from 
+  the server.
+
+  TODO - future optimization: not all int32/uint32 data series require
+  promotion to float64.  We COULD simply look at the data to decide. 
+  */
+  if (isFpTypedArray(o) || Array.isArray(o)) return o;
+
+  let TyepdArrayCtor;
+  switch (o.constructor) {
+    case Int8Array:
+    case Uint8Array:
+    case Uint8ClampedArray:
+    case Int16Array:
+    case Uint16Array:
+      TyepdArrayCtor = Float32Array;
+      break;
+
+    case Int32Array:
+    case Uint32Array:
+      TyepdArrayCtor = Float64Array;
+      break;
+
+    default:
+      throw new Error("Unexpected data type returned from server.");
+  }
+  if (o.constructor === TyepdArrayCtor) return o;
+  return new TyepdArrayCtor(o);
+}
+
+export function matrixFBSToDataframe(arrayBuffers) {
+  /*
+  Convert array of Matrix FBS to a Dataframe.
+
+  The application has strong assumptions that all scalar data will be
+  stored as a float32 or float64 (regardless of underlying data types).
+  For example, clipping of value ranges (eg, user-selected percentiles)
+  depends on the ability to use NaN in any numeric type.
+
+  All float data from the server is left as is.  All non-float is promoted
+  to an appropriate float.
+  */
+  if (!Array.isArray(arrayBuffers)) {
+    arrayBuffers = [arrayBuffers];
+  }
+  if (arrayBuffers.length === 0) {
+    return Dataframe.Dataframe.empty();
+  }
+
+  const fbs = arrayBuffers.map((ab) => decodeMatrixFBS(ab, true)); // leave in place
+  /* check that all FBS have same row dimensionality */
+  const { nRows } = fbs[0];
+  fbs.forEach((b) => {
+    if (b.nRows !== nRows)
+      throw new Error("FBS with inconsistent dimensionality");
+  });
+  const columns = fbs
+    .map((fb) =>
+      fb.columns.map((c) => {
+        if (isFpTypedArray(c) || Array.isArray(c)) return c;
+        return promoteTypedArray(c);
+      })
+    )
+    .flat();
+  // colIdx may be TypedArray or Array
+  const colIdx = fbs
+    .map((b) => (Array.isArray(b.colIdx) ? b.colIdx : Array.from(b.colIdx)))
+    .flat();
+  const nCols = columns.length;
+
+  const df = new Dataframe([nRows, nCols], columns, null, new KeyIndex(colIdx));
+  return df;
+}

client/src/util/stateManager/universe.js

@@ -40,84 +40,6 @@ These functions are used exclusively by the actions and reducers to
 build an internal POJO for use by the rendering components.
 */
 
-function promoteTypedArray(o) {
-  /*
-  Decide what internal data type to use for the data returned from 
-  the server.
-
-  TODO - future optimization: not all int32/uint32 data series require
-  promotion to float64.  We COULD simply look at the data to decide. 
-  */
-  if (isFpTypedArray(o) || Array.isArray(o)) return o;
-
-  let TyepdArrayCtor;
-  switch (o.constructor) {
-    case Int8Array:
-    case Uint8Array:
-    case Uint8ClampedArray:
-    case Int16Array:
-    case Uint16Array:
-      TyepdArrayCtor = Float32Array;
-      break;
-
-    case Int32Array:
-    case Uint32Array:
-      TyepdArrayCtor = Float64Array;
-      break;
-
-    default:
-      throw new Error("Unexpected data type returned from server.");
-  }
-  if (o.constructor === TyepdArrayCtor) return o;
-  return new TyepdArrayCtor(o);
-}
-
-export function matrixFBSToDataframe(arrayBuffers) {
-  /*
-  Convert array of Matrix FBS to a Dataframe.
-
-  The application has strong assumptions that all scalar data will be
-  stored as a float32 or float64 (regardless of underlying data types).
-  For example, clipping of value ranges (eg, user-selected percentiles)
-  depends on the ability to use NaN in any numeric type.
-
-  All float data from the server is left as is.  All non-float is promoted
-  to an appropriate float.
-  */
-  if (!Array.isArray(arrayBuffers)) {
-    arrayBuffers = [arrayBuffers];
-  }
-  if (arrayBuffers.length === 0) {
-    return Dataframe.Dataframe.empty();
-  }
-
-  const fbs = arrayBuffers.map((ab) => decodeMatrixFBS(ab, true)); // leave in place
-  /* check that all FBS have same row dimensionality */
-  const { nRows } = fbs[0];
-  fbs.forEach((b) => {
-    if (b.nRows !== nRows)
-      throw new Error("FBS with inconsistent dimensionality");
-  });
-  const columns = fbs
-    .map((fb) =>
-      fb.columns.map((c) => {
-        if (isFpTypedArray(c) || Array.isArray(c)) return c;
-        return promoteTypedArray(c);
-      })
-    )
-    .flat();
-  const colIdx = fbs.map((b) => b.colIdx).flat();
-  const nCols = columns.length;
-
-  const df = new Dataframe.Dataframe(
-    [nRows, nCols],
-    columns,
-    null,
-    new Dataframe.KeyIndex(colIdx)
-  );
-  return df;
-}
-
 export function createUniverseFromResponse(configResponse, schemaResponse) {
   /*
   build & return universe from a REST 0.2 /config, /schema and /annotations/obs response
@@ -178,7 +100,7 @@ export function addObsAnnotations(universe, df) {
 
   // for all of the new data, reconcile with schema and sort categories.
   const dfs = Array.isArray(df) ? df : [df];
-  const keys = dfs.map((d) => d.colIndex.keys()).flat();
+  const keys = dfs.map((d) => d.colIndex.labels()).flat();
   const { schema } = universe;
   keys.forEach((k) => {
     const colSchema = schema.annotations.obsByName[k];

client/src/util/stateManager/world.js

@@ -99,7 +99,7 @@ function clipDataframe(
   if (upperQuantile > 1) upperQuantile = 1;
   if (lowerQuantile === 0 && upperQuantile === 1) return df;
 
-  const keys = df.colIndex.keys();
+  const keys = df.colIndex.labels();
   return df.mapColumns((col, colIdx) => {
     const colLabel = keys[colIdx];
     if (!clipPredicate(df, colIdx, colLabel)) return col;
@@ -277,7 +277,7 @@ export function addObsDimensions(crossfilter, world) {
   but not yet in the crossfilter
   */
   const schema = world.schema.annotations.obsByName;
-  const dimsWeNeed = world.obsAnnotations.colIndex.keys();
+  const dimsWeNeed = world.obsAnnotations.colIndex.labels();
   crossfilter = dimsWeNeed.reduce((xfltr, name) => {
     const dimName = obsAnnoDimensionName(name);
     if (xfltr.hasDimension(dimName)) return xfltr;
@@ -321,7 +321,7 @@ export function getSelectedByIndex(crossfilter) {
   return array of obsIndex, containing all selected obs/cells.
   */
   const selected = crossfilter.allSelectedMask(); // array of bool-ish
-  const keys = crossfilter.data.rowIndex.keys(); // row keys, aka universe rowIndex
+  const keys = crossfilter.data.rowIndex.labels(); // row keys, aka universe rowIndex
 
   const set = new Int32Array(selected.length);
   let numElems = 0;

client/src/util/typedCrossfilter/crossfilter.js

@@ -60,9 +60,7 @@ export default class ImmutableTypedCrossfilter {
   }
 
   setData(data) {
-    const { selectionCache } = this;
+    return new ImmutableTypedCrossfilter(data, this.dimensions);
-    this.selectionCache = {};
-    return new ImmutableTypedCrossfilter(data, this.dimensions, selectionCache);
   }
 
   dimensionNames() {

dev_docs/developer_guidelines.md

@@ -51,11 +51,17 @@ JEST_ENV=prod make pydist install-dist dev-env smoke-test
 
 ## Server dev
 ### Install
+
+To install from the source tree
 * Build the client and put static files in place: `make build-for-server-dev`
 * Install from local files: `make install-dev`
 
+To install from a candidate python distribution
+* Make the distribution: `make pydist`
+* Install it: `make install-dist`
+
 ### Launch
-* `cellxgene launch [options] <datafile>`
+* `cellxgene launch [options] <datafile>` or `make start-server`
 
 ### Reloading
 If you install cellxgene using `make install-dev` the server will be restarted every time you make changes on the server code. If changes affects the client, the browser must be reloaded.
@@ -78,13 +84,13 @@ If you would like to run the server tests individually, follow the steps below
 ## Client dev
 ### Install
 1. Install prereqs for client: `make dev-env`
-2. Install cellxgene server: `make install-dev` Caveat: this will not build the production client package - you must use the [server install](#install) instructions above to serve web assets.
+2. Install cellxgene server as described in the [server install](#install) instructions above.
 
 ### Launch
-To launch with hot reloading you need to launch the server and the client separately. Node's hot reloading starts the client on its own node server and auto-refreshes when changes are made.
+To launch with hot reloading, you need to launch the server and the client separately. Node's hot reloading starts the client on its own node server and auto-refreshes when changes are made to source files.
-1. Launch server (the client relies on the REST API being available): `cellxgene launch [options] <datafile>`
+1. Launch server (the client relies on the REST API being available): `cellxgene launch --debug [other_options] <datafile>` or `make start-server`
-2. Launch client: in `client/` directory run `npm run start`
+2. Launch client: in `client/` directory run `make start-frontend`
-3. Client will be served on localhost:3000
+3. Client will be served on `localhost:3000`
 
 ### Build
 To build only the client: `make build-client`
@@ -95,10 +101,13 @@ We use `eslint` to lint the code and `prettier` as our code formatter.
 ### Test
 
 If you would like to run the client tests individually, follow the steps below in the `client` directory
-1. For unit tests run `npm run unit-test` or `make unit-test`
+1. For unit tests run `make unit-test`
-1. For the smoke test run `npm run smoke-test` or `make smoke-test`
+1. For the smoke test run `make smoke-test` for the standard smoke test suite and `make smoke-test-annotations` for the annotations test suite.
+
+If you would like to run the smoke tests against a hot-reloaded version of the client:
+1. Start the hot-reloading servers as described in the [Client dev section](#client-dev). If you plan to run the standard test suite (without annotations), you'll have to start the backend server with annotations disabled (e.g. `CXG_OPTIONS='--debug --disable-annotations' make start-server`).
+1. From the project root, `cd client`
+1. Run either the standard E2E test suite with `CXG_CLIENT_PORT=3000 make e2e` or the annotations test suite with `CXG_CLIENT_PORT=3000 make e2e-annotations`
 
 ### Tips
 * You can also install/launch the server side code from npm scrips (requires python3.6 with virtualenv) with the `scripts/backend_dev` script.
-
-

docs/_site/index.html

@@ -5,7 +5,7 @@
     <meta http-equiv="X-UA-Compatible" content="IE=edge">
     <meta name="viewport" content="width=device-width, initial-scale=1">
 
-<!-- Begin Jekyll SEO tag v2.6.1 -->
+<!-- Begin Jekyll SEO tag v2.5.0 -->
 <title>Index | cellxgene</title>
 <meta name="generator" content="Jekyll v3.8.5" />
 <meta property="og:title" content="Index" />
@@ -16,10 +16,10 @@
 <meta property="og:url" content="https://chanzuckerberg.github.io/cellxgene/" />
 <meta property="og:site_name" content="cellxgene" />
 <script type="application/ld+json">
-{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"An interactive explorer for single-cell transcriptomics data","@type":"WebSite","url":"https://chanzuckerberg.github.io/cellxgene/","headline":"Index","name":"cellxgene","@context":"https://schema.org"}</script>
+{"description":"An interactive explorer for single-cell transcriptomics data","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"@type":"WebSite","headline":"Index","url":"https://chanzuckerberg.github.io/cellxgene/","name":"cellxgene","@context":"http://schema.org"}</script>
 <!-- End Jekyll SEO tag -->
 
-    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=b45f000ecb36779dde84b689d463d17a077275d6">
+    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=994a7b19bb6437b8e48d92683aab7fd8077477f7">
     <!--[if lt IE 9]>
     <script src="https://cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv.min.js"></script>
     <![endif]-->

docs/_site/posts/annotations.html

@@ -5,7 +5,7 @@
     <meta http-equiv="X-UA-Compatible" content="IE=edge">
     <meta name="viewport" content="width=device-width, initial-scale=1">
 
-<!-- Begin Jekyll SEO tag v2.6.1 -->
+<!-- Begin Jekyll SEO tag v2.5.0 -->
 <title>annotations | cellxgene</title>
 <meta name="generator" content="Jekyll v3.8.5" />
 <meta property="og:title" content="annotations" />
@@ -16,10 +16,10 @@
 <meta property="og:url" content="https://chanzuckerberg.github.io/cellxgene/posts/annotations.html" />
 <meta property="og:site_name" content="cellxgene" />
 <script type="application/ld+json">
-{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"Creating annotations","@type":"WebPage","url":"https://chanzuckerberg.github.io/cellxgene/posts/annotations.html","headline":"annotations","@context":"https://schema.org"}</script>
+{"description":"Creating annotations","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"@type":"WebPage","headline":"annotations","url":"https://chanzuckerberg.github.io/cellxgene/posts/annotations.html","@context":"http://schema.org"}</script>
 <!-- End Jekyll SEO tag -->
 
-    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=b45f000ecb36779dde84b689d463d17a077275d6">
+    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=994a7b19bb6437b8e48d92683aab7fd8077477f7">
     <!--[if lt IE 9]>
     <script src="https://cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv.min.js"></script>
     <![endif]-->

docs/_site/posts/contact.html

@@ -5,7 +5,7 @@
     <meta http-equiv="X-UA-Compatible" content="IE=edge">
     <meta name="viewport" content="width=device-width, initial-scale=1">
 
-<!-- Begin Jekyll SEO tag v2.6.1 -->
+<!-- Begin Jekyll SEO tag v2.5.0 -->
 <title>Contact | cellxgene</title>
 <meta name="generator" content="Jekyll v3.8.5" />
 <meta property="og:title" content="Contact" />
@@ -16,10 +16,10 @@
 <meta property="og:url" content="https://chanzuckerberg.github.io/cellxgene/posts/contact.html" />
 <meta property="og:site_name" content="cellxgene" />
 <script type="application/ld+json">
-{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"Contact","@type":"WebPage","url":"https://chanzuckerberg.github.io/cellxgene/posts/contact.html","headline":"Contact","@context":"https://schema.org"}</script>
+{"description":"Contact","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"@type":"WebPage","headline":"Contact","url":"https://chanzuckerberg.github.io/cellxgene/posts/contact.html","@context":"http://schema.org"}</script>
 <!-- End Jekyll SEO tag -->
 
-    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=b45f000ecb36779dde84b689d463d17a077275d6">
+    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=994a7b19bb6437b8e48d92683aab7fd8077477f7">
     <!--[if lt IE 9]>
     <script src="https://cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv.min.js"></script>
     <![endif]-->

docs/_site/posts/contribute.html

@@ -5,7 +5,7 @@
     <meta http-equiv="X-UA-Compatible" content="IE=edge">
     <meta name="viewport" content="width=device-width, initial-scale=1">
 
-<!-- Begin Jekyll SEO tag v2.6.1 -->
+<!-- Begin Jekyll SEO tag v2.5.0 -->
 <title>Code of conduct | cellxgene</title>
 <meta name="generator" content="Jekyll v3.8.5" />
 <meta property="og:title" content="Code of conduct" />
@@ -16,10 +16,10 @@
 <meta property="og:url" content="https://chanzuckerberg.github.io/cellxgene/posts/contribute.html" />
 <meta property="og:site_name" content="cellxgene" />
 <script type="application/ld+json">
-{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"An interactive explorer for single-cell transcriptomics data","@type":"WebPage","url":"https://chanzuckerberg.github.io/cellxgene/posts/contribute.html","headline":"Code of conduct","@context":"https://schema.org"}</script>
+{"description":"An interactive explorer for single-cell transcriptomics data","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"@type":"WebPage","headline":"Code of conduct","url":"https://chanzuckerberg.github.io/cellxgene/posts/contribute.html","@context":"http://schema.org"}</script>
 <!-- End Jekyll SEO tag -->
 
-    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=b45f000ecb36779dde84b689d463d17a077275d6">
+    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=994a7b19bb6437b8e48d92683aab7fd8077477f7">
     <!--[if lt IE 9]>
     <script src="https://cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv.min.js"></script>
     <![endif]-->

docs/_site/posts/demo-data.html

@@ -5,7 +5,7 @@
     <meta http-equiv="X-UA-Compatible" content="IE=edge">
     <meta name="viewport" content="width=device-width, initial-scale=1">
 
-<!-- Begin Jekyll SEO tag v2.6.1 -->
+<!-- Begin Jekyll SEO tag v2.5.0 -->
 <title>demo-data | cellxgene</title>
 <meta name="generator" content="Jekyll v3.8.5" />
 <meta property="og:title" content="demo-data" />
@@ -16,10 +16,10 @@
 <meta property="og:url" content="https://chanzuckerberg.github.io/cellxgene/posts/demo-data.html" />
 <meta property="og:site_name" content="cellxgene" />
 <script type="application/ld+json">
-{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"Demo datasets","@type":"WebPage","url":"https://chanzuckerberg.github.io/cellxgene/posts/demo-data.html","headline":"demo-data","@context":"https://schema.org"}</script>
+{"description":"Demo datasets","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"@type":"WebPage","headline":"demo-data","url":"https://chanzuckerberg.github.io/cellxgene/posts/demo-data.html","@context":"http://schema.org"}</script>
 <!-- End Jekyll SEO tag -->
 
-    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=b45f000ecb36779dde84b689d463d17a077275d6">
+    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=994a7b19bb6437b8e48d92683aab7fd8077477f7">
     <!--[if lt IE 9]>
     <script src="https://cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv.min.js"></script>
     <![endif]-->

docs/_site/posts/gallery.html

@@ -5,7 +5,7 @@
     <meta http-equiv="X-UA-Compatible" content="IE=edge">
     <meta name="viewport" content="width=device-width, initial-scale=1">
 
-<!-- Begin Jekyll SEO tag v2.6.1 -->
+<!-- Begin Jekyll SEO tag v2.5.0 -->
 <title>Gallery | cellxgene</title>
 <meta name="generator" content="Jekyll v3.8.5" />
 <meta property="og:title" content="Gallery" />
@@ -16,10 +16,10 @@
 <meta property="og:url" content="https://chanzuckerberg.github.io/cellxgene/posts/gallery.html" />
 <meta property="og:site_name" content="cellxgene" />
 <script type="application/ld+json">
-{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"An interactive explorer for single-cell transcriptomics data","@type":"WebPage","url":"https://chanzuckerberg.github.io/cellxgene/posts/gallery.html","headline":"Gallery","@context":"https://schema.org"}</script>
+{"description":"An interactive explorer for single-cell transcriptomics data","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"@type":"WebPage","headline":"Gallery","url":"https://chanzuckerberg.github.io/cellxgene/posts/gallery.html","@context":"http://schema.org"}</script>
 <!-- End Jekyll SEO tag -->
 
-    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=b45f000ecb36779dde84b689d463d17a077275d6">
+    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=994a7b19bb6437b8e48d92683aab7fd8077477f7">
     <!--[if lt IE 9]>
     <script src="https://cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv.min.js"></script>
     <![endif]-->

docs/_site/posts/hosted.html

@@ -5,7 +5,7 @@
     <meta http-equiv="X-UA-Compatible" content="IE=edge">
     <meta name="viewport" content="width=device-width, initial-scale=1">
 
-<!-- Begin Jekyll SEO tag v2.6.1 -->
+<!-- Begin Jekyll SEO tag v2.5.0 -->
 <title>Hosting cellxgene on the web | cellxgene</title>
 <meta name="generator" content="Jekyll v3.8.5" />
 <meta property="og:title" content="Hosting cellxgene on the web" />
@@ -16,10 +16,10 @@
 <meta property="og:url" content="https://chanzuckerberg.github.io/cellxgene/posts/hosted.html" />
 <meta property="og:site_name" content="cellxgene" />
 <script type="application/ld+json">
-{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"An interactive explorer for single-cell transcriptomics data","@type":"WebPage","url":"https://chanzuckerberg.github.io/cellxgene/posts/hosted.html","headline":"Hosting cellxgene on the web","@context":"https://schema.org"}</script>
+{"description":"An interactive explorer for single-cell transcriptomics data","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"@type":"WebPage","headline":"Hosting cellxgene on the web","url":"https://chanzuckerberg.github.io/cellxgene/posts/hosted.html","@context":"http://schema.org"}</script>
 <!-- End Jekyll SEO tag -->
 
-    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=b45f000ecb36779dde84b689d463d17a077275d6">
+    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=994a7b19bb6437b8e48d92683aab7fd8077477f7">
     <!--[if lt IE 9]>
     <script src="https://cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv.min.js"></script>
     <![endif]-->

docs/_site/posts/install.html

@@ -5,7 +5,7 @@
     <meta http-equiv="X-UA-Compatible" content="IE=edge">
     <meta name="viewport" content="width=device-width, initial-scale=1">
 
-<!-- Begin Jekyll SEO tag v2.6.1 -->
+<!-- Begin Jekyll SEO tag v2.5.0 -->
 <title>Install | cellxgene</title>
 <meta name="generator" content="Jekyll v3.8.5" />
 <meta property="og:title" content="Install" />
@@ -16,10 +16,10 @@
 <meta property="og:url" content="https://chanzuckerberg.github.io/cellxgene/posts/install.html" />
 <meta property="og:site_name" content="cellxgene" />
 <script type="application/ld+json">
-{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"An interactive explorer for single-cell transcriptomics data","@type":"WebPage","url":"https://chanzuckerberg.github.io/cellxgene/posts/install.html","headline":"Install","@context":"https://schema.org"}</script>
+{"description":"An interactive explorer for single-cell transcriptomics data","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"@type":"WebPage","headline":"Install","url":"https://chanzuckerberg.github.io/cellxgene/posts/install.html","@context":"http://schema.org"}</script>
 <!-- End Jekyll SEO tag -->
 
-    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=b45f000ecb36779dde84b689d463d17a077275d6">
+    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=994a7b19bb6437b8e48d92683aab7fd8077477f7">
     <!--[if lt IE 9]>
     <script src="https://cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv.min.js"></script>
     <![endif]-->

docs/_site/posts/launch.html

@@ -5,7 +5,7 @@
     <meta http-equiv="X-UA-Compatible" content="IE=edge">
     <meta name="viewport" content="width=device-width, initial-scale=1">
 
-<!-- Begin Jekyll SEO tag v2.6.1 -->
+<!-- Begin Jekyll SEO tag v2.5.0 -->
 <title>demo-data | cellxgene</title>
 <meta name="generator" content="Jekyll v3.8.5" />
 <meta property="og:title" content="demo-data" />
@@ -16,10 +16,10 @@
 <meta property="og:url" content="https://chanzuckerberg.github.io/cellxgene/posts/launch.html" />
 <meta property="og:site_name" content="cellxgene" />
 <script type="application/ld+json">
-{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"Demo datasets","@type":"WebPage","url":"https://chanzuckerberg.github.io/cellxgene/posts/launch.html","headline":"demo-data","@context":"https://schema.org"}</script>
+{"description":"Demo datasets","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"@type":"WebPage","headline":"demo-data","url":"https://chanzuckerberg.github.io/cellxgene/posts/launch.html","@context":"http://schema.org"}</script>
 <!-- End Jekyll SEO tag -->
 
-    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=b45f000ecb36779dde84b689d463d17a077275d6">
+    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=994a7b19bb6437b8e48d92683aab7fd8077477f7">
     <!--[if lt IE 9]>
     <script src="https://cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv.min.js"></script>
     <![endif]-->

docs/_site/posts/methods.html

@@ -5,7 +5,7 @@
     <meta http-equiv="X-UA-Compatible" content="IE=edge">
     <meta name="viewport" content="width=device-width, initial-scale=1">
 
-<!-- Begin Jekyll SEO tag v2.6.1 -->
+<!-- Begin Jekyll SEO tag v2.5.0 -->
 <title>Methods | cellxgene</title>
 <meta name="generator" content="Jekyll v3.8.5" />
 <meta property="og:title" content="Methods" />
@@ -16,10 +16,10 @@
 <meta property="og:url" content="https://chanzuckerberg.github.io/cellxgene/posts/methods.html" />
 <meta property="og:site_name" content="cellxgene" />
 <script type="application/ld+json">
-{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"An interactive explorer for single-cell transcriptomics data","@type":"WebPage","url":"https://chanzuckerberg.github.io/cellxgene/posts/methods.html","headline":"Methods","@context":"https://schema.org"}</script>
+{"description":"An interactive explorer for single-cell transcriptomics data","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"@type":"WebPage","headline":"Methods","url":"https://chanzuckerberg.github.io/cellxgene/posts/methods.html","@context":"http://schema.org"}</script>
 <!-- End Jekyll SEO tag -->
 
-    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=b45f000ecb36779dde84b689d463d17a077275d6">
+    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=994a7b19bb6437b8e48d92683aab7fd8077477f7">
     <!--[if lt IE 9]>
     <script src="https://cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv.min.js"></script>
     <![endif]-->

docs/_site/posts/prepare.html

@@ -5,7 +5,7 @@
     <meta http-equiv="X-UA-Compatible" content="IE=edge">
     <meta name="viewport" content="width=device-width, initial-scale=1">
 
-<!-- Begin Jekyll SEO tag v2.6.1 -->
+<!-- Begin Jekyll SEO tag v2.5.0 -->
 <title>prepare | cellxgene</title>
 <meta name="generator" content="Jekyll v3.8.5" />
 <meta property="og:title" content="prepare" />
@@ -16,10 +16,10 @@
 <meta property="og:url" content="https://chanzuckerberg.github.io/cellxgene/posts/prepare.html" />
 <meta property="og:site_name" content="cellxgene" />
 <script type="application/ld+json">
-{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"Preparing your data","@type":"WebPage","url":"https://chanzuckerberg.github.io/cellxgene/posts/prepare.html","headline":"prepare","@context":"https://schema.org"}</script>
+{"description":"Preparing your data","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"@type":"WebPage","headline":"prepare","url":"https://chanzuckerberg.github.io/cellxgene/posts/prepare.html","@context":"http://schema.org"}</script>
 <!-- End Jekyll SEO tag -->
 
-    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=b45f000ecb36779dde84b689d463d17a077275d6">
+    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=994a7b19bb6437b8e48d92683aab7fd8077477f7">
     <!--[if lt IE 9]>
     <script src="https://cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv.min.js"></script>
     <![endif]-->

docs/_site/posts/roadmap.html

@@ -5,7 +5,7 @@
     <meta http-equiv="X-UA-Compatible" content="IE=edge">
     <meta name="viewport" content="width=device-width, initial-scale=1">
 
-<!-- Begin Jekyll SEO tag v2.6.1 -->
+<!-- Begin Jekyll SEO tag v2.5.0 -->
 <title>roadmap | cellxgene</title>
 <meta name="generator" content="Jekyll v3.8.5" />
 <meta property="og:title" content="roadmap" />
@@ -16,10 +16,10 @@
 <meta property="og:url" content="https://chanzuckerberg.github.io/cellxgene/posts/roadmap.html" />
 <meta property="og:site_name" content="cellxgene" />
 <script type="application/ld+json">
-{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"Roadmap","@type":"WebPage","url":"https://chanzuckerberg.github.io/cellxgene/posts/roadmap.html","headline":"roadmap","@context":"https://schema.org"}</script>
+{"description":"Roadmap","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"@type":"WebPage","headline":"roadmap","url":"https://chanzuckerberg.github.io/cellxgene/posts/roadmap.html","@context":"http://schema.org"}</script>
 <!-- End Jekyll SEO tag -->
 
-    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=b45f000ecb36779dde84b689d463d17a077275d6">
+    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=994a7b19bb6437b8e48d92683aab7fd8077477f7">
     <!--[if lt IE 9]>
     <script src="https://cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv.min.js"></script>
     <![endif]-->

docs/_site/posts/troubleshooting.html

@@ -5,7 +5,7 @@
     <meta http-equiv="X-UA-Compatible" content="IE=edge">
     <meta name="viewport" content="width=device-width, initial-scale=1">
 
-<!-- Begin Jekyll SEO tag v2.6.1 -->
+<!-- Begin Jekyll SEO tag v2.5.0 -->
 <title>Troubleshooting | cellxgene</title>
 <meta name="generator" content="Jekyll v3.8.5" />
 <meta property="og:title" content="Troubleshooting" />
@@ -16,10 +16,10 @@
 <meta property="og:url" content="https://chanzuckerberg.github.io/cellxgene/posts/troubleshooting.html" />
 <meta property="og:site_name" content="cellxgene" />
 <script type="application/ld+json">
-{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"Troubleshooting","@type":"WebPage","url":"https://chanzuckerberg.github.io/cellxgene/posts/troubleshooting.html","headline":"Troubleshooting","@context":"https://schema.org"}</script>
+{"description":"Troubleshooting","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"@type":"WebPage","headline":"Troubleshooting","url":"https://chanzuckerberg.github.io/cellxgene/posts/troubleshooting.html","@context":"http://schema.org"}</script>
 <!-- End Jekyll SEO tag -->
 
-    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=b45f000ecb36779dde84b689d463d17a077275d6">
+    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=994a7b19bb6437b8e48d92683aab7fd8077477f7">
     <!--[if lt IE 9]>
     <script src="https://cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv.min.js"></script>
     <![endif]-->

docs/posts/cellxgene_cziscience_com.md

@@ -125,13 +125,71 @@ with a link to embed on your own site, please drop us a note at <mailto:cellxgen
       </td>
     </tr>
     <tr>
-      <td><a href="https://cellxgene.cziscience.com/d/Single_cell_atlas_of_peripheral_immune_response_to_SARS_CoV_2_infection-25.cxg/" target="_blank">A single-cell atlas of the peripheral immune response to severe COVID-19
+      <td><a href="https://cellxgene.cziscience.com/d/Single_cell_atlas_of_peripheral_immune_response_to_SARS_CoV_2_infection-25.cxg/" target="_blank">A single-cell atlas of the peripheral immune response to severe COVID-19</a></td>
-</a></td>
       <td>
         <a href="https://blishlab.sites.stanford.edu/">Blish Lab</a>,
         <a href="https://www.medrxiv.org/content/10.1101/2020.04.17.20069930v1">medRxiv preprint</a>
       </td>
     </tr>
-
+    <tr>
+      <td><a href="https://cellxgene.cziscience.com/d/Atlas_of_Healthy_and_SHIV_Infected_Non_Human_Primate_Lung_and_Ileum_ACE2+_Cells_ileum-12.cxg/" target="_blank">Atlas of Healthy and SHIV-Infected Non-Human Primate Lung and Ileum ACE2+ Cells - Ileum</a></td>
+      <td>
+        <a href="https://singlecell.broadinstitute.org/single_cell/study/SCP807/atlas-of-healthy-and-shiv-infected-non-human-primate-lung-and-ileum-ace2-cells?scpbr=the-alexandria-project">Single Cell Portal</a>
+      </td>
+    </tr>
+    <tr>
+      <td><a href="https://cellxgene.cziscience.com/d/Atlas_of_Healthy_and_SHIV_Infected_Non_Human_Primate_Lung_and_Ileum_ACE2+_Cells_lung-11.cxg/" target="_blank">Atlas of Healthy and SHIV-Infected Non-Human Primate Lung and Ileum ACE2+ Cells - Lung</a></td>
+      <td>
+        <a href="https://singlecell.broadinstitute.org/single_cell/study/SCP807/atlas-of-healthy-and-shiv-infected-non-human-primate-lung-and-ileum-ace2-cells?scpbr=the-alexandria-project">Single Cell Portal</a>
+      </td>
+    </tr>
+    <tr>
+      <td><a href="https://cellxgene.cziscience.com/d/Allergic_inflammatory_memory_in_human_respiratory_epithelial_progenitor_cells_epithelial-10.cxg/" target="_blank">Allergic inflammatory memory in human respiratory epithelial progenitor cells - epithelial cells</a></td>
+      <td>
+        <a href="https://singlecell.broadinstitute.org/single_cell/study/SCP253/allergic-inflammatory-memory-in-human-respiratory-epithelial-progenitor-cells?scpbr=the-alexandria-project">Single Cell Portal</a>
+      </td>
+    </tr>
+    <tr>
+      <td><a href="https://cellxgene.cziscience.com/d/Allergic_inflammatory_memory_in_human_respiratory_epithelial_progenitor_cells_scraping-9.cxg/" target="_blank">Allergic inflammatory memory in human respiratory epithelial progenitor cells - nasal scrapings</a></td>
+      <td>
+        <a href="https://singlecell.broadinstitute.org/single_cell/study/SCP253/allergic-inflammatory-memory-in-human-respiratory-epithelial-progenitor-cells?scpbr=the-alexandria-project">Single Cell Portal</a>
+      </td>
+    </tr>
+    <tr>
+      <td><a href="https://cellxgene.cziscience.com/d/Allergic_inflammatory_memory_in_human_respiratory_epithelial_progenitor_cells_surgical-8.cxg/" target="_blank">Allergic inflammatory memory in human respiratory epithelial progenitor cells - surgical</a></td>
+      <td>
+        <a href="https://singlecell.broadinstitute.org/single_cell/study/SCP253/allergic-inflammatory-memory-in-human-respiratory-epithelial-progenitor-cells?scpbr=the-alexandria-project">Single Cell Portal</a>
+      </td>
+    </tr>
+    <tr>
+      <td><a href="https://cellxgene.cziscience.com/d/Allergic_inflammatory_memory_in_human_respiratory_epithelial_progenitor_cells_nasalsss-26.cxg/" target="_blank">Allergic inflammatory memory in human respiratory epithelial progenitor cells - nasal SSS</a></td>
+      <td>
+        <a href="https://singlecell.broadinstitute.org/single_cell/study/SCP253/allergic-inflammatory-memory-in-human-respiratory-epithelial-progenitor-cells?scpbr=the-alexandria-project">Single Cell Portal</a>
+      </td>
+    </tr>
+    <tr>
+      <td><a href="https://cellxgene.cziscience.com/d/ACE2_and_TMPRSS2_expression_in_human_non_inflamed_terminal_ileum_epithelial-7.cxg/" target="_blank">ACE2 and TMPRSS2 expression in human non-inflamed terminal ileum - epithelial cells</a></td>
+      <td>
+        <a href="https://singlecell.broadinstitute.org/single_cell/study/SCP812/ace2-and-tmprss2-expression-in-human-non-inflamed-terminal-ileum?scpbr=the-alexandria-project">Single Cell Portal</a>
+      </td>
+    </tr>
+    <tr>
+      <td><a href="https://cellxgene.cziscience.com/d/ACE2_and_TMPRSS2_expression_in_human_non_inflamed_terminal_ileum-6.cxg/" target="_blank">ACE2 and TMPRSS2 expression in human non-inflamed terminal ileum</a></td>
+      <td>
+        <a href="https://singlecell.broadinstitute.org/single_cell/study/SCP812/ace2-and-tmprss2-expression-in-human-non-inflamed-terminal-ileum?scpbr=the-alexandria-project">Single Cell Portal</a>
+      </td>
+    </tr>
+    <tr>
+      <td><a href="https://cellxgene.cziscience.com/d/Human_Lung_HIV_TB_Co_infection_ACE2+_Cells-5.cxg/" target="_blank">Human Lung HIV-TB Co-infection ACE2+ Cells</a></td>
+      <td>
+        <a href="https://singlecell.broadinstitute.org/single_cell/study/SCP814/human-lung-hiv-tb-co-infection-ace2-cells?scpbr=the-alexandria-project">Single Cell Portal</a>
+      </td>
+    </tr>
+    <tr>
+      <td><a href="https://cellxgene.cziscience.com/d/Epithelial_Cells_in_NHP_mTB_Granuloma_and_Uninvolved_Lung-4.cxg/" target="_blank">Epithelial Cells in NHP mTB Granuloma and Uninvolved Lung</a></td>
+      <td>
+        <a href="https://singlecell.broadinstitute.org/single_cell/study/SCP806/epithelial-cells-in-nhp-mtb-granuloma-and-uninvolved-lung?scpbr=the-alexandria-project">Single Cell Portal</a>
+      </td>
+    </tr>
   </tbody>
 </table>

server/app/app.py

@@ -58,7 +58,7 @@ def cache_control_always(**cache_kwargs):
 
 @webbp.route("/", methods=["GET"])
 @cache_control(public=True, max_age=ONE_WEEK)
-def dataset_index(dataset=None):
+def dataset_index(url_dataroot=None, dataset=None):
     config = current_app.app_config
     if dataset is None:
         if config.single_dataset__datapath:
@@ -66,7 +66,10 @@ def dataset_index(dataset=None):
         else:
             return dataroot_index()
     else:
-        location = path_join(config.multi_dataset__dataroot, dataset)
+        dataroot = config.multi_dataset__dataroot.get(url_dataroot)
+        if dataroot is None:
+            abort(HTTPStatus.NOT_FOUND)
+        location = path_join(dataroot, dataset)
 
     scripts = config.server__scripts
     inline_scripts = config.server__inline_scripts
@@ -91,18 +94,21 @@ def health():
     return health_check(config)
 
 
-def get_data_adaptor(dataset=None):
+def get_data_adaptor(url_dataroot=None, dataset=None):
     config = current_app.app_config
 
     if dataset is None:
         datapath = config.single_dataset__datapath
     else:
-        datapath = path_join(config.multi_dataset__dataroot, dataset)
+        dataroot = config.multi_dataset__dataroot.get(url_dataroot)
+        if dataroot is None:
+            raise DatasetAccessError(f"Invalid dataset {url_dataroot}/{dataset}")
+        datapath = path_join(dataroot, dataset)
         # path_join returns a normalized path.  Therefore it is
         # sufficient to check that the datapath starts with the
         # dataroot to determine that the datapath is under the dataroot.
-        if not datapath.startswith(config.multi_dataset__dataroot):
+        if not datapath.startswith(dataroot):
-            raise DatasetAccessError("Invalid dataset {dataset}")
+            raise DatasetAccessError("Invalid dataset {url_dataroot}/{dataset}")
 
     if datapath is None:
         return common_rest.abort_and_log(HTTPStatus.BAD_REQUEST, "Invalid dataset NONE", loglevel=logging.INFO)
@@ -115,7 +121,7 @@ def rest_get_data_adaptor(func):
     @wraps(func)
     def wrapped_function(self, dataset=None):
         try:
-            with get_data_adaptor(dataset) as data_adaptor:
+            with get_data_adaptor(self.url_dataroot, dataset) as data_adaptor:
                 return func(self, data_adaptor)
         except DatasetAccessError:
             return common_rest.abort_and_log(
@@ -132,22 +138,23 @@ def dataroot_test_index():
     data += "<body><H1>Welcome to cellxgene</H1>"
 
     config = current_app.app_config
-    locator = DataLocator(config.multi_dataset__dataroot, region_name=config.data_locator__s3__region_name)
     datasets = []
-    for fname in locator.ls():
+    for url_dataroot, dataroot in config.multi_dataset__dataroot.items():
-        location = path_join(config.multi_dataset__dataroot, fname)
+        locator = DataLocator(dataroot, region_name=config.data_locator__s3__region_name)
-        try:
+        for fname in locator.ls():
-            MatrixDataLoader(location, app_config=config)
+            location = path_join(dataroot, fname)
-            datasets.append(fname)
+            try:
-        except DatasetAccessError:
+                MatrixDataLoader(location, app_config=config)
-            # skip over invalid datasets
+                datasets.append((url_dataroot, fname))
-            pass
+            except DatasetAccessError:
+                # skip over invalid datasets
+                pass
 
     data += "<br/>Select one of these datasets...<br/>"
     data += "<ul>"
     datasets.sort()
-    for dataset in datasets:
+    for url_dataroot, dataset in datasets:
-        data += f"<li><a href=d/{dataset}>{dataset}</a></li>"
+        data += f"<li><a href={url_dataroot}/{dataset}>{dataset}</a></li>"
     data += "</ul>"
     data += "</body></html>"
 
@@ -165,21 +172,29 @@ def dataroot_index():
         return redirect(config.multi_dataset__index)
 
 
-class SchemaAPI(Resource):
+class DatasetResource(Resource):
+    """Base class for all Resources that act on datasets."""
+
+    def __init__(self, url_dataroot):
+        super().__init__()
+        self.url_dataroot = url_dataroot
+
+
+class SchemaAPI(DatasetResource):
     @cache_control(public=True, max_age=ONE_WEEK)
     @rest_get_data_adaptor
     def get(self, data_adaptor):
         return common_rest.schema_get(data_adaptor, current_app.annotations)
 
 
-class ConfigAPI(Resource):
+class ConfigAPI(DatasetResource):
     @cache_control(public=True, max_age=ONE_WEEK)
     @rest_get_data_adaptor
     def get(self, data_adaptor):
         return common_rest.config_get(current_app.app_config, data_adaptor, current_app.annotations)
 
 
-class AnnotationsObsAPI(Resource):
+class AnnotationsObsAPI(DatasetResource):
     @cache_control(public=True, max_age=ONE_WEEK)
     @rest_get_data_adaptor
     def get(self, data_adaptor):
@@ -191,14 +206,14 @@ class AnnotationsObsAPI(Resource):
         return common_rest.annotations_obs_put(request, data_adaptor, current_app.annotations)
 
 
-class AnnotationsVarAPI(Resource):
+class AnnotationsVarAPI(DatasetResource):
     @cache_control(public=True, max_age=ONE_WEEK)
     @rest_get_data_adaptor
     def get(self, data_adaptor):
         return common_rest.annotations_var_get(request, data_adaptor, current_app.annotations)
 
 
-class DataVarAPI(Resource):
+class DataVarAPI(DatasetResource):
     @cache_control(no_store=True)
     @rest_get_data_adaptor
     def put(self, data_adaptor):
@@ -210,21 +225,21 @@ class DataVarAPI(Resource):
         return common_rest.data_var_get(request, data_adaptor)
 
 
-class ColorsAPI(Resource):
+class ColorsAPI(DatasetResource):
     @cache_control(public=True, max_age=ONE_WEEK)
     @rest_get_data_adaptor
     def get(self, data_adaptor):
         return common_rest.colors_get(data_adaptor)
 
 
-class DiffExpObsAPI(Resource):
+class DiffExpObsAPI(DatasetResource):
     @cache_control(no_store=True)
     @rest_get_data_adaptor
     def post(self, data_adaptor):
         return common_rest.diffexp_obs_post(request, data_adaptor)
 
 
-class LayoutObsAPI(Resource):
+class LayoutObsAPI(DatasetResource):
     @cache_control(public=True, max_age=ONE_WEEK)
     @rest_get_data_adaptor
     def get(self, data_adaptor):
@@ -236,20 +251,25 @@ class LayoutObsAPI(Resource):
         return common_rest.layout_obs_put(request, data_adaptor)
 
 
-def get_api_resources(bp_api):
+def get_api_resources(bp_api, url_dataroot=None):
     api = Api(bp_api)
+
+    def add_resource(resource, url):
+        """convenience function to make the outer function less verbose"""
+        api.add_resource(resource, url, resource_class_args=(url_dataroot,))
+
     # Initialization routes
-    api.add_resource(SchemaAPI, "/schema")
+    add_resource(SchemaAPI, "/schema")
-    api.add_resource(ConfigAPI, "/config")
+    add_resource(ConfigAPI, "/config")
     # Data routes
-    api.add_resource(AnnotationsObsAPI, "/annotations/obs")
+    add_resource(AnnotationsObsAPI, "/annotations/obs")
-    api.add_resource(AnnotationsVarAPI, "/annotations/var")
+    add_resource(AnnotationsVarAPI, "/annotations/var")
-    api.add_resource(DataVarAPI, "/data/var")
+    add_resource(DataVarAPI, "/data/var")
     # Display routes
-    api.add_resource(ColorsAPI, "/colors")
+    add_resource(ColorsAPI, "/colors")
     # Computation routes
-    api.add_resource(DiffExpObsAPI, "/diffexp/obs")
+    add_resource(DiffExpObsAPI, "/diffexp/obs")
-    api.add_resource(LayoutObsAPI, "/layout/obs")
+    add_resource(LayoutObsAPI, "/layout/obs")
     return api
 
 
@@ -285,10 +305,18 @@ class Server:
             # NOTE:  These routes only allow the dataset to be in the directory
             # of the dataroot, and not a subdirectory.  We may want to change
             # the route format at some point
-            bp_api = Blueprint("api_dataset", __name__, url_prefix="/d/<dataset>" + api_version)
+            for url_dataroot in app_config.multi_dataset__dataroot.keys():
-            resources = get_api_resources(bp_api)
+                bp_api = Blueprint(
-            self.app.register_blueprint(resources.blueprint)
+                    f"api_dataset_{url_dataroot}", __name__, url_prefix=f"/{url_dataroot}/<dataset>" + api_version
-            self.app.add_url_rule("/d/<dataset>/", "dataset_index", dataset_index, methods=["GET"])
+                )
+                resources = get_api_resources(bp_api, url_dataroot)
+                self.app.register_blueprint(resources.blueprint)
+                self.app.add_url_rule(
+                    f"/{url_dataroot}/<dataset>/",
+                    f"dataset_index_{url_dataroot}",
+                    lambda dataset: dataset_index(url_dataroot, dataset),
+                    methods=["GET"],
+                )
         self.app.matrix_data_cache_manager = app_config.matrix_data_cache_manager
         self.app.annotations = app_config.user_annotations
         self.app.app_config = app_config

server/common/app_config.py

@@ -1,9 +1,9 @@
 from server import __version__ as cellxgene_version
-from flatten_dict import flatten
+from flatten_dict import flatten, unflatten
 import os
 from os.path import splitext, basename, isdir
 import sys
-from urllib.parse import urlparse
+from urllib.parse import urlparse, quote_plus
 import yaml
 import copy
 
@@ -125,17 +125,23 @@ class AppConfig(object):
         dc = copy.deepcopy(config)
         mapping = {}
 
-        # special case for tiledb_ctx whose value is a dict.
+        # special cases where the value could be a dict.
-        val = config.get("adaptor", {}).get("cxg_adaptor", {}).get("tiledb_ctx")
+        # If its value is not None, the entry is added to the mapping, and not included
-        if val is not None:
+        # in the flattening below.
-            mapping["adaptor__cxg_adaptor__tiledb_ctx"] = (("adaptor", "cxg_adaptor", "tiledb_ctx"), val)
+        dictval_cases = [
-            del dc["adaptor"]["cxg_adaptor"]["tiledb_ctx"]
+            ("adaptor", "cxg_adaptor", "tiledb_ctx"),
-
+            ("server", "csp_directives"),
-        # special case for csp_directives whose value is a dict.
+            ("multi_dataset", "dataroot"),
-        val = config.get("server", {}).get("csp_directives")
+        ]
-        if val is not None:
+        for dictval_case in dictval_cases:
-            mapping["server__csp_directives"] = (("server", "csp_directives"), val)
+            cur = dc
-            del dc["server"]["csp_directives"]
+            for part in dictval_case[:-1]:
+                cur = cur.get(part, {})
+            val = cur.get(dictval_case[-1])
+            if val is not None:
+                key = "__".join(dictval_case)
+                mapping[key] = (dictval_case, val)
+                del cur[dictval_case[-1]]
 
         flat_config = flatten(dc)
         for key, value in flat_config.items():
@@ -162,6 +168,14 @@ class AppConfig(object):
 
         self.is_completed = False
 
+    def write_config(self, config_file):
+        """output the config to a yaml file"""
+        mapping = self.__mapping(self.default_config)
+        for attrname in mapping.keys():
+            mapping[attrname] = getattr(self, attrname)
+        config = unflatten(mapping, splitter=lambda key: key.split("__"))
+        yaml.dump(config, open(config_file, "w"))
+
     def update(self, **kw):
         for key, value in kw.items():
             if not hasattr(self, key):
@@ -302,6 +316,14 @@ class AppConfig(object):
         self.__check_attr("data_locator__s3__region_name", (type(None), bool, str))
         if self.data_locator__s3__region_name is True:
             path = self.single_dataset__datapath or self.multi_dataset__dataroot
+            if type(path) == dict:
+                # if multi_dataset__dataroot is a dict, then use the first key
+                # that is in s3.   NOTE:  it is not supported to have dataroots
+                # in different regions.
+                paths = path.values()
+                for path in paths:
+                    if path.startswith("s3://"):
+                        break
             if path.startswith("s3://"):
                 region_name = discover_s3_region_name(path)
                 if region_name is None:
@@ -366,7 +388,7 @@ class AppConfig(object):
                 )
 
     def handle_multi_dataset(self, context):
-        self.__check_attr("multi_dataset__dataroot", (type(None), str))
+        self.__check_attr("multi_dataset__dataroot", (type(None), dict, str))
         self.__check_attr("multi_dataset__index", (type(None), bool, str))
         self.__check_attr("multi_dataset__allowed_matrix_types", list)
         self.__check_attr("multi_dataset__matrix_cache__max_datasets", int)
@@ -375,6 +397,18 @@ class AppConfig(object):
         if self.multi_dataset__dataroot is None:
             return
 
+        if type(self.multi_dataset__dataroot) == str:
+            self.multi_dataset__dataroot = dict(d=self.multi_dataset__dataroot)
+
+        for key in self.multi_dataset__dataroot.keys():
+            # sanity check for well formed keys
+            if type(key) != str:
+                raise ConfigurationError(f"error in multi_dataset__dataroot {key}")
+            if quote_plus(key) != key:
+                raise ConfigurationError(f"error in multi_dataset__dataroot {key}")
+            if os.path.split(os.path.normpath(key))[-1] != key:
+                raise ConfigurationError(f"error in multi_dataset__dataroot {key}")
+
         # error checking
         for mtype in self.multi_dataset__allowed_matrix_types:
             try:

server/common/default_config.py

@@ -29,6 +29,23 @@ presentation:
   custom_colors: true
 
 multi_dataset:
+  # If dataroot is set, then cellxgene may serve multiple datasets.  This parameter is not
+  # compatable with single_dataset/datapath.
+  # dataroot may be a string, representing the path to a directory or S3 prefix.  In this
+  # case the datasets in that location are accessed from <server>/d/<datasetname>.
+  # example:
+  #     dataroot:  /path/to/datasets/
+  # or
+  #     dataroot:  s3://bucket/prefix/
+  #
+  # As an alternative, dataroot can be a dictionary, mapping url prefixes to dataroot paths.
+  # example:
+  #     dataroot:
+  #         set1 : /path/to/set1_datasets/
+  #         set2 : /path/to/set2_datasets/
+  # In this case, datasets can be accessed from <server>/set1/<datasetname> or
+  # <server>/set2/<datasetname>.
+
   dataroot: null
 
   # The index page when in multi-dataset mode:

server/common/health.py

@@ -23,12 +23,13 @@ def health_check(config):
     """
     health = {"status": None, "version": "1", "releaseID": cellxgene_version}
 
-    checks = [
+    checks = False
-        (config.single_dataset__datapath is not None or config.multi_dataset__dataroot is not None),
+    if config.single_dataset__datapath is not None:
-        _is_accessible(config.single_dataset__datapath, config),
+        checks = _is_accessible(config.single_dataset__datapath, config)
-        _is_accessible(config.multi_dataset__dataroot, config),
+    elif config.multi_dataset__dataroot is not None:
-    ]
+        checks = all([_is_accessible(datapath, config) for datapath in config.multi_dataset__dataroot.values()])
-    health["status"] = "pass" if all(checks) else "fail"
+
+    health["status"] = "pass" if checks else "fail"
     code = HTTPStatus.OK if health["status"] == "pass" else HTTPStatus.BAD_REQUEST
     response = make_response(jsonify(health), code)
     response.headers["Content-Type"] = "application/health+json"

server/compute/diffexp_cxg.py

@@ -3,6 +3,8 @@ import numpy as np
 from server.compute.diffexp_generic import diffexp_ttest_from_mean_var, mean_var_n
 from server.data_cxg.cxg_util import pack_selector_from_indices
 from server.common.errors import ComputeError
+from numba import jit
+
 
 """
 See the comments in diffexp_generic for a description of this algorithm
@@ -33,26 +35,33 @@ def get_thread_executor():
 
 
 def diffexp_ttest(adaptor, maskA, maskB, top_n=8, diffexp_lfc_cutoff=0.01):
+
+    matrix = adaptor.open_array("X")
     row_selector_A = np.where(maskA)[0]
     row_selector_B = np.where(maskB)[0]
     nA = len(row_selector_A)
     nB = len(row_selector_B)
-    matrix = adaptor.open_array("X")
 
     dtype = matrix.dtype
     cols = matrix.shape[1]
     tile_extent = [dim.tile for dim in matrix.schema.domain]
 
-    # The rows from both row_selector_A and row_selector_B are gathered at the
+    is_sparse = matrix.schema.sparse
-    # same time, then the mean and variance are computed by subsetting on that
-    # combined submatrix.  Combining the gather reduces number of requests/bandwidth
-    # to the data source.
-    row_selector_AB = np.union1d(row_selector_A, row_selector_B)
-    row_selector_A_in_AB = np.in1d(row_selector_AB, row_selector_A, assume_unique=True)
-    row_selector_B_in_AB = np.in1d(row_selector_AB, row_selector_B, assume_unique=True)
-    row_selector_AB = pack_selector_from_indices(row_selector_AB)
 
-    # because all IO is done per-tile, and we are always dense and col-major,
+    if is_sparse:
+        row_selector_A = pack_selector_from_indices(row_selector_A)
+        row_selector_B = pack_selector_from_indices(row_selector_B)
+    else:
+        # The rows from both row_selector_A and row_selector_B are gathered at the
+        # same time, then the mean and variance are computed by subsetting on that
+        # combined submatrix.  Combining the gather reduces number of requests/bandwidth
+        # to the data source.
+        row_selector_AB = np.union1d(row_selector_A, row_selector_B)
+        row_selector_A_in_AB = np.in1d(row_selector_AB, row_selector_A, assume_unique=True)
+        row_selector_B_in_AB = np.in1d(row_selector_AB, row_selector_B, assume_unique=True)
+        row_selector_AB = pack_selector_from_indices(row_selector_AB)
+
+    # because all IO is done per-tile, and we are always col-major,
     # use the tile column size as the unit of partition.  Possibly access
     # more than one column tile at a time based on the target_workunit.
     # Revisit partitioning if we change the X layout, or start using a non-local execution environment
@@ -62,6 +71,7 @@ def diffexp_ttest(adaptor, maskA, maskB, top_n=8, diffexp_lfc_cutoff=0.01):
     # the target_workunit.  A potential improvement would be to partition by both columns and rows.
     # However partitioning the rows is slightly more complex due to the arbitrary distribution
     # of row selections that are passed into this algorithm.
+
     cells_per_coltile = (nA + nB) * tile_extent[1]
     cols_per_partition = max(1, int(target_workunit / cells_per_coltile)) * tile_extent[1]
     col_partitions = [(c, min(c + cols_per_partition, cols)) for c in range(0, cols, cols_per_partition)]
@@ -73,10 +83,15 @@ def diffexp_ttest(adaptor, maskA, maskB, top_n=8, diffexp_lfc_cutoff=0.01):
 
     executor = get_thread_executor()
     futures = []
-    for cols in col_partitions:
+
-        futures.append(
+    if is_sparse:
-            executor.submit(_mean_var_ab, matrix, row_selector_AB, row_selector_A_in_AB, row_selector_B_in_AB, cols)
+        for cols in col_partitions:
-        )
+            futures.append(executor.submit(_mean_var_sparse_ab, matrix, row_selector_A, nA, row_selector_B, nB, cols))
+    else:
+        for cols in col_partitions:
+            futures.append(
+                executor.submit(_mean_var_ab, matrix, row_selector_AB, row_selector_A_in_AB, row_selector_B_in_AB, cols)
+            )
 
     for future in futures:
         # returns tuple: (meanA, varA, meanB, varB, cols)
@@ -92,6 +107,12 @@ def diffexp_ttest(adaptor, maskA, maskB, top_n=8, diffexp_lfc_cutoff=0.01):
                 future.cancel()
             raise ComputeError(str(e))
 
+    if is_sparse:
+        if adaptor.has_array("X_col_shift"):
+            X_col_shift = adaptor.open_array("X_col_shift")[:]
+            meanA += X_col_shift
+            meanB += X_col_shift
+
     r = diffexp_ttest_from_mean_var(
         meanA.astype(dtype),
         varA.astype(dtype),
@@ -111,3 +132,67 @@ def _mean_var_ab(matrix, row_selector_AB, row_selector_A_in_AB, row_selector_B_i
     meanA, varA, n = mean_var_n(X[row_selector_A_in_AB])
     meanB, varB, n = mean_var_n(X[row_selector_B_in_AB])
     return (meanA, varA, meanB, varB, col_range)
+
+
+def _mean_var_sparse_ab(matrix, row_selector_A, nrows_A, row_selector_B, nrows_B, col_range):
+    meanA, varA = _mean_var_sparse(matrix, row_selector_A, nrows_A, col_range)
+    meanB, varB = _mean_var_sparse(matrix, row_selector_B, nrows_B, col_range)
+    return (meanA, varA, meanB, varB, col_range)
+
+
+@jit(nopython=True)
+def _mean_var_sparse_numba(x, var, nrows, ncols):
+    """Kernel to compute the mean and variance.  It was not clear if this function
+    could be written using numpy, thus avoiding the loops.  Therefore numba is
+    used here to speed things up.  With numba, this function takes a negligible amount
+    of time compared to reading in the sparse matrix"""
+    mean = np.zeros((ncols,), dtype=np.float64)
+    for col, val in zip(var, x):
+        mean[col] += val
+    mean /= nrows
+
+    # optimize the sumsq computation.
+    # since most entries in a sparse matrix are 0, then start by assuming
+    # all values are 0, so fill the sumsq array with nrows * (0 - mean)**2.
+    # as non-zero values are encountered, subtract off the (mean*mean) value
+    # and replace with (val-mean)**2.  Simplifying the expression
+    # gives the following code.
+    sumsq = nrows * np.multiply(mean, mean)
+    for col, val in zip(var, x):
+        sumsq[col] += val * (val - 2 * mean[col])
+    v = sumsq / (nrows - 1)
+    return mean, v
+
+
+def _mean_var_sparse(matrix, selector, nrows, col_range):
+    data = matrix.multi_index[selector, col_range[0] : col_range[1] - 1]
+    x = data[""]
+
+    # tiledb < 0.6.0 and >= 0.6.0 have slightly different interfaces.
+    # the following takes care of both cases:
+    #   older:  data["coords]["var"]
+    #   newer:  data["var"]
+    var = data.get("coords", data)["var"]
+
+    # shift the column indices to start at 0, this
+    # will become the index into the mean and var arrays.
+    var -= col_range[0]
+
+    fp_err_occurred = False
+
+    def fp_err_set(err, flag):
+        nonlocal fp_err_occurred
+        fp_err_occurred = True
+
+    ncols = col_range[1] - col_range[0]
+    with np.errstate(divide="call", invalid="call", call=fp_err_set):
+        mean, v = _mean_var_sparse_numba(x, var, nrows, ncols)
+
+    if fp_err_occurred:
+        mean[np.isfinite(mean) == False] = 0  # noqa: E712
+        v[np.isfinite(v) == False] = 0  # noqa: E712
+    else:
+        mean[np.isnan(mean)] = 0
+        v[np.isnan(v)] = 0
+
+    return mean, v

server/converters/cxgtool.py

@@ -8,8 +8,11 @@ The organization of the TileDB structure is:
     ├─ obs                          TileDB array containing cell (row) attributes, one attribute per
     │                               dataframe column, shape (n_obs,)
     ├─ var                          TileDB array containing gene (column) attributes, with one attribute per
-    │                               dataframe column, shape (n_obs,)
+    │                               dataframe column, shape (n_var,)
     ├─ X                            Main count matrix as a 2D TileDB array, single unnamed numeric attribute
+    ├─ X_col_shift                  TilebDB Array used in column shift encoding, shape (n_var,), dtype = X.dtype.
+    │                               Single unnamed numeric attribute.  If this array is sparse, and X_col_shift exists,
+    │                               then all values in the i'th column were subtracted by X_col_shift[i].
     ├─ emb                          TileDB group, storing optional embeddings (group may be empty)
     │  └─ <name1>                   TileDB Array, single anon attribute, ND numeric array, shape (n_obs, N)
     └─ cxg_group_metadata           Empty array used only to stash metadata about the overall object.
@@ -70,6 +73,7 @@ import argparse
 import numpy as np
 from os.path import splitext, basename
 import json
+from scipy.stats import mode
 
 from server.common.colors import convert_anndata_category_colors_to_cxg_category_colors
 from server.common.errors import ColorFormatException
@@ -114,6 +118,13 @@ def main():
         help="URL providing more information about the dataset (hint: must be a fully specified absolute URL).",
     )
     parser.add_argument("--out", "--output", "-o", help="output CXG file name")
+    parser.add_argument(
+        "--sparse-threshold",
+        "-s",
+        type=float,
+        default=0.0,  # force dense by default
+        help="The X array will be sparse if the percent of non-zeros falls below this value",
+    )
     args = parser.parse_args()
 
     global log_level
@@ -135,12 +146,15 @@ def main():
         obs_names=args.obs_names,
         about=args.about,
         extract_colors=not args.disable_custom_colors,
+        sparse_threshold=args.sparse_threshold,
     )
 
     log(1, "done")
 
 
-def write_cxg(adata, container, title, var_names=None, obs_names=None, about=None, extract_colors=False):
+def write_cxg(
+    adata, container, title, var_names=None, obs_names=None, about=None, extract_colors=False, sparse_threshold=5.0
+):
     if not adata.var.index.is_unique:
         raise ValueError("Variable index is not unique - unable to convert.")
     if not adata.obs.index.is_unique:
@@ -195,7 +209,7 @@ def write_cxg(adata, container, title, var_names=None, obs_names=None, about=Non
     log(1, "\t...embeddings created")
 
     # X matrix
-    save_X(container, adata, ctx)
+    save_X(container, adata.X, ctx, sparse_threshold)
     log(1, "\t...X created")
 
 
@@ -366,7 +380,7 @@ def create_emb(e_name, emb):
     dims = []
     for d in range(emb.ndim):
         shape = emb.shape
-        dims.append(tiledb.Dim("", domain=(0, shape[d] - 1), tile=min(shape[d], 1000), dtype=np.uint32))
+        dims.append(tiledb.Dim(domain=(0, shape[d] - 1), tile=min(shape[d], 1000), dtype=np.uint32))
     domain = tiledb.Domain(*dims)
     schema = tiledb.ArraySchema(
         domain=domain, sparse=False, attrs=attrs, capacity=1_000_000, cell_order="row-major", tile_order="row-major"
@@ -398,46 +412,175 @@ def save_embeddings(container, adata, ctx):
             log(1, f"\t\t...{name} embedding created")
 
 
-def create_X(X_name, shape):
+def create_X(X_name, shape, is_sparse):
     """
-    Dense, always.  Future task: explore if sparse encoding is worth the trouble
+    The X matrix is accessed in both row and column oriented patterns, depending on the
-    below a sparsity threshold.
-
-    The X matrix is access in both row and column oriented patterns, depending on the
     particular operation.  Because of the data type, default compression works best.
-    The tile size (50, 100) and global layout (row/col) was choosen empirically, by benchmarking
+    The tile size, (50, 100) for dense, and (512,2048) for sparse,
+    and global layout (row/col) was chosen empirically, by benchmarking
     the current cellxgene backend.
     """
     filters = tiledb.FilterList([tiledb.ZstdFilter()])
     attrs = [tiledb.Attr(dtype=np.float32, filters=filters)]
-    domain = tiledb.Domain(
+    if is_sparse:
-        tiledb.Dim(name="obs", domain=(0, shape[0] - 1), tile=min(shape[0], 50), dtype=np.uint32),
+        domain = tiledb.Domain(
-        tiledb.Dim(name="var", domain=(0, shape[1] - 1), tile=min(shape[1], 100), dtype=np.uint32),
+            tiledb.Dim(name="obs", domain=(0, shape[0] - 1), tile=min(shape[0], 512), dtype=np.uint32),
-    )
+            tiledb.Dim(name="var", domain=(0, shape[1] - 1), tile=min(shape[1], 2048), dtype=np.uint32),
+        )
+    else:
+        domain = tiledb.Domain(
+            tiledb.Dim(name="obs", domain=(0, shape[0] - 1), tile=min(shape[0], 50), dtype=np.uint32),
+            tiledb.Dim(name="var", domain=(0, shape[1] - 1), tile=min(shape[1], 100), dtype=np.uint32),
+        )
     schema = tiledb.ArraySchema(
-        domain=domain, sparse=False, attrs=attrs, cell_order="row-major", tile_order="col-major"
+        domain=domain, sparse=is_sparse, attrs=attrs, cell_order="row-major", tile_order="col-major"
     )
-    tiledb.DenseArray.create(X_name, schema)
+    if is_sparse:
+        tiledb.SparseArray.create(X_name, schema)
+    else:
+        tiledb.DenseArray.create(X_name, schema)
 
 
-def save_X(container, adata, ctx):
+def evaluate_for_sparse_encoding(xdata, sparse_threshold):
+    """
+    This function determines if the X matrix has a sparsity below the sparse_threshold.
+    This function also returns the number of non-zeros encountered and number
+    of elements evaluated.  This function may return before evaluating the whole X matrix
+    if it can be determined that X is not sparse enough.
+    """
+    shape = xdata.shape
+    stride = min(int(np.power(10, np.around(np.log10(1e9 / shape[1])))), 10_000)
+    nnz = 0
+    maxnnz = int(shape[0] * shape[1] * sparse_threshold / 100)
+    for row in range(0, shape[0], stride):
+        lim = min(row + stride, shape[0])
+        a = xdata[row:lim, :]
+        if type(a) is not np.ndarray:
+            a = a.toarray()
+        nnz += np.count_nonzero(a)
+        if nnz > maxnnz:
+            return (False, nnz, lim * shape[1])
+        log(2, "\t...rows", lim, "of", shape[0], "nnz", nnz, "nnz percent %5.2f%%" % (100 * nnz / (lim * shape[1])))
+
+    is_sparse = (100.0 * nnz / (shape[0] * shape[1])) < sparse_threshold
+    return (is_sparse, nnz, shape[0] * shape[1])
+
+
+def evaluate_for_sparse_column_shift_encoding(xdata, sparse_threshold):
+    """Column shift encoding works by taking the most common value in each column, then
+    subtracting that value from each element of the column.  If each column mostly contains
+    its most common value, then the resulting matrix can be very sparse.
+
+    This function determines if column shift encoding can be used to transform
+    the X matrix into a sparse matrix with a sparsity below the sparse_threshold.
+    If so, return the col_shift array that stores this encoding.
+    This function also returns the number of non-zeros encountered and number
+    of elements evaluated.  This function may return before evaluating the whole X matrix
+    if it can be determined that X cannot benefit from column shift encoding.
+    """
+    shape = xdata.shape
+    stride = max(1, 128_000_000 // shape[0])
+    col_shift = np.zeros(shape[1])
+    nnz = 0
+    maxnnz = int(shape[0] * shape[1] * sparse_threshold / 100)
+    for col in range(0, shape[1], stride):
+        lim = min(col + stride, shape[1])
+        a = xdata[:, col:lim]
+        if type(a) is not np.ndarray:
+            a = a.toarray()
+        m = mode(a)
+        col_shift[col:lim] = m.mode
+        nnz += shape[0] * (lim - col) - np.sum(m.count)
+        if nnz > maxnnz:
+            return (None, nnz, shape[0] * lim)
+        log(2, "\t...cols", lim, "of", shape[1], "nnz",
+            nnz, "nnz percent %5.2f%%" % (100 * nnz / (lim * shape[0])))
+
+    is_sparse = (100.0 * nnz / (shape[0] * shape[1])) < sparse_threshold
+    return (col_shift if is_sparse else None, nnz, shape[0] * shape[1])
+
+
+def save_X(container, xdata, ctx, sparse_threshold, expect_sparse=False):
     # Save X count matrix
     X_name = f"{container}/X"
-    shape = adata.X.shape
-    create_X(X_name, shape)
 
+    shape = xdata.shape
+    log(1, "\t...shape:", str(shape))
+
+    col_shift = None
+    if sparse_threshold == 100:
+        is_sparse = True
+    elif sparse_threshold == 0:
+        is_sparse = False
+    else:
+        is_sparse, nnz, nelem = evaluate_for_sparse_encoding(xdata, sparse_threshold)
+        percent = 100.0 * nnz / nelem
+        if nelem != shape[0] * shape[1]:
+            log(1, "\t...sparse=", is_sparse, "non-zeros percent (estimate): %6.2f" % percent)
+        else:
+            log(1, "\t...sparse=", is_sparse, "non-zeros:", nnz, "percent: %6.2f" % percent)
+
+        is_sparse = percent < sparse_threshold
+        if not is_sparse:
+            col_shift, nnz, nelem = evaluate_for_sparse_column_shift_encoding(xdata, sparse_threshold)
+            is_sparse = col_shift is not None
+            percent = 100.0 * nnz / nelem
+            if nelem != shape[0] * shape[1]:
+                log(1, "\t...sparse=", is_sparse, "col shift non-zeros percent (estimate): %6.2f" % percent)
+            else:
+                log(1, "\t...sparse=", is_sparse, "col shift non-zeros:", nnz, "percent: %6.2f" % percent)
+
+    if expect_sparse is True and is_sparse is False:
+        return False
+
+    create_X(X_name, shape, is_sparse)
     stride = min(int(np.power(10, np.around(np.log10(1e9 / shape[1])))), 10_000)
-    with tiledb.DenseArray(X_name, mode="w", ctx=ctx) as X:
+    if is_sparse:
-        for row in range(0, shape[0], stride):
+        if col_shift is not None:
-            lim = min(row + stride, shape[0])
+            log(1, "\t...output X as sparse matrix with column shift encoding")
-            a = adata.X[row:lim, :]
+            X_col_shift_name = f"{container}/X_col_shift"
-            if type(a) is not np.ndarray:
+            filters = tiledb.FilterList([tiledb.ZstdFilter()])
-                a = a.toarray()
+            attrs = [tiledb.Attr(dtype=np.float32, filters=filters)]
-            X[row:lim, :] = a
+            domain = tiledb.Domain(tiledb.Dim(domain=(0, shape[1] - 1), tile=min(shape[1], 5000), dtype=np.uint32))
-            log(2, "\t...rows", row, "to", lim)
+            schema = tiledb.ArraySchema(domain=domain, attrs=attrs)
-        tiledb.consolidate(X_name, ctx=ctx)
+            tiledb.DenseArray.create(X_col_shift_name, schema)
+            with tiledb.DenseArray(X_col_shift_name, mode="w", ctx=ctx) as X_col_shift:
+                X_col_shift[:] = col_shift
+            tiledb.consolidate(X_col_shift_name, ctx=ctx)
+        else:
+            log(1, "\t...output X as sparse matrix")
+
+        with tiledb.SparseArray(X_name, mode="w", ctx=ctx) as X:
+            nnz = 0
+            for row in range(0, shape[0], stride):
+                lim = min(row + stride, shape[0])
+                a = xdata[row:lim, :]
+                if type(a) is not np.ndarray:
+                    a = a.toarray()
+                if col_shift is not None:
+                    a = a - col_shift
+                indices = np.nonzero(a)
+                trow = indices[0] + row
+                nnz += indices[0].shape[0]
+                X[trow, indices[1]] = a[indices[0], indices[1]]
+                log(2, "\t...rows", lim, "of", shape[0], "nnz", nnz, "sparse", nnz / (lim * shape[1]))
+
+    else:
+        log(1, "\t...output X as dense matrix")
+        with tiledb.DenseArray(X_name, mode="w", ctx=ctx) as X:
+            for row in range(0, shape[0], stride):
+                lim = min(row + stride, shape[0])
+                a = xdata[row:lim, :]
+                if type(a) is not np.ndarray:
+                    a = a.toarray()
+                X[row:lim, :] = a
+                log(2, "\t...rows", row, "to", lim)
 
     tiledb.consolidate(X_name, ctx=ctx)
+    if hasattr(tiledb, "vacuum"):
+        tiledb.vacuum(X_name)
+
+    return is_sparse
 
 
 def save_metadata(container, metadata_dict):

server/converters/to_sparse.py

@@ -0,0 +1,62 @@
+"""
+Script to create a sparse dataset in CXG format based on an input dataset in CXG format.
+The input dataset is not modified.
+"""
+import os
+import shutil
+import tiledb
+import argparse
+import sys
+import server.converters.cxgtool as cxgtool
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("input", help="input cxg directory")
+    parser.add_argument("output", help="output cxg directory")
+    parser.add_argument("--overwrite", action="store_true", help="replace output cxg directory")
+    parser.add_argument("--verbose", "-v", action="count", default=0, help="verbose output")
+    parser.add_argument(
+        "--sparse-threshold",
+        "-s",
+        type=float,
+        default=5.0,  # default is 5% non-zero values
+        help="The X array will be sparse if the percent of non-zeros falls below this value",
+    )
+    args = parser.parse_args()
+
+    if os.path.exists(args.output):
+        print("output dir exists:", args.output)
+        if args.overwrite:
+            print("output dir removed:", args.output)
+            shutil.rmtree(args.output)
+        else:
+            print("use the overwrite option to remove the output directory")
+            sys.exit(1)
+
+    if not os.path.isdir(args.input):
+        print("input is not a directory", args.input)
+        sys.exit(1)
+
+    shutil.copytree(args.input, args.output,
+                    ignore=shutil.ignore_patterns("X", "X_col_shift"))
+
+    ctx = tiledb.Ctx(
+        {
+            "sm.num_reader_threads": 32,
+            "sm.num_writer_threads": 32,
+            "sm.consolidation.buffer_size": 1 * 1024 * 1024 * 1024,
+        }
+    )
+
+    with tiledb.DenseArray(os.path.join(args.input, "X"), mode="r", ctx=ctx) as X_in:
+        is_sparse = cxgtool.save_X(args.output, X_in, ctx, args.sparse_threshold, expect_sparse=True)
+
+    if is_sparse is False:
+        print("The array is not sparse, cleaning up, abort.")
+        shutil.rmtree(args.output)
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

server/data_common/data_adaptor.py

@@ -241,8 +241,7 @@ class DataAdaptor(metaclass=ABCMeta):
         duplicate_columns = list(set(labels_df.columns) & set(obs_columns))
         if len(duplicate_columns) > 0:
             raise KeyError(
-                "Labels file may not contain column names which overlap "
+                "Labels file may not contain column names which overlap " f"with h5ad obs columns {duplicate_columns}"
-                f"with h5ad obs columns {duplicate_columns}"
             )
 
         # labels must have same count as obs annotations

server/data_cxg/cxg_adaptor.py

@@ -133,6 +133,10 @@ class CxgAdaptor(DataAdaptor):
             return False
         return True
 
+    def has_array(self, name):
+        a_type = tiledb.object_type(path_join(self.url, name), ctx=self.tiledb_ctx)
+        return a_type == "array"
+
     def _validate_and_initialize(self):
         """
         remember, preload_validation() has already been called, so
@@ -147,13 +151,7 @@ class CxgAdaptor(DataAdaptor):
         * version 0.1 -- metadata attache to cxg_group_metadata array.
           Same as 0, except it adds group metadata.
         """
-        a_type = tiledb.object_type(path_join(self.url, "cxg_group_metadata"), ctx=self.tiledb_ctx)
+        if self.has_array("cxg_group_metadata"):
-        if a_type is None:
-            # version 0
-            cxg_version = "0.0"
-            title = None
-            about = None
-        elif a_type == "array":
             # version >0
             gmd = self.open_array("cxg_group_metadata")
             cxg_version = gmd.meta["cxg_version"]
@@ -161,6 +159,11 @@ class CxgAdaptor(DataAdaptor):
                 cxg_properties = json.loads(gmd.meta["cxg_properties"])
                 title = cxg_properties.get("title", None)
                 about = cxg_properties.get("about", None)
+        else:
+            # version 0
+            cxg_version = "0.0"
+            title = None
+            about = None
 
         if cxg_version not in ["0.0", "0.1"]:
             raise DatasetAccessError(f"cxg matrix is not valid: {self.url}")
@@ -171,7 +174,11 @@ class CxgAdaptor(DataAdaptor):
 
     @staticmethod
     def _open_array(uri, tiledb_ctx):
-        return tiledb.DenseArray(uri, mode="r", ctx=tiledb_ctx)
+        with tiledb.Array(uri, mode="r", ctx=tiledb_ctx) as array:
+            if array.schema.sparse:
+                return tiledb.SparseArray(uri, mode="r", ctx=tiledb_ctx)
+            else:
+                return tiledb.DenseArray(uri, mode="r", ctx=tiledb_ctx)
 
     def open_array(self, name):
         try:
@@ -200,15 +207,73 @@ class CxgAdaptor(DataAdaptor):
         meta = self.open_array("cxg_group_metadata").meta
         return json.loads(meta["cxg_category_colors"]) if "cxg_category_colors" in meta else dict()
 
+    def __remap_indices(self, coord_range, coord_mask, coord_data):
+        """
+        This function maps the indices in coord_data, which could be in the range [0,coord_range), to
+        a range that only includes the number of indices encoded in coord_mask.
+        coord_range is the maxinum size of the range (e.g. get_shape()[0] or get_shape()[1])
+        coord_mask is a mask passed into the get_X_array, of size coord_range
+        coord_data are indices representing locations of non-zero values, in the range [0,coord_range).
+
+        For example, say
+        coord_mask = [1,0,1,0,0,1]
+        coord_data = [2,0,2,2,5]
+
+        The function computes the following:
+        indices = [0,2,5]
+        ncoord = 3
+        maprange = [0,1,2]
+        mapindex = [0,0,1,0,0,2]
+        coordindices = [1,0,1,1,2]
+        """
+        if coord_mask is None:
+            return coord_range, coord_data
+
+        indices = np.where(coord_mask)[0]
+        ncoord = indices.shape[0]
+        maprange = np.arange(ncoord)
+        mapindex = np.zeros(indices[-1] + 1, dtype=int)
+        mapindex[indices] = maprange
+        coordindices = mapindex[coord_data]
+        return ncoord, coordindices
+
     def get_X_array(self, obs_mask=None, var_mask=None):
         obs_items = pack_selector_from_mask(obs_mask)
         var_items = pack_selector_from_mask(var_mask)
+        if obs_items is None or var_items is None:
+            # If either zero rows or zero columns were selected, return an empty 2d array.
+            shape = self.get_shape()
+            obs_size = 0 if obs_items is None else shape[0] if obs_mask is None else np.count_nonzero(obs_mask)
+            var_size = 0 if var_items is None else shape[1] if var_mask is None else np.count_nonzero(var_mask)
+            return np.ndarray((obs_size, var_size))
+
         X = self.open_array("X")
-        if obs_items == slice(None) and var_items == slice(None):
+
-            data = X[:, :]
+        if X.schema.sparse:
+            if obs_items == slice(None) and var_items == slice(None):
+                data = X[:, :]
+            else:
+                data = X.multi_index[obs_items, var_items]
+
+            nrows, obsindices = self.__remap_indices(X.shape[0], obs_mask, data.get("coords", data)["obs"])
+            ncols, varindices = self.__remap_indices(X.shape[1], var_mask, data.get("coords", data)["var"])
+            densedata = np.zeros((nrows, ncols), dtype=self.get_X_array_dtype())
+            densedata[obsindices, varindices] = data[""]
+            if self.has_array("X_col_shift"):
+                X_col_shift = self.open_array("X_col_shift")
+                if var_items == slice(None):
+                    densedata += X_col_shift[:]
+                else:
+                    densedata += X_col_shift.multi_index[var_items][""]
+
+            return densedata
+
         else:
-            data = X.multi_index[obs_items, var_items][""]
+            if obs_items == slice(None) and var_items == slice(None):
-        return data
+                data = X[:, :]
+            else:
+                data = X.multi_index[obs_items, var_items][""]
+            return data
 
     def get_shape(self):
         X = self.open_array("X")

server/data_cxg/cxg_util.py

@@ -20,7 +20,7 @@ def pack_selector_from_mask(boolarray):
 def pack_selector_from_indices(selector):
 
     if len(selector) == 0:
-        return slice(None)
+        return None
 
     result = []
     current = slice(selector[0], selector[0])

server/eb/.gitignore

@@ -0,0 +1,5 @@
+
+# Elastic Beanstalk Files
+.elasticbeanstalk/*
+!.elasticbeanstalk/*.cfg.yml
+!.elasticbeanstalk/*.global.yml

server/eb/app.py

@@ -67,7 +67,6 @@ class WSGIServer(Server):
             "object-src": ["'none'"],
             "base-uri": ["'none'"],
             "frame-ancestors": ["'none'"],
-            "require-trusted-types-for": ["'script'"],
         }
 
         if not app.debug:

server/requirements.txt

@@ -11,12 +11,13 @@ flask-talisman>=0.7.0
 flatbuffers>=1.10.0
 flatten-dict>=0.2.0
 fsspec>=0.4.4
+numba>=0.49.1
 numpy>=1.16.0
 packaging>=20.0
 pandas>=0.24.2
 PyYAML>=5.3
 scipy>=1.3.0
 requests>=2.22.0
-tiledb==0.5.9
+tiledb>=0.5.9,!=0.6.0
 s3fs>=0.4.2
 gunicorn>=20.0.4

server/test/__init__.py

@@ -2,13 +2,19 @@ import random
 import shutil
 import string
 import tempfile
+import requests
+import time
+import os
+from subprocess import Popen
 from os import path, popen
+from contextlib import contextmanager
 
 import pandas as pd
 
 from server.common.annotations import AnnotationsLocalFile
 from server.common.data_locator import DataLocator
-from server.common.app_config import AppConfig
+from server.common.app_config import AppConfig, DEFAULT_SERVER_PORT
+from server.common.utils import find_available_port
 from server.data_common.fbs.matrix import encode_matrix_fbs
 from server.data_common.matrix_loader import MatrixDataLoader, MatrixDataType
 
@@ -60,7 +66,7 @@ def skip_if(condition, reason: str):
     return decorator
 
 
-def app_config(data_locator, backed=False):
+def app_config(data_locator, backed=False, extra={}):
     args = {
         "embeddings__names": ["umap", "tsne", "pca"],
         "presentation__max_categories": 100,
@@ -74,9 +80,61 @@ def app_config(data_locator, backed=False):
     }
     config = AppConfig()
     config.update(**args)
+    config.update(**extra)
     config.complete_config()
     return config
 
 
 def random_string(n):
     return "".join(random.choice(string.ascii_letters) for _ in range(n))
+
+
+@contextmanager
+def test_server(command_line_args=[], app_config=None):
+    """A context to run the cellxgene server.
+    Command line arguments can be passed in, as well as an app_config.
+    This function is meant to be used like this, for example:
+
+    with test_server(...) as server:
+       r = requests.get(f"{server}/...")
+       // check r
+
+    where the server can be accessed within the context, and is terminated when
+    the context is exited.
+    The port is automatically set using find_available_port.
+    The verbose flag is automatically set to True.
+    If an app_config is provided, then this function writes a temporary
+    yaml config file, which this server will read and parse.
+    """
+
+    port = DEFAULT_SERVER_PORT
+    port = find_available_port("localhost", port)
+    command = ["cellxgene", "--no-upgrade-check", "launch", "--verbose", "--port=%d" % port] + command_line_args
+
+    tempdir = None
+    if app_config:
+        tempdir = tempfile.TemporaryDirectory()
+        config_file = os.path.join(tempdir.name, "config.yaml")
+        app_config.write_config(config_file)
+        command.extend(["-c", config_file])
+
+    server = f"http://localhost:{port}"
+    ps = Popen(command)
+
+    for _ in range(10):
+        try:
+            requests.get(f"{server}/health")
+            break
+        except requests.exceptions.ConnectionError:
+            time.sleep(1)
+
+    if tempdir:
+        tempdir.cleanup()
+
+    try:
+        yield server
+    finally:
+        try:
+            ps.terminate()
+        except ProcessLookupError:
+            pass

server/test/create_test_matrix.py

@@ -0,0 +1,44 @@
+import anndata
+import argparse
+import random
+import scipy
+import numpy as np
+
+
+def main():
+    parser = argparse.ArgumentParser("A command to generate test h5ad files")
+    parser.add_argument("output", help="Name of the output file")
+    parser.add_argument("nobs", type=int, help="Number of observations (rows)")
+    parser.add_argument("nvar", type=int, help="Number of variables (columns)")
+    parser.add_argument("-n", "--nnz-percent", type=float, default=100, help="percent of non-zeros")
+    parser.add_argument("-c", "--col-shift", action="store_true", help="add a random value to each column")
+    parser.add_argument("--seed", type=int, default=None, help="add a random value to each column")
+
+    args = parser.parse_args()
+    create_test_h5ad(args.output, args.nobs, args.nvar, args.nnz_percent, args.col_shift, args.seed)
+
+
+def create_test_h5ad(outfile, nobs, nvar, nnz_percent=100, apply_col_shift=False, seed=None):
+    random.seed(seed)
+    np.random.seed(seed)
+    x = create_X_array(nobs, nvar, nnz_percent, apply_col_shift)
+    obsm = {"X_random": np.random.rand(nobs, 2).astype(np.float32)}
+    adata = anndata.AnnData(x, obsm=obsm)
+    adata.write(outfile)
+
+
+def create_X_array(nobs, nvar, nnz_percent, apply_col_shift):
+    if nnz_percent < 100:
+        array = scipy.sparse.random(nobs, nvar, nnz_percent * 0.01, dtype=np.float32, format="csc")
+    else:
+        array = np.random.rand(nobs, nvar).astype(np.float32)
+
+    if apply_col_shift:
+        col_shift = np.random.rand((nvar))
+        array += col_shift
+
+    return array
+
+
+if __name__ == "__main__":
+    main()

server/test/run_diffexp.py

@@ -15,8 +15,10 @@ from server.data_cxg.cxg_adaptor import CxgAdaptor
 def main():
     parser = argparse.ArgumentParser("A command to test diffexp")
     parser.add_argument("dataset", help="name of a dataset to load")
-    parser.add_argument("-na", "--numA", type=int, required=True, help="number of rows in group A")
+    parser.add_argument("-na", "--numA", type=int, help="number of rows in group A")
-    parser.add_argument("-nb", "--numB", type=int, required=True, help="number of rows in group B")
+    parser.add_argument("-nb", "--numB", type=int, help="number of rows in group B")
+    parser.add_argument("-va", "--varA", help="obs variable:value to use for group A")
+    parser.add_argument("-vb", "--varB", help="obs variable:value to use for group B")
     parser.add_argument("-t", "--trials", default=1, type=int, help="number of trials")
     parser.add_argument(
         "-a", "--alg", choices=("default", "generic", "cxg"), default="default", help="algorithm to use"
@@ -41,22 +43,34 @@ def main():
         if isinstance(adaptor, CxgAdaptor):
             adaptor.open_array("X").schema.dump()
 
-    numA = args.numA
+    random.seed(args.seed)
-    numB = args.numB
+    np.random.seed(args.seed)
     rows = adaptor.get_shape()[0]
 
-    random.seed(args.seed)
+    if args.numA:
+        filterA = random.sample(range(rows), args.numA)
+    elif args.varA:
+        vname, vval = args.varA.split(":")
+        filterA = get_filter_from_obs(adaptor, vname, vval)
+    else:
+        print("must supply numA or varA")
+        sys.exit(1)
 
-    if not args.new_selection:
+    if args.numB:
-        samples = random.sample(range(rows), numA + numB)
+        filterB = random.sample(range(rows), args.numB)
-        filterA = samples[:numA]
+    elif args.varB:
-        filterB = samples[numA:]
+        vname, vval = args.varB.split(":")
+        filterB = get_filter_from_obs(adaptor, vname, vval)
+    else:
+        print("must supply numB or varB")
+        sys.exit(1)
 
     for i in range(args.trials):
         if args.new_selection:
-            samples = random.sample(range(rows), numA + numB)
+            if args.numA:
-            filterA = samples[:numA]
+                filterA = random.sample(range(rows), args.numA)
-            filterB = samples[numA:]
+            if args.numB:
+                filterB = random.sample(range(rows), args.numB)
 
         maskA = np.zeros(rows, dtype=bool)
         maskA[filterA] = True
@@ -82,5 +96,22 @@ def main():
             print(res)
 
 
+def get_filter_from_obs(adaptor, obsname, obsval):
+    attrs = adaptor.get_obs_columns()
+    if obsname not in attrs:
+        print(f"Unknown obs attr {obsname}: expected on of {attrs}")
+        sys.exit(1)
+    obsvals = adaptor.query_obs_array(obsname)[:]
+    obsval = type(obsvals[0])(obsval)
+
+    vfilter = np.where(obsvals == obsval)[0]
+    if len(vfilter) == 0:
+        u = np.unique(obsvals)
+        print(f"Unknown value in variable {obsname}:{obsval}: expected one of {list(u)}")
+        sys.exit(1)
+
+    return vfilter
+
+
 if __name__ == "__main__":
     main()

server/test/test_api.py

@@ -244,6 +244,19 @@ class EndPoints(object):
         self.assertEqual(df["n_rows"], 2638)
         self.assertEqual(df["n_cols"], 1)
 
+    def test_data_get_unknown_filter_fbs(self):
+        index_col_name = self.schema["schema"]["annotations"]["var"]["index"]
+        endpoint = "data/var"
+        query = f"var:{index_col_name}=UNKNOWN"
+        url = f"{self.URL_BASE}{endpoint}?{query}"
+        header = {"Accept": "application/octet-stream"}
+        result = self.session.get(url, headers=header)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
+        df = decode_fbs.decode_matrix_FBS(result.content)
+        self.assertEqual(df["n_rows"], 2638)
+        self.assertEqual(df["n_cols"], 0)
+
     def test_data_put_single_var(self):
         endpoint = "data/var"
         url = f"{self.URL_BASE}{endpoint}"

server/test/test_app_config.py

@@ -1,5 +1,8 @@
 import unittest
 from server.common.app_config import AppConfig
+from server.common.errors import ConfigurationError
+from server.test import PROJECT_ROOT, test_server
+import requests
 
 # NOTE, there are more tests that should be written for AppConfig.
 # this is just a start.
@@ -26,3 +29,43 @@ class AppConfigTest(unittest.TestCase):
         c.update(server__scripts=("a", "b"), server__inline_scripts=["c", "d"])
         v = c.changes_from_default()
         self.assertCountEqual(v, [("server__scripts", ["a", "b"], []), ("server__inline_scripts", ["c", "d"], [])])
+
+    def test_multi_dataset(self):
+
+        c = AppConfig()
+        # test for illegal url_dataroots
+        for illegal in ("a/b", "../b", "!$*", "\\n", "", "(bad)"):
+            c.update(multi_dataset__dataroot={illegal: f"{PROJECT_ROOT}/example-dataset"})
+            with self.assertRaises(ConfigurationError):
+                c.complete_config()
+
+        # test for legal url_dataroots
+        for legal in (
+            "d",
+            "this.is-okay_",
+        ):
+            c.update(multi_dataset__dataroot={legal: f"{PROJECT_ROOT}/example-dataset"})
+            c.complete_config()
+
+        # test that multi dataroots work end to end
+        c.update(
+            multi_dataset__dataroot=dict(
+                set1=f"{PROJECT_ROOT}/example-dataset",
+                set2=f"{PROJECT_ROOT}/server/test/test_datasets"
+            )
+        )
+        c.complete_config()
+
+        with test_server(app_config=c) as server:
+            session = requests.Session()
+
+            r = session.get(f"{server}/set1/pbmc3k.h5ad/api/v0.2/config")
+            data_config = r.json()
+            assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
+
+            r = session.get(f"{server}/set2/pbmc3k.cxg/api/v0.2/config")
+            data_config = r.json()
+            assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
+
+            r = session.get(f"{server}/health")
+            assert r.json()["status"] == "pass"

server/test/test_diffexp.py

@@ -1,24 +1,24 @@
 import unittest
 from server.data_common.matrix_loader import MatrixDataLoader
-from server.common.app_config import AppConfig
+from server.test import PROJECT_ROOT, app_config
 import server.compute.diffexp_cxg as diffexp_cxg
 import server.compute.diffexp_generic as diffexp_generic
+from server.converters.cxgtool import write_cxg
+from server.test.create_test_matrix import create_test_h5ad
+from server.data_common.fbs.matrix import encode_matrix_fbs, decode_matrix_fbs
 import numpy as np
-
+import tempfile
-from server.test import PROJECT_ROOT
+import os
 
 
 class DiffExpTest(unittest.TestCase):
     """Tests the diffexp returns the expected results for one test case, using different
     adaptor types and different algorithms."""
 
-    def load_dataset(self, path):
+    def load_dataset(self, path, extra={}):
-        app_config = AppConfig()
+        config = app_config(path, extra=extra)
-        app_config.single_dataset__datapath = path
-        app_config.server__verbose = True
-        app_config.complete_config()
         loader = MatrixDataLoader(path)
-        adaptor = loader.open(app_config)
+        adaptor = loader.open(config)
         return adaptor
 
     def get_mask(self, adaptor, start, stride):
@@ -29,6 +29,14 @@ class DiffExpTest(unittest.TestCase):
         mask[sel] = True
         return mask
 
+    def compare_diffexp_results(self, results, expects):
+        self.assertEqual(len(results), len(expects))
+        for result, expect in zip(results, expects):
+            self.assertEqual(result[0], expect[0])
+            self.assertTrue(np.isclose(result[1], expect[1], 1e-6, 1e-4))
+            self.assertTrue(np.isclose(result[2], expect[2], 1e-6, 1e-4))
+            self.assertTrue(np.isclose(result[3], expect[3], 1e-6, 1e-4))
+
     def check_1_10_2_10(self, results):
         """Checks the results for a specific set of rows selections"""
         expects = [
@@ -43,12 +51,12 @@ class DiffExpTest(unittest.TestCase):
             [1575, 1.0317602, 0.007830310753043345, 1.0],
             [576, 0.97873515, 0.008272092578813124, 1.0],
         ]
-        self.assertEqual(len(results), len(expects))
+        self.compare_diffexp_results(results, expects)
-        for result, expect in zip(results, expects):
+
-            self.assertEqual(result[0], expect[0])
+    def get_X_col(self, adaptor, cols):
-            self.assertAlmostEqual(result[1], expect[1])
+        varmask = np.zeros(adaptor.get_shape()[1], dtype=bool)
-            self.assertAlmostEqual(result[2], expect[2])
+        varmask[cols] = True
-            self.assertAlmostEqual(result[3], expect[3])
+        return adaptor.get_X_array(None, varmask)
 
     def test_anndata_default(self):
         """Test an anndata adaptor with its default diffexp algorithm (diffexp_generic)"""
@@ -80,3 +88,64 @@ class DiffExpTest(unittest.TestCase):
         # run it directly
         results = diffexp_generic.diffexp_ttest(adaptor, maskA, maskB, 10)
         self.check_1_10_2_10(results)
+
+    def test_cxg_sparse(self):
+        self.sparse_diffexp(False)
+
+    def test_cxg_sparse_col_shift(self):
+        self.sparse_diffexp(True)
+
+    def sparse_diffexp(self, apply_col_shift):
+        with tempfile.TemporaryDirectory() as dirname:
+            # create a sparse matrix
+            h5adfile = os.path.join(dirname, "sparse.h5ad")
+            create_test_h5ad(h5adfile, 2000, 2000, 10, apply_col_shift)
+            adaptor_anndata = self.load_dataset(h5adfile, extra=dict(embeddings__names=[]))
+            adata = adaptor_anndata.data
+
+            sparsename = os.path.join(dirname, "sparse.cxg")
+            write_cxg(adata=adata, container=sparsename, title="sparse", sparse_threshold=11)
+            adaptor_sparse = self.load_dataset(sparsename)
+            assert adaptor_sparse.open_array("X").schema.sparse
+            assert adaptor_sparse.has_array("X_col_shift") == apply_col_shift
+
+            densename = os.path.join(dirname, "dense.cxg")
+            write_cxg(adata=adata, container=densename, title="dense", sparse_threshold=0)
+            adaptor_dense = self.load_dataset(densename)
+            assert not adaptor_dense.open_array("X").schema.sparse
+            assert not adaptor_dense.has_array("X_col_shift")
+
+            maskA = self.get_mask(adaptor_anndata, 1, 10)
+            maskB = self.get_mask(adaptor_anndata, 2, 10)
+
+            diffexp_results_anndata = diffexp_generic.diffexp_ttest(adaptor_anndata, maskA, maskB, 10)
+            diffexp_results_sparse = diffexp_cxg.diffexp_ttest(adaptor_sparse, maskA, maskB, 10)
+            diffexp_results_dense = diffexp_cxg.diffexp_ttest(adaptor_dense, maskA, maskB, 10)
+
+            self.compare_diffexp_results(diffexp_results_anndata, diffexp_results_sparse)
+            self.compare_diffexp_results(diffexp_results_anndata, diffexp_results_dense)
+
+            topcols = np.array([x[0] for x in diffexp_results_anndata])
+            cols_anndata = self.get_X_col(adaptor_anndata, topcols)
+            cols_sparse = self.get_X_col(adaptor_sparse, topcols)
+            cols_dense = self.get_X_col(adaptor_dense, topcols)
+            assert cols_anndata.shape[0] == adaptor_sparse.get_shape()[0]
+            assert cols_anndata.shape[1] == len(diffexp_results_anndata)
+
+            def convert(mat, cols):
+                return decode_matrix_fbs(encode_matrix_fbs(mat, col_idx=cols)).to_numpy()
+
+            cols_anndata = convert(cols_anndata, topcols)
+            cols_sparse = convert(cols_sparse, topcols)
+            cols_dense = convert(cols_dense, topcols)
+
+            x = adaptor_sparse.get_X_array()
+            assert x.shape == adaptor_sparse.get_shape()
+
+            for row in range(cols_anndata.shape[0]):
+                for col in range(cols_anndata.shape[1]):
+                    vanndata = cols_anndata[row][col]
+                    vsparse = cols_sparse[row][col]
+                    vdense = cols_dense[row][col]
+                    self.assertTrue(np.isclose(vanndata, vsparse, 1e-6, 1e-6))
+                    self.assertTrue(np.isclose(vanndata, vdense, 1e-6, 1e-6))