release 0.10.0 (#794)

* WIP

* add --developer; fix incompatibility of --port and --debug

* put REST tests on separate ports

* PR review

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.9.1
+current_version = 0.10.0
 
 [bumpversion:file:setup.py]
 search = version="{current_version}"

.travis.yml

@@ -9,8 +9,8 @@ cache:
 install:
   - set -eo pipefail
   - pip install flake8
-  - make build
-  - make install
+  - make pydist
+  - make install-dist
   - pip install -r server/requirements-dev.txt
 
 jobs:
@@ -27,6 +27,5 @@ jobs:
     script: docker build .
   - name: "Smoke Tests"
     python: "3.6"
-    if: branch = master AND type = cron
     script:
       - npm run --prefix client/ smoke-test

client/__tests__/e2e/data.js

@@ -1,6 +1,6 @@
 export const datasets = {
   pbmc3k: {
-    title: "cellxgene: pbmc3k",
+    title: "pbmc3k",
     dataframe: {
       nObs: "2638",
       nVar: "1838",
@@ -26,8 +26,8 @@ export const datasets = {
     cellsets: {
       lasso: [
         {
-          "coordinates-as-percent": { x1: 0.25, y1: 0.25, x2: 0.35, y2: 0.35 },
-          count: "26"
+          "coordinates-as-percent": { x1: 0.05, y1: 0.25, x2: 0.15, y2: 0.35 },
+          count: "104"
         }
       ],
       categorical: [
@@ -91,8 +91,8 @@ export const datasets = {
         }
       },
       lasso: {
-        "coordinates-as-percent": { x1: 0.45, y1: 0.45, x2: 0.5, y2: 0.5 },
-        count: "67"
+        "coordinates-as-percent": { x1: 0.45, y1: 0.05, x2: 0.5, y2: 0.1 },
+        count: "76"
       }
     },
     scatter: {
@@ -108,6 +108,15 @@ export const datasets = {
           count: "24"
         }
       }
+    },
+    clip: {
+      min: "30",
+      max: "70",
+      metadata: "n_genes",
+      gene: "S100A8",
+      "coordinates-as-percent": { x1: 0.25, y1: 0.5, x2: 0.55, y2: 0.5 },
+      count: "392",
+      "gene-cell-count": "421"
     }
   }
 };

client/__tests__/e2e/e2e.test.js

@@ -182,7 +182,6 @@ describe("diffexp", async () => {
     );
   });
 });
-//
 
 describe("subset/reset", async () => {
   test("subset - cell count matches", async () => {
@@ -271,6 +270,35 @@ describe("scatter plot", async () => {
   });
 });
 
+describe("clipping", async () => {
+  test("clip continuous", async () => {
+    await cxgActions.clip(data.clip.min, data.clip.max);
+    const histId = `histogram-${data.clip.metadata}-plot-brush`;
+    const coords = await cxgActions.calcDragCoordinates(
+      histId,
+      data.clip["coordinates-as-percent"]
+    );
+    await cxgActions.drag(histId, coords.start, coords.end);
+    const cellCount = await cxgActions.cellSet(1);
+    expect(cellCount).toBe(data.clip.count);
+  });
+
+  test("clip gene", async () => {
+    await utils.typeInto("gene-search", data.clip.gene);
+    await page.keyboard.press("Enter");
+    await page.waitForSelector(`[data-testid='histogram-${data.clip.gene}']`);
+    await cxgActions.clip(data.clip.min, data.clip.max);
+    const histId = `histogram-${data.clip.gene}-plot-brush`;
+    const coords = await cxgActions.calcDragCoordinates(
+      histId,
+      data.clip["coordinates-as-percent"]
+    );
+    await cxgActions.drag(histId, coords.start, coords.end);
+    const cellCount = await cxgActions.cellSet(1);
+    expect(cellCount).toBe(data.clip["gene-cell-count"]);
+  });
+});
+
 // interact with UI elements just that they do not break
 describe("ui elements don't error", async () => {
   test("color by", async () => {

client/__tests__/e2e/puppeteerUtils.js

@@ -16,10 +16,25 @@ export const puppeteerUtils = puppeteerPage => ({
   async typeInto(testid, text) {
     // only works for text without special characters
     await this.waitByID(testid);
+    const selector = `[data-testid='${testid}']`;
     // type ahead can be annoying if you don't pause before you type
-    await puppeteerPage.click(`[data-testid='${testid}']`);
+    await puppeteerPage.click(selector);
     await puppeteerPage.waitFor(200);
-    await puppeteerPage.type(`[data-testid='${testid}']`, text);
+    await puppeteerPage.type(selector, text);
+  },
+
+  async clearInputAndTypeInto(testid, text) {
+    await this.waitByID(testid);
+    const selector = `[data-testid='${testid}']`;
+    // only works for text without special characters
+    // type ahead can be annoying if you don't pause before you type
+    await puppeteerPage.click(selector);
+    await puppeteerPage.waitFor(200);
+    // select all
+
+    await puppeteerPage.click(selector, {clickCount: 3})
+    await puppeteerPage.keyboard.type("Backspace")
+    await puppeteerPage.type(selector, text);
   },
 
   async clickOn(testid) {
@@ -29,11 +44,13 @@ export const puppeteerUtils = puppeteerPage => ({
   },
 
   async getOneElementInnerHTML(selector) {
+    await puppeteerPage.waitForSelector(selector);
     let text = await puppeteerPage.$eval(selector, el => el.innerHTML);
     return text;
   },
 
   async getOneElementInnerText(selector) {
+    await puppeteerPage.waitForSelector(selector);
     let text = await puppeteerPage.$eval(selector, el => el.innerText);
     return text;
   }
@@ -161,5 +178,13 @@ export const cellxgeneActions = puppeteerPage => ({
     await puppeteerUtils(puppeteerPage).clickOn("reset");
     // loading state never actually happens, reset is too fast
     await page.waitFor(200);
+  },
+
+  async clip(min = 0, max = 100) { 
+    await puppeteerUtils(puppeteerPage).clickOn("visualization-settings");
+    await puppeteerUtils(puppeteerPage).clearInputAndTypeInto("clip-min-input", min);
+    await puppeteerUtils(puppeteerPage).clearInputAndTypeInto("clip-max-input", max);
+    await puppeteerUtils(puppeteerPage).clickOn("clip-commit");
   }
+
 });

client/__tests__/util/range.test.js

@@ -0,0 +1,42 @@
+import { range, rangeFill } from "../../src/util/range";
+
+describe("range", () => {
+	test("no defaults", () => {
+		expect(range(0, 3, 1)).toMatchObject([0, 1, 2]);
+	});
+
+	test("range(stop)", () => {
+		expect(range(3)).toMatchObject([0, 1, 2]);
+		expect(range(0)).toMatchObject([]);
+		expect(range(1)).toMatchObject([0]);
+	});
+
+	test("range(start,stop)", () => {
+		expect(range(0, 0)).toMatchObject([]);
+		expect(range(0, 2)).toMatchObject([0, 1]);
+		expect(range(4, 8)).toMatchObject([4, 5, 6, 7]);
+	});
+
+	test("range(start, stop, step", () => {
+		expect(range(4, 0, -1)).toMatchObject([4, 3, 2, 1]);
+		expect(range(0, 4, 2)).toMatchObject([0, 2]);
+	});
+});
+
+describe("rangefill", () => {
+	test("rangeFill(arr)", () => {
+		expect(rangeFill(new Int32Array(3))).toMatchObject(
+			new Int32Array([0, 1, 2])
+		);
+	});
+	test("rangeFill(arr, start)", () => {
+		expect(rangeFill(new Int32Array(2), 1)).toMatchObject(
+			new Int32Array([1, 2])
+		);
+	});
+	test("rangeFill(arr, start, step)", () => {
+		expect(rangeFill(new Int32Array(3), 2, -1)).toMatchObject(
+			new Int32Array([2, 1, 0])
+		);
+	});
+});

client/__tests__/util/stateManager/sampleResponses.js

@@ -34,28 +34,38 @@ const aSchemaResponse = {
       type: "float32"
     },
     annotations: {
-      obs: [
-        { name: "name", type: "string" },
-        { name: "field1", type: "int32" },
-        { name: "field2", type: "float32" },
-        { name: "field3", type: "boolean" },
-        {
-          name: "field4",
-          type: "categorical",
-          categories: field4Categories
-        }
-      ],
-      var: [
-        { name: "name", type: "string" },
-        { name: "fieldA", type: "int32" },
-        { name: "fieldB", type: "float32" },
-        { name: "fieldC", type: "boolean" },
-        {
-          name: "fieldD",
-          type: "categorical",
-          categories: fieldDCategories
-        }
-      ]
+      obs: {
+        index: "name",
+        columns: [
+          { name: "name", type: "string" },
+          { name: "field1", type: "int32" },
+          { name: "field2", type: "float32" },
+          { name: "field3", type: "boolean" },
+          {
+            name: "field4",
+            type: "categorical",
+            categories: field4Categories
+          }
+        ]
+      },
+      var: {
+        index: "name",
+        columns: [
+          { name: "name", type: "string" },
+          { name: "fieldA", type: "int32" },
+          { name: "fieldB", type: "float32" },
+          { name: "fieldC", type: "boolean" },
+          {
+            name: "fieldD",
+            type: "categorical",
+            categories: fieldDCategories
+          }
+        ]
+      }
+    },
+    layout: {
+      obs: [{ name: "umap", type: "float32", dims: ["umap_0", "umap_1"] }],
+      var: []
     }
   }
 };
@@ -162,29 +172,7 @@ const aLayoutFBSResponse = (() => {
     new Float32Array(nObs).fill(Math.random()),
     new Float32Array(nObs).fill(Math.random())
   ];
-  const builder = new flatbuffers.Builder(1024);
-
-  const cols = _.map(coords, carr => {
-    const cdv = NetEncoding.Float32Array.createDataVector(builder, carr);
-    NetEncoding.Float32Array.startFloat32Array(builder);
-    NetEncoding.Float32Array.addData(builder, cdv);
-    const floatArr = NetEncoding.Float32Array.endFloat32Array(builder);
-
-    NetEncoding.Column.startColumn(builder);
-    NetEncoding.Column.addUType(builder, NetEncoding.TypedArray.Float32Array);
-    NetEncoding.Column.addU(builder, floatArr);
-    return NetEncoding.Column.endColumn(builder);
-  });
-
-  const columns = NetEncoding.Matrix.createColumnsVector(builder, cols);
-
-  NetEncoding.Matrix.startMatrix(builder);
-  NetEncoding.Matrix.addNRows(builder, nObs);
-  NetEncoding.Matrix.addNCols(builder, coords.length);
-  NetEncoding.Matrix.addColumns(builder, columns);
-  const matrix = NetEncoding.Matrix.endMatrix(builder);
-  builder.finish(matrix);
-  return builder.asUint8Array();
+  return encodeMatrix(coords, ["umap_0", "umap_1"]);
 })();
 
 const aDataObsResponse = {

client/__tests__/util/stateManager/universe.test.js

@@ -53,13 +53,15 @@ describe("createUniverseFromResponse", () => {
 
     expect(universe.obsAnnotations.dims).toEqual([
       nObs,
-      REST.schema.schema.annotations.obs.length
+      REST.schema.schema.annotations.obs.columns.length
     ]);
     expect(universe.obsLayout.dims).toEqual([nObs, 2]);
-    expect(universe.obsLayout.colIndex.keys()).toEqual(["X", "Y"]);
+    expect(universe.obsLayout.colIndex.keys()).toEqual(
+      universe.schema.layout.obs[0].dims
+    );
     expect(universe.varAnnotations.dims).toEqual([
       nVar,
-      REST.schema.schema.annotations.var.length
+      REST.schema.schema.annotations.var.columns.length
     ]);
     expect(universe.varData.isEmpty()).toBeTruthy();
   });

client/__tests__/util/stateManager/world.test.js

@@ -29,7 +29,8 @@ const defaultBigBang = () => {
   /* create crossfilter */
   const crossfilter = World.createObsDimensions(
     new Crossfilter(world.obsAnnotations),
-    world
+    world,
+    REST.schema.schema.layout.obs[0].dims
   );
 
   return {
@@ -138,7 +139,9 @@ describe("createWorldFromCurrentSelection", () => {
     expect(world.obsLayout.rowIndex.keys()).toEqual(
       new Int32Array(matchingIndices)
     );
-    expect(world.obsLayout.colIndex.keys()).toEqual(["X", "Y"]);
+    expect(world.obsLayout.colIndex.keys()).toEqual(
+      world.schema.layout.obs[0].dims
+    );
   });
 });
 
@@ -152,14 +155,18 @@ describe("createObsDimensionMap", () => {
 
     const { crossfilter } = defaultBigBang();
     const annotationNames = _.map(
-      REST.schema.schema.annotations.obs,
+      REST.schema.schema.annotations.obs.columns,
       c => c.name
     );
-    const schemaByObsName = _.keyBy(REST.schema.schema.annotations.obs, "name");
+    const obsIndexColName = REST.schema.schema.annotations.obs.index;
+    const schemaByObsName = _.keyBy(
+      REST.schema.schema.annotations.obs.columns,
+      "name"
+    );
     expect(crossfilter).toBeDefined();
     annotationNames.forEach(name => {
       const dim = crossfilter.dimensions[obsAnnoDimensionName(name)];
-      if (name === "name") {
+      if (name === obsIndexColName) {
         expect(dim).toBeUndefined();
       } else {
         const { type } = schemaByObsName[name];

client/__tests__/util/typedCrossfilter/util.test.js

@@ -1,8 +1,8 @@
 import {
-  fillRange,
   sliceByIndex,
   makeSortIndex
 } from "../../../src/util/typedCrossfilter/util";
+import { rangeFill as fillRange } from "../../../src/util/range";
 
 describe("fillRange", () => {
   test("Array", () => {

client/configuration/babel/babel.dev.js

@@ -7,8 +7,8 @@ module.exports = {
   ],
   plugins: [
     "@babel/plugin-proposal-function-bind",
-    "@babel/plugin-proposal-class-properties",
     ["@babel/plugin-proposal-decorators", { legacy: true }],
+    ["@babel/plugin-proposal-class-properties", { loose: true }],
     "@babel/plugin-proposal-export-namespace-from",
     "@babel/plugin-proposal-optional-chaining",
     "@babel/plugin-proposal-nullish-coalescing-operator"

client/configuration/babel/babel.prod.js

@@ -6,8 +6,8 @@ module.exports = {
   ],
   plugins: [
     "@babel/plugin-proposal-function-bind",
-    "@babel/plugin-proposal-class-properties",
     ["@babel/plugin-proposal-decorators", { legacy: true }],
+    ["@babel/plugin-proposal-class-properties", { loose: true }],
     "@babel/plugin-proposal-export-namespace-from",
     "@babel/plugin-transform-react-constant-elements",
     "@babel/plugin-transform-runtime",

client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cellxgene",
-  "version": "0.9.1",
+  "version": "0.10.0",
   "license": "MIT",
   "description": "cellxgene is a web application for the interactive exploration of single cell sequence data.",
   "repository": "https://github.com/chanzuckerberg/cellxgene",
@@ -9,13 +9,13 @@
     "build": "npm run clean && webpack --config configuration/webpack/webpack.config.prod.js",
     "clean": "rimraf build",
     "dev": "npm run clean && webpack --config configuration/webpack/webpack.config.dev.js",
-    "e2e": "jest --verbose false --config __tests__/e2e/e2eJestConfig.json e2e/e2e.test.js",
+    "e2e": "node node_modules/jest/bin/jest.js --verbose false --config __tests__/e2e/e2eJestConfig.json e2e/e2e.test.js",
     "lint": "eslint src",
     "smoke-test": "start-server-and-test start-server-for-test :5000 e2e",
     "start": "node server/development.js",
     "start-server-for-test": "cellxgene launch -p 5000 ../example-dataset/pbmc3k.h5ad",
-    "test": "jest",
-    "unit-test": "jest --testPathIgnorePatterns e2e"
+    "test": "node node_modules/jest/bin/jest.js",
+    "unit-test": "node node_modules/jest/bin/jest.js --testPathIgnorePatterns e2e"
   },
   "engineStrict": true,
   "engines": {
@@ -31,8 +31,8 @@
     "eslint-scope": "3.7.1"
   },
   "dependencies": {
-    "@blueprintjs/core": "^3.15.0",
-    "@blueprintjs/icons": "^3.3.0",
+    "@blueprintjs/core": "^3.15.1",
+    "@blueprintjs/icons": "^3.8.0",
     "@blueprintjs/select": "^3.8.0",
     "canvas-fit": "^1.5.0",
     "d3": "^4.10.0",
@@ -41,79 +41,77 @@
     "font-color-contrast": "^1.0.3",
     "fuzzysort": "^1.1.4",
     "gl-mat4": "^1.1.4",
-    "gl-matrix": "^2.7.1",
+    "gl-vec3": "^1.1.3",
+    "gl-matrix": "^3.0.0",
     "is-number": "^7.0.0",
     "key-pressed": "0.0.1",
     "lodash": "^4.17.4",
-    "memoize-one": "^4.0.0",
+    "memoize-one": "^5.0.4",
     "mouse-position": "^2.0.1",
     "mouse-pressed": "^1.0.0",
-    "normalize.css": "^8.0.0",
     "orbit-camera": "^1.0.0",
-    "query-string": "^6.1.0",
-    "react": "^16.6.0",
+    "query-string": "^6.5.0",
+    "react": "^16.8.6",
     "react-autocomplete": "^1.7.2",
-    "react-dom": "^16.6.0",
-    "react-helmet": "^5.2.0",
-    "react-icons": "^3.2.2",
-    "react-redux": "^5.1.0",
+    "react-dom": "^16.8.6",
+    "react-helmet": "^5.2.1",
+    "react-icons": "^3.7.0",
+    "react-redux": "^7.0.3",
     "redux": "^4.0.1",
-    "redux-devtools-extension": "^2.13.5",
     "redux-thunk": "^2.2.0",
-    "regl": "^1.3.9",
+    "regl": "^1.3.11",
     "scroll-speed": "^1.0.0",
     "urijs": "^1.19.0"
   },
   "devDependencies": {
-    "@babel/core": "^7.1.5",
-    "@babel/plugin-proposal-class-properties": "^7.0.0",
-    "@babel/plugin-proposal-decorators": "^7.0.0",
-    "@babel/plugin-proposal-export-namespace-from": "^7.0.0",
-    "@babel/plugin-proposal-function-bind": "^7.0.0",
-    "@babel/plugin-proposal-nullish-coalescing-operator": "^7.2.0",
+    "@babel/core": "^7.4.4",
+    "@babel/plugin-proposal-class-properties": "^7.4.4",
+    "@babel/plugin-proposal-decorators": "^7.4.4",
+    "@babel/plugin-proposal-export-namespace-from": "^7.2.0",
+    "@babel/plugin-proposal-function-bind": "^7.2.0",
+    "@babel/plugin-proposal-nullish-coalescing-operator": "^7.4.4",
     "@babel/plugin-proposal-optional-chaining": "^7.2.0",
-    "@babel/plugin-transform-react-constant-elements": "^7.0.0",
-    "@babel/plugin-transform-runtime": "^7.1.0",
-    "@babel/preset-env": "^7.1.5",
+    "@babel/plugin-transform-react-constant-elements": "^7.2.0",
+    "@babel/plugin-transform-runtime": "^7.4.4",
+    "@babel/preset-env": "^7.4.4",
     "@babel/preset-react": "^7.0.0",
-    "@babel/register": "^7.0.0",
-    "@babel/runtime": "^7.1.5",
-    "babel-core": "^7.0.0-bridge.0",
+    "@babel/register": "^7.4.4",
+    "@babel/runtime": "^7.4.4",
     "babel-eslint": "^10.0.1",
-    "babel-jest": "^23.6.0",
-    "babel-loader": "^8.0.0",
-    "babel-preset-modern-browsers": "^12.0.0",
+    "babel-jest": "^24.8.0",
+    "babel-loader": "^8.0.6",
+    "babel-preset-modern-browsers": "^14.0.0",
     "chalk": "^2.4.2",
     "connect-history-api-fallback": "^1.6.0",
-    "copy-webpack-plugin": "^4.6.0",
-    "css-loader": "^1.0.1",
-    "eslint": "^5.13.0",
+    "copy-webpack-plugin": "^5.0.3",
+    "css-loader": "^2.1.1",
+    "eslint": "^5.16.0",
     "eslint-config-airbnb": "^17.1.0",
-    "eslint-config-prettier": "^4.0.0",
+    "eslint-config-prettier": "^4.2.0",
     "eslint-loader": "^2.1.2",
     "eslint-plugin-filenames": "^1.3.2",
-    "eslint-plugin-import": "^2.16.0",
-    "eslint-plugin-jest": "^22.2.2",
+    "eslint-plugin-import": "^2.17.2",
+    "eslint-plugin-jest": "^22.5.1",
     "eslint-plugin-jsx-a11y": "^6.2.1",
-    "eslint-plugin-react": "^7.12.4",
+    "eslint-plugin-react": "^7.13.0",
     "express": "^4.14.0",
-    "file-loader": "^2.0.0",
+    "file-loader": "^3.0.1",
     "html-webpack-inline-source-plugin": "0.0.10",
     "html-webpack-plugin": "^3.2.0",
-    "jest": "^24.1.0",
-    "jest-puppeteer": "^4.1.0",
+    "jest": "^24.8.0",
+    "jest-puppeteer": "^4.1.1",
     "json-loader": "^0.5.4",
-    "mini-css-extract-plugin": "^0.4.1",
-    "puppeteer": "^1.12.1",
+    "mini-css-extract-plugin": "^0.6.0",
+    "puppeteer": "^1.16.0",
     "rimraf": "^2.6.3",
     "serve-favicon": "^2.3.0",
-    "start-server-and-test": "^1.7.11",
+    "start-server-and-test": "^1.9.0",
     "style-loader": "^0.23.1",
     "sw-precache-webpack-plugin": "^0.11.5",
     "url-loader": "^1.1.0",
-    "webpack": "^4.25.1",
-    "webpack-cli": "^3.1.0",
-    "webpack-dev-middleware": "^3.1.3"
+    "webpack": "^4.31.0",
+    "webpack-cli": "^3.3.2",
+    "webpack-dev-middleware": "^3.6.2"
   },
   "jest": {
     "testMatch": [
@@ -133,16 +131,23 @@
         ],
         "plugins": [
           "@babel/plugin-proposal-function-bind",
-          "@babel/plugin-proposal-class-properties",
           [
             "@babel/plugin-proposal-decorators",
             {
               "legacy": true
             }
           ],
+          [
+            "@babel/plugin-proposal-class-properties",
+            {
+              "loose": true
+            }
+          ],
           "@babel/plugin-proposal-export-namespace-from",
           "@babel/plugin-transform-react-constant-elements",
-          "@babel/plugin-transform-runtime"
+          "@babel/plugin-transform-runtime",
+          "@babel/plugin-proposal-optional-chaining",
+          "@babel/plugin-proposal-nullish-coalescing-operator"
         ]
       }
     }

client/src/actions/index.js

@@ -21,24 +21,32 @@ const doInitialDataLoad = () =>
     dispatch({ type: "initial data load start" });
 
     try {
-      const requestJson = _(["config", "schema"])
+      /*
+      Step 1 - config & schema, all JSON
+      */
+      const requestJson = ["config", "schema"]
         .map(r => `${globals.API.prefix}${globals.API.version}${r}`)
-        .map(url => doJsonRequest(url))
-        .value();
-      const requestBinary = _([
-        "annotations/obs",
-        "annotations/var?annotation-name=name",
-        "layout/obs"
-      ])
-        .map(r => `${globals.API.prefix}${globals.API.version}${r}`)
-        .map(url => doBinaryRequest(url))
-        .value();
-
-      const results = await Promise.all(_.concat(requestJson, requestBinary));
-
+        .map(url => doJsonRequest(url));
+      const stepOneResults = await Promise.all(requestJson);
       /* set config defaults */
-      const config = { ...globals.configDefaults, ...results[0].config };
-      const [, schema, obsAnno, varAnno, obsLayout] = [...results];
+      const config = { ...globals.configDefaults, ...stepOneResults[0].config };
+      const schema = stepOneResults[1];
+
+      /*
+      Step 2 - dataframes, all binary.  NOTE: uses results of step 1.
+      */
+      /* only load names for var annotations, if possible*/
+      const varIndexName = schema?.schema?.annotations?.var?.index;
+      const varAnnotationsQuery = varIndexName
+        ? `?annotation-name=${varIndexName}`
+        : "";
+      const varAnnotationsURL = `annotations/var${varAnnotationsQuery}`;
+      const requestBinary = ["annotations/obs", varAnnotationsURL, "layout/obs"]
+        .map(r => `${globals.API.prefix}${globals.API.version}${r}`)
+        .map(url => doBinaryRequest(url));
+      const stepTwoResults = await Promise.all(requestBinary);
+      const [obsAnno, varAnno, obsLayout] = [...stepTwoResults];
+
       const universe = Universe.createUniverseFromResponse(
         config,
         schema,
@@ -91,6 +99,10 @@ needs expression data.
 Transparently utilizes cached data if it is already present.
 */
 async function _doRequestExpressionData(dispatch, getState, genes) {
+  const state = getState();
+  const { universe } = state;
+  const varIndexName = universe.schema.annotations.var.index;
+
   /* helper for this function only */
   const fetchData = async geneNames => {
     const res = await fetch(
@@ -100,7 +112,7 @@ async function _doRequestExpressionData(dispatch, getState, genes) {
         body: JSON.stringify({
           filter: {
             var: {
-              annotation_value: [{ name: "name", values: geneNames }]
+              annotation_value: [{ name: varIndexName, values: geneNames }]
             }
           }
         }),
@@ -123,8 +135,6 @@ async function _doRequestExpressionData(dispatch, getState, genes) {
     return Universe.convertDataFBStoObject(universe, data);
   };
 
-  const state = getState();
-  const { universe } = state;
   /* preload data already in cache */
   let expressionData = _.transform(
     genes,
@@ -241,6 +251,7 @@ const requestDifferentialExpression = (set1, set2, num_genes = 10) => async (
     */
     const state = getState();
     const { universe } = state;
+    const varIndexName = universe.schema.annotations.var.index;
 
     // Legal values are null, Array or TypedArray.  Null is initial state.
     if (!set1) set1 = [];
@@ -277,7 +288,7 @@ const requestDifferentialExpression = (set1, set2, num_genes = 10) => async (
     const data = await res.json();
     // result is [ [varIdx, ...], ... ]
     const topNGenes = _.map(data, r =>
-      universe.varAnnotations.at(r[0], "name")
+      universe.varAnnotations.at(r[0], varIndexName)
     );
 
     /*

client/src/components/categorical/category.js

@@ -27,10 +27,10 @@ class Category extends React.Component {
       const cat = categoricalSelection[metadataField];
       const categoryCount = {
         // total number of categories in this dimension
-        totalCatCount: cat.numCategories,
+        totalCatCount: cat.numCategoryValues,
         // number of selected options in this category
         selectedCatCount: _.reduce(
-          cat.categorySelected,
+          cat.categoryValueSelected,
           (res, cond) => (cond ? res + 1 : res),
           0
         )
@@ -91,7 +91,7 @@ class Category extends React.Component {
     const { categoricalSelection, metadataField } = this.props;
 
     const cat = categoricalSelection[metadataField];
-    const optTuples = sortedCategoryValues([...cat.categoryIndices]);
+    const optTuples = sortedCategoryValues([...cat.categoryValueIndices]);
     return _.map(optTuples, (tuple, i) => (
       <Value
         optTuples={optTuples}

client/src/components/categorical/occupancy.js

@@ -1,25 +1,16 @@
 // jshint esversion: 6
 import React from "react";
-import _ from "lodash";
 import { connect } from "react-redux";
 import * as d3 from "d3";
 
 @connect()
 class Occupancy extends React.Component {
   render() {
-    const {
-      occupancy,
-      colorScale,
-      categoricalSelection,
-      colorAccessor,
-      schema
-    } = this.props;
+    const { occupancy, colorScale, colorAccessor, schema, world } = this.props;
     const width = 100;
     const height = 11;
 
-    const categories = _.filter(schema.annotations.obs, {
-      name: colorAccessor
-    })[0].categories;
+    const categories = schema.annotations.obsByName[colorAccessor]?.categories;
 
     const x = d3
       .scaleLinear()
@@ -28,8 +19,9 @@ class Occupancy extends React.Component {
       .range([0, width]);
 
     let currentOffset = 0;
-
-    const stacks = categoricalSelection[colorAccessor].categoryValues.map(d => {
+    const dfColumn = world.obsAnnotations.col(colorAccessor);
+    const categoryValues = dfColumn.summarize().categories;
+    const stacks = categoryValues.map(d => {
       const o = occupancy.get(d);
 
       const scaledValue = x(o);

client/src/components/categorical/util.js

@@ -5,7 +5,6 @@
 // return sorted index
 
 import isNumber from "is-number";
-import _ from "lodash";
 
 const sortedCategoryValues = values => {
   /* this sort could be memoized for perf */
@@ -13,7 +12,7 @@ const sortedCategoryValues = values => {
   const strings = [];
   const ints = [];
 
-  _.forEach(values, v => {
+  values.forEach(v => {
     if (isNumber(v[0])) {
       ints.push(v);
     } else {

client/src/components/categorical/value.js

@@ -1,7 +1,6 @@
 // jshint esversion: 6
 import { connect } from "react-redux";
 import React from "react";
-import _ from "lodash";
 import Occupancy from "./occupancy";
 import { countCategoryValues2D } from "../../util/stateManager/worldUtil";
 import * as globals from "../../globals";
@@ -10,7 +9,7 @@ import * as globals from "../../globals";
   categoricalSelection: state.categoricalSelection,
   colorScale: state.colors.scale,
   colorAccessor: state.colors.colorAccessor,
-  schema: _.get(state.world, "schema", null),
+  schema: state.world?.schema,
   world: state.world
 }))
 class CategoryValue extends React.Component {
@@ -47,8 +46,8 @@ class CategoryValue extends React.Component {
     if (!categoricalSelection) return null;
 
     const category = categoricalSelection[metadataField];
-    const selected = category.categorySelected[categoryIndex];
-    const count = category.categoryCounts[categoryIndex];
+    const selected = category.categoryValueSelected[categoryIndex];
+    const count = category.categoryValueCounts[categoryIndex];
     const value = category.categoryValues[categoryIndex];
     const displayString = String(
       category.categoryValues[categoryIndex]
@@ -60,9 +59,7 @@ class CategoryValue extends React.Component {
     let occupancy = null;
 
     if (isColorBy && schema) {
-      categories = _.filter(schema.annotations.obs, {
-        name: colorAccessor
-      })[0].categories;
+      categories = schema.annotations.obsByName[colorAccessor]?.categories;
     }
 
     if (colorAccessor && !isColorBy && categoricalSelection[colorAccessor]) {

client/src/components/continuous/continuous.js

@@ -9,10 +9,10 @@ import * as globals from "../../globals";
 import HistogramBrush from "../brushableHistogram";
 
 @connect(state => ({
-  obsAnnotations: _.get(state.world, "obsAnnotations", null),
+  obsAnnotations: state.world?.obsAnnotations,
   colorAccessor: state.colors.colorAccessor,
   colorScale: state.colors.scale,
-  schema: _.get(state.world, "schema", null)
+  schema: state.world?.schema
 }))
 class Continuous extends React.Component {
   constructor(props) {
@@ -66,7 +66,8 @@ class Continuous extends React.Component {
           ? _.map(obsAnnotations.colIndex.keys(), key => {
               const isColorField =
                 key.includes("color") || key.includes("Color");
-              if (key === "name" || isColorField) return null;
+              if (key === schema.annotations.obs.index || isColorField)
+                return null;
 
               const summary = obsAnnotations.col(key).summarize();
               const nonFiniteExtent =

client/src/components/framework/logo.js

@@ -0,0 +1,18 @@
+import React from "react";
+import * as globals from "../../globals";
+
+const Logo = props => {
+  const { size } = props;
+  return (
+    <svg width={size} height={size} viewBox={`0 0 48 48`} fill="none">
+      <rect width="48" height="48" fill="white" />
+      <rect width="48" height="48" fill={globals.logoColor} />
+      <rect x="19" y="19" width="22" height="22" fill="white" />
+      <rect x="24" y="24" width="12" height="12" fill={globals.logoColor} />
+      <rect x="7" y="19" width="7" height="22" fill="white" />
+      <rect x="19" y="7" width="22" height="7" fill="white" />
+    </svg>
+  );
+};
+
+export default Logo;

client/src/components/geneExpression/index.js

@@ -57,7 +57,7 @@ const filterGenes = (query, genes) =>
 
 @connect(state => {
   return {
-    obsAnnotations: _.get(state.world, "obsAnnotations", null),
+    obsAnnotations: state.world?.obsAnnotations,
     userDefinedGenes: state.controls.userDefinedGenes,
     userDefinedGenesLoading: state.controls.userDefinedGenesLoading,
     world: state.world,
@@ -85,7 +85,8 @@ class GeneExpression extends React.Component {
     */
     const { world } = this.props;
     const { varAnnotations } = world;
-    const geneNames = varAnnotations.col("name").asArray();
+    const varIndexName = world.schema.annotations.var.index;
+    const geneNames = varAnnotations.col(varIndexName).asArray();
     if (geneNames.length > 0) {
       const placeholder = [];
       let len = geneNames.length;
@@ -107,6 +108,7 @@ class GeneExpression extends React.Component {
 
   handleClick(g) {
     const { world, dispatch, userDefinedGenes } = this.props;
+    const varIndexName = world.schema.annotations.var.index;
     const gene = g.target;
     if (userDefinedGenes.indexOf(gene) !== -1) {
       postUserErrorToast("That gene already exists");
@@ -114,7 +116,9 @@ class GeneExpression extends React.Component {
       postUserErrorToast(
         "That's too many genes, you can have at most 15 user defined genes"
       );
-    } else if (world.varAnnotations.col("name").indexOf(gene) === undefined) {
+    } else if (
+      world.varAnnotations.col(varIndexName).indexOf(gene) === undefined
+    ) {
       postUserErrorToast("That doesn't appear to be a valid gene name.");
     } else {
       dispatch({ type: "single user defined gene start" });
@@ -127,6 +131,7 @@ class GeneExpression extends React.Component {
 
   handleBulkAddClick() {
     const { world, dispatch, userDefinedGenes } = this.props;
+    const varIndexName = world.schema.annotations.var.index;
     const { bulkAdd } = this.state;
 
     /*
@@ -145,7 +150,9 @@ class GeneExpression extends React.Component {
           if (userDefinedGenes.indexOf(gene) !== -1) {
             return keepAroundErrorToast("That gene already exists");
           }
-          if (world.varAnnotations.col("name").indexOf(gene) === undefined) {
+          if (
+            world.varAnnotations.col(varIndexName).indexOf(gene) === undefined
+          ) {
             return keepAroundErrorToast(
               `${gene} doesn't appear to be a valid gene name.`
             );
@@ -168,7 +175,7 @@ class GeneExpression extends React.Component {
       userDefinedGenesLoading,
       differential
     } = this.props;
-
+    const varIndexName = world?.schema?.annotations?.var?.index;
     const { tab, bulkAdd } = this.state;
 
     return (
@@ -243,7 +250,7 @@ class GeneExpression extends React.Component {
                 itemRenderer={renderGene.bind(this)}
                 items={
                   world && world.varAnnotations
-                    ? world.varAnnotations.col("name").asArray()
+                    ? world.varAnnotations.col(varIndexName).asArray()
                     : ["No genes"]
                 }
                 popoverProps={{ minimal: true }}
@@ -322,7 +329,7 @@ class GeneExpression extends React.Component {
           <ExpressionButtons />
           {differential.diffExp
             ? _.map(differential.diffExp, (value, index) => {
-                const name = world.varAnnotations.at(value[0], "name");
+                const name = world.varAnnotations.at(value[0], varIndexName);
                 const values = world.varData.col(name);
                 if (!values) {
                   return null;

client/src/components/graph/drawPointsRegl.js

@@ -1,5 +1,6 @@
 // jshint esversion: 6
 const mat4 = require("gl-mat4");
+const vec3 = require("gl-vec3");
 
 // opacity: https://github.com/spacetx/starfish/blob/master/viz/draw/regions.js
 
@@ -38,7 +39,20 @@ export default function(regl) {
     uniforms: {
       distance: regl.prop("distance"),
       view: regl.prop("view"),
-      projection: ({viewportWidth, viewportHeight}) => mat4.perspective([], Math.PI / 2, viewportWidth / viewportHeight, 0.01, 1000)
+      projection: ({ viewportWidth, viewportHeight }) => {
+        const aspectRatio = viewportWidth / viewportHeight;
+        let m = mat4.perspective(
+          [],
+          Math.PI / 2,
+          viewportWidth / viewportHeight,
+          0.01,
+          1000
+        );
+        if (aspectRatio < 1) {
+          m = mat4.scale(m, m, vec3.fromValues(1, 1, 1 / aspectRatio));
+        }
+        return m;
+      }
     },
 
     count: regl.prop("count"),

client/src/components/graph/graph.js

@@ -3,7 +3,9 @@ import React from "react";
 import * as d3 from "d3";
 import { connect } from "react-redux";
 import mat4 from "gl-mat4";
+import vec3 from "gl-vec3";
 import _regl from "regl";
+import memoize from "memoize-one";
 import {
   Button,
   AnchorButton,
@@ -13,7 +15,9 @@ import {
   MenuItem,
   Position,
   NumericInput,
-  Icon
+  Icon,
+  RadioGroup,
+  Radio
 } from "@blueprintjs/core";
 
 import * as globals from "../../globals";
@@ -47,7 +51,8 @@ import { World } from "../../util/stateManager";
   undoDisabled: state["@@undoable/past"].length === 0,
   redoDisabled: state["@@undoable/future"].length === 0,
   selectionTool: state.graphSelection.tool,
-  currentSelection: state.graphSelection.selection
+  currentSelection: state.graphSelection.selection,
+  layoutChoice: state.layoutChoice
 }))
 class Graph extends React.Component {
   static isValidDigitKeyEvent(e) {
@@ -74,6 +79,38 @@ class Graph extends React.Component {
     return key >= 0 && key <= 9;
   }
 
+  computePointPositions = memoize((X, Y, scaleX, scaleY) => {
+    /*
+    compute webgl coordinate buffer for each point
+    */
+    const positions = new Float32Array(2 * X.length);
+    for (let i = 0, len = X.length; i < len; i += 1) {
+      positions[2 * i] = scaleX(X[i]);
+      positions[2 * i + 1] = scaleY(Y[i]);
+    }
+    return positions;
+  });
+
+  computePointColors = memoize(rgb => {
+    /*
+    compute webgl colors for each point
+    */
+    const colors = new Float32Array(3 * rgb.length);
+    for (let i = 0, len = rgb.length; i < len; i += 1) {
+      colors.set(rgb[i], 3 * i);
+    }
+    return colors;
+  });
+
+  computePointSizes = memoize((len, crossfilter) => {
+    /*
+    compute webgl dot size for each point
+    */
+    const sizes = new Float32Array(len);
+    crossfilter.fillByIsSelected(sizes, 4, 0.2);
+    return sizes;
+  });
+
   constructor(props) {
     super(props);
     this.count = 0;
@@ -81,6 +118,8 @@ class Graph extends React.Component {
     this.graphPaddingBottom = 45;
     this.graphPaddingRight = globals.leftSidebarWidth;
     this.renderCache = {
+      X: null,
+      Y: null,
       positions: null,
       colors: null,
       sizes: null
@@ -106,6 +145,12 @@ class Graph extends React.Component {
     const colorBuffer = regl.buffer();
     const sizeBuffer = regl.buffer();
 
+    // preallocate coordinate system transformation between data and gl
+    const transform = {
+      glScaleX: scaleLinear([0, 1], [-1, 1]),
+      glScaleY: scaleLinear([0, 1], [1, -1])
+    };
+
     /* first time, but this duplicates above function, should be possile to avoid this */
     const reglRender = regl.frame(() => {
       this.reglDraw(
@@ -128,7 +173,8 @@ class Graph extends React.Component {
       colorBuffer,
       sizeBuffer,
       camera,
-      reglRender
+      reglRender,
+      transform
     });
   }
 
@@ -140,19 +186,10 @@ class Graph extends React.Component {
       colorRGB,
       responsive,
       selectionTool,
-      currentSelection
+      currentSelection,
+      layoutChoice
     } = this.props;
-    const {
-      reglRender,
-      mode,
-      regl,
-      drawPoints,
-      camera,
-      pointBuffer,
-      colorBuffer,
-      sizeBuffer,
-      svg
-    } = this.state;
+    const { reglRender, mode, regl, svg } = this.state;
     let stateChanges = {};
 
     if (reglRender && this.reglRenderState === "rendering" && mode !== "zoom") {
@@ -163,53 +200,40 @@ class Graph extends React.Component {
     if (regl && world) {
       /* update the regl state */
       const { obsLayout, nObs } = world;
-      const X = obsLayout.col("X").asArray();
-      const Y = obsLayout.col("Y").asArray();
+      const {
+        drawPoints,
+        transform,
+        camera,
+        pointBuffer,
+        colorBuffer,
+        sizeBuffer
+      } = this.state;
 
-      // X/Y positions for each point - a cached value that only
-      // changes if we have loaded entirely new cell data
-      //
-      if (!renderCache.positions || world !== prevProps.world) {
-        renderCache.positions = new Float32Array(2 * nObs);
-
-        const glScaleX = scaleLinear([0, 1], [-1, 1]);
-        const glScaleY = scaleLinear([0, 1], [1, -1]);
-
-        const offset = [d3.mean(X) - 0.5, d3.mean(Y) - 0.5];
-
-        for (let i = 0, { positions } = renderCache; i < nObs; i += 1) {
-          positions[2 * i] = glScaleX(X[i] - offset[0]);
-          positions[2 * i + 1] = glScaleY(Y[i] - offset[1]);
-        }
-        pointBuffer({
-          data: renderCache.positions,
-          dimension: 2
-        });
-
-        stateChanges.offset = offset;
+      /* coordinates for each point */
+      const { glScaleX, glScaleY } = transform;
+      const X = obsLayout.col(layoutChoice.currentDimNames[0]).asArray();
+      const Y = obsLayout.col(layoutChoice.currentDimNames[1]).asArray();
+      const newPositions = this.computePointPositions(X, Y, glScaleX, glScaleY);
+      if (renderCache.positions !== newPositions) {
+        /* update our cache & GL if the buffer changes */
+        renderCache.positions = newPositions;
+        pointBuffer({ data: newPositions, dimension: 2 });
       }
 
-      // Colors for each point - a cached value that only changes when
-      // the cell metadata changes.
-      if (!renderCache.colors || colorRGB !== prevProps.colorRGB) {
-        const rgb = colorRGB;
-        if (!renderCache.colors) {
-          renderCache.colors = new Float32Array(3 * rgb.length);
-        }
-        for (let i = 0, { colors } = renderCache; i < rgb.length; i += 1) {
-          colors.set(rgb[i], 3 * i);
-        }
-        colorBuffer({ data: renderCache.colors, dimension: 3 });
+      /* colors for each point */
+      const newColors = this.computePointColors(colorRGB);
+      if (renderCache.colors !== newColors) {
+        /* update our cache & GL if the buffer changes */
+        renderCache.colors = newColors;
+        colorBuffer({ data: newColors, dimension: 3 });
       }
 
-      // Sizes for each point - updates are triggered only when selected
-      // obs change
-      if (!renderCache.sizes || crossfilter !== prevProps.crossfilter) {
-        if (!renderCache.sizes) {
-          renderCache.sizes = new Float32Array(nObs);
-        }
-        crossfilter.fillByIsSelected(renderCache.sizes, 4, 0.2);
-        sizeBuffer({ data: renderCache.sizes, dimension: 1 });
+      /* sizes for each point */
+      const newSizes = this.computePointSizes(nObs, crossfilter);
+      if (renderCache.sizes !== newSizes) {
+        /* update our cache & GL if the buffer changes */
+        renderCache.size = newSizes;
+        sizeBuffer({ data: newSizes, dimension: 1 });
       }
 
       this.count = nObs;
@@ -271,11 +295,10 @@ class Graph extends React.Component {
       mode !== prevState.mode ||
       stateChanges.svg
     ) {
-      const { tool, container, offset } = this.state;
+      const { tool, container } = this.state;
       this.selectionToolUpdate(
         stateChanges.tool ? stateChanges.tool : tool,
-        stateChanges.container ? stateChanges.container : container,
-        stateChanges.offset ? stateChanges.offset : offset
+        stateChanges.container ? stateChanges.container : container
       );
     }
 
@@ -435,7 +458,15 @@ class Graph extends React.Component {
     this.setState({ pendingClipPercentiles: null });
   };
 
-  brushToolUpdate(tool, container, offset) {
+  handleLayoutChoiceChange = e => {
+    const { dispatch } = this.props;
+    dispatch({
+      type: "set layout choice",
+      layoutChoice: e.currentTarget.value
+    });
+  };
+
+  brushToolUpdate(tool, container) {
     /*
     this is called from componentDidUpdate(), so be very careful using
     anything from this.state, which may be updated asynchronously.
@@ -449,8 +480,8 @@ class Graph extends React.Component {
         if there is a selection, make sure the brush tool matches
         */
         const screenCoords = [
-          this.mapPointToScreen(currentSelection.brushCoords.northwest, offset),
-          this.mapPointToScreen(currentSelection.brushCoords.southeast, offset)
+          this.mapPointToScreen(currentSelection.brushCoords.northwest),
+          this.mapPointToScreen(currentSelection.brushCoords.southeast)
         ];
         if (!toolCurrentSelection) {
           /* tool is not selected, so just move the brush */
@@ -477,7 +508,7 @@ class Graph extends React.Component {
     }
   }
 
-  lassoToolUpdate(tool, container, offset) {
+  lassoToolUpdate(tool, container) {
     /*
     this is called from componentDidUpdate(), so be very careful using
     anything from this.state, which may be updated asynchronously.
@@ -488,7 +519,7 @@ class Graph extends React.Component {
       if there is a current selection, make sure the lasso tool matches
       */
       const polygon = currentSelection.polygon.map(p =>
-        this.mapPointToScreen(p, offset)
+        this.mapPointToScreen(p)
       );
       tool.move(polygon);
     } else {
@@ -496,7 +527,7 @@ class Graph extends React.Component {
     }
   }
 
-  selectionToolUpdate(tool, container, offset) {
+  selectionToolUpdate(tool, container) {
     /*
     this is called from componentDidUpdate(), so be very careful using
     anything from this.state, which may be updated asynchronously.
@@ -504,10 +535,10 @@ class Graph extends React.Component {
     const { selectionTool } = this.props;
     switch (selectionTool) {
       case "brush":
-        this.brushToolUpdate(tool, container, offset);
+        this.brushToolUpdate(tool, container);
         break;
       case "lasso":
-        this.lassoToolUpdate(tool, container, offset);
+        this.lassoToolUpdate(tool, container);
         break;
       default:
         /* punt? */
@@ -564,12 +595,14 @@ class Graph extends React.Component {
     accounting for current pan/zoom camera.
     */
     const { responsive } = this.props;
-    const { regl, camera, offset } = this.state;
+    const { regl, camera, transform } = this.state;
+    const { glScaleX, glScaleY } = transform;
 
     const gl = regl._gl;
 
     // get aspect ratio
     const aspect = gl.drawingBufferWidth / gl.drawingBufferHeight;
+    const scale = aspect < 1 ? 1 / aspect : 1;
 
     // compute inverse view matrix
     const inverse = mat4.invert([], camera.view());
@@ -578,37 +611,37 @@ class Graph extends React.Component {
     const x = (2 * pin[0]) / (responsive.width - this.graphPaddingRight) - 1;
     const y = 2 * (1 - pin[1] / (responsive.height - this.graphPaddingTop)) - 1;
     const pout = [
-      x * inverse[14] * aspect + inverse[12],
-      y * inverse[14] + inverse[13]
+      x * inverse[14] * aspect * scale + inverse[12],
+      -(y * inverse[14] * scale + inverse[13])
     ];
 
-    return [(pout[0] + 1) / 2 + offset[0], (pout[1] + 1) / 2 + offset[1]];
+    const xy = [glScaleX.invert(pout[0]), glScaleY.invert(pout[1])];
+    return xy;
   }
 
-  mapPointToScreen(xyCell, offset) {
+  mapPointToScreen(xyCell) {
     /*
     Map an XY coordinate from cell/point domain to screen range.  Inverse
     of mapScreenToPoint()
     */
     const { responsive } = this.props;
-    const { regl, camera } = this.state;
+    const { regl, camera, transform } = this.state;
+    const { glScaleX, glScaleY } = transform;
 
     const gl = regl._gl;
 
     // get aspect ratio
     const aspect = gl.drawingBufferWidth / gl.drawingBufferHeight;
+    const scale = aspect < 1 ? 1 / aspect : 1;
 
     // compute inverse view matrix
-    const inverse = mat4.invert([], camera.view());
+    let inverse = mat4.invert([], camera.view());
 
     // variable names are choosen to reflect inverse of those used
     // in mapScreenToPoint().
-    const pout = [
-      (xyCell[0] - offset[0]) * 2 - 1,
-      (xyCell[1] - offset[1]) * 2 - 1
-    ];
-    const x = (pout[0] - inverse[12]) / aspect / inverse[14];
-    const y = (pout[1] - inverse[13]) / inverse[14];
+    const pout = [glScaleX(xyCell[0]), glScaleY(xyCell[1])];
+    const x = (pout[0] - inverse[12]) / aspect / scale / inverse[14];
+    const y = (-pout[1] - inverse[13]) / scale / inverse[14];
 
     const pin = [
       Math.round(((x + 1) * (responsive.width - this.graphPaddingRight)) / 2),
@@ -745,7 +778,8 @@ class Graph extends React.Component {
       redoDisabled,
       selectionTool,
       clipPercentileMin,
-      clipPercentileMax
+      clipPercentileMax,
+      layoutChoice
     } = this.props;
     const { mode, pendingClipPercentiles } = this.state;
 
@@ -888,6 +922,7 @@ class Graph extends React.Component {
                 />
               </Tooltip>
             </div>
+
             <div
               className="bp3-button-group"
               style={{
@@ -898,6 +933,49 @@ class Graph extends React.Component {
                 target={
                   <Button
                     type="button"
+                    data-testid="layout-choice"
+                    className="bp3-button bp3-icon-heatmap"
+                    style={{
+                      cursor: "pointer"
+                    }}
+                  />
+                }
+                position={Position.BOTTOM_RIGHT}
+                content={
+                  <div
+                    style={{
+                      display: "flex",
+                      justifyContent: "flex-start",
+                      alignItems: "flex-start",
+                      flexDirection: "column",
+                      padding: 10
+                    }}
+                  >
+                    <RadioGroup
+                      label="Layout Choice"
+                      onChange={this.handleLayoutChoiceChange}
+                      selectedValue={layoutChoice.current}
+                    >
+                      {layoutChoice.available.map(name => (
+                        <Radio label={name} value={name} key={name} />
+                      ))}
+                    </RadioGroup>
+                  </div>
+                }
+              />
+            </div>
+
+            <div
+              className="bp3-button-group"
+              style={{
+                marginLeft: 10
+              }}
+            >
+              <Popover
+                target={
+                  <Button
+                    type="button"
+                    data-testid="visualization-settings"
                     className={`bp3-button bp3-icon-timeline-bar-chart ${activeClipClass}`}
                     style={{
                       cursor: "pointer"
@@ -929,6 +1007,7 @@ class Graph extends React.Component {
                     >
                       <NumericInput
                         style={{ width: 50 }}
+                        data-testid={"clip-min-input"}
                         onValueChange={this.handleClipPercentileMinValueChange}
                         onKeyPress={this.handleClipOnKeyPress}
                         value={clipMin}
@@ -949,6 +1028,7 @@ class Graph extends React.Component {
                       <span style={{ marginRight: 5, marginLeft: 5 }}> - </span>
                       <NumericInput
                         style={{ width: 50 }}
+                        data-testid={"clip-max-input"}
                         onValueChange={this.handleClipPercentileMaxValueChange}
                         onKeyPress={this.handleClipOnKeyPress}
                         value={clipMax}
@@ -968,6 +1048,7 @@ class Graph extends React.Component {
                       />
                       <Button
                         type="button"
+                        data-testid="clip-commit"
                         className="bp3-button"
                         disabled={this.isClipDisabled()}
                         style={{

client/src/components/leftsidebar.js

@@ -1,5 +1,4 @@
 // jshint esversion: 6
-import _ from "lodash";
 import React from "react";
 import { connect } from "react-redux";
 import Categorical from "./categorical/categorical";
@@ -7,10 +6,11 @@ import Continuous from "./continuous/continuous";
 import GeneExpression from "./geneExpression";
 import * as globals from "../globals";
 import DynamicScatterplot from "./scatterplot/scatterplot";
+import Logo from "./framework/logo.js";
 
 @connect(state => ({
   responsive: state.responsive,
-  datasetTitle: _.get(state.config, "displayNames.dataset"),
+  datasetTitle: state.config?.displayNames?.dataset,
   scatterplotXXaccessor: state.controls.scatterplotXXaccessor,
   scatterplotYYaccessor: state.controls.scatterplotYYaccessor
 }))
@@ -28,7 +28,6 @@ class LeftSideBar extends React.Component {
     if cellxgene logo or tabs change, this must as well
     */
     const metadataSectionPadding = 0;
-    // scatterplotXXaccessor && scatterplotYYaccessor ? 450 : 0;
 
     return (
       <div
@@ -40,20 +39,49 @@ class LeftSideBar extends React.Component {
         }}
       >
         <p
-          data-testid="header"
           style={{
             position: "fixed",
             top: globals.cellxgeneTitleTopPadding,
             left: globals.leftSidebarWidth + globals.cellxgeneTitleLeftPadding,
-            margin: 0,
-            fontSize: globals.largestFontSize,
-            color: globals.darkerGrey,
-            width: "100%"
+            margin: 0
           }}
         >
-          cellxgene: {datasetTitle}
+          <Logo size={32} />
+          <span
+            style={{
+              fontSize: 28,
+              position: "relative",
+              top: -4,
+              fontWeight: "bold",
+              marginLeft: 5,
+              color: globals.logoColor,
+              userSelect: "none"
+            }}
+          >
+            cell<span
+              style={{
+                position: "relative",
+                top: 1,
+                fontWeight: 300,
+                fontSize: 24
+              }}
+            >
+              ×
+            </span>gene
+          </span>
+          <span
+            data-testid="header"
+            style={{
+              fontSize: 16,
+              display: "block",
+              position: "relative",
+              marginTop: 10,
+              top: -4
+            }}
+          >
+            {datasetTitle}
+          </span>
         </p>
-
         <div
           style={{
             height: responsive.height - metadataSectionPadding,

client/src/globals.js

@@ -31,6 +31,7 @@ export const darkGreen = "#448C4D";
 
 export const nonFiniteCellColor = lightGrey;
 export const defaultCellColor = "rgb(0,0,0,1)";
+export const logoColor = "black"; /* logo pink: "#E9429A" */
 
 /* typography constants */
 

client/src/reducers/categoricalSelection.js

@@ -1,12 +1,9 @@
-import _ from "lodash";
-
 import { ControlsHelpers } from "../util/stateManager";
 import * as globals from "../globals";
 
 function maxCategoryItems(state) {
-  return _.get(
-    state.config,
-    "parameters.max-category-items",
+  return (
+    state.config.parameters?.["max-category-items"] ??
     globals.configDefaults.parameters["max-category-items"]
   );
 }
@@ -33,15 +30,15 @@ const CategoricalSelection = (
       /*
       Set the specific category in this field to false
       */
-      const newCategorySelected = Array.from(
-        state[action.metadataField].categorySelected
+      const newCategoryValueSelected = Array.from(
+        state[action.metadataField].categoryValueSelected
       );
-      newCategorySelected[action.categoryIndex] = true;
+      newCategoryValueSelected[action.categoryIndex] = true;
       const newCategoricalSelection = {
         ...state,
         [action.metadataField]: {
           ...state[action.metadataField],
-          categorySelected: newCategorySelected
+          categoryValueSelected: newCategoryValueSelected
         }
       };
       return newCategoricalSelection;
@@ -51,15 +48,15 @@ const CategoricalSelection = (
       /*
       Set the specific category in this field to false
       */
-      const newCategorySelected = Array.from(
-        state[action.metadataField].categorySelected
+      const newCategoryValueSelected = Array.from(
+        state[action.metadataField].categoryValueSelected
       );
-      newCategorySelected[action.categoryIndex] = false;
+      newCategoryValueSelected[action.categoryIndex] = false;
       const newCategoricalSelection = {
         ...state,
         [action.metadataField]: {
           ...state[action.metadataField],
-          categorySelected: newCategorySelected
+          categoryValueSelected: newCategoryValueSelected
         }
       };
       return newCategoricalSelection;
@@ -73,8 +70,9 @@ const CategoricalSelection = (
         ...state,
         [action.metadataField]: {
           ...state[action.metadataField],
-          categorySelected: Array.from(
-            state[action.metadataField].categorySelected
+          categorySelected: false,
+          categoryValueSelected: Array.from(
+            state[action.metadataField].categoryValueSelected
           ).fill(false)
         }
       };
@@ -89,8 +87,9 @@ const CategoricalSelection = (
         ...state,
         [action.metadataField]: {
           ...state[action.metadataField],
-          categorySelected: Array.from(
-            state[action.metadataField].categorySelected
+          categorySelected: true,
+          categoryValueSelected: Array.from(
+            state[action.metadataField].categoryValueSelected
           ).fill(true)
         }
       };

client/src/reducers/controls.js

@@ -93,9 +93,10 @@ const Controls = (
     }
     case "request differential expression success": {
       const { world } = prevSharedState;
+      const varIndexName = world.schema.annotations.var.index;
       const _diffexpGenes = [];
       action.data.forEach(d => {
-        _diffexpGenes.push(world.varAnnotations.at(d[0], "name"));
+        _diffexpGenes.push(world.varAnnotations.at(d[0], varIndexName));
       });
       return {
         ...state,

client/src/reducers/crossfilter.js

@@ -20,10 +20,11 @@ const CrossfilterReducer = (
 ) => {
   switch (action.type) {
     case "initial data load complete (universe exists)": {
-      const { world } = nextSharedState;
+      const { world, layoutChoice } = nextSharedState;
       const crossfilter = World.createObsDimensions(
         new Crossfilter(world.obsAnnotations),
-        world
+        world,
+        layoutChoice.currentDimNames
       );
       return crossfilter;
     }
@@ -43,9 +44,13 @@ const CrossfilterReducer = (
     case "set clip quantiles":
     case "set World to current selection": {
       const { userDefinedGenes, diffexpGenes } = prevSharedState.controls;
-      const { world } = nextSharedState;
+      const { world, layoutChoice } = nextSharedState;
       let crossfilter = new Crossfilter(world.obsAnnotations);
-      crossfilter = World.createObsDimensions(crossfilter, world);
+      crossfilter = World.createObsDimensions(
+        crossfilter,
+        world,
+        layoutChoice.currentDimNames
+      );
       crossfilter = ControlsHelpers.createGeneDimensions(
         userDefinedGenes,
         diffexpGenes,
@@ -55,6 +60,23 @@ const CrossfilterReducer = (
       return crossfilter;
     }
 
+    case "set layout choice": {
+      /*
+      when switching layouts:
+      - delete the existing XY index
+      - add the new XY index (which implicitly selects all on it)
+      */
+      const { world, layoutChoice } = nextSharedState;
+      return state
+        .delDimension(layoutDimensionName("XY"))
+        .addDimension(
+          layoutDimensionName("XY"),
+          "spatial",
+          world.obsLayout.col(layoutChoice.currentDimNames[0]).asArray(),
+          world.obsLayout.col(layoutChoice.currentDimNames[1]).asArray()
+        );
+    }
+
     case "request user defined gene success": {
       const { world } = prevSharedState;
       const gene = action.data.genes[0];
@@ -68,8 +90,9 @@ const CrossfilterReducer = (
 
     case "request differential expression success": {
       const { world } = prevSharedState;
+      const varIndexName = world.schema.annotations.var.index;
       const genes = _.map(action.data, d =>
-        world.varAnnotations.at(d[0], "name")
+        world.varAnnotations.at(d[0], varIndexName)
       );
       const crossfilter = _.reduce(
         genes,
@@ -87,10 +110,11 @@ const CrossfilterReducer = (
 
     case "clear differential expression": {
       const { world } = prevSharedState;
+      const varIndexName = world.schema.annotations.var.index;
       const crossfilter = _.reduce(
         action.diffExp,
         (xfltr, values) => {
-          const name = world.varAnnotations.at(values[0], "name");
+          const name = world.varAnnotations.at(values[0], varIndexName);
           return xfltr.delDimension(diffexpDimensionName(name));
         },
         state
@@ -161,10 +185,12 @@ const CrossfilterReducer = (
     case "categorical metadata filter select":
     case "categorical metadata filter deselect": {
       const { categoricalSelection } = nextSharedState;
+      const { world } = prevSharedState;
       const cat = categoricalSelection[action.metadataField];
+      const col = world.obsAnnotations.col(action.metadataField);
       return state.select(obsAnnoDimensionName(action.metadataField), {
         mode: "exact",
-        values: ControlsHelpers.selectedValuesForCategory(cat)
+        values: ControlsHelpers.selectedValuesForCategory(cat, col)
       });
     }
 

client/src/reducers/graphSelection.js

@@ -7,7 +7,8 @@ const GraphSelection = (
 ) => {
   switch (action.type) {
     case "set clip quantiles":
-    case "reset World to eq Universe": {
+    case "reset World to eq Universe":
+    case "set layout choice": {
       return {
         ...state,
         selection: {

client/src/reducers/index.js

@@ -12,6 +12,7 @@ import graphSelection from "./graphSelection";
 import crossfilter from "./crossfilter";
 import colors from "./colors";
 import differential from "./differential";
+import layoutChoice from "./layoutChoice";
 import responsive from "./responsive";
 import controls from "./controls";
 import resetCache from "./resetCache";
@@ -19,31 +20,33 @@ import resetCache from "./resetCache";
 import undoableConfig from "./undoableConfig";
 
 const Reducer = undoable(
-  cascadeReducers([
-    ["config", config],
-    ["universe", universe],
-    ["world", world],
-    ["categoricalSelection", categoricalSelection],
-    ["continuousSelection", continuousSelection],
-    ["graphSelection", graphSelection],
-    ["crossfilter", crossfilter],
-    ["colors", colors],
-    ["controls", controls],
-    ["differential", differential],
-    ["responsive", responsive],
-    ["resetCache", resetCache]
-  ]),
-  [
-    "world",
-    "categoricalSelection",
-    "continuousSelection",
-    "graphSelection",
-    "crossfilter",
-    "colors",
-    "controls",
-    "differential"
-  ],
-  undoableConfig
+    cascadeReducers([
+        ["config", config],
+        ["universe", universe],
+        ["world", world],
+        ["layoutChoice", layoutChoice],
+        ["categoricalSelection", categoricalSelection],
+        ["continuousSelection", continuousSelection],
+        ["graphSelection", graphSelection],
+        ["crossfilter", crossfilter],
+        ["colors", colors],
+        ["controls", controls],
+        ["differential", differential],
+        ["responsive", responsive],
+        ["resetCache", resetCache]
+    ]),
+    [
+        "world",
+        "categoricalSelection",
+        "continuousSelection",
+        "graphSelection",
+        "crossfilter",
+        "colors",
+        "controls",
+        "differential",
+        "layoutChoice"
+    ],
+    undoableConfig
 );
 
 const store = createStore(Reducer, applyMiddleware(thunk));

client/src/reducers/layoutChoice.js

@@ -0,0 +1,50 @@
+/*
+we have a UI heuristic to pick the default layout, based on assumptions
+about commonly used names.  Preferentially, pick in the following order:
+
+  1. "umap"
+  2. "tsne"
+  3. "pca"
+  4. give up, use the first available
+*/
+function bestDefaultLayout(layouts) {
+  const preferredNames = ["umap", "tsne", "pca"];
+  const idx = preferredNames.findIndex(name => layouts.indexOf(name) !== -1);
+  if (idx !== -1) return preferredNames[idx];
+  return layouts[0];
+}
+
+const LayoutChoice = (
+  state = {
+    available: [], // all available choices
+    current: undefined, // name of the current layout, eg, 'umap'
+    currentDimNames: [] // dimension name
+  },
+  action,
+  nextSharedState
+) => {
+  switch (action.type) {
+    case "initial data load complete (universe exists)":
+    case "reset World to eq Universe": {
+      // set default to default
+      const { schema } = nextSharedState.world;
+      const available = schema.layout.obs.map(v => v.name);
+      const current = bestDefaultLayout(available);
+      const currentDimNames = schema.layout.obsByName[current].dims;
+      return { available, current, currentDimNames };
+    }
+
+    case "set layout choice": {
+      const { schema } = nextSharedState.world;
+      const current = action.layoutChoice;
+      const currentDimNames = schema.layout.obsByName[current].dims;
+      return { ...state, current, currentDimNames };
+    }
+
+    default: {
+      return state;
+    }
+  }
+};
+
+export default LayoutChoice;

client/src/reducers/undoableConfig.js

@@ -59,7 +59,7 @@ const saveOnActions = new Set([
   "categorical metadata filter select",
   "categorical metadata filter deselect",
   "categorical metadata filter all of these",
-  "categorical metadata none of these",
+  "categorical metadata filter none of these",
 
   "color by categorical metadata",
   "color by continuous metadata",
@@ -72,7 +72,9 @@ const saveOnActions = new Set([
   "store current cell selection as differential set 2",
 
   "set World to current selection",
-  "set clip quantiles"
+  "set clip quantiles",
+
+  "set layout choice"
 ]);
 
 /**
@@ -98,9 +100,15 @@ const applyPending = () => ({
   [actionKey]: "applyPending",
   [stateKey]: { fsm: null }
 });
-const skip = fsm => ({ [actionKey]: "skip", [stateKey]: { fsm } });
+const skip = (fsm, transition) => ({
+  [actionKey]: "skip",
+  [stateKey]: { fsm: transition.to !== "done" ? fsm : null }
+});
 const clear = () => ({ [actionKey]: "clear", [stateKey]: { fsm: null } });
-const save = fsm => ({ [actionKey]: "save", [stateKey]: { fsm } });
+const save = (fsm, transition) => ({
+  [actionKey]: "save",
+  [stateKey]: { fsm: transition.to !== "done" ? fsm : null }
+});
 
 /*
 Error handler for state transitions that are unexpected.  Called by

client/src/util/camera.js

@@ -5,10 +5,9 @@ const mp = require("mouse-position");
 const mb = require("mouse-pressed");
 const key = require("key-pressed");
 
-const panSpeed = 0.4;
+const panSpeed = 1.0; // changed from 0.4 to 1.0 per issue #722
 const scaleSpeed = 0.5;
 const scaleMax = 3;
-// const scaleMin = 1.15
 const scaleMin = 1.03;
 
 function attachCamera(canvas, opts) {

client/src/util/dataframe/labelIndex.js

@@ -3,6 +3,8 @@ Label indexing - map a label to & from an integer offset.  See Dataframe
 for how this is used.
 **/
 
+import { rangeFill as fillRange } from "../range";
+
 /*
 Private utility functions
 */
@@ -21,14 +23,6 @@ function extent(tarr) {
   return [min, max];
 }
 
-function fillRange(arr, start = 0) {
-  const larr = arr;
-  for (let i = 0, l = larr.length; i < l; i += 1) {
-    larr[i] = i + start;
-  }
-  return larr;
-}
-
 /* eslint-disable class-methods-use-this */
 class IdentityInt32Index {
   /*

client/src/util/range.js

@@ -0,0 +1,45 @@
+/*
+Array range creation
+
+range(start, stop, step) -> Array
+	This is identical to https://docs.python.org/3/library/functions.html#func-range
+	Returns new array filled with a range of numbers.
+
+	Usage:
+
+		range(stop) - start defaults to zero, step defaults to 1
+		range(start, stop, [step]) - step defaults to 1
+
+	Examples:
+		range(3) -> [0, 1, 2]
+		range(1, 3) -> [1, 2]
+		range(1, 5, 2) -> [1, 3]
+
+
+rangeFill(array, start, step) -> array
+	Fill entire array with values, from start, by step.  Returns first array.
+	start defaults to zero, step defaults to one.
+
+*/
+
+function _doFill(arr, start, step, count) {
+	for (let idx = 0, val = start; idx < count; idx += 1, val += step) {
+		arr[idx] = val;
+	}
+	return arr;
+}
+
+export function rangeFill(arr, start = 0, step = 1) {
+	return _doFill(arr, start, step, arr.length);
+}
+
+export function range(start, stop, step) {
+	if (start === undefined) return [];
+	if (stop === undefined) {
+		stop = start;
+		start = 0;
+	}
+	step = step || 1; // catch undefind and zero
+	const len = Math.max(Math.ceil((stop - start) / step), 0);
+	return _doFill(new Array(len), start, step, len);
+}

client/src/util/scaleLinear.js

@@ -9,8 +9,14 @@
 // this is is equivalent to d3.scaleLinear().domain([0,1]).range([-1,1])
 
 export default (domain, range) => {
-  const domainStart = domain[0];
-  const scale = (range[1] - range[0]) / (domain[1] - domain[0]);
-  const rangeStart = range[0];
-  return value => (value - domainStart) * scale + rangeStart;
+	const domainStart = domain[0];
+	const scale = (range[1] - range[0]) / (domain[1] - domain[0]);
+	const invScale = 1 / scale;
+	const rangeStart = range[0];
+	const f = value => (value - domainStart) * scale + rangeStart;
+
+	// inverter
+	f.invert = value => (value - rangeStart) * invScale + domainStart;
+
+	return f;
 };

client/src/util/stateManager/colorHelpers.js

@@ -1,12 +1,12 @@
 /*
 Helper functions for the embedded graph colors
 */
-import _ from "lodash";
 import * as d3 from "d3";
 import { interpolateRainbow, interpolateCool } from "d3-scale-chromatic";
 import * as globals from "../../globals";
 import parseRGB from "../parseRGB";
 import finiteExtent from "../finiteExtent";
+import { range } from "../range";
 
 /*
 create new colors state object.   Paramters:
@@ -37,9 +37,7 @@ function createColors(world, colorMode = null, colorAccessor = null) {
 }
 
 function createColorsByCategoricalMetadata(world, accessor) {
-  const { categories } = _.filter(world.schema.annotations.obs, {
-    name: accessor
-  })[0];
+  const { categories } = world.schema.annotations.obsByName[accessor];
 
   const scale = d3
     .scaleSequential(interpolateRainbow)
@@ -67,7 +65,7 @@ function createColorsByContinuousMetadata(world, accessor) {
   const scale = d3
     .scaleQuantile()
     .domain([min, max])
-    .range(_.range(colorBins - 1, -1, -1));
+    .range(range(colorBins - 1, -1, -1));
 
   /* pre-create colors - much faster than doing it for each obs */
   const colors = new Array(colorBins);
@@ -97,7 +95,7 @@ function createColorsByExpression(world, accessor) {
   const scale = d3
     .scaleQuantile()
     .domain([min, max])
-    .range(_.range(colorBins - 1, -1, -1));
+    .range(range(colorBins - 1, -1, -1));
 
   /* pre-create colors - much faster than doing it for each obs */
   const colors = new Array(colorBins);

client/src/util/stateManager/controlsHelpers.js

@@ -5,7 +5,7 @@ Helper functions for the controls reducer
 import _ from "lodash";
 
 import * as globals from "../../globals";
-import { fillRange } from "../typedCrossfilter/util";
+import { rangeFill as fillRange } from "../range";
 import {
   userDefinedDimensionName,
   diffexpDimensionName
@@ -21,16 +21,16 @@ Remember that option values can be ANY js type, except undefined/null.
   {
     _category_name_1: {
       // map of option value to index
-      categoryIndices: Map([
+      categoryValueIndices: Map([
         catval1: index,
         ...
       ])
 
       // index->selection true/false state
-      categorySelected: [ true/false, true/false, ... ]
+      categoryValueSelected: [ true/false, true/false, ... ]
 
       // number of options
-      numCategories: number,
+      numCategoryValues: number,
 
       // isTruncated - true if the options for selection has
       // been truncated (ie, was too large to implement)
@@ -56,27 +56,31 @@ function topNCategories(summary) {
 
 export function createCategoricalSelection(maxCategoryItems, world) {
   const res = {};
+  const obsIndexName = world.schema.annotations.obs.index;
   _.forEach(world.obsAnnotations.colIndex.keys(), key => {
     const summary = world.obsAnnotations.col(key).summarize();
     if (summary.categories) {
       const isColorField = key.includes("color") || key.includes("Color");
       const isSelectableCategory =
         !isColorField &&
-        key !== "name" &&
+        key !== obsIndexName &&
         summary.categories.length < maxCategoryItems;
       if (isSelectableCategory) {
-        const [categoryValues, categoryCounts] = topNCategories(summary);
-        const categoryIndices = new Map(categoryValues.map((v, i) => [v, i]));
-        const numCategories = categoryIndices.size;
-        const categorySelected = new Array(numCategories).fill(true);
+        const [categoryValues, categoryValueCounts] = topNCategories(summary);
+        const categoryValueIndices = new Map(
+          categoryValues.map((v, i) => [v, i])
+        );
+        const numCategoryValues = categoryValueIndices.size;
+        const categoryValueSelected = new Array(numCategoryValues).fill(true);
         const isTruncated = categoryValues.length < summary.numCategories;
         res[key] = {
           categoryValues, // array: of natively typed category values
-          categoryIndices, // map: category value (native type) -> category index
-          categorySelected, // array: t/f selection state
-          numCategories, // number: of categories
+          categoryValueIndices, // map: category value (native type) -> category index
+          categoryValueSelected, // array: t/f selection state
+          numCategoryValues, // number: of values in the category
           isTruncated, // bool: true if list was truncated
-          categoryCounts // array: cardinality of each category
+          categoryValueCounts, // array: cardinality of each category,
+          categorySelected: true // bool - default state for entire category
         };
       }
     }
@@ -88,12 +92,26 @@ export function createCategoricalSelection(maxCategoryItems, world) {
 given a categoricalSelection, return the list of all category values
 where selection state is true (ie, they are selected).
 */
-export function selectedValuesForCategory(categorySelectionState) {
-  const selectedValues = _([...categorySelectionState.categoryIndices])
-    .filter(tuple => categorySelectionState.categorySelected[tuple[1]])
-    .map(tuple => tuple[0])
-    .value();
-  return selectedValues;
+export function selectedValuesForCategory(categorySelectionState, dfColumn) {
+  const {
+    categorySelected,
+    categoryValueSelected,
+    categoryValueIndices
+  } = categorySelectionState;
+  let selectedValues;
+  if (categorySelected) {
+    selectedValues = new Set(dfColumn.summarize().categories);
+  } else {
+    selectedValues = new Set();
+  }
+  categoryValueIndices.forEach((catIndex, catValue) => {
+    if (!categoryValueSelected[catIndex]) {
+      selectedValues.delete(catValue);
+    } else {
+      selectedValues.add(catValue);
+    }
+  });
+  return [...selectedValues.values()];
 }
 
 /*

client/src/util/stateManager/universe.js

@@ -78,6 +78,7 @@ function AnnotationsFBSToDataframe(arrayBuffer) {
   The application has strong assumptions that all scalar data will be
   stored as a float32 or float64 (regardless of underlying data types).
   For example, clipping of value ranges (eg, user-selected percentiles)
+  depends on the ability to use NaN in any numeric type.
 
   All float data from the server is left as is.  All non-float is promoted
   to an appropriate float.
@@ -98,15 +99,16 @@ function AnnotationsFBSToDataframe(arrayBuffer) {
 
 function LayoutFBSToDataframe(arrayBuffer) {
   const fbs = decodeMatrixFBS(arrayBuffer, true);
-  if (fbs.columns.length !== 2 || !fbs.columns.every(isFpTypedArray)) {
+  if (fbs.columns.length < 2 || !fbs.columns.every(isFpTypedArray)) {
     // We have strong assumptions about the shape & type of layout data.
     throw new Error("Unexpected layout data type returned from server");
   }
+
   const df = new Dataframe.Dataframe(
     [fbs.nRows, fbs.nCols],
     fbs.columns,
     null,
-    new Dataframe.KeyIndex(["X", "Y"])
+    new Dataframe.KeyIndex(fbs.colIdx)
   );
   return df;
 }
@@ -122,15 +124,15 @@ function reconcileSchemaCategoriesWithSummary(universe) {
   cases, add a 'categories' field to the schema so it is accessible.
   */
 
-  _.forEach(universe.schema.annotations.obs, s => {
+  universe.schema.annotations.obs.columns.forEach(s => {
     if (
       s.type === "string" ||
       s.type === "boolean" ||
       s.type === "categorical"
     ) {
       const categories = _.union(
-        _.get(s, "categories", []),
-        _.get(universe.obsAnnotations.col(s.name).summarize(), "categories", [])
+        s.categories ?? [],
+        universe.obsAnnotations.col(s.name).summarize().categories ?? []
       );
       s.categories = categories;
     }
@@ -154,6 +156,9 @@ export function createUniverseFromResponse(
   universe.schema = schema;
   universe.nObs = schema.dataframe.nObs;
   universe.nVar = schema.dataframe.nVar;
+  /* add defaults, as we can't assume back-end will fully populate schema */
+  if (!schema.layout.var) schema.layout.var = [];
+  if (!schema.layout.obs) schema.layout.obs = [];
 
   /* annotations */
   universe.obsAnnotations = AnnotationsFBSToDataframe(annotationsObsResponse);
@@ -174,10 +179,16 @@ export function createUniverseFromResponse(
 
   /* Index schema for ease of use */
   universe.schema.annotations.obsByName = fromEntries(
-    universe.schema.annotations.obs.map(v => [v.name, v])
+    universe.schema.annotations.obs.columns.map(v => [v.name, v])
   );
   universe.schema.annotations.varByName = fromEntries(
-    universe.schema.annotations.var.map(v => [v.name, v])
+    universe.schema.annotations.var.columns.map(v => [v.name, v])
+  );
+  universe.schema.layout.obsByName = fromEntries(
+    universe.schema.layout.obs.map(v => [v.name, v])
+  );
+  universe.schema.layout.varByName = fromEntries(
+    universe.schema.layout.var.map(v => [v.name, v])
   );
   return universe;
 }
@@ -202,8 +213,9 @@ export function convertDataFBStoObject(universe, arrayBuffer) {
     throw new Error("Unexpected non-floating point response from server.");
   }
 
+  const varIndexName = universe.schema.annotations.var.index;
   for (let c = 0; c < colIdx.length; c += 1) {
-    const varName = universe.varAnnotations.at(colIdx[c], "name");
+    const varName = universe.varAnnotations.at(colIdx[c], varIndexName);
     result[varName] = columns[c];
   }
   return result;

client/src/util/stateManager/world.js

@@ -260,13 +260,17 @@ function deduceDimensionType(attributes, fieldName) {
   return dimensionType;
 }
 
-export function createObsDimensions(crossfilter, world) {
+export function createObsDimensions(crossfilter, world, XYdimNames) {
   /*
   create and return a crossfilter with a dimension for every obs annotation
-  for which we have a supported type, *except* 'name'
+  for which we have a supported type, *except* for the index column, indicated
+  by schema.annotations.obs.index.
   */
   const { schema, obsLayout, obsAnnotations } = world;
-  const annoList = schema.annotations.obs.filter(anno => anno.name !== "name");
+  const indexName = schema.annotations.obs.index;
+  const annoList = schema.annotations.obs.columns.filter(
+    anno => anno.name !== indexName
+  );
   crossfilter = annoList.reduce((xfltr, anno) => {
     const dimType = deduceDimensionType(anno, anno.name);
     const colData = obsAnnotations.col(anno.name).asArray();
@@ -283,8 +287,8 @@ export function createObsDimensions(crossfilter, world) {
   return crossfilter.addDimension(
     layoutDimensionName("XY"),
     "spatial",
-    obsLayout.col("X").asArray(),
-    obsLayout.col("Y").asArray()
+    obsLayout.col(XYdimNames[0]).asArray(),
+    obsLayout.col(XYdimNames[1]).asArray()
   );
 }
 

client/src/util/typedCrossfilter/util.js

@@ -1,22 +1,12 @@
 // jshint esversion: 6
 
 import { sortIndex } from "./sort";
+import { rangeFill as fillRange } from "../range";
 
 /*
     Utility functions, private to this module.
 */
 
-// fill an array or typedarray with a sequential range of numbers,
-// starting with `start`
-//
-export function fillRange(arr, start = 0) {
-  const larr = arr;
-  for (let i = 0, len = larr.length; i < len; i += 1) {
-    larr[i] = i + start;
-  }
-  return larr;
-}
-
 // slice out of one array into another, using an index array
 //
 export function sliceByIndex(src, index) {

dev_docs/developer_guidelines.md

@@ -0,0 +1,91 @@
+# Developer guidelines
+
+### Requirements
+- npm
+- Python 3.6+
+- Chrome
+
+[See dev section of README](../README.md)
+
+**All instructions are expected to be run from the top level cellxgene directory unless otherwise specified.**
+
+## Server dev
+### Install
+* Build the client and put static files in place: `make build-for-server-dev`
+* Install from local files: `make install-dev`
+
+### Launch
+* `cellxgene launch [options] <datafile>`
+
+### Reloading
+If you install cellxgene using `make install-dev` the server will be restarted every time you make changes on the server code. If changes affects the client, the browser must be reloaded.
+
+### Linter
+We use `flake8` to lint code. Travis CI runs `flake8 server`.
+
+### Test
+1. Install development requirements `pip install -r server/requirements-dev.txt`
+2. Run tests `pytest server/test`
+
+### Tips
+* Install in a virtualenv
+* May need to rebuild/reinstall when you make client changes
+
+## Client dev
+### Install
+1. Install prereqs for client: `npm install --prefix client/ client`
+2. Install cellxgene server: `pip install -e .` Caveat: this will not build the production client package - you must use the [server install](#install) instructions above to serve web assets.
+
+### Launch
+To launch with hot reloading you need to launch the server and the client separately. Node's hot reloading starts the client on its own node server and auto-refreshes when changes are made.
+1. Launch server (the client relies on the REST API being available): `cellxgene launch [options] <datafile>`
+2. Launch client: in `client/` directory run `npm run start`
+3. Client will be served on localhost:3000
+
+### Build
+To build only the client: `make build-client`
+
+### Linter
+We use `eslint` to lint the code and `prettier` as our code formatter.
+
+### Test
+In `client/` directory run `npm run unit-test`
+
+### Tips
+* You can also install/launch the server side code from npm scrips (requires python3.6 with virtualenv) in `client/` directory run `npm run backend-dev`
+
+## Running tests
+Client and server tests run on Travis CI for every push, PR, and commit to master on github. End to end tests run nightly on master only. 
+
+### Server unit tests
+Install development requirements `pip install -r server/requirements-dev.txt`
+Run tests `pytest server/test`
+
+### Client unit tests
+In `client/` directory run `npm run unit-test`
+
+### End to end tests
+
+End to end tests use two env variables:
+* `JEST_ENV` - environment to run end to end tests. Default `dev`
+    * `prod` - run headless with no slowdown, chromium will not open.
+    * `dev` - opens chromimum, runs tests with minimal slowdown, close on exit.
+    * `debug` - opens chromium, runs tests with 100ms slowdown, dev tools open, chrome stays open on exit.
+* `JEST_CXG_PORT` - port that end to end tests are being run on. Default `3000` (client hosted port).
+
+On CI the end to end tests are run with `JEST_ENV` set to `prod` using the `smoke-test` npm script
+
+To run end to end tests as they will be run on CI
+1. cellxgene should be built and installed as [specified in server dev](#install)
+2. `export JEST_ENV='prod'`
+3. `export JEST_CXG_PORT='5000'`
+4. Run `npm run --prefix client/ smoke-test`
+
+Run end to end tests interactively during development
+1. cellxgene should be installed as [specified in client dev](#install-1)
+2. Follow [launch](#launch-1) instructions for client dev with dataset `example-dataset/pbmc3k`
+3. Run `npm run --prefix client/ e2e`
+4. To debug a failing test `export JEST_ENV='debug'` and re-run.
+
+
+

dev_docs/release_process.md

@@ -43,24 +43,53 @@ Follow these steps to create a release.
 8. Publish to pypi by performing the following steps (assumes you that you have registered for pypi, 
 and that you have write access to the cellxgene pypi package):
     - Build the distribution and upload to test pypi `make release-stage-2`
-    - [optional] Test the test installation in a fresh virtual environment using `make install-release-test`
+    - Test the test installation in a fresh virtual environment using `make install-release-test`
     - Upload the package to real pypi using `make release-stage-final`
-    - [optional] Test the installation in a fresh virtual environment using
+    - Test the installation in a fresh virtual environment using
       `pip install cellxgene`
-    - **Troubleshooting**:
-        - Fails to upload to test.pypi: pypi doesn't allow you to reupload a release with the same version number, 
-        if you accidentally burned a release number you want to use on prod, you have a couple options. 
-            1) OPTION 1: Create distribution `make pydist`; test release locally `pip install dist/<release tarball>`;
-             then upload to prod `make release-stage-final`. 
-            2) OPTION 2: (DANGER) release directly to prod: `make release-burned`.  
-            3) OPTION 3: If the release was burned on prod as well run from Step 3 again with option
-            PART=patch until you get to an unburned version.
-        - The release doesn't install or fails your tests when you install it: Delete it from pypi - Go to pypi.org, sign in, 
-        go to the cellxgene package, click manage, then in the options drop down, click delete and 
-        follow the instructions. You will not be able to use that release number again. If it is a minor bug 
-        and not a major regression, you can just release a patch. 
-    
 
 The optional steps are for testing purposes, and are recommended
 for publishing any major releases, and any releases that significantly
 change the packaging (e.g. new bundled files, new dependencies, etc.)
+
+## Troubleshooting
+### Fails to upload to test.pypi
+ 
+ _PyPi doesn't allow you to reupload a release with the same version number_ 
+If you accidentally burned a release number you want to use on prod, you have a few options:
+1) OPTION 1: Create distribution `make pydist`; test release locally `pip install dist/<release tarball>`;
+ then upload to prod `make release-stage-final`. 
+2) OPTION 2: (DANGER) release directly to prod: `make release-directly-to-prod`.  
+3) OPTION 3: If the release was burned on prod as well run from Step 3 again with option
+PART=patch until you get to an unburned version.
+    
+### The release doesn't install or fails your tests when you install it
+ 
+Delete it from pypi - Go to pypi.org -> sign in -> go to the cellxgene package -> click manage -> then in the options drop down click delete -> follow the instructions. You will not be able to use that release number again. If it is a minor bug and not a major regression, you can just release a patch. 
+### If you need to run stage final on a different computer than stage 2
+ If you run stage final without running stage 2 first, the dist will not have been build on the computer running stage final. The solution is to run `make release-directly-to-prod`. This both builds the distribution files and then releases directly to prod pypi.org. 
+
+## Stage Details
+### Stage 1 - `make release-stage-1`
+    1. Pip installs requirements-dev
+    2. Bumps version by [PART]
+    3. Deletes build directory, client/build, dist and cellxgene.egg-info
+    4. Creates the package-lock.json
+
+### Stage 2 - `make release-stage-2`
+    1. Pip installs requirements-dev
+    2. Builds client and server
+    3. Creates distribution release (sdist)
+    4. Uploads to test.pypi.org
+
+### Stage final - `make release-stage-final`
+    ** Does not build distribution **
+    1. Uploads to pypi.org
+    
+### (DANGER) Release directly to prod `make release-directly-to-prod`
+    ** builds distribution and uploads directly to prod **
+    Only use this if you are directed to by the troubleshooting guide
+    1. Pip installs requirements-dev
+    2. Builds client and server
+    3. Creates distribution release (sdist)
+    4. Uploads to pypi.org

docs/faq.md

@@ -38,7 +38,7 @@ Currently this is not supported directly, but you should be able to do this your
 
 - `.obs` and `.var` annotations are use to extract metadata for filtering
 - `.X` is used to display expression (histograms, scatterplot & colorscale) and to compute differential expression
-- `.obsm` is used for layout
+- `.obsm` is used for layout. If an embedding has more than two components, the first two will be used for visualization.
 
 #### I have a BIG dataset - how can I make cellxgene run as fast as possible?
 

makefile

@@ -76,7 +76,7 @@ release-stage-final: twine-prod
 
 # DANGER: releases directly to prod
 # use this if you accidently burned a test release version number,
-release-burned : dev-env pydist twine-prod
+release-directly-to-prod : dev-env pydist twine-prod
 	@echo "Dist built and uploaded to pypi.org"
 	@echo "Test the install:"
 	@echo "    make install-release"
@@ -114,14 +114,18 @@ install-dev : uninstall
 
 # install from test.pypi to test your release
 install-release-test : uninstall
-	pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple cellxgene
+	pip install --no-cache-dir --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple cellxgene
 	@echo "Installed cellxgene from test.pypi.org, now run and smoke test"
 
 # install from pypi to test your release
 install-release : uninstall
-	pip install cellxgene
+	pip install --no-cache-dir cellxgene
 	@echo "Installed cellxgene from pypi.org"
 
+# install from dist
+install-dist : uninstall
+	pip install dist/cellxgene*.tar.gz
+
 uninstall :
 	pip uninstall -y cellxgene || :
 

server/app/scanpy_engine/scanpy_engine.py

@@ -1,12 +1,13 @@
 import warnings
 
 import numpy as np
+import pandas
 from pandas.core.dtypes.dtypes import CategoricalDtype
-import scanpy as sc
+import anndata
 from scipy import sparse
 
 from server.app.driver.driver import CXGDriver
-from server.app.util.constants import Axis, DEFAULT_TOP_N
+from server.app.util.constants import Axis, DEFAULT_TOP_N, MAX_LAYOUTS
 from server.app.util.errors import (
     FilterError,
     JSONEncodingValueError,
@@ -41,7 +42,7 @@ class ScanpyEngine(CXGDriver):
     @staticmethod
     def _get_default_config():
         return {
-            "layout": "umap",
+            "layout": [],
             "diffexp": "ttest",
             "max_category_items": 100,
             "obs_names": None,
@@ -49,41 +50,61 @@ class ScanpyEngine(CXGDriver):
             "diffexp_lfc_cutoff": 0.01,
         }
 
-    def _alias_annotation_names(self, axis, name):
-        """
-        Do all user-specified annotation aliasing.
+    @staticmethod
+    def _create_unique_column_name(df, col_name_prefix):
+        """ given the columns of a dataframe, and a name prefix, return a column name which
+            does not exist in the dataframe, AND which is prefixed by `prefix`
 
-        As a *critical* side-effect, ensure the indices are simple number ranges
-        (accomplished by calling pandas.DataFrame.reset_index())
+            The approach is to append a numeric suffix, starting at zero and increasing by
+            one, until an unused name is found (eg, prefix_0, prefix_1, ...).
         """
-        if name == "name":
-            # a noop, so skip it
-            return
+        suffix = 0
+        while f"{col_name_prefix}{suffix}" in df:
+            suffix += 1
+        return f"{col_name_prefix}{suffix}"
 
-        ax_name = str(axis)
-        df_axis = getattr(self.data, ax_name)
-        if name is None:
-            # reset index to simple range; alias "name" to point at the
-            # previously specified index.
-            df_axis.reset_index(inplace=True)
-            df_axis.rename(inplace=True, columns={"index": "name"})
-        elif name in df_axis.columns:
-            if name not in df_axis.columns:
+    def _alias_annotation_names(self):
+        """
+        The front-end relies on the existance of a unique, human-readable
+        index for obs & var (eg, var is typically gene name, obs the cell name).
+        The user can specify these via the --obs-names and --var-names config.
+        If they are not specified, use the existing index to create them, giving
+        the resulting column a unique name (eg, "name").
+
+        In both cases, enforce that the result is unique, and communicate the
+        index column name to the front-end via the obs_names and var_names config
+        (which is incorporated into the schema).
+        """
+        for (ax_name, config_name) in ((Axis.OBS, "obs_names"), (Axis.VAR, "var_names")):
+            name = self.config[config_name]
+            df_axis = getattr(self.data, str(ax_name))
+            if name is None:
+                # Default: create unique names from index
+                if not df_axis.index.is_unique:
+                    raise KeyError(
+                        f"Values in {ax_name}.index must be unique. "
+                        "Please prepare data to contain unique index values, or specify an "
+                        "alternative with --{ax_name}-name."
+                    )
+                name = self._create_unique_column_name(df_axis.columns, "name_")
+                self.config[config_name] = name
+                # reset index to simple range; alias name to point at the
+                # previously specified index.
+                df_axis.rename_axis(name, inplace=True)
+                df_axis.reset_index(inplace=True)
+            elif name in df_axis.columns:
+                # User has specified alternative column for unique names, and it exists
+                if not df_axis[name].is_unique:
+                    raise KeyError(
+                        f"Values in {ax_name}.{name} must be unique. "
+                        "Please prepare data to contain unique values."
+                    )
+                df_axis.reset_index(drop=True, inplace=True)
+            else:
+                # user specified a non-existent column name
                 raise KeyError(
                     f"Annotation name {name}, specified in --{ax_name}-name does not exist."
                 )
-            if not df_axis[name].is_unique:
-                raise KeyError(
-                    f"Values in -{ax_name}-name must be unique. "
-                    "Please prepare data to contain unique values."
-                )
-            # reset index to simple range; alias user-specified annotation to "name"
-            df_axis.reset_index(drop=True, inplace=True)
-            df_axis.rename(inplace=True, columns={name: "name"})
-        else:
-            raise KeyError(
-                f"Annotation name {name}, specified in --{ax_name}_name does not exist."
-            )
 
     @staticmethod
     def _can_cast_to_float32(ann):
@@ -113,7 +134,17 @@ class ScanpyEngine(CXGDriver):
                 "nVar": self.gene_count,
                 "type": str(self.data.X.dtype),
             },
-            "annotations": {"obs": [], "var": []},
+            "annotations": {
+                "obs": {
+                    "index": self.config["obs_names"],
+                    "columns": []
+                },
+                "var": {
+                    "index": self.config["var_names"],
+                    "columns": []
+                }
+            },
+            "layout": {"obs": []}
         }
         for ax in Axis:
             curr_axis = getattr(self.data, str(ax))
@@ -137,14 +168,21 @@ class ScanpyEngine(CXGDriver):
                     raise TypeError(
                         f"Annotations of type {curr_axis[ann].dtype} are unsupported by cellxgene."
                     )
-                self.schema["annotations"][ax].append(ann_schema)
+                self.schema["annotations"][ax]["columns"].append(ann_schema)
+
+        for layout in self.config['layout']:
+            layout_schema = {
+                "name": layout,
+                "type": "float32",
+                "dims": [f"{layout}_0", f"{layout}_1"]
+            }
+            self.schema["layout"]["obs"].append(layout_schema)
 
     def _load_data(self, data):
-        # Based on benchmarking, cache=True has no impact on perf.
-        # Note: as of current scanpy/anndata release, setting backed='r' will
-        # result in an error.  https://github.com/theislab/anndata/issues/79
+        # as of AnnData 0.6.19, backed mode performs initial load fast, but at the
+        # cost of significantly slower access to X data.
         try:
-            self.data = sc.read(data, cache=True)
+            self.data = anndata.read_h5ad(data)
         except ValueError:
             raise ScanpyFileError(
                 "File must be in the .h5ad format. Please read "
@@ -164,16 +202,68 @@ class ScanpyEngine(CXGDriver):
 
     @requires_data
     def _validate_and_initialize(self):
-        self._alias_annotation_names(Axis.OBS, self.config["obs_names"])
-        self._alias_annotation_names(Axis.VAR, self.config["var_names"])
+        # var and obs column names must be unique
+        if not self.data.obs.columns.is_unique or not self.data.var.columns.is_unique:
+            raise KeyError(f"All annotation column names must be unique.")
+
+        self._alias_annotation_names()
         self._validate_data_types()
-        self._validate_data_calculations()
         self.cell_count = self.data.shape[0]
         self.gene_count = self.data.shape[1]
+        self._default_and_validate_layouts()
         self._create_schema()
 
+    @requires_data
+    def _default_and_validate_layouts(self):
+        """ function:
+            a) generate list of default layouts, if not already user specified
+            b) validate layouts are legal.  remove/warn on any that are not
+            c) cap total list of layouts at global const MAX_LAYOUTS
+        """
+        layouts = self.config['layout']
+        # handle default
+        if layouts is None or len(layouts) == 0:
+            # load default layouts from the data.
+            layouts = [key[2:] for key in self.data.obsm_keys() if type(key) == str and key.startswith("X_")]
+            if len(layouts) == 0:
+                raise PrepareError(f"Unable to find any precomputed layouts within the dataset.")
+
+        # remove invalid layouts
+        valid_layouts = []
+        obsm_keys = self.data.obsm_keys()
+        for layout in layouts:
+            layout_name = f"X_{layout}"
+            if layout_name not in obsm_keys:
+                warnings.warn(f"Ignoring unknown layout name: {layout}.")
+            elif not self._is_valid_layout(self.data.obsm[layout_name]):
+                warnings.warn(f"Ignoring layout due to malformed shape or data type: {layout}")
+            else:
+                valid_layouts.append(layout)
+
+        if len(valid_layouts) == 0:
+            raise PrepareError(f"No valid layout data.")
+
+        # cap layouts to MAX_LAYOUTS
+        self.config['layout'] = valid_layouts[0:MAX_LAYOUTS]
+
+    @requires_data
+    def _is_valid_layout(self, arr):
+        """ return True if this layout data is a valid array for front-end presentation:
+            * ndarray, with shape (n_obs, >= 2), dtype float/int/uint
+            * contains only finite values
+        """
+        is_valid = type(arr) == np.ndarray and arr.dtype.kind in "fiu"
+        is_valid = is_valid and arr.shape[0] == self.data.n_obs and arr.shape[1] >= 2
+        is_valid = is_valid and np.all(np.isfinite(arr))
+        return is_valid
+
     @requires_data
     def _validate_data_types(self):
+        if sparse.isspmatrix(self.data.X) and not sparse.isspmatrix_csc(self.data.X):
+            warnings.warn(
+                f"Scanpy data matrix is sparse, but not a CSC (columnar) matrix.  "
+                f"Performance may be improved by using CSC."
+            )
         if self.data.X.dtype != "float32":
             warnings.warn(
                 f"Scanpy data matrix is in {self.data.X.dtype} format not float32. "
@@ -204,20 +294,6 @@ class ScanpyEngine(CXGDriver):
                             f"annotations with more than 500 categories in the UI"
                         )
 
-    @requires_data
-    def _validate_data_calculations(self):
-        layout_key = f"X_{self.config['layout']}"
-        try:
-            assert layout_key in self.data.obsm_keys()
-        except AssertionError:
-            raise PrepareError(
-                f"Cannot find a field with coordinates for the {self.config['layout']} layout requested. A different"
-                f" layout may have been computed. The requested layout must be pre-calculated and saved "
-                f"back in the h5ad file. You can run "
-                f"`cellxgene prepare --layout {self.config['layout']} <datafile>` "
-                f"to solve this problem. "
-            )
-
     @staticmethod
     def _annotation_filter_to_mask(filter, d_axis, count):
         mask = np.ones((count,), dtype=bool)
@@ -304,7 +380,7 @@ class ScanpyEngine(CXGDriver):
         if sparse.issparse(X):  # use tuned getcol/hstack for performance
             indices = np.nonzero(var_mask)[0]
             cols = [X.getcol(i) for i in indices]
-            return sparse.hstack(cols)
+            return sparse.hstack(cols, format="csc")
         else:   # else, just use standard slicing, which is fine for dense arrays
             return X[:, var_mask]
 
@@ -368,15 +444,18 @@ class ScanpyEngine(CXGDriver):
         * only returns Matrix in columnar layout
         """
         try:
-            full_embedding = self.data.obsm[f"X_{self.config['layout']}"]
-            if full_embedding.shape[1] > 2:
-                warnings.warn(f"Warning: found {full_embedding.shape[1]} \
-                components of embedding. Using the first two for layout display.")
-            df_layout = full_embedding[:, :2]
+            layout_data = []
+            for layout in self.config["layout"]:
+                full_embedding = self.data.obsm[f"X_{layout}"]
+                embedding = full_embedding[:, :2]
+                normalized_layout = (embedding - embedding.min()) / (embedding.max() - embedding.min())
+                normalized_layout = normalized_layout.astype(dtype=np.float32)
+                layout_data.append(pandas.DataFrame(normalized_layout, columns=[f"{layout}_0", f"{layout}_1"]))
+
         except ValueError as e:
             raise PrepareError(
                 f"Layout has not been calculated using {self.config['layout']}, "
                 f"please prepare your datafile and relaunch cellxgene") from e
 
-        normalized_layout = (df_layout - df_layout.min()) / (df_layout.max() - df_layout.min())
-        return encode_matrix_fbs(normalized_layout.astype(dtype=np.float32), col_idx=None, row_idx=None)
+        df = pandas.concat(layout_data, axis=1, copy=False)
+        return encode_matrix_fbs(df, col_idx=df.columns, row_idx=None)

server/app/util/constants.py

@@ -31,3 +31,5 @@ JSON_NaN_to_num_warning_msg = (
     "JSON encoding failure - please verify all data are finite values (no NaN or Infinities)"
 )
 REACTIVE_LIMIT = 1_000_000
+
+MAX_LAYOUTS = 30

server/cli/cli.py

@@ -5,7 +5,7 @@ from .prepare import prepare
 
 
 @click.group(name="cellxgene", context_settings=dict(max_content_width=85))
-@click.version_option(version="0.9.1", prog_name="cellxgene", message="[%(prog)s] Version %(version)s")
+@click.version_option(version="0.10.0", prog_name="cellxgene", message="[%(prog)s] Version %(version)s")
 def cli():
     pass
 

server/cli/launch.py

@@ -1,6 +1,7 @@
+import errno
 import logging
 from os import devnull
-from os.path import splitext, basename
+from os.path import splitext, basename, getsize
 import sys
 import warnings
 import webbrowser
@@ -10,7 +11,11 @@ import click
 from server.app.app import Server
 from server.app.util.errors import ScanpyFileError
 from server.app.util.utils import custom_format_warning
-from server.utils.constants import MODES
+from server.utils.utils import find_available_port, is_port_available
+
+
+# anything bigger than this will generate a special message
+BIG_FILE_SIZE_THRESHOLD = 100 * 2**20  # 100MB
 
 
 @click.command()
@@ -18,10 +23,10 @@ from server.utils.constants import MODES
 @click.option(
     "--layout",
     "-l",
-    type=click.Choice(MODES),
-    default="umap",
+    default=[],
+    multiple=True,
     show_default=True,
-    help="Method for layout."
+    help="Layout name, eg, 'umap'."
 )
 @click.option(
     "--diffexp",
@@ -50,7 +55,8 @@ from server.utils.constants import MODES
     show_default=True,
     help="Open the web browser after launch.",
 )
-@click.option("--port", "-p", help="Port to run server on.", metavar="", default=5005, show_default=True)
+@click.option("--port", "-p", help="Port to run server on, if not specified cellxgene will find an available port.",
+              metavar="", show_default=True)
 @click.option("--obs-names", default=None, metavar="", help="Name of annotation field to use for observations.")
 @click.option("--var-names", default=None, metavar="", help="Name of annotation to use for variables.")
 @click.option("--host", default="127.0.0.1", help="Host IP address")
@@ -135,6 +141,16 @@ security risk by including the --scripts flag. Make sure you trust the scripts t
         file_parts = splitext(basename(data))
         title = file_parts[0]
 
+    if port:
+        if debug:
+            raise click.ClickException("--port and --debug may not be used together (try --verbose for error logging).")
+        if not is_port_available(host, int(port)):
+            raise click.ClickException(
+                f"The port selected {port} is in use, please specify an open port using the --port flag."
+            )
+    else:
+        port = find_available_port(host)
+
     # Setup app
     cellxgene_url = f"http://{host}:{port}"
 
@@ -148,7 +164,13 @@ security risk by including the --scripts flag. Make sure you trust the scripts t
         log = logging.getLogger("werkzeug")
         log.setLevel(logging.ERROR)
 
-    click.echo(f"[cellxgene] Loading data from {basename(data)}, this may take awhile...")
+    file_size = getsize(data)
+
+    # if a big file, let the user know it may take a while to load.
+    if file_size > BIG_FILE_SIZE_THRESHOLD:
+        click.echo(f"[cellxgene] Loading data from {basename(data)}, this may take awhile...")
+    else:
+        click.echo(f"[cellxgene] Loading data from {basename(data)}.")
 
     # Fix for anaconda python. matplotlib typically expects python to be installed as a framework TKAgg is usually
     # available and fixes this issue. See https://matplotlib.org/faq/virtualenv_faq.html
@@ -183,4 +205,9 @@ security risk by including the --scripts flag. Make sure you trust the scripts t
         f = open(devnull, "w")
         sys.stdout = f
 
-    server.app.run(host=host, debug=debug, port=port, threaded=True)
+    try:
+        server.app.run(host=host, debug=debug, port=port, threaded=True, use_debugger=False)
+    except OSError as e:
+        if e.errno == errno.EADDRINUSE:
+            raise click.ClickException("Port is in use, please specify an open port using the --port flag.") from e
+        raise

server/requirements.txt

@@ -1,3 +1,4 @@
+anndata>=0.6.20
 click>=6.7
 Flask>=1.0.2
 Flask-Caching>=1.4.0

server/test/schema.json

@@ -5,46 +5,61 @@
     "type": "float32"
   },
   "annotations": {
+    "obs": {
+      "index": "name_0",
+      "columns": [
+        {
+          "name": "name_0",
+          "type": "string"
+        },
+        {
+          "name": "n_genes",
+          "type": "int32"
+        },
+        {
+          "name": "percent_mito",
+          "type": "float32"
+        },
+        {
+          "name": "n_counts",
+          "type": "float32"
+        },
+        {
+          "name": "louvain",
+          "type": "categorical",
+          "categories": [
+            "CD4 T cells",
+            "CD14+ Monocytes",
+            "B cells",
+            "CD8 T cells",
+            "NK cells",
+            "FCGR3A+ Monocytes",
+            "Dendritic cells",
+            "Megakaryocytes"
+          ]
+        }
+      ]
+    },
+    "var": {
+      "index": "name_0",
+      "columns": [
+        {
+          "name": "name_0",
+          "type": "string"
+        },
+        {
+          "name": "n_cells",
+          "type": "int32"
+        }
+      ]
+    }
+  },
+  "layout": {
     "obs": [
       {
-        "name": "name",
-        "type": "string"
-      },
-      {
-        "name": "n_genes",
-        "type": "int32"
-      },
-      {
-        "name": "percent_mito",
-        "type": "float32"
-      },
-      {
-        "name": "n_counts",
-        "type": "float32"
-      },
-      {
-        "name": "louvain",
-        "type": "categorical",
-        "categories": [
-          "CD4 T cells",
-          "CD14+ Monocytes",
-          "B cells",
-          "CD8 T cells",
-          "NK cells",
-          "FCGR3A+ Monocytes",
-          "Dendritic cells",
-          "Megakaryocytes"
-        ]
-      }
-    ],
-    "var": [
-      {
-        "name": "name",
-        "type": "string"
-      },
-      {
-        "name": "n_cells",
-        "type": "int32"
+        "name": "umap",
+        "type": "float32",
+        "dims": ["umap_0", "umap_1"]
       }
     ]
   }

server/test/test_api.py

@@ -19,11 +19,12 @@ class EndPoints(unittest.TestCase):
 
     @classmethod
     def setUpClass(cls):
-        cls.ps = Popen(["cellxgene", "launch", "example-dataset/pbmc3k.h5ad", "--debug"])
+        cls.ps = Popen(["cellxgene", "launch", "example-dataset/pbmc3k.h5ad", "--verbose", "--port", "5005"])
         session = requests.Session()
         for i in range(90):
             try:
-                session.get(f"{URL_BASE}schema")
+                result = session.get(f"{URL_BASE}schema")
+                cls.schema = result.json()
             except requests.exceptions.ConnectionError:
                 time.sleep(1)
 
@@ -45,7 +46,8 @@ class EndPoints(unittest.TestCase):
         self.assertEqual(result.headers["Content-Type"], "application/json")
         result_data = result.json()
         self.assertEqual(result_data["schema"]["dataframe"]["nObs"], 2638)
-        self.assertEqual(len(result_data["schema"]["annotations"]["obs"]), 5)
+        self.assertEqual(len(result_data["schema"]["annotations"]["obs"]), 2)
+        self.assertEqual(len(result_data["schema"]["annotations"]["obs"]["columns"]), 5)
 
     def test_config(self):
         endpoint = "config"
@@ -67,9 +69,11 @@ class EndPoints(unittest.TestCase):
         self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
         df = decode_fbs.decode_matrix_FBS(result.content)
         self.assertEqual(df['n_rows'], 2638)
-        self.assertEqual(df['n_cols'], 2)
+        self.assertEqual(df['n_cols'], 8)
         self.assertIsNotNone(df['columns'])
-        self.assertIsNone(df['col_idx'])
+        self.assertListEqual(df['col_idx'], [
+            'pca_0', 'pca_1', 'tsne_0', 'tsne_1', 'umap_0', 'umap_1', 'draw_graph_fr_0', 'draw_graph_fr_1'
+        ])
         self.assertIsNone(df['row_idx'])
         self.assertEqual(len(df['columns']), df['n_cols'])
 
@@ -93,7 +97,8 @@ class EndPoints(unittest.TestCase):
         self.assertIsNotNone(df['col_idx'])
         self.assertIsNone(df['row_idx'])
         self.assertEqual(len(df['columns']), df['n_cols'])
-        self.assertListEqual(df['col_idx'], ['name', 'n_genes', 'percent_mito', 'n_counts', 'louvain'])
+        obs_index_col_name = self.schema["schema"]["annotations"]["obs"]["index"]
+        self.assertListEqual(df['col_idx'], [obs_index_col_name, 'n_genes', 'percent_mito', 'n_counts', 'louvain'])
 
     def test_get_annotations_obs_keys_fbs(self):
         endpoint = "annotations/obs"
@@ -163,7 +168,8 @@ class EndPoints(unittest.TestCase):
         self.assertIsNotNone(df['col_idx'])
         self.assertIsNone(df['row_idx'])
         self.assertEqual(len(df['columns']), df['n_cols'])
-        self.assertListEqual(df['col_idx'], ['name', 'n_cells'])
+        var_index_col_name = self.schema["schema"]["annotations"]["var"]["index"]
+        self.assertListEqual(df['col_idx'], [var_index_col_name, 'n_cells'])
 
     def test_get_annotations_var_keys_fbs(self):
         endpoint = "annotations/var"
@@ -245,7 +251,8 @@ class EndPoints(unittest.TestCase):
         endpoint = f"data/var"
         url = f"{URL_BASE}{endpoint}"
         header = {"Accept": "application/octet-stream"}
-        var_filter = {"filter": {"var": {"annotation_value": [{"name": "name", "values": ["RER1"]}]}}}
+        index_col_name = self.schema["schema"]["annotations"]["var"]["index"]
+        var_filter = {"filter": {"var": {"annotation_value": [{"name": index_col_name, "values": ["RER1"]}]}}}
         result = self.session.put(url, headers=header, json=var_filter)
         self.assertEqual(result.status_code, HTTPStatus.OK)
         self.assertEqual(result.headers["Content-Type"], "application/octet-stream")

server/test/test_nan_rest.py

@@ -8,7 +8,7 @@ import decode_fbs
 
 import requests
 
-LOCAL_URL = "http://127.0.0.1:5005/"
+LOCAL_URL = "http://127.0.0.1:5006/"
 VERSION = "v0.2"
 URL_BASE = f"{LOCAL_URL}api/{VERSION}/"
 
@@ -21,7 +21,7 @@ class WithNaNs(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
         cls.ps = Popen(
-            ["cellxgene", "launch", "server/test/test_datasets/nan.h5ad", "--debug"]
+            ["cellxgene", "launch", "server/test/test_datasets/nan.h5ad", "--verbose", "--port", "5006"]
         )
         session = requests.Session()
         for i in range(90):

server/test/test_nan_scanpy_engine.py

@@ -12,7 +12,7 @@ from server.app.util.errors import FilterError
 class NaNTest(unittest.TestCase):
     def setUp(self):
         self.args = {
-            "layout": "umap",
+            "layout": ["umap"],
             "diffexp": "ttest",
             "max_category_items": 100,
             "obs_names": None,
@@ -58,14 +58,16 @@ class NaNTest(unittest.TestCase):
 
     def test_annotation(self):
         annotations = decode_fbs.decode_matrix_FBS(self.data.annotation_to_fbs_matrix("obs"))
+        obs_index_col_name = self.data.schema["annotations"]["obs"]["index"]
         self.assertEqual(
             annotations["col_idx"],
-            ["name", "n_genes", "percent_mito", "n_counts", "louvain"]
+            [obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain"]
         )
         self.assertEqual(annotations["n_rows"], 100)
         self.assertTrue(math.isnan(annotations["columns"][2][0]))
 
         annotations = decode_fbs.decode_matrix_FBS(self.data.annotation_to_fbs_matrix("var"))
-        self.assertEqual(annotations["col_idx"], ["name", "n_cells", "var_with_nans"])
+        var_index_col_name = self.data.schema["annotations"]["var"]["index"]
+        self.assertEqual(annotations["col_idx"], [var_index_col_name, "n_cells", "var_with_nans"])
         self.assertEqual(annotations["n_rows"], 100)
         self.assertTrue(math.isnan(annotations["columns"][2][0]))

server/test/test_scanpy_engine.py

@@ -15,7 +15,7 @@ from server.app.util.errors import FilterError
 class EngineTest(unittest.TestCase):
     def setUp(self):
         args = {
-            "layout": "umap",
+            "layout": ["umap"],
             "diffexp": "ttest",
             "max_category_items": 100,
             "obs_names": None,
@@ -31,9 +31,11 @@ class EngineTest(unittest.TestCase):
         self.assertTrue(self.data.data.X[0, 0] - -0.171_469_51 < epsilon)
 
     def test_mandatory_annotations(self):
-        self.assertIn("name", self.data.data.obs)
+        obs_index_col_name = self.data.schema["annotations"]["obs"]["index"]
+        self.assertIn(obs_index_col_name, self.data.data.obs)
         self.assertEqual(list(self.data.data.obs.index), list(range(2638)))
-        self.assertIn("name", self.data.data.var)
+        var_index_col_name = self.data.schema["annotations"]["var"]["index"]
+        self.assertIn(var_index_col_name, self.data.data.var)
         self.assertEqual(list(self.data.data.var.index), list(range(1838)))
 
     @pytest.mark.filterwarnings("ignore:Scanpy data matrix")
@@ -70,12 +72,14 @@ class EngineTest(unittest.TestCase):
         self.assertEqual(data["n_cols"], 91)
 
     def test_obs_and_var_names(self):
-        self.assertEqual(np.sum(self.data.data.var["name"].isna()), 0)
-        self.assertEqual(np.sum(self.data.data.obs["name"].isna()), 0)
+        self.assertEqual(np.sum(self.data.data.var[self.data.schema["annotations"]["var"]["index"]].isna()), 0)
+        self.assertEqual(np.sum(self.data.data.obs[self.data.schema["annotations"]["obs"]["index"]].isna()), 0)
 
     def test_schema(self):
         with open(path.join(path.dirname(__file__), "schema.json")) as fh:
             schema = json.load(fh)
+            print(schema)
+            print(self.data.schema)
             self.assertEqual(self.data.schema, schema)
 
     def test_schema_produces_error(self):
@@ -108,16 +112,18 @@ class EngineTest(unittest.TestCase):
         annotations = decode_fbs.decode_matrix_FBS(fbs)
         self.assertEqual(annotations["n_rows"], 2638)
         self.assertEqual(annotations["n_cols"], 5)
+        obs_index_col_name = self.data.schema["annotations"]["obs"]["index"]
         self.assertEqual(
             annotations["col_idx"],
-            ["name", "n_genes", "percent_mito", "n_counts", "louvain"],
+            [obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain"],
         )
 
         fbs = self.data.annotation_to_fbs_matrix("var")
         annotations = decode_fbs.decode_matrix_FBS(fbs)
         self.assertEqual(annotations['n_rows'], 1838)
         self.assertEqual(annotations['n_cols'], 2)
-        self.assertEqual(annotations["col_idx"], ["name", "n_cells"])
+        var_index_col_name = self.data.schema["annotations"]["var"]["index"]
+        self.assertEqual(annotations["col_idx"], [var_index_col_name, "n_cells"])
 
     def test_annotation_fields(self):
         fbs = self.data.annotation_to_fbs_matrix("obs", ["n_genes", "n_counts"])
@@ -125,7 +131,8 @@ class EngineTest(unittest.TestCase):
         self.assertEqual(annotations["n_rows"], 2638)
         self.assertEqual(annotations['n_cols'], 2)
 
-        fbs = self.data.annotation_to_fbs_matrix("var", ["name"])
+        var_index_col_name = self.data.schema["annotations"]["var"]["index"]
+        fbs = self.data.annotation_to_fbs_matrix("var", [var_index_col_name])
         annotations = decode_fbs.decode_matrix_FBS(fbs)
         self.assertEqual(annotations['n_rows'], 1838)
         self.assertEqual(annotations['n_cols'], 1)
@@ -163,9 +170,10 @@ class EngineTest(unittest.TestCase):
             self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
 
     def test_data_named_gene(self):
+        var_index_col_name = self.data.schema["annotations"]["var"]["index"]
         filter_ = {
             "filter": {
-                "var": {"annotation_value": [{"name": "name", "values": ["RER1"]}]}
+                "var": {"annotation_value": [{"name": var_index_col_name, "values": ["RER1"]}]}
             }
         }
         fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
@@ -176,7 +184,7 @@ class EngineTest(unittest.TestCase):
 
         filter_ = {
             "filter": {
-                "var": {"annotation_value": [{"name": "name", "values": ["SPEN", "TYMP", "PRMT2"]}]}
+                "var": {"annotation_value": [{"name": var_index_col_name, "values": ["SPEN", "TYMP", "PRMT2"]}]}
             }
         }
         fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")

server/test/test_scanpy_engine_data_load.py

@@ -15,7 +15,7 @@ class DataLoadEngineTest(unittest.TestCase):
 
     def test_delayed_load_args(self):
         args = {
-            "layout": "tsne",
+            "layout": ["tsne"],
             "diffexp": "ttest",
             "max_category_items": 1000,
             "obs_names": "foo",

server/utils/utils.py

@@ -0,0 +1,26 @@
+import contextlib
+import errno
+import socket
+
+
+def find_available_port(host, port=5005):
+    """
+    Helper method to find open port on host. Tries 5000 ports incremented from the specified port
+    """
+    # Takes approx 2 seconds to do a scan of 5000 ports on my laptop
+    num_ports_to_try = 5000
+    for port_to_try in range(port, port + num_ports_to_try):
+        if is_port_available(host, port_to_try):
+            return port_to_try
+    raise socket.error(errno.EADDRINUSE, f"No port in range {port} - {port + num_ports_to_try - 1} available.")
+
+
+def is_port_available(host, port):
+    is_available = False
+    with contextlib.closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s:
+        try:
+            s.bind((host, port))
+            is_available = True
+        except socket.error:
+            pass
+    return is_available

setup.py

@@ -8,7 +8,7 @@ with open("server/requirements.txt") as fh:
 
 setup(
     name="cellxgene",
-    version="0.9.1",
+    version="0.10.0",
     packages=find_packages(),
     url="https://github.com/chanzuckerberg/cellxgene",
     license="MIT",