patch release 0.9.2

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.10.1
+current_version = 0.9.2
 
 [bumpversion:file:setup.py]
 search = version="{current_version}"

.travis.yml

@@ -9,8 +9,8 @@ cache:
 install:
   - set -eo pipefail
   - pip install flake8
-  - make pydist
+  - make build
-  - make install-dist
+  - make install
   - pip install -r server/requirements-dev.txt
 
 jobs:
@@ -27,5 +27,6 @@ jobs:
     script: docker build .
   - name: "Smoke Tests"
     python: "3.6"
+    if: branch = master AND type = cron
     script:
       - npm run --prefix client/ smoke-test

README.md

@@ -2,8 +2,6 @@
 
 > an interactive explorer for single-cell transcriptomics data
 
-[![DOI](https://zenodo.org/badge/105615409.svg)](https://zenodo.org/badge/latestdoi/105615409)
-
 _cellxgene_ (pronounced "sell-by-jean") is an interactive data explorer for single-cell transcriptomics datasets, such as those coming from the [Human Cell Atlas](https://humancellatlas.org). Leveraging modern web development techniques to enable fast visualizations of at least 1 million cells, we hope to enable biologists and computational researchers to explore their data, and to demonstrate general, scalable, and reusable patterns for scientific data visualization.
 
 <img src="https://raw.githubusercontent.com/chanzuckerberg/cellxgene/master/docs/cellxgene-demo-1.gif" width="200" height="200" hspace="30"><img src="https://raw.githubusercontent.com/chanzuckerberg/cellxgene/master/docs/cellxgene-demo-2.gif" width="200" height="200" hspace="30"><img src="https://raw.githubusercontent.com/chanzuckerberg/cellxgene/master/docs/cellxgene-demo-3.gif" width="200" height="200" hspace="30">

client/__tests__/e2e/data.js

@@ -1,6 +1,6 @@
 export const datasets = {
   pbmc3k: {
-    title: "pbmc3k",
+    title: "cellxgene: pbmc3k",
     dataframe: {
       nObs: "2638",
       nVar: "1838",
@@ -26,8 +26,8 @@ export const datasets = {
     cellsets: {
       lasso: [
         {
-          "coordinates-as-percent": { x1: 0.05, y1: 0.25, x2: 0.15, y2: 0.35 },
+          "coordinates-as-percent": { x1: 0.25, y1: 0.25, x2: 0.35, y2: 0.35 },
-          count: "101"
+          count: "26"
         }
       ],
       categorical: [
@@ -91,8 +91,8 @@ export const datasets = {
         }
       },
       lasso: {
-        "coordinates-as-percent": { x1: 0.45, y1: 0.05, x2: 0.65, y2: 0.15 },
+        "coordinates-as-percent": { x1: 0.45, y1: 0.45, x2: 0.5, y2: 0.5 },
-        count: "46"
+        count: "67"
       }
     },
     scatter: {
@@ -108,15 +108,6 @@ export const datasets = {
           count: "24"
         }
       }
-    },
-    clip: {
-      min: "30",
-      max: "70",
-      metadata: "n_genes",
-      gene: "S100A8",
-      "coordinates-as-percent": { x1: 0.25, y1: 0.5, x2: 0.55, y2: 0.5 },
-      count: "392",
-      "gene-cell-count": "421"
     }
   }
 };

client/__tests__/e2e/e2e.test.js

@@ -182,6 +182,7 @@ describe("diffexp", async () => {
     );
   });
 });
+//
 
 describe("subset/reset", async () => {
   test("subset - cell count matches", async () => {
@@ -270,35 +271,6 @@ describe("scatter plot", async () => {
   });
 });
 
-describe("clipping", async () => {
-  test("clip continuous", async () => {
-    await cxgActions.clip(data.clip.min, data.clip.max);
-    const histId = `histogram-${data.clip.metadata}-plot-brush`;
-    const coords = await cxgActions.calcDragCoordinates(
-      histId,
-      data.clip["coordinates-as-percent"]
-    );
-    await cxgActions.drag(histId, coords.start, coords.end);
-    const cellCount = await cxgActions.cellSet(1);
-    expect(cellCount).toBe(data.clip.count);
-  });
-
-  test("clip gene", async () => {
-    await utils.typeInto("gene-search", data.clip.gene);
-    await page.keyboard.press("Enter");
-    await page.waitForSelector(`[data-testid='histogram-${data.clip.gene}']`);
-    await cxgActions.clip(data.clip.min, data.clip.max);
-    const histId = `histogram-${data.clip.gene}-plot-brush`;
-    const coords = await cxgActions.calcDragCoordinates(
-      histId,
-      data.clip["coordinates-as-percent"]
-    );
-    await cxgActions.drag(histId, coords.start, coords.end);
-    const cellCount = await cxgActions.cellSet(1);
-    expect(cellCount).toBe(data.clip["gene-cell-count"]);
-  });
-});
-
 // interact with UI elements just that they do not break
 describe("ui elements don't error", async () => {
   test("color by", async () => {

client/__tests__/e2e/puppeteerUtils.js

@@ -16,25 +16,10 @@ export const puppeteerUtils = puppeteerPage => ({
   async typeInto(testid, text) {
     // only works for text without special characters
     await this.waitByID(testid);
-    const selector = `[data-testid='${testid}']`;
     // type ahead can be annoying if you don't pause before you type
-    await puppeteerPage.click(selector);
+    await puppeteerPage.click(`[data-testid='${testid}']`);
     await puppeteerPage.waitFor(200);
-    await puppeteerPage.type(selector, text);
+    await puppeteerPage.type(`[data-testid='${testid}']`, text);
-  },
-
-  async clearInputAndTypeInto(testid, text) {
-    await this.waitByID(testid);
-    const selector = `[data-testid='${testid}']`;
-    // only works for text without special characters
-    // type ahead can be annoying if you don't pause before you type
-    await puppeteerPage.click(selector);
-    await puppeteerPage.waitFor(200);
-    // select all
-
-    await puppeteerPage.click(selector, {clickCount: 3})
-    await puppeteerPage.keyboard.type("Backspace")
-    await puppeteerPage.type(selector, text);
   },
 
   async clickOn(testid) {
@@ -44,13 +29,11 @@ export const puppeteerUtils = puppeteerPage => ({
   },
 
   async getOneElementInnerHTML(selector) {
-    await puppeteerPage.waitForSelector(selector);
     let text = await puppeteerPage.$eval(selector, el => el.innerHTML);
     return text;
   },
 
   async getOneElementInnerText(selector) {
-    await puppeteerPage.waitForSelector(selector);
     let text = await puppeteerPage.$eval(selector, el => el.innerText);
     return text;
   }
@@ -178,13 +161,5 @@ export const cellxgeneActions = puppeteerPage => ({
     await puppeteerUtils(puppeteerPage).clickOn("reset");
     // loading state never actually happens, reset is too fast
     await page.waitFor(200);
-  },
-
-  async clip(min = 0, max = 100) { 
-    await puppeteerUtils(puppeteerPage).clickOn("visualization-settings");
-    await puppeteerUtils(puppeteerPage).clearInputAndTypeInto("clip-min-input", min);
-    await puppeteerUtils(puppeteerPage).clearInputAndTypeInto("clip-max-input", max);
-    await puppeteerUtils(puppeteerPage).clickOn("clip-commit");
   }
-
 });

client/__tests__/util/range.test.js

@@ -1,42 +0,0 @@
-import { range, rangeFill } from "../../src/util/range";
-
-describe("range", () => {
-	test("no defaults", () => {
-		expect(range(0, 3, 1)).toMatchObject([0, 1, 2]);
-	});
-
-	test("range(stop)", () => {
-		expect(range(3)).toMatchObject([0, 1, 2]);
-		expect(range(0)).toMatchObject([]);
-		expect(range(1)).toMatchObject([0]);
-	});
-
-	test("range(start,stop)", () => {
-		expect(range(0, 0)).toMatchObject([]);
-		expect(range(0, 2)).toMatchObject([0, 1]);
-		expect(range(4, 8)).toMatchObject([4, 5, 6, 7]);
-	});
-
-	test("range(start, stop, step", () => {
-		expect(range(4, 0, -1)).toMatchObject([4, 3, 2, 1]);
-		expect(range(0, 4, 2)).toMatchObject([0, 2]);
-	});
-});
-
-describe("rangefill", () => {
-	test("rangeFill(arr)", () => {
-		expect(rangeFill(new Int32Array(3))).toMatchObject(
-			new Int32Array([0, 1, 2])
-		);
-	});
-	test("rangeFill(arr, start)", () => {
-		expect(rangeFill(new Int32Array(2), 1)).toMatchObject(
-			new Int32Array([1, 2])
-		);
-	});
-	test("rangeFill(arr, start, step)", () => {
-		expect(rangeFill(new Int32Array(3), 2, -1)).toMatchObject(
-			new Int32Array([2, 1, 0])
-		);
-	});
-});

client/__tests__/util/stateManager/sampleResponses.js

@@ -34,38 +34,28 @@ const aSchemaResponse = {
       type: "float32"
     },
     annotations: {
-      obs: {
+      obs: [
-        index: "name",
+        { name: "name", type: "string" },
-        columns: [
+        { name: "field1", type: "int32" },
-          { name: "name", type: "string" },
+        { name: "field2", type: "float32" },
-          { name: "field1", type: "int32" },
+        { name: "field3", type: "boolean" },
-          { name: "field2", type: "float32" },
+        {
-          { name: "field3", type: "boolean" },
+          name: "field4",
-          {
+          type: "categorical",
-            name: "field4",
+          categories: field4Categories
-            type: "categorical",
+        }
-            categories: field4Categories
+      ],
-          }
+      var: [
-        ]
+        { name: "name", type: "string" },
-      },
+        { name: "fieldA", type: "int32" },
-      var: {
+        { name: "fieldB", type: "float32" },
-        index: "name",
+        { name: "fieldC", type: "boolean" },
-        columns: [
+        {
-          { name: "name", type: "string" },
+          name: "fieldD",
-          { name: "fieldA", type: "int32" },
+          type: "categorical",
-          { name: "fieldB", type: "float32" },
+          categories: fieldDCategories
-          { name: "fieldC", type: "boolean" },
+        }
-          {
+      ]
-            name: "fieldD",
-            type: "categorical",
-            categories: fieldDCategories
-          }
-        ]
-      }
-    },
-    layout: {
-      obs: [{ name: "umap", type: "float32", dims: ["umap_0", "umap_1"] }],
-      var: []
     }
   }
 };
@@ -172,7 +162,29 @@ const aLayoutFBSResponse = (() => {
     new Float32Array(nObs).fill(Math.random()),
     new Float32Array(nObs).fill(Math.random())
   ];
-  return encodeMatrix(coords, ["umap_0", "umap_1"]);
+  const builder = new flatbuffers.Builder(1024);
+
+  const cols = _.map(coords, carr => {
+    const cdv = NetEncoding.Float32Array.createDataVector(builder, carr);
+    NetEncoding.Float32Array.startFloat32Array(builder);
+    NetEncoding.Float32Array.addData(builder, cdv);
+    const floatArr = NetEncoding.Float32Array.endFloat32Array(builder);
+
+    NetEncoding.Column.startColumn(builder);
+    NetEncoding.Column.addUType(builder, NetEncoding.TypedArray.Float32Array);
+    NetEncoding.Column.addU(builder, floatArr);
+    return NetEncoding.Column.endColumn(builder);
+  });
+
+  const columns = NetEncoding.Matrix.createColumnsVector(builder, cols);
+
+  NetEncoding.Matrix.startMatrix(builder);
+  NetEncoding.Matrix.addNRows(builder, nObs);
+  NetEncoding.Matrix.addNCols(builder, coords.length);
+  NetEncoding.Matrix.addColumns(builder, columns);
+  const matrix = NetEncoding.Matrix.endMatrix(builder);
+  builder.finish(matrix);
+  return builder.asUint8Array();
 })();
 
 const aDataObsResponse = {

client/__tests__/util/stateManager/universe.test.js

@@ -53,15 +53,13 @@ describe("createUniverseFromResponse", () => {
 
     expect(universe.obsAnnotations.dims).toEqual([
       nObs,
-      REST.schema.schema.annotations.obs.columns.length
+      REST.schema.schema.annotations.obs.length
     ]);
     expect(universe.obsLayout.dims).toEqual([nObs, 2]);
-    expect(universe.obsLayout.colIndex.keys()).toEqual(
+    expect(universe.obsLayout.colIndex.keys()).toEqual(["X", "Y"]);
-      universe.schema.layout.obs[0].dims
-    );
     expect(universe.varAnnotations.dims).toEqual([
       nVar,
-      REST.schema.schema.annotations.var.columns.length
+      REST.schema.schema.annotations.var.length
     ]);
     expect(universe.varData.isEmpty()).toBeTruthy();
   });

client/__tests__/util/stateManager/world.test.js

@@ -29,8 +29,7 @@ const defaultBigBang = () => {
   /* create crossfilter */
   const crossfilter = World.createObsDimensions(
     new Crossfilter(world.obsAnnotations),
-    world,
+    world
-    REST.schema.schema.layout.obs[0].dims
   );
 
   return {
@@ -139,9 +138,7 @@ describe("createWorldFromCurrentSelection", () => {
     expect(world.obsLayout.rowIndex.keys()).toEqual(
       new Int32Array(matchingIndices)
     );
-    expect(world.obsLayout.colIndex.keys()).toEqual(
+    expect(world.obsLayout.colIndex.keys()).toEqual(["X", "Y"]);
-      world.schema.layout.obs[0].dims
-    );
   });
 });
 
@@ -155,18 +152,14 @@ describe("createObsDimensionMap", () => {
 
     const { crossfilter } = defaultBigBang();
     const annotationNames = _.map(
-      REST.schema.schema.annotations.obs.columns,
+      REST.schema.schema.annotations.obs,
       c => c.name
     );
-    const obsIndexColName = REST.schema.schema.annotations.obs.index;
+    const schemaByObsName = _.keyBy(REST.schema.schema.annotations.obs, "name");
-    const schemaByObsName = _.keyBy(
-      REST.schema.schema.annotations.obs.columns,
-      "name"
-    );
     expect(crossfilter).toBeDefined();
     annotationNames.forEach(name => {
       const dim = crossfilter.dimensions[obsAnnoDimensionName(name)];
-      if (name === obsIndexColName) {
+      if (name === "name") {
         expect(dim).toBeUndefined();
       } else {
         const { type } = schemaByObsName[name];

client/__tests__/util/typedCrossfilter/util.test.js

@@ -1,8 +1,8 @@
 import {
+  fillRange,
   sliceByIndex,
   makeSortIndex
 } from "../../../src/util/typedCrossfilter/util";
-import { rangeFill as fillRange } from "../../../src/util/range";
 
 describe("fillRange", () => {
   test("Array", () => {

client/configuration/babel/babel.dev.js

@@ -7,8 +7,8 @@ module.exports = {
   ],
   plugins: [
     "@babel/plugin-proposal-function-bind",
+    "@babel/plugin-proposal-class-properties",
     ["@babel/plugin-proposal-decorators", { legacy: true }],
-    ["@babel/plugin-proposal-class-properties", { loose: true }],
     "@babel/plugin-proposal-export-namespace-from",
     "@babel/plugin-proposal-optional-chaining",
     "@babel/plugin-proposal-nullish-coalescing-operator"

client/configuration/babel/babel.prod.js

@@ -6,8 +6,8 @@ module.exports = {
   ],
   plugins: [
     "@babel/plugin-proposal-function-bind",
+    "@babel/plugin-proposal-class-properties",
     ["@babel/plugin-proposal-decorators", { legacy: true }],
-    ["@babel/plugin-proposal-class-properties", { loose: true }],
     "@babel/plugin-proposal-export-namespace-from",
     "@babel/plugin-transform-react-constant-elements",
     "@babel/plugin-transform-runtime",

client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cellxgene",
-  "version": "0.10.1",
+  "version": "0.9.2",
   "license": "MIT",
   "description": "cellxgene is a web application for the interactive exploration of single cell sequence data.",
   "repository": "https://github.com/chanzuckerberg/cellxgene",
@@ -9,13 +9,13 @@
     "build": "npm run clean && webpack --config configuration/webpack/webpack.config.prod.js",
     "clean": "rimraf build",
     "dev": "npm run clean && webpack --config configuration/webpack/webpack.config.dev.js",
-    "e2e": "node node_modules/jest/bin/jest.js --verbose false --config __tests__/e2e/e2eJestConfig.json e2e/e2e.test.js",
+    "e2e": "jest --verbose false --config __tests__/e2e/e2eJestConfig.json e2e/e2e.test.js",
     "lint": "eslint src",
     "smoke-test": "start-server-and-test start-server-for-test :5000 e2e",
     "start": "node server/development.js",
     "start-server-for-test": "cellxgene launch -p 5000 ../example-dataset/pbmc3k.h5ad",
-    "test": "node node_modules/jest/bin/jest.js",
+    "test": "jest",
-    "unit-test": "node node_modules/jest/bin/jest.js --testPathIgnorePatterns e2e"
+    "unit-test": "jest --testPathIgnorePatterns e2e"
   },
   "engineStrict": true,
   "engines": {
@@ -31,8 +31,8 @@
     "eslint-scope": "3.7.1"
   },
   "dependencies": {
-    "@blueprintjs/core": "^3.15.1",
+    "@blueprintjs/core": "^3.15.0",
-    "@blueprintjs/icons": "^3.8.0",
+    "@blueprintjs/icons": "^3.3.0",
     "@blueprintjs/select": "^3.8.0",
     "canvas-fit": "^1.5.0",
     "d3": "^4.10.0",
@@ -41,77 +41,79 @@
     "font-color-contrast": "^1.0.3",
     "fuzzysort": "^1.1.4",
     "gl-mat4": "^1.1.4",
-    "gl-vec3": "^1.1.3",
+    "gl-matrix": "^2.7.1",
-    "gl-matrix": "^3.0.0",
     "is-number": "^7.0.0",
     "key-pressed": "0.0.1",
     "lodash": "^4.17.4",
-    "memoize-one": "^5.0.4",
+    "memoize-one": "^4.0.0",
     "mouse-position": "^2.0.1",
     "mouse-pressed": "^1.0.0",
+    "normalize.css": "^8.0.0",
     "orbit-camera": "^1.0.0",
-    "query-string": "^6.5.0",
+    "query-string": "^6.1.0",
-    "react": "^16.8.6",
+    "react": "^16.6.0",
     "react-autocomplete": "^1.7.2",
-    "react-dom": "^16.8.6",
+    "react-dom": "^16.6.0",
-    "react-helmet": "^5.2.1",
+    "react-helmet": "^5.2.0",
-    "react-icons": "^3.7.0",
+    "react-icons": "^3.2.2",
-    "react-redux": "^7.0.3",
+    "react-redux": "^5.1.0",
     "redux": "^4.0.1",
+    "redux-devtools-extension": "^2.13.5",
     "redux-thunk": "^2.2.0",
-    "regl": "^1.3.11",
+    "regl": "^1.3.9",
     "scroll-speed": "^1.0.0",
     "urijs": "^1.19.0"
   },
   "devDependencies": {
-    "@babel/core": "^7.4.4",
+    "@babel/core": "^7.1.5",
-    "@babel/plugin-proposal-class-properties": "^7.4.4",
+    "@babel/plugin-proposal-class-properties": "^7.0.0",
-    "@babel/plugin-proposal-decorators": "^7.4.4",
+    "@babel/plugin-proposal-decorators": "^7.0.0",
-    "@babel/plugin-proposal-export-namespace-from": "^7.2.0",
+    "@babel/plugin-proposal-export-namespace-from": "^7.0.0",
-    "@babel/plugin-proposal-function-bind": "^7.2.0",
+    "@babel/plugin-proposal-function-bind": "^7.0.0",
-    "@babel/plugin-proposal-nullish-coalescing-operator": "^7.4.4",
+    "@babel/plugin-proposal-nullish-coalescing-operator": "^7.2.0",
     "@babel/plugin-proposal-optional-chaining": "^7.2.0",
-    "@babel/plugin-transform-react-constant-elements": "^7.2.0",
+    "@babel/plugin-transform-react-constant-elements": "^7.0.0",
-    "@babel/plugin-transform-runtime": "^7.4.4",
+    "@babel/plugin-transform-runtime": "^7.1.0",
-    "@babel/preset-env": "^7.4.4",
+    "@babel/preset-env": "^7.1.5",
     "@babel/preset-react": "^7.0.0",
-    "@babel/register": "^7.4.4",
+    "@babel/register": "^7.0.0",
-    "@babel/runtime": "^7.4.4",
+    "@babel/runtime": "^7.1.5",
+    "babel-core": "^7.0.0-bridge.0",
     "babel-eslint": "^10.0.1",
-    "babel-jest": "^24.8.0",
+    "babel-jest": "^23.6.0",
-    "babel-loader": "^8.0.6",
+    "babel-loader": "^8.0.0",
-    "babel-preset-modern-browsers": "^14.0.0",
+    "babel-preset-modern-browsers": "^12.0.0",
     "chalk": "^2.4.2",
     "connect-history-api-fallback": "^1.6.0",
-    "copy-webpack-plugin": "^5.0.3",
+    "copy-webpack-plugin": "^4.6.0",
-    "css-loader": "^2.1.1",
+    "css-loader": "^1.0.1",
-    "eslint": "^5.16.0",
+    "eslint": "^5.13.0",
     "eslint-config-airbnb": "^17.1.0",
-    "eslint-config-prettier": "^4.2.0",
+    "eslint-config-prettier": "^4.0.0",
     "eslint-loader": "^2.1.2",
     "eslint-plugin-filenames": "^1.3.2",
-    "eslint-plugin-import": "^2.17.2",
+    "eslint-plugin-import": "^2.16.0",
-    "eslint-plugin-jest": "^22.5.1",
+    "eslint-plugin-jest": "^22.2.2",
     "eslint-plugin-jsx-a11y": "^6.2.1",
-    "eslint-plugin-react": "^7.13.0",
+    "eslint-plugin-react": "^7.12.4",
     "express": "^4.14.0",
-    "file-loader": "^3.0.1",
+    "file-loader": "^2.0.0",
     "html-webpack-inline-source-plugin": "0.0.10",
     "html-webpack-plugin": "^3.2.0",
-    "jest": "^24.8.0",
+    "jest": "^24.1.0",
-    "jest-puppeteer": "^4.1.1",
+    "jest-puppeteer": "^4.1.0",
     "json-loader": "^0.5.4",
-    "mini-css-extract-plugin": "^0.6.0",
+    "mini-css-extract-plugin": "^0.4.1",
-    "puppeteer": "^1.16.0",
+    "puppeteer": "^1.12.1",
     "rimraf": "^2.6.3",
     "serve-favicon": "^2.3.0",
-    "start-server-and-test": "^1.9.0",
+    "start-server-and-test": "^1.7.11",
     "style-loader": "^0.23.1",
     "sw-precache-webpack-plugin": "^0.11.5",
     "url-loader": "^1.1.0",
-    "webpack": "^4.31.0",
+    "webpack": "^4.25.1",
-    "webpack-cli": "^3.3.2",
+    "webpack-cli": "^3.1.0",
-    "webpack-dev-middleware": "^3.6.2"
+    "webpack-dev-middleware": "^3.1.3"
   },
   "jest": {
     "testMatch": [
@@ -131,23 +133,16 @@
         ],
         "plugins": [
           "@babel/plugin-proposal-function-bind",
+          "@babel/plugin-proposal-class-properties",
           [
             "@babel/plugin-proposal-decorators",
             {
               "legacy": true
             }
           ],
-          [
-            "@babel/plugin-proposal-class-properties",
-            {
-              "loose": true
-            }
-          ],
           "@babel/plugin-proposal-export-namespace-from",
           "@babel/plugin-transform-react-constant-elements",
-          "@babel/plugin-transform-runtime",
+          "@babel/plugin-transform-runtime"
-          "@babel/plugin-proposal-optional-chaining",
-          "@babel/plugin-proposal-nullish-coalescing-operator"
         ]
       }
     }

client/src/actions/index.js

@@ -21,32 +21,24 @@ const doInitialDataLoad = () =>
     dispatch({ type: "initial data load start" });
 
     try {
-      /*
+      const requestJson = _(["config", "schema"])
-      Step 1 - config & schema, all JSON
-      */
-      const requestJson = ["config", "schema"]
         .map(r => `${globals.API.prefix}${globals.API.version}${r}`)
-        .map(url => doJsonRequest(url));
+        .map(url => doJsonRequest(url))
-      const stepOneResults = await Promise.all(requestJson);
+        .value();
+      const requestBinary = _([
+        "annotations/obs",
+        "annotations/var?annotation-name=name",
+        "layout/obs"
+      ])
+        .map(r => `${globals.API.prefix}${globals.API.version}${r}`)
+        .map(url => doBinaryRequest(url))
+        .value();
+
+      const results = await Promise.all(_.concat(requestJson, requestBinary));
+
       /* set config defaults */
-      const config = { ...globals.configDefaults, ...stepOneResults[0].config };
+      const config = { ...globals.configDefaults, ...results[0].config };
-      const schema = stepOneResults[1];
+      const [, schema, obsAnno, varAnno, obsLayout] = [...results];
-
-      /*
-      Step 2 - dataframes, all binary.  NOTE: uses results of step 1.
-      */
-      /* only load names for var annotations, if possible*/
-      const varIndexName = schema?.schema?.annotations?.var?.index;
-      const varAnnotationsQuery = varIndexName
-        ? `?annotation-name=${varIndexName}`
-        : "";
-      const varAnnotationsURL = `annotations/var${varAnnotationsQuery}`;
-      const requestBinary = ["annotations/obs", varAnnotationsURL, "layout/obs"]
-        .map(r => `${globals.API.prefix}${globals.API.version}${r}`)
-        .map(url => doBinaryRequest(url));
-      const stepTwoResults = await Promise.all(requestBinary);
-      const [obsAnno, varAnno, obsLayout] = [...stepTwoResults];
-
       const universe = Universe.createUniverseFromResponse(
         config,
         schema,
@@ -99,10 +91,6 @@ needs expression data.
 Transparently utilizes cached data if it is already present.
 */
 async function _doRequestExpressionData(dispatch, getState, genes) {
-  const state = getState();
-  const { universe } = state;
-  const varIndexName = universe.schema.annotations.var.index;
-
   /* helper for this function only */
   const fetchData = async geneNames => {
     const res = await fetch(
@@ -112,7 +100,7 @@ async function _doRequestExpressionData(dispatch, getState, genes) {
         body: JSON.stringify({
           filter: {
             var: {
-              annotation_value: [{ name: varIndexName, values: geneNames }]
+              annotation_value: [{ name: "name", values: geneNames }]
             }
           }
         }),
@@ -135,6 +123,8 @@ async function _doRequestExpressionData(dispatch, getState, genes) {
     return Universe.convertDataFBStoObject(universe, data);
   };
 
+  const state = getState();
+  const { universe } = state;
   /* preload data already in cache */
   let expressionData = _.transform(
     genes,
@@ -251,7 +241,6 @@ const requestDifferentialExpression = (set1, set2, num_genes = 10) => async (
     */
     const state = getState();
     const { universe } = state;
-    const varIndexName = universe.schema.annotations.var.index;
 
     // Legal values are null, Array or TypedArray.  Null is initial state.
     if (!set1) set1 = [];
@@ -288,7 +277,7 @@ const requestDifferentialExpression = (set1, set2, num_genes = 10) => async (
     const data = await res.json();
     // result is [ [varIdx, ...], ... ]
     const topNGenes = _.map(data, r =>
-      universe.varAnnotations.at(r[0], varIndexName)
+      universe.varAnnotations.at(r[0], "name")
     );
 
     /*

client/src/components/categorical/category.js

@@ -27,10 +27,10 @@ class Category extends React.Component {
       const cat = categoricalSelection[metadataField];
       const categoryCount = {
         // total number of categories in this dimension
-        totalCatCount: cat.numCategoryValues,
+        totalCatCount: cat.numCategories,
         // number of selected options in this category
         selectedCatCount: _.reduce(
-          cat.categoryValueSelected,
+          cat.categorySelected,
           (res, cond) => (cond ? res + 1 : res),
           0
         )
@@ -91,7 +91,7 @@ class Category extends React.Component {
     const { categoricalSelection, metadataField } = this.props;
 
     const cat = categoricalSelection[metadataField];
-    const optTuples = sortedCategoryValues([...cat.categoryValueIndices]);
+    const optTuples = sortedCategoryValues([...cat.categoryIndices]);
     return _.map(optTuples, (tuple, i) => (
       <Value
         optTuples={optTuples}

client/src/components/categorical/occupancy.js

@@ -1,16 +1,25 @@
 // jshint esversion: 6
 import React from "react";
+import _ from "lodash";
 import { connect } from "react-redux";
 import * as d3 from "d3";
 
 @connect()
 class Occupancy extends React.Component {
   render() {
-    const { occupancy, colorScale, colorAccessor, schema, world } = this.props;
+    const {
+      occupancy,
+      colorScale,
+      categoricalSelection,
+      colorAccessor,
+      schema
+    } = this.props;
     const width = 100;
     const height = 11;
 
-    const categories = schema.annotations.obsByName[colorAccessor]?.categories;
+    const categories = _.filter(schema.annotations.obs, {
+      name: colorAccessor
+    })[0].categories;
 
     const x = d3
       .scaleLinear()
@@ -19,9 +28,8 @@ class Occupancy extends React.Component {
       .range([0, width]);
 
     let currentOffset = 0;
-    const dfColumn = world.obsAnnotations.col(colorAccessor);
+
-    const categoryValues = dfColumn.summarize().categories;
+    const stacks = categoricalSelection[colorAccessor].categoryValues.map(d => {
-    const stacks = categoryValues.map(d => {
       const o = occupancy.get(d);
 
       const scaledValue = x(o);

client/src/components/categorical/util.js

@@ -5,6 +5,7 @@
 // return sorted index
 
 import isNumber from "is-number";
+import _ from "lodash";
 
 const sortedCategoryValues = values => {
   /* this sort could be memoized for perf */
@@ -12,7 +13,7 @@ const sortedCategoryValues = values => {
   const strings = [];
   const ints = [];
 
-  values.forEach(v => {
+  _.forEach(values, v => {
     if (isNumber(v[0])) {
       ints.push(v);
     } else {

client/src/components/categorical/value.js

@@ -1,6 +1,7 @@
 // jshint esversion: 6
 import { connect } from "react-redux";
 import React from "react";
+import _ from "lodash";
 import Occupancy from "./occupancy";
 import { countCategoryValues2D } from "../../util/stateManager/worldUtil";
 import * as globals from "../../globals";
@@ -9,7 +10,7 @@ import * as globals from "../../globals";
   categoricalSelection: state.categoricalSelection,
   colorScale: state.colors.scale,
   colorAccessor: state.colors.colorAccessor,
-  schema: state.world?.schema,
+  schema: _.get(state.world, "schema", null),
   world: state.world
 }))
 class CategoryValue extends React.Component {
@@ -46,8 +47,8 @@ class CategoryValue extends React.Component {
     if (!categoricalSelection) return null;
 
     const category = categoricalSelection[metadataField];
-    const selected = category.categoryValueSelected[categoryIndex];
+    const selected = category.categorySelected[categoryIndex];
-    const count = category.categoryValueCounts[categoryIndex];
+    const count = category.categoryCounts[categoryIndex];
     const value = category.categoryValues[categoryIndex];
     const displayString = String(
       category.categoryValues[categoryIndex]
@@ -59,7 +60,9 @@ class CategoryValue extends React.Component {
     let occupancy = null;
 
     if (isColorBy && schema) {
-      categories = schema.annotations.obsByName[colorAccessor]?.categories;
+      categories = _.filter(schema.annotations.obs, {
+        name: colorAccessor
+      })[0].categories;
     }
 
     if (colorAccessor && !isColorBy && categoricalSelection[colorAccessor]) {

client/src/components/continuous/continuous.js

@@ -9,10 +9,10 @@ import * as globals from "../../globals";
 import HistogramBrush from "../brushableHistogram";
 
 @connect(state => ({
-  obsAnnotations: state.world?.obsAnnotations,
+  obsAnnotations: _.get(state.world, "obsAnnotations", null),
   colorAccessor: state.colors.colorAccessor,
   colorScale: state.colors.scale,
-  schema: state.world?.schema
+  schema: _.get(state.world, "schema", null)
 }))
 class Continuous extends React.Component {
   constructor(props) {
@@ -66,8 +66,7 @@ class Continuous extends React.Component {
           ? _.map(obsAnnotations.colIndex.keys(), key => {
               const isColorField =
                 key.includes("color") || key.includes("Color");
-              if (key === schema.annotations.obs.index || isColorField)
+              if (key === "name" || isColorField) return null;
-                return null;
 
               const summary = obsAnnotations.col(key).summarize();
               const nonFiniteExtent =

client/src/components/framework/logo.js

@@ -1,18 +0,0 @@
-import React from "react";
-import * as globals from "../../globals";
-
-const Logo = props => {
-  const { size } = props;
-  return (
-    <svg width={size} height={size} viewBox={`0 0 48 48`} fill="none">
-      <rect width="48" height="48" fill="white" />
-      <rect width="48" height="48" fill={globals.logoColor} />
-      <rect x="19" y="19" width="22" height="22" fill="white" />
-      <rect x="24" y="24" width="12" height="12" fill={globals.logoColor} />
-      <rect x="7" y="19" width="7" height="22" fill="white" />
-      <rect x="19" y="7" width="22" height="7" fill="white" />
-    </svg>
-  );
-};
-
-export default Logo;

client/src/components/geneExpression/index.js

@@ -57,7 +57,7 @@ const filterGenes = (query, genes) =>
 
 @connect(state => {
   return {
-    obsAnnotations: state.world?.obsAnnotations,
+    obsAnnotations: _.get(state.world, "obsAnnotations", null),
     userDefinedGenes: state.controls.userDefinedGenes,
     userDefinedGenesLoading: state.controls.userDefinedGenesLoading,
     world: state.world,
@@ -85,8 +85,7 @@ class GeneExpression extends React.Component {
     */
     const { world } = this.props;
     const { varAnnotations } = world;
-    const varIndexName = world.schema.annotations.var.index;
+    const geneNames = varAnnotations.col("name").asArray();
-    const geneNames = varAnnotations.col(varIndexName).asArray();
     if (geneNames.length > 0) {
       const placeholder = [];
       let len = geneNames.length;
@@ -108,7 +107,6 @@ class GeneExpression extends React.Component {
 
   handleClick(g) {
     const { world, dispatch, userDefinedGenes } = this.props;
-    const varIndexName = world.schema.annotations.var.index;
     const gene = g.target;
     if (userDefinedGenes.indexOf(gene) !== -1) {
       postUserErrorToast("That gene already exists");
@@ -116,9 +114,7 @@ class GeneExpression extends React.Component {
       postUserErrorToast(
         "That's too many genes, you can have at most 15 user defined genes"
       );
-    } else if (
+    } else if (world.varAnnotations.col("name").indexOf(gene) === undefined) {
-      world.varAnnotations.col(varIndexName).indexOf(gene) === undefined
-    ) {
       postUserErrorToast("That doesn't appear to be a valid gene name.");
     } else {
       dispatch({ type: "single user defined gene start" });
@@ -131,7 +127,6 @@ class GeneExpression extends React.Component {
 
   handleBulkAddClick() {
     const { world, dispatch, userDefinedGenes } = this.props;
-    const varIndexName = world.schema.annotations.var.index;
     const { bulkAdd } = this.state;
 
     /*
@@ -150,9 +145,7 @@ class GeneExpression extends React.Component {
           if (userDefinedGenes.indexOf(gene) !== -1) {
             return keepAroundErrorToast("That gene already exists");
           }
-          if (
+          if (world.varAnnotations.col("name").indexOf(gene) === undefined) {
-            world.varAnnotations.col(varIndexName).indexOf(gene) === undefined
-          ) {
             return keepAroundErrorToast(
               `${gene} doesn't appear to be a valid gene name.`
             );
@@ -175,7 +168,7 @@ class GeneExpression extends React.Component {
       userDefinedGenesLoading,
       differential
     } = this.props;
-    const varIndexName = world?.schema?.annotations?.var?.index;
+
     const { tab, bulkAdd } = this.state;
 
     return (
@@ -250,7 +243,7 @@ class GeneExpression extends React.Component {
                 itemRenderer={renderGene.bind(this)}
                 items={
                   world && world.varAnnotations
-                    ? world.varAnnotations.col(varIndexName).asArray()
+                    ? world.varAnnotations.col("name").asArray()
                     : ["No genes"]
                 }
                 popoverProps={{ minimal: true }}
@@ -329,7 +322,7 @@ class GeneExpression extends React.Component {
           <ExpressionButtons />
           {differential.diffExp
             ? _.map(differential.diffExp, (value, index) => {
-                const name = world.varAnnotations.at(value[0], varIndexName);
+                const name = world.varAnnotations.at(value[0], "name");
                 const values = world.varData.col(name);
                 if (!values) {
                   return null;

client/src/components/graph/drawPointsRegl.js

@@ -1,6 +1,5 @@
 // jshint esversion: 6
 const mat4 = require("gl-mat4");
-const vec3 = require("gl-vec3");
 
 // opacity: https://github.com/spacetx/starfish/blob/master/viz/draw/regions.js
 
@@ -39,20 +38,7 @@ export default function(regl) {
     uniforms: {
       distance: regl.prop("distance"),
       view: regl.prop("view"),
-      projection: ({ viewportWidth, viewportHeight }) => {
+      projection: ({viewportWidth, viewportHeight}) => mat4.perspective([], Math.PI / 2, viewportWidth / viewportHeight, 0.01, 1000)
-        const aspectRatio = viewportWidth / viewportHeight;
-        let m = mat4.perspective(
-          [],
-          Math.PI / 2,
-          viewportWidth / viewportHeight,
-          0.01,
-          1000
-        );
-        if (aspectRatio < 1) {
-          m = mat4.scale(m, m, vec3.fromValues(1, 1, 1 / aspectRatio));
-        }
-        return m;
-      }
     },
 
     count: regl.prop("count"),

client/src/components/graph/graph.js

@@ -3,9 +3,7 @@ import React from "react";
 import * as d3 from "d3";
 import { connect } from "react-redux";
 import mat4 from "gl-mat4";
-import vec3 from "gl-vec3";
 import _regl from "regl";
-import memoize from "memoize-one";
 import {
   Button,
   AnchorButton,
@@ -15,9 +13,7 @@ import {
   MenuItem,
   Position,
   NumericInput,
-  Icon,
+  Icon
-  RadioGroup,
-  Radio
 } from "@blueprintjs/core";
 
 import * as globals from "../../globals";
@@ -51,8 +47,7 @@ import { World } from "../../util/stateManager";
   undoDisabled: state["@@undoable/past"].length === 0,
   redoDisabled: state["@@undoable/future"].length === 0,
   selectionTool: state.graphSelection.tool,
-  currentSelection: state.graphSelection.selection,
+  currentSelection: state.graphSelection.selection
-  layoutChoice: state.layoutChoice
 }))
 class Graph extends React.Component {
   static isValidDigitKeyEvent(e) {
@@ -79,38 +74,6 @@ class Graph extends React.Component {
     return key >= 0 && key <= 9;
   }
 
-  computePointPositions = memoize((X, Y, scaleX, scaleY) => {
-    /*
-    compute webgl coordinate buffer for each point
-    */
-    const positions = new Float32Array(2 * X.length);
-    for (let i = 0, len = X.length; i < len; i += 1) {
-      positions[2 * i] = scaleX(X[i]);
-      positions[2 * i + 1] = scaleY(Y[i]);
-    }
-    return positions;
-  });
-
-  computePointColors = memoize(rgb => {
-    /*
-    compute webgl colors for each point
-    */
-    const colors = new Float32Array(3 * rgb.length);
-    for (let i = 0, len = rgb.length; i < len; i += 1) {
-      colors.set(rgb[i], 3 * i);
-    }
-    return colors;
-  });
-
-  computePointSizes = memoize((len, crossfilter) => {
-    /*
-    compute webgl dot size for each point
-    */
-    const sizes = new Float32Array(len);
-    crossfilter.fillByIsSelected(sizes, 4, 0.2);
-    return sizes;
-  });
-
   constructor(props) {
     super(props);
     this.count = 0;
@@ -118,8 +81,6 @@ class Graph extends React.Component {
     this.graphPaddingBottom = 45;
     this.graphPaddingRight = globals.leftSidebarWidth;
     this.renderCache = {
-      X: null,
-      Y: null,
       positions: null,
       colors: null,
       sizes: null
@@ -145,13 +106,6 @@ class Graph extends React.Component {
     const colorBuffer = regl.buffer();
     const sizeBuffer = regl.buffer();
 
-    // preallocate coordinate system transformation between data and gl
-    const fractionToUse = 0.98; // fraction of dimension to use
-    const transform = {
-      glScaleX: scaleLinear([0, 1], [-1 * fractionToUse, 1 * fractionToUse]),
-      glScaleY: scaleLinear([0, 1], [1 * fractionToUse, -1 * fractionToUse])
-    };
-
     /* first time, but this duplicates above function, should be possile to avoid this */
     const reglRender = regl.frame(() => {
       this.reglDraw(
@@ -174,8 +128,7 @@ class Graph extends React.Component {
       colorBuffer,
       sizeBuffer,
       camera,
-      reglRender,
+      reglRender
-      transform
     });
   }
 
@@ -187,10 +140,19 @@ class Graph extends React.Component {
       colorRGB,
       responsive,
       selectionTool,
-      currentSelection,
+      currentSelection
-      layoutChoice
     } = this.props;
-    const { reglRender, mode, regl, svg } = this.state;
+    const {
+      reglRender,
+      mode,
+      regl,
+      drawPoints,
+      camera,
+      pointBuffer,
+      colorBuffer,
+      sizeBuffer,
+      svg
+    } = this.state;
     let stateChanges = {};
 
     if (reglRender && this.reglRenderState === "rendering" && mode !== "zoom") {
@@ -201,40 +163,53 @@ class Graph extends React.Component {
     if (regl && world) {
       /* update the regl state */
       const { obsLayout, nObs } = world;
-      const {
+      const X = obsLayout.col("X").asArray();
-        drawPoints,
+      const Y = obsLayout.col("Y").asArray();
-        transform,
-        camera,
-        pointBuffer,
-        colorBuffer,
-        sizeBuffer
-      } = this.state;
 
-      /* coordinates for each point */
+      // X/Y positions for each point - a cached value that only
-      const { glScaleX, glScaleY } = transform;
+      // changes if we have loaded entirely new cell data
-      const X = obsLayout.col(layoutChoice.currentDimNames[0]).asArray();
+      //
-      const Y = obsLayout.col(layoutChoice.currentDimNames[1]).asArray();
+      if (!renderCache.positions || world !== prevProps.world) {
-      const newPositions = this.computePointPositions(X, Y, glScaleX, glScaleY);
+        renderCache.positions = new Float32Array(2 * nObs);
-      if (renderCache.positions !== newPositions) {
+
-        /* update our cache & GL if the buffer changes */
+        const glScaleX = scaleLinear([0, 1], [-1, 1]);
-        renderCache.positions = newPositions;
+        const glScaleY = scaleLinear([0, 1], [1, -1]);
-        pointBuffer({ data: newPositions, dimension: 2 });
+
+        const offset = [d3.mean(X) - 0.5, d3.mean(Y) - 0.5];
+
+        for (let i = 0, { positions } = renderCache; i < nObs; i += 1) {
+          positions[2 * i] = glScaleX(X[i] - offset[0]);
+          positions[2 * i + 1] = glScaleY(Y[i] - offset[1]);
+        }
+        pointBuffer({
+          data: renderCache.positions,
+          dimension: 2
+        });
+
+        stateChanges.offset = offset;
       }
 
-      /* colors for each point */
+      // Colors for each point - a cached value that only changes when
-      const newColors = this.computePointColors(colorRGB);
+      // the cell metadata changes.
-      if (renderCache.colors !== newColors) {
+      if (!renderCache.colors || colorRGB !== prevProps.colorRGB) {
-        /* update our cache & GL if the buffer changes */
+        const rgb = colorRGB;
-        renderCache.colors = newColors;
+        if (!renderCache.colors) {
-        colorBuffer({ data: newColors, dimension: 3 });
+          renderCache.colors = new Float32Array(3 * rgb.length);
+        }
+        for (let i = 0, { colors } = renderCache; i < rgb.length; i += 1) {
+          colors.set(rgb[i], 3 * i);
+        }
+        colorBuffer({ data: renderCache.colors, dimension: 3 });
       }
 
-      /* sizes for each point */
+      // Sizes for each point - updates are triggered only when selected
-      const newSizes = this.computePointSizes(nObs, crossfilter);
+      // obs change
-      if (renderCache.sizes !== newSizes) {
+      if (!renderCache.sizes || crossfilter !== prevProps.crossfilter) {
-        /* update our cache & GL if the buffer changes */
+        if (!renderCache.sizes) {
-        renderCache.size = newSizes;
+          renderCache.sizes = new Float32Array(nObs);
-        sizeBuffer({ data: newSizes, dimension: 1 });
+        }
+        crossfilter.fillByIsSelected(renderCache.sizes, 4, 0.2);
+        sizeBuffer({ data: renderCache.sizes, dimension: 1 });
       }
 
       this.count = nObs;
@@ -296,10 +271,11 @@ class Graph extends React.Component {
       mode !== prevState.mode ||
       stateChanges.svg
     ) {
-      const { tool, container } = this.state;
+      const { tool, container, offset } = this.state;
       this.selectionToolUpdate(
         stateChanges.tool ? stateChanges.tool : tool,
-        stateChanges.container ? stateChanges.container : container
+        stateChanges.container ? stateChanges.container : container,
+        stateChanges.offset ? stateChanges.offset : offset
       );
     }
 
@@ -459,15 +435,7 @@ class Graph extends React.Component {
     this.setState({ pendingClipPercentiles: null });
   };
 
-  handleLayoutChoiceChange = e => {
+  brushToolUpdate(tool, container, offset) {
-    const { dispatch } = this.props;
-    dispatch({
-      type: "set layout choice",
-      layoutChoice: e.currentTarget.value
-    });
-  };
-
-  brushToolUpdate(tool, container) {
     /*
     this is called from componentDidUpdate(), so be very careful using
     anything from this.state, which may be updated asynchronously.
@@ -481,8 +449,8 @@ class Graph extends React.Component {
         if there is a selection, make sure the brush tool matches
         */
         const screenCoords = [
-          this.mapPointToScreen(currentSelection.brushCoords.northwest),
+          this.mapPointToScreen(currentSelection.brushCoords.northwest, offset),
-          this.mapPointToScreen(currentSelection.brushCoords.southeast)
+          this.mapPointToScreen(currentSelection.brushCoords.southeast, offset)
         ];
         if (!toolCurrentSelection) {
           /* tool is not selected, so just move the brush */
@@ -509,7 +477,7 @@ class Graph extends React.Component {
     }
   }
 
-  lassoToolUpdate(tool, container) {
+  lassoToolUpdate(tool, container, offset) {
     /*
     this is called from componentDidUpdate(), so be very careful using
     anything from this.state, which may be updated asynchronously.
@@ -520,7 +488,7 @@ class Graph extends React.Component {
       if there is a current selection, make sure the lasso tool matches
       */
       const polygon = currentSelection.polygon.map(p =>
-        this.mapPointToScreen(p)
+        this.mapPointToScreen(p, offset)
       );
       tool.move(polygon);
     } else {
@@ -528,7 +496,7 @@ class Graph extends React.Component {
     }
   }
 
-  selectionToolUpdate(tool, container) {
+  selectionToolUpdate(tool, container, offset) {
     /*
     this is called from componentDidUpdate(), so be very careful using
     anything from this.state, which may be updated asynchronously.
@@ -536,10 +504,10 @@ class Graph extends React.Component {
     const { selectionTool } = this.props;
     switch (selectionTool) {
       case "brush":
-        this.brushToolUpdate(tool, container);
+        this.brushToolUpdate(tool, container, offset);
         break;
       case "lasso":
-        this.lassoToolUpdate(tool, container);
+        this.lassoToolUpdate(tool, container, offset);
         break;
       default:
         /* punt? */
@@ -596,14 +564,12 @@ class Graph extends React.Component {
     accounting for current pan/zoom camera.
     */
     const { responsive } = this.props;
-    const { regl, camera, transform } = this.state;
+    const { regl, camera, offset } = this.state;
-    const { glScaleX, glScaleY } = transform;
 
     const gl = regl._gl;
 
     // get aspect ratio
     const aspect = gl.drawingBufferWidth / gl.drawingBufferHeight;
-    const scale = aspect < 1 ? 1 / aspect : 1;
 
     // compute inverse view matrix
     const inverse = mat4.invert([], camera.view());
@@ -612,37 +578,37 @@ class Graph extends React.Component {
     const x = (2 * pin[0]) / (responsive.width - this.graphPaddingRight) - 1;
     const y = 2 * (1 - pin[1] / (responsive.height - this.graphPaddingTop)) - 1;
     const pout = [
-      x * inverse[14] * aspect * scale + inverse[12],
+      x * inverse[14] * aspect + inverse[12],
-      -(y * inverse[14] * scale + inverse[13])
+      y * inverse[14] + inverse[13]
     ];
 
-    const xy = [glScaleX.invert(pout[0]), glScaleY.invert(pout[1])];
+    return [(pout[0] + 1) / 2 + offset[0], (pout[1] + 1) / 2 + offset[1]];
-    return xy;
   }
 
-  mapPointToScreen(xyCell) {
+  mapPointToScreen(xyCell, offset) {
     /*
     Map an XY coordinate from cell/point domain to screen range.  Inverse
     of mapScreenToPoint()
     */
     const { responsive } = this.props;
-    const { regl, camera, transform } = this.state;
+    const { regl, camera } = this.state;
-    const { glScaleX, glScaleY } = transform;
 
     const gl = regl._gl;
 
     // get aspect ratio
     const aspect = gl.drawingBufferWidth / gl.drawingBufferHeight;
-    const scale = aspect < 1 ? 1 / aspect : 1;
 
     // compute inverse view matrix
-    let inverse = mat4.invert([], camera.view());
+    const inverse = mat4.invert([], camera.view());
 
     // variable names are choosen to reflect inverse of those used
     // in mapScreenToPoint().
-    const pout = [glScaleX(xyCell[0]), glScaleY(xyCell[1])];
+    const pout = [
-    const x = (pout[0] - inverse[12]) / aspect / scale / inverse[14];
+      (xyCell[0] - offset[0]) * 2 - 1,
-    const y = (-pout[1] - inverse[13]) / scale / inverse[14];
+      (xyCell[1] - offset[1]) * 2 - 1
+    ];
+    const x = (pout[0] - inverse[12]) / aspect / inverse[14];
+    const y = (pout[1] - inverse[13]) / inverse[14];
 
     const pin = [
       Math.round(((x + 1) * (responsive.width - this.graphPaddingRight)) / 2),
@@ -779,8 +745,7 @@ class Graph extends React.Component {
       redoDisabled,
       selectionTool,
       clipPercentileMin,
-      clipPercentileMax,
+      clipPercentileMax
-      layoutChoice
     } = this.props;
     const { mode, pendingClipPercentiles } = this.state;
 
@@ -923,7 +888,6 @@ class Graph extends React.Component {
                 />
               </Tooltip>
             </div>
-
             <div
               className="bp3-button-group"
               style={{
@@ -934,49 +898,6 @@ class Graph extends React.Component {
                 target={
                   <Button
                     type="button"
-                    data-testid="layout-choice"
-                    className="bp3-button bp3-icon-heatmap"
-                    style={{
-                      cursor: "pointer"
-                    }}
-                  />
-                }
-                position={Position.BOTTOM_RIGHT}
-                content={
-                  <div
-                    style={{
-                      display: "flex",
-                      justifyContent: "flex-start",
-                      alignItems: "flex-start",
-                      flexDirection: "column",
-                      padding: 10
-                    }}
-                  >
-                    <RadioGroup
-                      label="Layout Choice"
-                      onChange={this.handleLayoutChoiceChange}
-                      selectedValue={layoutChoice.current}
-                    >
-                      {layoutChoice.available.map(name => (
-                        <Radio label={name} value={name} key={name} />
-                      ))}
-                    </RadioGroup>
-                  </div>
-                }
-              />
-            </div>
-
-            <div
-              className="bp3-button-group"
-              style={{
-                marginLeft: 10
-              }}
-            >
-              <Popover
-                target={
-                  <Button
-                    type="button"
-                    data-testid="visualization-settings"
                     className={`bp3-button bp3-icon-timeline-bar-chart ${activeClipClass}`}
                     style={{
                       cursor: "pointer"
@@ -1008,7 +929,6 @@ class Graph extends React.Component {
                     >
                       <NumericInput
                         style={{ width: 50 }}
-                        data-testid={"clip-min-input"}
                         onValueChange={this.handleClipPercentileMinValueChange}
                         onKeyPress={this.handleClipOnKeyPress}
                         value={clipMin}
@@ -1029,7 +949,6 @@ class Graph extends React.Component {
                       <span style={{ marginRight: 5, marginLeft: 5 }}> - </span>
                       <NumericInput
                         style={{ width: 50 }}
-                        data-testid={"clip-max-input"}
                         onValueChange={this.handleClipPercentileMaxValueChange}
                         onKeyPress={this.handleClipOnKeyPress}
                         value={clipMax}
@@ -1049,7 +968,6 @@ class Graph extends React.Component {
                       />
                       <Button
                         type="button"
-                        data-testid="clip-commit"
                         className="bp3-button"
                         disabled={this.isClipDisabled()}
                         style={{

client/src/components/leftsidebar.js

@@ -1,4 +1,5 @@
 // jshint esversion: 6
+import _ from "lodash";
 import React from "react";
 import { connect } from "react-redux";
 import Categorical from "./categorical/categorical";
@@ -6,11 +7,10 @@ import Continuous from "./continuous/continuous";
 import GeneExpression from "./geneExpression";
 import * as globals from "../globals";
 import DynamicScatterplot from "./scatterplot/scatterplot";
-import Logo from "./framework/logo.js";
 
 @connect(state => ({
   responsive: state.responsive,
-  datasetTitle: state.config?.displayNames?.dataset,
+  datasetTitle: _.get(state.config, "displayNames.dataset"),
   scatterplotXXaccessor: state.controls.scatterplotXXaccessor,
   scatterplotYYaccessor: state.controls.scatterplotYYaccessor
 }))
@@ -28,6 +28,7 @@ class LeftSideBar extends React.Component {
     if cellxgene logo or tabs change, this must as well
     */
     const metadataSectionPadding = 0;
+    // scatterplotXXaccessor && scatterplotYYaccessor ? 450 : 0;
 
     return (
       <div
@@ -39,49 +40,20 @@ class LeftSideBar extends React.Component {
         }}
       >
         <p
+          data-testid="header"
           style={{
             position: "fixed",
             top: globals.cellxgeneTitleTopPadding,
             left: globals.leftSidebarWidth + globals.cellxgeneTitleLeftPadding,
-            margin: 0
+            margin: 0,
+            fontSize: globals.largestFontSize,
+            color: globals.darkerGrey,
+            width: "100%"
           }}
         >
-          <Logo size={32} />
+          cellxgene: {datasetTitle}
-          <span
-            style={{
-              fontSize: 28,
-              position: "relative",
-              top: -4,
-              fontWeight: "bold",
-              marginLeft: 5,
-              color: globals.logoColor,
-              userSelect: "none"
-            }}
-          >
-            cell<span
-              style={{
-                position: "relative",
-                top: 1,
-                fontWeight: 300,
-                fontSize: 24
-              }}
-            >
-              ×
-            </span>gene
-          </span>
-          <span
-            data-testid="header"
-            style={{
-              fontSize: 16,
-              display: "block",
-              position: "relative",
-              marginTop: 10,
-              top: -4
-            }}
-          >
-            {datasetTitle}
-          </span>
         </p>
+
         <div
           style={{
             height: responsive.height - metadataSectionPadding,

client/src/globals.js

@@ -31,7 +31,6 @@ export const darkGreen = "#448C4D";
 
 export const nonFiniteCellColor = lightGrey;
 export const defaultCellColor = "rgb(0,0,0,1)";
-export const logoColor = "black"; /* logo pink: "#E9429A" */
 
 /* typography constants */
 

client/src/reducers/categoricalSelection.js

@@ -1,9 +1,12 @@
+import _ from "lodash";
+
 import { ControlsHelpers } from "../util/stateManager";
 import * as globals from "../globals";
 
 function maxCategoryItems(state) {
-  return (
+  return _.get(
-    state.config.parameters?.["max-category-items"] ??
+    state.config,
+    "parameters.max-category-items",
     globals.configDefaults.parameters["max-category-items"]
   );
 }
@@ -30,15 +33,15 @@ const CategoricalSelection = (
       /*
       Set the specific category in this field to false
       */
-      const newCategoryValueSelected = Array.from(
+      const newCategorySelected = Array.from(
-        state[action.metadataField].categoryValueSelected
+        state[action.metadataField].categorySelected
       );
-      newCategoryValueSelected[action.categoryIndex] = true;
+      newCategorySelected[action.categoryIndex] = true;
       const newCategoricalSelection = {
         ...state,
         [action.metadataField]: {
           ...state[action.metadataField],
-          categoryValueSelected: newCategoryValueSelected
+          categorySelected: newCategorySelected
         }
       };
       return newCategoricalSelection;
@@ -48,15 +51,15 @@ const CategoricalSelection = (
       /*
       Set the specific category in this field to false
       */
-      const newCategoryValueSelected = Array.from(
+      const newCategorySelected = Array.from(
-        state[action.metadataField].categoryValueSelected
+        state[action.metadataField].categorySelected
       );
-      newCategoryValueSelected[action.categoryIndex] = false;
+      newCategorySelected[action.categoryIndex] = false;
       const newCategoricalSelection = {
         ...state,
         [action.metadataField]: {
           ...state[action.metadataField],
-          categoryValueSelected: newCategoryValueSelected
+          categorySelected: newCategorySelected
         }
       };
       return newCategoricalSelection;
@@ -70,9 +73,8 @@ const CategoricalSelection = (
         ...state,
         [action.metadataField]: {
           ...state[action.metadataField],
-          categorySelected: false,
+          categorySelected: Array.from(
-          categoryValueSelected: Array.from(
+            state[action.metadataField].categorySelected
-            state[action.metadataField].categoryValueSelected
           ).fill(false)
         }
       };
@@ -87,9 +89,8 @@ const CategoricalSelection = (
         ...state,
         [action.metadataField]: {
           ...state[action.metadataField],
-          categorySelected: true,
+          categorySelected: Array.from(
-          categoryValueSelected: Array.from(
+            state[action.metadataField].categorySelected
-            state[action.metadataField].categoryValueSelected
           ).fill(true)
         }
       };

client/src/reducers/controls.js

@@ -93,10 +93,9 @@ const Controls = (
     }
     case "request differential expression success": {
       const { world } = prevSharedState;
-      const varIndexName = world.schema.annotations.var.index;
       const _diffexpGenes = [];
       action.data.forEach(d => {
-        _diffexpGenes.push(world.varAnnotations.at(d[0], varIndexName));
+        _diffexpGenes.push(world.varAnnotations.at(d[0], "name"));
       });
       return {
         ...state,

client/src/reducers/crossfilter.js

@@ -20,11 +20,10 @@ const CrossfilterReducer = (
 ) => {
   switch (action.type) {
     case "initial data load complete (universe exists)": {
-      const { world, layoutChoice } = nextSharedState;
+      const { world } = nextSharedState;
       const crossfilter = World.createObsDimensions(
         new Crossfilter(world.obsAnnotations),
-        world,
+        world
-        layoutChoice.currentDimNames
       );
       return crossfilter;
     }
@@ -44,13 +43,9 @@ const CrossfilterReducer = (
     case "set clip quantiles":
     case "set World to current selection": {
       const { userDefinedGenes, diffexpGenes } = prevSharedState.controls;
-      const { world, layoutChoice } = nextSharedState;
+      const { world } = nextSharedState;
       let crossfilter = new Crossfilter(world.obsAnnotations);
-      crossfilter = World.createObsDimensions(
+      crossfilter = World.createObsDimensions(crossfilter, world);
-        crossfilter,
-        world,
-        layoutChoice.currentDimNames
-      );
       crossfilter = ControlsHelpers.createGeneDimensions(
         userDefinedGenes,
         diffexpGenes,
@@ -60,23 +55,6 @@ const CrossfilterReducer = (
       return crossfilter;
     }
 
-    case "set layout choice": {
-      /*
-      when switching layouts:
-      - delete the existing XY index
-      - add the new XY index (which implicitly selects all on it)
-      */
-      const { world, layoutChoice } = nextSharedState;
-      return state
-        .delDimension(layoutDimensionName("XY"))
-        .addDimension(
-          layoutDimensionName("XY"),
-          "spatial",
-          world.obsLayout.col(layoutChoice.currentDimNames[0]).asArray(),
-          world.obsLayout.col(layoutChoice.currentDimNames[1]).asArray()
-        );
-    }
-
     case "request user defined gene success": {
       const { world } = prevSharedState;
       const gene = action.data.genes[0];
@@ -90,9 +68,8 @@ const CrossfilterReducer = (
 
     case "request differential expression success": {
       const { world } = prevSharedState;
-      const varIndexName = world.schema.annotations.var.index;
       const genes = _.map(action.data, d =>
-        world.varAnnotations.at(d[0], varIndexName)
+        world.varAnnotations.at(d[0], "name")
       );
       const crossfilter = _.reduce(
         genes,
@@ -110,11 +87,10 @@ const CrossfilterReducer = (
 
     case "clear differential expression": {
       const { world } = prevSharedState;
-      const varIndexName = world.schema.annotations.var.index;
       const crossfilter = _.reduce(
         action.diffExp,
         (xfltr, values) => {
-          const name = world.varAnnotations.at(values[0], varIndexName);
+          const name = world.varAnnotations.at(values[0], "name");
           return xfltr.delDimension(diffexpDimensionName(name));
         },
         state
@@ -185,12 +161,10 @@ const CrossfilterReducer = (
     case "categorical metadata filter select":
     case "categorical metadata filter deselect": {
       const { categoricalSelection } = nextSharedState;
-      const { world } = prevSharedState;
       const cat = categoricalSelection[action.metadataField];
-      const col = world.obsAnnotations.col(action.metadataField);
       return state.select(obsAnnoDimensionName(action.metadataField), {
         mode: "exact",
-        values: ControlsHelpers.selectedValuesForCategory(cat, col)
+        values: ControlsHelpers.selectedValuesForCategory(cat)
       });
     }
 

client/src/reducers/graphSelection.js

@@ -7,8 +7,7 @@ const GraphSelection = (
 ) => {
   switch (action.type) {
     case "set clip quantiles":
-    case "reset World to eq Universe":
+    case "reset World to eq Universe": {
-    case "set layout choice": {
       return {
         ...state,
         selection: {

client/src/reducers/index.js

@@ -12,7 +12,6 @@ import graphSelection from "./graphSelection";
 import crossfilter from "./crossfilter";
 import colors from "./colors";
 import differential from "./differential";
-import layoutChoice from "./layoutChoice";
 import responsive from "./responsive";
 import controls from "./controls";
 import resetCache from "./resetCache";
@@ -20,33 +19,31 @@ import resetCache from "./resetCache";
 import undoableConfig from "./undoableConfig";
 
 const Reducer = undoable(
-    cascadeReducers([
+  cascadeReducers([
-        ["config", config],
+    ["config", config],
-        ["universe", universe],
+    ["universe", universe],
-        ["world", world],
+    ["world", world],
-        ["layoutChoice", layoutChoice],
+    ["categoricalSelection", categoricalSelection],
-        ["categoricalSelection", categoricalSelection],
+    ["continuousSelection", continuousSelection],
-        ["continuousSelection", continuousSelection],
+    ["graphSelection", graphSelection],
-        ["graphSelection", graphSelection],
+    ["crossfilter", crossfilter],
-        ["crossfilter", crossfilter],
+    ["colors", colors],
-        ["colors", colors],
+    ["controls", controls],
-        ["controls", controls],
+    ["differential", differential],
-        ["differential", differential],
+    ["responsive", responsive],
-        ["responsive", responsive],
+    ["resetCache", resetCache]
-        ["resetCache", resetCache]
+  ]),
-    ]),
+  [
-    [
+    "world",
-        "world",
+    "categoricalSelection",
-        "categoricalSelection",
+    "continuousSelection",
-        "continuousSelection",
+    "graphSelection",
-        "graphSelection",
+    "crossfilter",
-        "crossfilter",
+    "colors",
-        "colors",
+    "controls",
-        "controls",
+    "differential"
-        "differential",
+  ],
-        "layoutChoice"
+  undoableConfig
-    ],
-    undoableConfig
 );
 
 const store = createStore(Reducer, applyMiddleware(thunk));

client/src/reducers/layoutChoice.js

@@ -1,50 +0,0 @@
-/*
-we have a UI heuristic to pick the default layout, based on assumptions
-about commonly used names.  Preferentially, pick in the following order:
-
-  1. "umap"
-  2. "tsne"
-  3. "pca"
-  4. give up, use the first available
-*/
-function bestDefaultLayout(layouts) {
-  const preferredNames = ["umap", "tsne", "pca"];
-  const idx = preferredNames.findIndex(name => layouts.indexOf(name) !== -1);
-  if (idx !== -1) return preferredNames[idx];
-  return layouts[0];
-}
-
-const LayoutChoice = (
-  state = {
-    available: [], // all available choices
-    current: undefined, // name of the current layout, eg, 'umap'
-    currentDimNames: [] // dimension name
-  },
-  action,
-  nextSharedState
-) => {
-  switch (action.type) {
-    case "initial data load complete (universe exists)":
-    case "reset World to eq Universe": {
-      // set default to default
-      const { schema } = nextSharedState.world;
-      const available = schema.layout.obs.map(v => v.name);
-      const current = bestDefaultLayout(available);
-      const currentDimNames = schema.layout.obsByName[current].dims;
-      return { available, current, currentDimNames };
-    }
-
-    case "set layout choice": {
-      const { schema } = nextSharedState.world;
-      const current = action.layoutChoice;
-      const currentDimNames = schema.layout.obsByName[current].dims;
-      return { ...state, current, currentDimNames };
-    }
-
-    default: {
-      return state;
-    }
-  }
-};
-
-export default LayoutChoice;

client/src/reducers/undoableConfig.js

@@ -59,7 +59,7 @@ const saveOnActions = new Set([
   "categorical metadata filter select",
   "categorical metadata filter deselect",
   "categorical metadata filter all of these",
-  "categorical metadata filter none of these",
+  "categorical metadata none of these",
 
   "color by categorical metadata",
   "color by continuous metadata",
@@ -72,9 +72,7 @@ const saveOnActions = new Set([
   "store current cell selection as differential set 2",
 
   "set World to current selection",
-  "set clip quantiles",
+  "set clip quantiles"
-
-  "set layout choice"
 ]);
 
 /**
@@ -100,15 +98,9 @@ const applyPending = () => ({
   [actionKey]: "applyPending",
   [stateKey]: { fsm: null }
 });
-const skip = (fsm, transition) => ({
+const skip = fsm => ({ [actionKey]: "skip", [stateKey]: { fsm } });
-  [actionKey]: "skip",
-  [stateKey]: { fsm: transition.to !== "done" ? fsm : null }
-});
 const clear = () => ({ [actionKey]: "clear", [stateKey]: { fsm: null } });
-const save = (fsm, transition) => ({
+const save = fsm => ({ [actionKey]: "save", [stateKey]: { fsm } });
-  [actionKey]: "save",
-  [stateKey]: { fsm: transition.to !== "done" ? fsm : null }
-});
 
 /*
 Error handler for state transitions that are unexpected.  Called by

client/src/util/camera.js

@@ -5,9 +5,10 @@ const mp = require("mouse-position");
 const mb = require("mouse-pressed");
 const key = require("key-pressed");
 
-const panSpeed = 1.0; // changed from 0.4 to 1.0 per issue #722
+const panSpeed = 0.4;
 const scaleSpeed = 0.5;
 const scaleMax = 3;
+// const scaleMin = 1.15
 const scaleMin = 1.03;
 
 function attachCamera(canvas, opts) {

client/src/util/dataframe/labelIndex.js

@@ -3,8 +3,6 @@ Label indexing - map a label to & from an integer offset.  See Dataframe
 for how this is used.
 **/
 
-import { rangeFill as fillRange } from "../range";
-
 /*
 Private utility functions
 */
@@ -23,6 +21,14 @@ function extent(tarr) {
   return [min, max];
 }
 
+function fillRange(arr, start = 0) {
+  const larr = arr;
+  for (let i = 0, l = larr.length; i < l; i += 1) {
+    larr[i] = i + start;
+  }
+  return larr;
+}
+
 /* eslint-disable class-methods-use-this */
 class IdentityInt32Index {
   /*

client/src/util/range.js

@@ -1,45 +0,0 @@
-/*
-Array range creation
-
-range(start, stop, step) -> Array
-	This is identical to https://docs.python.org/3/library/functions.html#func-range
-	Returns new array filled with a range of numbers.
-
-	Usage:
-
-		range(stop) - start defaults to zero, step defaults to 1
-		range(start, stop, [step]) - step defaults to 1
-
-	Examples:
-		range(3) -> [0, 1, 2]
-		range(1, 3) -> [1, 2]
-		range(1, 5, 2) -> [1, 3]
-
-
-rangeFill(array, start, step) -> array
-	Fill entire array with values, from start, by step.  Returns first array.
-	start defaults to zero, step defaults to one.
-
-*/
-
-function _doFill(arr, start, step, count) {
-	for (let idx = 0, val = start; idx < count; idx += 1, val += step) {
-		arr[idx] = val;
-	}
-	return arr;
-}
-
-export function rangeFill(arr, start = 0, step = 1) {
-	return _doFill(arr, start, step, arr.length);
-}
-
-export function range(start, stop, step) {
-	if (start === undefined) return [];
-	if (stop === undefined) {
-		stop = start;
-		start = 0;
-	}
-	step = step || 1; // catch undefind and zero
-	const len = Math.max(Math.ceil((stop - start) / step), 0);
-	return _doFill(new Array(len), start, step, len);
-}

client/src/util/scaleLinear.js

@@ -9,14 +9,8 @@
 // this is is equivalent to d3.scaleLinear().domain([0,1]).range([-1,1])
 
 export default (domain, range) => {
-	const domainStart = domain[0];
+  const domainStart = domain[0];
-	const scale = (range[1] - range[0]) / (domain[1] - domain[0]);
+  const scale = (range[1] - range[0]) / (domain[1] - domain[0]);
-	const invScale = 1 / scale;
+  const rangeStart = range[0];
-	const rangeStart = range[0];
+  return value => (value - domainStart) * scale + rangeStart;
-	const f = value => (value - domainStart) * scale + rangeStart;
-
-	// inverter
-	f.invert = value => (value - rangeStart) * invScale + domainStart;
-
-	return f;
 };

client/src/util/stateManager/colorHelpers.js

@@ -1,12 +1,12 @@
 /*
 Helper functions for the embedded graph colors
 */
+import _ from "lodash";
 import * as d3 from "d3";
 import { interpolateRainbow, interpolateCool } from "d3-scale-chromatic";
 import * as globals from "../../globals";
 import parseRGB from "../parseRGB";
 import finiteExtent from "../finiteExtent";
-import { range } from "../range";
 
 /*
 create new colors state object.   Paramters:
@@ -37,7 +37,9 @@ function createColors(world, colorMode = null, colorAccessor = null) {
 }
 
 function createColorsByCategoricalMetadata(world, accessor) {
-  const { categories } = world.schema.annotations.obsByName[accessor];
+  const { categories } = _.filter(world.schema.annotations.obs, {
+    name: accessor
+  })[0];
 
   const scale = d3
     .scaleSequential(interpolateRainbow)
@@ -65,7 +67,7 @@ function createColorsByContinuousMetadata(world, accessor) {
   const scale = d3
     .scaleQuantile()
     .domain([min, max])
-    .range(range(colorBins - 1, -1, -1));
+    .range(_.range(colorBins - 1, -1, -1));
 
   /* pre-create colors - much faster than doing it for each obs */
   const colors = new Array(colorBins);
@@ -95,7 +97,7 @@ function createColorsByExpression(world, accessor) {
   const scale = d3
     .scaleQuantile()
     .domain([min, max])
-    .range(range(colorBins - 1, -1, -1));
+    .range(_.range(colorBins - 1, -1, -1));
 
   /* pre-create colors - much faster than doing it for each obs */
   const colors = new Array(colorBins);

client/src/util/stateManager/controlsHelpers.js

@@ -5,7 +5,7 @@ Helper functions for the controls reducer
 import _ from "lodash";
 
 import * as globals from "../../globals";
-import { rangeFill as fillRange } from "../range";
+import { fillRange } from "../typedCrossfilter/util";
 import {
   userDefinedDimensionName,
   diffexpDimensionName
@@ -21,16 +21,16 @@ Remember that option values can be ANY js type, except undefined/null.
   {
     _category_name_1: {
       // map of option value to index
-      categoryValueIndices: Map([
+      categoryIndices: Map([
         catval1: index,
         ...
       ])
 
       // index->selection true/false state
-      categoryValueSelected: [ true/false, true/false, ... ]
+      categorySelected: [ true/false, true/false, ... ]
 
       // number of options
-      numCategoryValues: number,
+      numCategories: number,
 
       // isTruncated - true if the options for selection has
       // been truncated (ie, was too large to implement)
@@ -56,31 +56,27 @@ function topNCategories(summary) {
 
 export function createCategoricalSelection(maxCategoryItems, world) {
   const res = {};
-  const obsIndexName = world.schema.annotations.obs.index;
   _.forEach(world.obsAnnotations.colIndex.keys(), key => {
     const summary = world.obsAnnotations.col(key).summarize();
     if (summary.categories) {
       const isColorField = key.includes("color") || key.includes("Color");
       const isSelectableCategory =
         !isColorField &&
-        key !== obsIndexName &&
+        key !== "name" &&
         summary.categories.length < maxCategoryItems;
       if (isSelectableCategory) {
-        const [categoryValues, categoryValueCounts] = topNCategories(summary);
+        const [categoryValues, categoryCounts] = topNCategories(summary);
-        const categoryValueIndices = new Map(
+        const categoryIndices = new Map(categoryValues.map((v, i) => [v, i]));
-          categoryValues.map((v, i) => [v, i])
+        const numCategories = categoryIndices.size;
-        );
+        const categorySelected = new Array(numCategories).fill(true);
-        const numCategoryValues = categoryValueIndices.size;
-        const categoryValueSelected = new Array(numCategoryValues).fill(true);
         const isTruncated = categoryValues.length < summary.numCategories;
         res[key] = {
           categoryValues, // array: of natively typed category values
-          categoryValueIndices, // map: category value (native type) -> category index
+          categoryIndices, // map: category value (native type) -> category index
-          categoryValueSelected, // array: t/f selection state
+          categorySelected, // array: t/f selection state
-          numCategoryValues, // number: of values in the category
+          numCategories, // number: of categories
           isTruncated, // bool: true if list was truncated
-          categoryValueCounts, // array: cardinality of each category,
+          categoryCounts // array: cardinality of each category
-          categorySelected: true // bool - default state for entire category
         };
       }
     }
@@ -92,26 +88,12 @@ export function createCategoricalSelection(maxCategoryItems, world) {
 given a categoricalSelection, return the list of all category values
 where selection state is true (ie, they are selected).
 */
-export function selectedValuesForCategory(categorySelectionState, dfColumn) {
+export function selectedValuesForCategory(categorySelectionState) {
-  const {
+  const selectedValues = _([...categorySelectionState.categoryIndices])
-    categorySelected,
+    .filter(tuple => categorySelectionState.categorySelected[tuple[1]])
-    categoryValueSelected,
+    .map(tuple => tuple[0])
-    categoryValueIndices
+    .value();
-  } = categorySelectionState;
+  return selectedValues;
-  let selectedValues;
-  if (categorySelected) {
-    selectedValues = new Set(dfColumn.summarize().categories);
-  } else {
-    selectedValues = new Set();
-  }
-  categoryValueIndices.forEach((catIndex, catValue) => {
-    if (!categoryValueSelected[catIndex]) {
-      selectedValues.delete(catValue);
-    } else {
-      selectedValues.add(catValue);
-    }
-  });
-  return [...selectedValues.values()];
 }
 
 /*

client/src/util/stateManager/universe.js

@@ -78,7 +78,6 @@ function AnnotationsFBSToDataframe(arrayBuffer) {
   The application has strong assumptions that all scalar data will be
   stored as a float32 or float64 (regardless of underlying data types).
   For example, clipping of value ranges (eg, user-selected percentiles)
-  depends on the ability to use NaN in any numeric type.
 
   All float data from the server is left as is.  All non-float is promoted
   to an appropriate float.
@@ -99,16 +98,15 @@ function AnnotationsFBSToDataframe(arrayBuffer) {
 
 function LayoutFBSToDataframe(arrayBuffer) {
   const fbs = decodeMatrixFBS(arrayBuffer, true);
-  if (fbs.columns.length < 2 || !fbs.columns.every(isFpTypedArray)) {
+  if (fbs.columns.length !== 2 || !fbs.columns.every(isFpTypedArray)) {
     // We have strong assumptions about the shape & type of layout data.
     throw new Error("Unexpected layout data type returned from server");
   }
-
   const df = new Dataframe.Dataframe(
     [fbs.nRows, fbs.nCols],
     fbs.columns,
     null,
-    new Dataframe.KeyIndex(fbs.colIdx)
+    new Dataframe.KeyIndex(["X", "Y"])
   );
   return df;
 }
@@ -124,15 +122,15 @@ function reconcileSchemaCategoriesWithSummary(universe) {
   cases, add a 'categories' field to the schema so it is accessible.
   */
 
-  universe.schema.annotations.obs.columns.forEach(s => {
+  _.forEach(universe.schema.annotations.obs, s => {
     if (
       s.type === "string" ||
       s.type === "boolean" ||
       s.type === "categorical"
     ) {
       const categories = _.union(
-        s.categories ?? [],
+        _.get(s, "categories", []),
-        universe.obsAnnotations.col(s.name).summarize().categories ?? []
+        _.get(universe.obsAnnotations.col(s.name).summarize(), "categories", [])
       );
       s.categories = categories;
     }
@@ -156,9 +154,6 @@ export function createUniverseFromResponse(
   universe.schema = schema;
   universe.nObs = schema.dataframe.nObs;
   universe.nVar = schema.dataframe.nVar;
-  /* add defaults, as we can't assume back-end will fully populate schema */
-  if (!schema.layout.var) schema.layout.var = [];
-  if (!schema.layout.obs) schema.layout.obs = [];
 
   /* annotations */
   universe.obsAnnotations = AnnotationsFBSToDataframe(annotationsObsResponse);
@@ -179,16 +174,10 @@ export function createUniverseFromResponse(
 
   /* Index schema for ease of use */
   universe.schema.annotations.obsByName = fromEntries(
-    universe.schema.annotations.obs.columns.map(v => [v.name, v])
+    universe.schema.annotations.obs.map(v => [v.name, v])
   );
   universe.schema.annotations.varByName = fromEntries(
-    universe.schema.annotations.var.columns.map(v => [v.name, v])
+    universe.schema.annotations.var.map(v => [v.name, v])
-  );
-  universe.schema.layout.obsByName = fromEntries(
-    universe.schema.layout.obs.map(v => [v.name, v])
-  );
-  universe.schema.layout.varByName = fromEntries(
-    universe.schema.layout.var.map(v => [v.name, v])
   );
   return universe;
 }
@@ -213,9 +202,8 @@ export function convertDataFBStoObject(universe, arrayBuffer) {
     throw new Error("Unexpected non-floating point response from server.");
   }
 
-  const varIndexName = universe.schema.annotations.var.index;
   for (let c = 0; c < colIdx.length; c += 1) {
-    const varName = universe.varAnnotations.at(colIdx[c], varIndexName);
+    const varName = universe.varAnnotations.at(colIdx[c], "name");
     result[varName] = columns[c];
   }
   return result;

client/src/util/stateManager/world.js

@@ -260,17 +260,13 @@ function deduceDimensionType(attributes, fieldName) {
   return dimensionType;
 }
 
-export function createObsDimensions(crossfilter, world, XYdimNames) {
+export function createObsDimensions(crossfilter, world) {
   /*
   create and return a crossfilter with a dimension for every obs annotation
-  for which we have a supported type, *except* for the index column, indicated
+  for which we have a supported type, *except* 'name'
-  by schema.annotations.obs.index.
   */
   const { schema, obsLayout, obsAnnotations } = world;
-  const indexName = schema.annotations.obs.index;
+  const annoList = schema.annotations.obs.filter(anno => anno.name !== "name");
-  const annoList = schema.annotations.obs.columns.filter(
-    anno => anno.name !== indexName
-  );
   crossfilter = annoList.reduce((xfltr, anno) => {
     const dimType = deduceDimensionType(anno, anno.name);
     const colData = obsAnnotations.col(anno.name).asArray();
@@ -287,8 +283,8 @@ export function createObsDimensions(crossfilter, world, XYdimNames) {
   return crossfilter.addDimension(
     layoutDimensionName("XY"),
     "spatial",
-    obsLayout.col(XYdimNames[0]).asArray(),
+    obsLayout.col("X").asArray(),
-    obsLayout.col(XYdimNames[1]).asArray()
+    obsLayout.col("Y").asArray()
   );
 }
 

client/src/util/typedCrossfilter/util.js

@@ -1,12 +1,22 @@
 // jshint esversion: 6
 
 import { sortIndex } from "./sort";
-import { rangeFill as fillRange } from "../range";
 
 /*
     Utility functions, private to this module.
 */
 
+// fill an array or typedarray with a sequential range of numbers,
+// starting with `start`
+//
+export function fillRange(arr, start = 0) {
+  const larr = arr;
+  for (let i = 0, len = larr.length; i < len; i += 1) {
+    larr[i] = i + start;
+  }
+  return larr;
+}
+
 // slice out of one array into another, using an index array
 //
 export function sliceByIndex(src, index) {

dev_docs/developer_guidelines.md

@@ -1,91 +0,0 @@
-# Developer guidelines
-
-### Requirements
-- npm
-- Python 3.6+
-- Chrome
-
-[See dev section of README](../README.md)
-
-**All instructions are expected to be run from the top level cellxgene directory unless otherwise specified.**
-
-## Server dev
-### Install
-* Build the client and put static files in place: `make build-for-server-dev`
-* Install from local files: `make install-dev`
-
-### Launch
-* `cellxgene launch [options] <datafile>`
-
-### Reloading
-If you install cellxgene using `make install-dev` the server will be restarted every time you make changes on the server code. If changes affects the client, the browser must be reloaded.
-
-### Linter
-We use `flake8` to lint code. Travis CI runs `flake8 server`.
-
-### Test
-1. Install development requirements `pip install -r server/requirements-dev.txt`
-2. Run tests `pytest server/test`
-
-### Tips
-* Install in a virtualenv
-* May need to rebuild/reinstall when you make client changes
-
-## Client dev
-### Install
-1. Install prereqs for client: `npm install --prefix client/ client`
-2. Install cellxgene server: `pip install -e .` Caveat: this will not build the production client package - you must use the [server install](#install) instructions above to serve web assets.
-
-### Launch
-To launch with hot reloading you need to launch the server and the client separately. Node's hot reloading starts the client on its own node server and auto-refreshes when changes are made.
-1. Launch server (the client relies on the REST API being available): `cellxgene launch [options] <datafile>`
-2. Launch client: in `client/` directory run `npm run start`
-3. Client will be served on localhost:3000
-
-### Build
-To build only the client: `make build-client`
-
-### Linter
-We use `eslint` to lint the code and `prettier` as our code formatter.
-
-### Test
-In `client/` directory run `npm run unit-test`
-
-### Tips
-* You can also install/launch the server side code from npm scrips (requires python3.6 with virtualenv) in `client/` directory run `npm run backend-dev`
-
-## Running tests
-Client and server tests run on Travis CI for every push, PR, and commit to master on github. End to end tests run nightly on master only. 
-
-### Server unit tests
-Install development requirements `pip install -r server/requirements-dev.txt`
-Run tests `pytest server/test`
-
-### Client unit tests
-In `client/` directory run `npm run unit-test`
-
-### End to end tests
-
-End to end tests use two env variables:
-* `JEST_ENV` - environment to run end to end tests. Default `dev`
-    * `prod` - run headless with no slowdown, chromium will not open.
-    * `dev` - opens chromimum, runs tests with minimal slowdown, close on exit.
-    * `debug` - opens chromium, runs tests with 100ms slowdown, dev tools open, chrome stays open on exit.
-* `JEST_CXG_PORT` - port that end to end tests are being run on. Default `3000` (client hosted port).
-
-On CI the end to end tests are run with `JEST_ENV` set to `prod` using the `smoke-test` npm script
-
-To run end to end tests as they will be run on CI
-1. cellxgene should be built and installed as [specified in server dev](#install)
-2. `export JEST_ENV='prod'`
-3. `export JEST_CXG_PORT='5000'`
-4. Run `npm run --prefix client/ smoke-test`
-
-Run end to end tests interactively during development
-1. cellxgene should be installed as [specified in client dev](#install-1)
-2. Follow [launch](#launch-1) instructions for client dev with dataset `example-dataset/pbmc3k`
-3. Run `npm run --prefix client/ e2e`
-4. To debug a failing test `export JEST_ENV='debug'` and re-run.
-
-
-

dev_docs/release_process.md

@@ -43,53 +43,24 @@ Follow these steps to create a release.
 8. Publish to pypi by performing the following steps (assumes you that you have registered for pypi, 
 and that you have write access to the cellxgene pypi package):
     - Build the distribution and upload to test pypi `make release-stage-2`
-    - Test the test installation in a fresh virtual environment using `make install-release-test`
+    - [optional] Test the test installation in a fresh virtual environment using `make install-release-test`
     - Upload the package to real pypi using `make release-stage-final`
-    - Test the installation in a fresh virtual environment using
+    - [optional] Test the installation in a fresh virtual environment using
       `pip install cellxgene`
+    - **Troubleshooting**:
+        - Fails to upload to test.pypi: pypi doesn't allow you to reupload a release with the same version number, 
+        if you accidentally burned a release number you want to use on prod, you have a couple options. 
+            1) OPTION 1: Create distribution `make pydist`; test release locally `pip install dist/<release tarball>`;
+             then upload to prod `make release-stage-final`. 
+            2) OPTION 2: (DANGER) release directly to prod: `make release-burned`.  
+            3) OPTION 3: If the release was burned on prod as well run from Step 3 again with option
+            PART=patch until you get to an unburned version.
+        - The release doesn't install or fails your tests when you install it: Delete it from pypi - Go to pypi.org, sign in, 
+        go to the cellxgene package, click manage, then in the options drop down, click delete and 
+        follow the instructions. You will not be able to use that release number again. If it is a minor bug 
+        and not a major regression, you can just release a patch. 
+    
 
 The optional steps are for testing purposes, and are recommended
 for publishing any major releases, and any releases that significantly
 change the packaging (e.g. new bundled files, new dependencies, etc.)
-
-## Troubleshooting
-### Fails to upload to test.pypi
- 
- _PyPi doesn't allow you to reupload a release with the same version number_ 
-If you accidentally burned a release number you want to use on prod, you have a few options:
-1) OPTION 1: Create distribution `make pydist`; test release locally `pip install dist/<release tarball>`;
- then upload to prod `make release-stage-final`. 
-2) OPTION 2: (DANGER) release directly to prod: `make release-directly-to-prod`.  
-3) OPTION 3: If the release was burned on prod as well run from Step 3 again with option
-PART=patch until you get to an unburned version.
-    
-### The release doesn't install or fails your tests when you install it
- 
-Delete it from pypi - Go to pypi.org -> sign in -> go to the cellxgene package -> click manage -> then in the options drop down click delete -> follow the instructions. You will not be able to use that release number again. If it is a minor bug and not a major regression, you can just release a patch. 
-### If you need to run stage final on a different computer than stage 2
- If you run stage final without running stage 2 first, the dist will not have been build on the computer running stage final. The solution is to run `make release-directly-to-prod`. This both builds the distribution files and then releases directly to prod pypi.org. 
-
-## Stage Details
-### Stage 1 - `make release-stage-1`
-    1. Pip installs requirements-dev
-    2. Bumps version by [PART]
-    3. Deletes build directory, client/build, dist and cellxgene.egg-info
-    4. Creates the package-lock.json
-
-### Stage 2 - `make release-stage-2`
-    1. Pip installs requirements-dev
-    2. Builds client and server
-    3. Creates distribution release (sdist)
-    4. Uploads to test.pypi.org
-
-### Stage final - `make release-stage-final`
-    ** Does not build distribution **
-    1. Uploads to pypi.org
-    
-### (DANGER) Release directly to prod `make release-directly-to-prod`
-    ** builds distribution and uploads directly to prod **
-    Only use this if you are directed to by the troubleshooting guide
-    1. Pip installs requirements-dev
-    2. Builds client and server
-    3. Creates distribution release (sdist)
-    4. Uploads to pypi.org

docs/faq.md

@@ -38,7 +38,7 @@ Currently this is not supported directly, but you should be able to do this your
 
 - `.obs` and `.var` annotations are use to extract metadata for filtering
 - `.X` is used to display expression (histograms, scatterplot & colorscale) and to compute differential expression
-- `.obsm` is used for layout. If an embedding has more than two components, the first two will be used for visualization.
+- `.obsm` is used for layout
 
 #### I have a BIG dataset - how can I make cellxgene run as fast as possible?
 

makefile

@@ -76,7 +76,7 @@ release-stage-final: twine-prod
 
 # DANGER: releases directly to prod
 # use this if you accidently burned a test release version number,
-release-directly-to-prod : dev-env pydist twine-prod
+release-burned : dev-env pydist twine-prod
 	@echo "Dist built and uploaded to pypi.org"
 	@echo "Test the install:"
 	@echo "    make install-release"
@@ -114,18 +114,14 @@ install-dev : uninstall
 
 # install from test.pypi to test your release
 install-release-test : uninstall
-	pip install --no-cache-dir --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple cellxgene
+	pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple cellxgene
 	@echo "Installed cellxgene from test.pypi.org, now run and smoke test"
 
 # install from pypi to test your release
 install-release : uninstall
-	pip install --no-cache-dir cellxgene
+	pip install cellxgene
 	@echo "Installed cellxgene from pypi.org"
 
-# install from dist
-install-dist : uninstall
-	pip install dist/cellxgene*.tar.gz
-
 uninstall :
 	pip uninstall -y cellxgene || :
 

server/app/scanpy_engine/scanpy_engine.py

@@ -1,13 +1,12 @@
 import warnings
 
 import numpy as np
-import pandas
 from pandas.core.dtypes.dtypes import CategoricalDtype
-import anndata
+import scanpy as sc
 from scipy import sparse
 
 from server.app.driver.driver import CXGDriver
-from server.app.util.constants import Axis, DEFAULT_TOP_N, MAX_LAYOUTS
+from server.app.util.constants import Axis, DEFAULT_TOP_N
 from server.app.util.errors import (
     FilterError,
     JSONEncodingValueError,
@@ -42,7 +41,7 @@ class ScanpyEngine(CXGDriver):
     @staticmethod
     def _get_default_config():
         return {
-            "layout": [],
+            "layout": "umap",
             "diffexp": "ttest",
             "max_category_items": 100,
             "obs_names": None,
@@ -50,61 +49,41 @@ class ScanpyEngine(CXGDriver):
             "diffexp_lfc_cutoff": 0.01,
         }
 
-    @staticmethod
+    def _alias_annotation_names(self, axis, name):
-    def _create_unique_column_name(df, col_name_prefix):
-        """ given the columns of a dataframe, and a name prefix, return a column name which
-            does not exist in the dataframe, AND which is prefixed by `prefix`
-
-            The approach is to append a numeric suffix, starting at zero and increasing by
-            one, until an unused name is found (eg, prefix_0, prefix_1, ...).
         """
-        suffix = 0
+        Do all user-specified annotation aliasing.
-        while f"{col_name_prefix}{suffix}" in df:
-            suffix += 1
-        return f"{col_name_prefix}{suffix}"
 
-    def _alias_annotation_names(self):
+        As a *critical* side-effect, ensure the indices are simple number ranges
+        (accomplished by calling pandas.DataFrame.reset_index())
         """
-        The front-end relies on the existance of a unique, human-readable
+        if name == "name":
-        index for obs & var (eg, var is typically gene name, obs the cell name).
+            # a noop, so skip it
-        The user can specify these via the --obs-names and --var-names config.
+            return
-        If they are not specified, use the existing index to create them, giving
-        the resulting column a unique name (eg, "name").
 
-        In both cases, enforce that the result is unique, and communicate the
+        ax_name = str(axis)
-        index column name to the front-end via the obs_names and var_names config
+        df_axis = getattr(self.data, ax_name)
-        (which is incorporated into the schema).
+        if name is None:
-        """
+            # reset index to simple range; alias "name" to point at the
-        for (ax_name, config_name) in ((Axis.OBS, "obs_names"), (Axis.VAR, "var_names")):
+            # previously specified index.
-            name = self.config[config_name]
+            df_axis.reset_index(inplace=True)
-            df_axis = getattr(self.data, str(ax_name))
+            df_axis.rename(inplace=True, columns={"index": "name"})
-            if name is None:
+        elif name in df_axis.columns:
-                # Default: create unique names from index
+            if name not in df_axis.columns:
-                if not df_axis.index.is_unique:
-                    raise KeyError(
-                        f"Values in {ax_name}.index must be unique. "
-                        "Please prepare data to contain unique index values, or specify an "
-                        "alternative with --{ax_name}-name."
-                    )
-                name = self._create_unique_column_name(df_axis.columns, "name_")
-                self.config[config_name] = name
-                # reset index to simple range; alias name to point at the
-                # previously specified index.
-                df_axis.rename_axis(name, inplace=True)
-                df_axis.reset_index(inplace=True)
-            elif name in df_axis.columns:
-                # User has specified alternative column for unique names, and it exists
-                if not df_axis[name].is_unique:
-                    raise KeyError(
-                        f"Values in {ax_name}.{name} must be unique. "
-                        "Please prepare data to contain unique values."
-                    )
-                df_axis.reset_index(drop=True, inplace=True)
-            else:
-                # user specified a non-existent column name
                 raise KeyError(
                     f"Annotation name {name}, specified in --{ax_name}-name does not exist."
                 )
+            if not df_axis[name].is_unique:
+                raise KeyError(
+                    f"Values in -{ax_name}-name must be unique. "
+                    "Please prepare data to contain unique values."
+                )
+            # reset index to simple range; alias user-specified annotation to "name"
+            df_axis.reset_index(drop=True, inplace=True)
+            df_axis.rename(inplace=True, columns={name: "name"})
+        else:
+            raise KeyError(
+                f"Annotation name {name}, specified in --{ax_name}_name does not exist."
+            )
 
     @staticmethod
     def _can_cast_to_float32(ann):
@@ -134,17 +113,7 @@ class ScanpyEngine(CXGDriver):
                 "nVar": self.gene_count,
                 "type": str(self.data.X.dtype),
             },
-            "annotations": {
+            "annotations": {"obs": [], "var": []},
-                "obs": {
-                    "index": self.config["obs_names"],
-                    "columns": []
-                },
-                "var": {
-                    "index": self.config["var_names"],
-                    "columns": []
-                }
-            },
-            "layout": {"obs": []}
         }
         for ax in Axis:
             curr_axis = getattr(self.data, str(ax))
@@ -168,21 +137,14 @@ class ScanpyEngine(CXGDriver):
                     raise TypeError(
                         f"Annotations of type {curr_axis[ann].dtype} are unsupported by cellxgene."
                     )
-                self.schema["annotations"][ax]["columns"].append(ann_schema)
+                self.schema["annotations"][ax].append(ann_schema)
-
-        for layout in self.config['layout']:
-            layout_schema = {
-                "name": layout,
-                "type": "float32",
-                "dims": [f"{layout}_0", f"{layout}_1"]
-            }
-            self.schema["layout"]["obs"].append(layout_schema)
 
     def _load_data(self, data):
-        # as of AnnData 0.6.19, backed mode performs initial load fast, but at the
+        # Based on benchmarking, cache=True has no impact on perf.
-        # cost of significantly slower access to X data.
+        # Note: as of current scanpy/anndata release, setting backed='r' will
+        # result in an error.  https://github.com/theislab/anndata/issues/79
         try:
-            self.data = anndata.read_h5ad(data)
+            self.data = sc.read(data, cache=True)
         except ValueError:
             raise ScanpyFileError(
                 "File must be in the .h5ad format. Please read "
@@ -202,68 +164,16 @@ class ScanpyEngine(CXGDriver):
 
     @requires_data
     def _validate_and_initialize(self):
-        # var and obs column names must be unique
+        self._alias_annotation_names(Axis.OBS, self.config["obs_names"])
-        if not self.data.obs.columns.is_unique or not self.data.var.columns.is_unique:
+        self._alias_annotation_names(Axis.VAR, self.config["var_names"])
-            raise KeyError(f"All annotation column names must be unique.")
-
-        self._alias_annotation_names()
         self._validate_data_types()
+        self._validate_data_calculations()
         self.cell_count = self.data.shape[0]
         self.gene_count = self.data.shape[1]
-        self._default_and_validate_layouts()
         self._create_schema()
 
-    @requires_data
-    def _default_and_validate_layouts(self):
-        """ function:
-            a) generate list of default layouts, if not already user specified
-            b) validate layouts are legal.  remove/warn on any that are not
-            c) cap total list of layouts at global const MAX_LAYOUTS
-        """
-        layouts = self.config['layout']
-        # handle default
-        if layouts is None or len(layouts) == 0:
-            # load default layouts from the data.
-            layouts = [key[2:] for key in self.data.obsm_keys() if type(key) == str and key.startswith("X_")]
-            if len(layouts) == 0:
-                raise PrepareError(f"Unable to find any precomputed layouts within the dataset.")
-
-        # remove invalid layouts
-        valid_layouts = []
-        obsm_keys = self.data.obsm_keys()
-        for layout in layouts:
-            layout_name = f"X_{layout}"
-            if layout_name not in obsm_keys:
-                warnings.warn(f"Ignoring unknown layout name: {layout}.")
-            elif not self._is_valid_layout(self.data.obsm[layout_name]):
-                warnings.warn(f"Ignoring layout due to malformed shape or data type: {layout}")
-            else:
-                valid_layouts.append(layout)
-
-        if len(valid_layouts) == 0:
-            raise PrepareError(f"No valid layout data.")
-
-        # cap layouts to MAX_LAYOUTS
-        self.config['layout'] = valid_layouts[0:MAX_LAYOUTS]
-
-    @requires_data
-    def _is_valid_layout(self, arr):
-        """ return True if this layout data is a valid array for front-end presentation:
-            * ndarray, with shape (n_obs, >= 2), dtype float/int/uint
-            * contains only finite values
-        """
-        is_valid = type(arr) == np.ndarray and arr.dtype.kind in "fiu"
-        is_valid = is_valid and arr.shape[0] == self.data.n_obs and arr.shape[1] >= 2
-        is_valid = is_valid and np.all(np.isfinite(arr))
-        return is_valid
-
     @requires_data
     def _validate_data_types(self):
-        if sparse.isspmatrix(self.data.X) and not sparse.isspmatrix_csc(self.data.X):
-            warnings.warn(
-                f"Scanpy data matrix is sparse, but not a CSC (columnar) matrix.  "
-                f"Performance may be improved by using CSC."
-            )
         if self.data.X.dtype != "float32":
             warnings.warn(
                 f"Scanpy data matrix is in {self.data.X.dtype} format not float32. "
@@ -294,6 +204,20 @@ class ScanpyEngine(CXGDriver):
                             f"annotations with more than 500 categories in the UI"
                         )
 
+    @requires_data
+    def _validate_data_calculations(self):
+        layout_key = f"X_{self.config['layout']}"
+        try:
+            assert layout_key in self.data.obsm_keys()
+        except AssertionError:
+            raise PrepareError(
+                f"Cannot find a field with coordinates for the {self.config['layout']} layout requested. A different"
+                f" layout may have been computed. The requested layout must be pre-calculated and saved "
+                f"back in the h5ad file. You can run "
+                f"`cellxgene prepare --layout {self.config['layout']} <datafile>` "
+                f"to solve this problem. "
+            )
+
     @staticmethod
     def _annotation_filter_to_mask(filter, d_axis, count):
         mask = np.ones((count,), dtype=bool)
@@ -442,33 +366,17 @@ class ScanpyEngine(CXGDriver):
         Caveats:
         * does not support filtering
         * only returns Matrix in columnar layout
-
-        All embeddings must be individually centered & scaled (isotropically)
-        to a [0, 1] range.
         """
         try:
-            layout_data = []
+            full_embedding = self.data.obsm[f"X_{self.config['layout']}"]
-            for layout in self.config["layout"]:
+            if full_embedding.shape[1] > 2:
-                full_embedding = self.data.obsm[f"X_{layout}"]
+                warnings.warn(f"Warning: found {full_embedding.shape[1]} \
-                embedding = full_embedding[:, :2]
+                components of embedding. Using the first two for layout display.")
-
+            df_layout = full_embedding[:, :2]
-                # scale isotropically
-                min = embedding.min(axis=0)
-                max = embedding.max(axis=0)
-                scale = np.amax(max - min)
-                normalized_layout = (embedding - min) / scale
-
-                # translate to center on both axis
-                translate = 0.5 - ((max - min) / scale / 2)
-                normalized_layout = normalized_layout + translate
-
-                normalized_layout = normalized_layout.astype(dtype=np.float32)
-                layout_data.append(pandas.DataFrame(normalized_layout, columns=[f"{layout}_0", f"{layout}_1"]))
-
         except ValueError as e:
             raise PrepareError(
                 f"Layout has not been calculated using {self.config['layout']}, "
                 f"please prepare your datafile and relaunch cellxgene") from e
 
-        df = pandas.concat(layout_data, axis=1, copy=False)
+        normalized_layout = (df_layout - df_layout.min()) / (df_layout.max() - df_layout.min())
-        return encode_matrix_fbs(df, col_idx=df.columns, row_idx=None)
+        return encode_matrix_fbs(normalized_layout.astype(dtype=np.float32), col_idx=None, row_idx=None)

server/app/util/constants.py

@@ -31,5 +31,3 @@ JSON_NaN_to_num_warning_msg = (
     "JSON encoding failure - please verify all data are finite values (no NaN or Infinities)"
 )
 REACTIVE_LIMIT = 1_000_000
-
-MAX_LAYOUTS = 30

server/cli/cli.py

@@ -5,7 +5,7 @@ from .prepare import prepare
 
 
 @click.group(name="cellxgene", context_settings=dict(max_content_width=85))
-@click.version_option(version="0.10.1", prog_name="cellxgene", message="[%(prog)s] Version %(version)s")
+@click.version_option(version="0.9.2", prog_name="cellxgene", message="[%(prog)s] Version %(version)s")
 def cli():
     pass
 

server/cli/launch.py

@@ -1,7 +1,6 @@
-import errno
 import logging
 from os import devnull
-from os.path import splitext, basename, getsize
+from os.path import splitext, basename
 import sys
 import warnings
 import webbrowser
@@ -11,11 +10,7 @@ import click
 from server.app.app import Server
 from server.app.util.errors import ScanpyFileError
 from server.app.util.utils import custom_format_warning
-from server.utils.utils import find_available_port, is_port_available
+from server.utils.constants import MODES
-
-
-# anything bigger than this will generate a special message
-BIG_FILE_SIZE_THRESHOLD = 100 * 2**20  # 100MB
 
 
 @click.command()
@@ -23,10 +18,10 @@ BIG_FILE_SIZE_THRESHOLD = 100 * 2**20  # 100MB
 @click.option(
     "--layout",
     "-l",
-    default=[],
+    type=click.Choice(MODES),
-    multiple=True,
+    default="umap",
     show_default=True,
-    help="Layout name, eg, 'umap'."
+    help="Method for layout."
 )
 @click.option(
     "--diffexp",
@@ -55,8 +50,7 @@ BIG_FILE_SIZE_THRESHOLD = 100 * 2**20  # 100MB
     show_default=True,
     help="Open the web browser after launch.",
 )
-@click.option("--port", "-p", help="Port to run server on, if not specified cellxgene will find an available port.",
+@click.option("--port", "-p", help="Port to run server on.", metavar="", default=5005, show_default=True)
-              metavar="", show_default=True)
 @click.option("--obs-names", default=None, metavar="", help="Name of annotation field to use for observations.")
 @click.option("--var-names", default=None, metavar="", help="Name of annotation to use for variables.")
 @click.option("--host", default="127.0.0.1", help="Host IP address")
@@ -141,16 +135,6 @@ security risk by including the --scripts flag. Make sure you trust the scripts t
         file_parts = splitext(basename(data))
         title = file_parts[0]
 
-    if port:
-        if debug:
-            raise click.ClickException("--port and --debug may not be used together (try --verbose for error logging).")
-        if not is_port_available(host, int(port)):
-            raise click.ClickException(
-                f"The port selected {port} is in use, please specify an open port using the --port flag."
-            )
-    else:
-        port = find_available_port(host)
-
     # Setup app
     cellxgene_url = f"http://{host}:{port}"
 
@@ -164,13 +148,7 @@ security risk by including the --scripts flag. Make sure you trust the scripts t
         log = logging.getLogger("werkzeug")
         log.setLevel(logging.ERROR)
 
-    file_size = getsize(data)
+    click.echo(f"[cellxgene] Loading data from {basename(data)}, this may take awhile...")
-
-    # if a big file, let the user know it may take a while to load.
-    if file_size > BIG_FILE_SIZE_THRESHOLD:
-        click.echo(f"[cellxgene] Loading data from {basename(data)}, this may take awhile...")
-    else:
-        click.echo(f"[cellxgene] Loading data from {basename(data)}.")
 
     # Fix for anaconda python. matplotlib typically expects python to be installed as a framework TKAgg is usually
     # available and fixes this issue. See https://matplotlib.org/faq/virtualenv_faq.html
@@ -205,9 +183,4 @@ security risk by including the --scripts flag. Make sure you trust the scripts t
         f = open(devnull, "w")
         sys.stdout = f
 
-    try:
+    server.app.run(host=host, debug=debug, port=port, threaded=True)
-        server.app.run(host=host, debug=debug, port=port, threaded=True, use_debugger=False)
-    except OSError as e:
-        if e.errno == errno.EADDRINUSE:
-            raise click.ClickException("Port is in use, please specify an open port using the --port flag.") from e
-        raise

server/requirements.txt

@@ -1,4 +1,3 @@
-anndata>=0.6.20
 click>=6.7
 Flask>=1.0.2
 Flask-Caching>=1.4.0
@@ -12,4 +11,4 @@ pandas>=0.23.1
 scanpy>=1.3.7
 scipy>=1.1.0
 scikit-learn>=0.19.1,!=0.20.0
-tables==3.5.1
+tables>=3.5.1

server/test/schema.json

@@ -5,61 +5,46 @@
     "type": "float32"
   },
   "annotations": {
-    "obs": {
-      "index": "name_0",
-      "columns": [
-        {
-          "name": "name_0",
-          "type": "string"
-        },
-        {
-          "name": "n_genes",
-          "type": "int32"
-        },
-        {
-          "name": "percent_mito",
-          "type": "float32"
-        },
-        {
-          "name": "n_counts",
-          "type": "float32"
-        },
-        {
-          "name": "louvain",
-          "type": "categorical",
-          "categories": [
-            "CD4 T cells",
-            "CD14+ Monocytes",
-            "B cells",
-            "CD8 T cells",
-            "NK cells",
-            "FCGR3A+ Monocytes",
-            "Dendritic cells",
-            "Megakaryocytes"
-          ]
-        }
-      ]
-    },
-    "var": {
-      "index": "name_0",
-      "columns": [
-        {
-          "name": "name_0",
-          "type": "string"
-        },
-        {
-          "name": "n_cells",
-          "type": "int32"
-        }
-      ]
-    }
-  },
-  "layout": {
     "obs": [
       {
-        "name": "umap",
+        "name": "name",
-        "type": "float32",
+        "type": "string"
-        "dims": ["umap_0", "umap_1"]
+      },
+      {
+        "name": "n_genes",
+        "type": "int32"
+      },
+      {
+        "name": "percent_mito",
+        "type": "float32"
+      },
+      {
+        "name": "n_counts",
+        "type": "float32"
+      },
+      {
+        "name": "louvain",
+        "type": "categorical",
+        "categories": [
+          "CD4 T cells",
+          "CD14+ Monocytes",
+          "B cells",
+          "CD8 T cells",
+          "NK cells",
+          "FCGR3A+ Monocytes",
+          "Dendritic cells",
+          "Megakaryocytes"
+        ]
+      }
+    ],
+    "var": [
+      {
+        "name": "name",
+        "type": "string"
+      },
+      {
+        "name": "n_cells",
+        "type": "int32"
       }
     ]
   }

server/test/test_api.py

@@ -19,12 +19,11 @@ class EndPoints(unittest.TestCase):
 
     @classmethod
     def setUpClass(cls):
-        cls.ps = Popen(["cellxgene", "launch", "example-dataset/pbmc3k.h5ad", "--verbose", "--port", "5005"])
+        cls.ps = Popen(["cellxgene", "launch", "example-dataset/pbmc3k.h5ad", "--debug"])
         session = requests.Session()
         for i in range(90):
             try:
-                result = session.get(f"{URL_BASE}schema")
+                session.get(f"{URL_BASE}schema")
-                cls.schema = result.json()
             except requests.exceptions.ConnectionError:
                 time.sleep(1)
 
@@ -46,8 +45,7 @@ class EndPoints(unittest.TestCase):
         self.assertEqual(result.headers["Content-Type"], "application/json")
         result_data = result.json()
         self.assertEqual(result_data["schema"]["dataframe"]["nObs"], 2638)
-        self.assertEqual(len(result_data["schema"]["annotations"]["obs"]), 2)
+        self.assertEqual(len(result_data["schema"]["annotations"]["obs"]), 5)
-        self.assertEqual(len(result_data["schema"]["annotations"]["obs"]["columns"]), 5)
 
     def test_config(self):
         endpoint = "config"
@@ -69,11 +67,9 @@ class EndPoints(unittest.TestCase):
         self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
         df = decode_fbs.decode_matrix_FBS(result.content)
         self.assertEqual(df['n_rows'], 2638)
-        self.assertEqual(df['n_cols'], 8)
+        self.assertEqual(df['n_cols'], 2)
         self.assertIsNotNone(df['columns'])
-        self.assertListEqual(df['col_idx'], [
+        self.assertIsNone(df['col_idx'])
-            'pca_0', 'pca_1', 'tsne_0', 'tsne_1', 'umap_0', 'umap_1', 'draw_graph_fr_0', 'draw_graph_fr_1'
-        ])
         self.assertIsNone(df['row_idx'])
         self.assertEqual(len(df['columns']), df['n_cols'])
 
@@ -97,8 +93,7 @@ class EndPoints(unittest.TestCase):
         self.assertIsNotNone(df['col_idx'])
         self.assertIsNone(df['row_idx'])
         self.assertEqual(len(df['columns']), df['n_cols'])
-        obs_index_col_name = self.schema["schema"]["annotations"]["obs"]["index"]
+        self.assertListEqual(df['col_idx'], ['name', 'n_genes', 'percent_mito', 'n_counts', 'louvain'])
-        self.assertListEqual(df['col_idx'], [obs_index_col_name, 'n_genes', 'percent_mito', 'n_counts', 'louvain'])
 
     def test_get_annotations_obs_keys_fbs(self):
         endpoint = "annotations/obs"
@@ -168,8 +163,7 @@ class EndPoints(unittest.TestCase):
         self.assertIsNotNone(df['col_idx'])
         self.assertIsNone(df['row_idx'])
         self.assertEqual(len(df['columns']), df['n_cols'])
-        var_index_col_name = self.schema["schema"]["annotations"]["var"]["index"]
+        self.assertListEqual(df['col_idx'], ['name', 'n_cells'])
-        self.assertListEqual(df['col_idx'], [var_index_col_name, 'n_cells'])
 
     def test_get_annotations_var_keys_fbs(self):
         endpoint = "annotations/var"
@@ -251,8 +245,7 @@ class EndPoints(unittest.TestCase):
         endpoint = f"data/var"
         url = f"{URL_BASE}{endpoint}"
         header = {"Accept": "application/octet-stream"}
-        index_col_name = self.schema["schema"]["annotations"]["var"]["index"]
+        var_filter = {"filter": {"var": {"annotation_value": [{"name": "name", "values": ["RER1"]}]}}}
-        var_filter = {"filter": {"var": {"annotation_value": [{"name": index_col_name, "values": ["RER1"]}]}}}
         result = self.session.put(url, headers=header, json=var_filter)
         self.assertEqual(result.status_code, HTTPStatus.OK)
         self.assertEqual(result.headers["Content-Type"], "application/octet-stream")

server/test/test_nan_rest.py

@@ -8,7 +8,7 @@ import decode_fbs
 
 import requests
 
-LOCAL_URL = "http://127.0.0.1:5006/"
+LOCAL_URL = "http://127.0.0.1:5005/"
 VERSION = "v0.2"
 URL_BASE = f"{LOCAL_URL}api/{VERSION}/"
 
@@ -21,7 +21,7 @@ class WithNaNs(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
         cls.ps = Popen(
-            ["cellxgene", "launch", "server/test/test_datasets/nan.h5ad", "--verbose", "--port", "5006"]
+            ["cellxgene", "launch", "server/test/test_datasets/nan.h5ad", "--debug"]
         )
         session = requests.Session()
         for i in range(90):

server/test/test_nan_scanpy_engine.py

@@ -12,7 +12,7 @@ from server.app.util.errors import FilterError
 class NaNTest(unittest.TestCase):
     def setUp(self):
         self.args = {
-            "layout": ["umap"],
+            "layout": "umap",
             "diffexp": "ttest",
             "max_category_items": 100,
             "obs_names": None,
@@ -58,16 +58,14 @@ class NaNTest(unittest.TestCase):
 
     def test_annotation(self):
         annotations = decode_fbs.decode_matrix_FBS(self.data.annotation_to_fbs_matrix("obs"))
-        obs_index_col_name = self.data.schema["annotations"]["obs"]["index"]
         self.assertEqual(
             annotations["col_idx"],
-            [obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain"]
+            ["name", "n_genes", "percent_mito", "n_counts", "louvain"]
         )
         self.assertEqual(annotations["n_rows"], 100)
         self.assertTrue(math.isnan(annotations["columns"][2][0]))
 
         annotations = decode_fbs.decode_matrix_FBS(self.data.annotation_to_fbs_matrix("var"))
-        var_index_col_name = self.data.schema["annotations"]["var"]["index"]
+        self.assertEqual(annotations["col_idx"], ["name", "n_cells", "var_with_nans"])
-        self.assertEqual(annotations["col_idx"], [var_index_col_name, "n_cells", "var_with_nans"])
         self.assertEqual(annotations["n_rows"], 100)
         self.assertTrue(math.isnan(annotations["columns"][2][0]))

server/test/test_scanpy_engine.py

@@ -15,7 +15,7 @@ from server.app.util.errors import FilterError
 class EngineTest(unittest.TestCase):
     def setUp(self):
         args = {
-            "layout": ["umap"],
+            "layout": "umap",
             "diffexp": "ttest",
             "max_category_items": 100,
             "obs_names": None,
@@ -31,11 +31,9 @@ class EngineTest(unittest.TestCase):
         self.assertTrue(self.data.data.X[0, 0] - -0.171_469_51 < epsilon)
 
     def test_mandatory_annotations(self):
-        obs_index_col_name = self.data.schema["annotations"]["obs"]["index"]
+        self.assertIn("name", self.data.data.obs)
-        self.assertIn(obs_index_col_name, self.data.data.obs)
         self.assertEqual(list(self.data.data.obs.index), list(range(2638)))
-        var_index_col_name = self.data.schema["annotations"]["var"]["index"]
+        self.assertIn("name", self.data.data.var)
-        self.assertIn(var_index_col_name, self.data.data.var)
         self.assertEqual(list(self.data.data.var.index), list(range(1838)))
 
     @pytest.mark.filterwarnings("ignore:Scanpy data matrix")
@@ -72,14 +70,12 @@ class EngineTest(unittest.TestCase):
         self.assertEqual(data["n_cols"], 91)
 
     def test_obs_and_var_names(self):
-        self.assertEqual(np.sum(self.data.data.var[self.data.schema["annotations"]["var"]["index"]].isna()), 0)
+        self.assertEqual(np.sum(self.data.data.var["name"].isna()), 0)
-        self.assertEqual(np.sum(self.data.data.obs[self.data.schema["annotations"]["obs"]["index"]].isna()), 0)
+        self.assertEqual(np.sum(self.data.data.obs["name"].isna()), 0)
 
     def test_schema(self):
         with open(path.join(path.dirname(__file__), "schema.json")) as fh:
             schema = json.load(fh)
-            print(schema)
-            print(self.data.schema)
             self.assertEqual(self.data.schema, schema)
 
     def test_schema_produces_error(self):
@@ -112,18 +108,16 @@ class EngineTest(unittest.TestCase):
         annotations = decode_fbs.decode_matrix_FBS(fbs)
         self.assertEqual(annotations["n_rows"], 2638)
         self.assertEqual(annotations["n_cols"], 5)
-        obs_index_col_name = self.data.schema["annotations"]["obs"]["index"]
         self.assertEqual(
             annotations["col_idx"],
-            [obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain"],
+            ["name", "n_genes", "percent_mito", "n_counts", "louvain"],
         )
 
         fbs = self.data.annotation_to_fbs_matrix("var")
         annotations = decode_fbs.decode_matrix_FBS(fbs)
         self.assertEqual(annotations['n_rows'], 1838)
         self.assertEqual(annotations['n_cols'], 2)
-        var_index_col_name = self.data.schema["annotations"]["var"]["index"]
+        self.assertEqual(annotations["col_idx"], ["name", "n_cells"])
-        self.assertEqual(annotations["col_idx"], [var_index_col_name, "n_cells"])
 
     def test_annotation_fields(self):
         fbs = self.data.annotation_to_fbs_matrix("obs", ["n_genes", "n_counts"])
@@ -131,8 +125,7 @@ class EngineTest(unittest.TestCase):
         self.assertEqual(annotations["n_rows"], 2638)
         self.assertEqual(annotations['n_cols'], 2)
 
-        var_index_col_name = self.data.schema["annotations"]["var"]["index"]
+        fbs = self.data.annotation_to_fbs_matrix("var", ["name"])
-        fbs = self.data.annotation_to_fbs_matrix("var", [var_index_col_name])
         annotations = decode_fbs.decode_matrix_FBS(fbs)
         self.assertEqual(annotations['n_rows'], 1838)
         self.assertEqual(annotations['n_cols'], 1)
@@ -170,10 +163,9 @@ class EngineTest(unittest.TestCase):
             self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
 
     def test_data_named_gene(self):
-        var_index_col_name = self.data.schema["annotations"]["var"]["index"]
         filter_ = {
             "filter": {
-                "var": {"annotation_value": [{"name": var_index_col_name, "values": ["RER1"]}]}
+                "var": {"annotation_value": [{"name": "name", "values": ["RER1"]}]}
             }
         }
         fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
@@ -184,7 +176,7 @@ class EngineTest(unittest.TestCase):
 
         filter_ = {
             "filter": {
-                "var": {"annotation_value": [{"name": var_index_col_name, "values": ["SPEN", "TYMP", "PRMT2"]}]}
+                "var": {"annotation_value": [{"name": "name", "values": ["SPEN", "TYMP", "PRMT2"]}]}
             }
         }
         fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")

server/test/test_scanpy_engine_data_load.py

@@ -15,7 +15,7 @@ class DataLoadEngineTest(unittest.TestCase):
 
     def test_delayed_load_args(self):
         args = {
-            "layout": ["tsne"],
+            "layout": "tsne",
             "diffexp": "ttest",
             "max_category_items": 1000,
             "obs_names": "foo",

server/utils/utils.py

@@ -1,26 +0,0 @@
-import contextlib
-import errno
-import socket
-
-
-def find_available_port(host, port=5005):
-    """
-    Helper method to find open port on host. Tries 5000 ports incremented from the specified port
-    """
-    # Takes approx 2 seconds to do a scan of 5000 ports on my laptop
-    num_ports_to_try = 5000
-    for port_to_try in range(port, port + num_ports_to_try):
-        if is_port_available(host, port_to_try):
-            return port_to_try
-    raise socket.error(errno.EADDRINUSE, f"No port in range {port} - {port + num_ports_to_try - 1} available.")
-
-
-def is_port_available(host, port):
-    is_available = False
-    with contextlib.closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s:
-        try:
-            s.bind((host, port))
-            is_available = True
-        except socket.error:
-            pass
-    return is_available

setup.py

@@ -8,7 +8,7 @@ with open("server/requirements.txt") as fh:
 
 setup(
     name="cellxgene",
-    version="0.10.1",
+    version="0.9.2",
     packages=find_packages(),
     url="https://github.com/chanzuckerberg/cellxgene",
     license="MIT",