release 0.10.0 (#794)

* WIP

* add --developer; fix incompatibility of --port and --debug

* put REST tests on separate ports

* PR review

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.9.1
+current_version = 0.10.0
 
 [bumpversion:file:setup.py]
 search = version="{current_version}"

.travis.yml

@@ -27,6 +27,5 @@ jobs:
     script: docker build .
   - name: "Smoke Tests"
     python: "3.6"
-    if: branch = master AND type = cron
     script:
       - npm run --prefix client/ smoke-test

client/__tests__/e2e/data.js

@@ -1,6 +1,6 @@
 export const datasets = {
   pbmc3k: {
-    title: "cellxgene: pbmc3k",
+    title: "pbmc3k",
     dataframe: {
       nObs: "2638",
       nVar: "1838",
@@ -26,8 +26,8 @@ export const datasets = {
     cellsets: {
       lasso: [
         {
-          "coordinates-as-percent": { x1: 0.25, y1: 0.25, x2: 0.35, y2: 0.35 },
+          "coordinates-as-percent": { x1: 0.05, y1: 0.25, x2: 0.15, y2: 0.35 },
-          count: "26"
+          count: "104"
         }
       ],
       categorical: [
@@ -91,8 +91,8 @@ export const datasets = {
         }
       },
       lasso: {
-        "coordinates-as-percent": { x1: 0.45, y1: 0.45, x2: 0.5, y2: 0.5 },
+        "coordinates-as-percent": { x1: 0.45, y1: 0.05, x2: 0.5, y2: 0.1 },
-        count: "67"
+        count: "76"
       }
     },
     scatter: {

client/__tests__/e2e/e2e.test.js

@@ -272,7 +272,7 @@ describe("scatter plot", async () => {
 
 describe("clipping", async () => {
   test("clip continuous", async () => {
-    await cxgActions.clip(data.clip.min, data.clip.max)
+    await cxgActions.clip(data.clip.min, data.clip.max);
     const histId = `histogram-${data.clip.metadata}-plot-brush`;
     const coords = await cxgActions.calcDragCoordinates(
       histId,
@@ -281,16 +281,13 @@ describe("clipping", async () => {
     await cxgActions.drag(histId, coords.start, coords.end);
     const cellCount = await cxgActions.cellSet(1);
     expect(cellCount).toBe(data.clip.count);
-    
   });
 
   test("clip gene", async () => {
     await utils.typeInto("gene-search", data.clip.gene);
     await page.keyboard.press("Enter");
-    await page.waitForSelector(
+    await page.waitForSelector(`[data-testid='histogram-${data.clip.gene}']`);
-      `[data-testid='histogram-${data.clip.gene}']`
+    await cxgActions.clip(data.clip.min, data.clip.max);
-    );
-    await cxgActions.clip(data.clip.min, data.clip.max)
     const histId = `histogram-${data.clip.gene}-plot-brush`;
     const coords = await cxgActions.calcDragCoordinates(
       histId,

client/__tests__/e2e/puppeteerUtils.js

@@ -44,11 +44,13 @@ export const puppeteerUtils = puppeteerPage => ({
   },
 
   async getOneElementInnerHTML(selector) {
+    await puppeteerPage.waitForSelector(selector);
     let text = await puppeteerPage.$eval(selector, el => el.innerHTML);
     return text;
   },
 
   async getOneElementInnerText(selector) {
+    await puppeteerPage.waitForSelector(selector);
     let text = await puppeteerPage.$eval(selector, el => el.innerText);
     return text;
   }

client/__tests__/util/stateManager/sampleResponses.js

@@ -34,28 +34,38 @@ const aSchemaResponse = {
       type: "float32"
     },
     annotations: {
-      obs: [
+      obs: {
-        { name: "name", type: "string" },
+        index: "name",
-        { name: "field1", type: "int32" },
+        columns: [
-        { name: "field2", type: "float32" },
+          { name: "name", type: "string" },
-        { name: "field3", type: "boolean" },
+          { name: "field1", type: "int32" },
-        {
+          { name: "field2", type: "float32" },
-          name: "field4",
+          { name: "field3", type: "boolean" },
-          type: "categorical",
+          {
-          categories: field4Categories
+            name: "field4",
-        }
+            type: "categorical",
-      ],
+            categories: field4Categories
-      var: [
+          }
-        { name: "name", type: "string" },
+        ]
-        { name: "fieldA", type: "int32" },
+      },
-        { name: "fieldB", type: "float32" },
+      var: {
-        { name: "fieldC", type: "boolean" },
+        index: "name",
-        {
+        columns: [
-          name: "fieldD",
+          { name: "name", type: "string" },
-          type: "categorical",
+          { name: "fieldA", type: "int32" },
-          categories: fieldDCategories
+          { name: "fieldB", type: "float32" },
-        }
+          { name: "fieldC", type: "boolean" },
-      ]
+          {
+            name: "fieldD",
+            type: "categorical",
+            categories: fieldDCategories
+          }
+        ]
+      }
+    },
+    layout: {
+      obs: [{ name: "umap", type: "float32", dims: ["umap_0", "umap_1"] }],
+      var: []
     }
   }
 };

client/__tests__/util/stateManager/universe.test.js

@@ -53,13 +53,15 @@ describe("createUniverseFromResponse", () => {
 
     expect(universe.obsAnnotations.dims).toEqual([
       nObs,
-      REST.schema.schema.annotations.obs.length
+      REST.schema.schema.annotations.obs.columns.length
     ]);
     expect(universe.obsLayout.dims).toEqual([nObs, 2]);
-    expect(universe.obsLayout.colIndex.keys()).toEqual(["X", "Y"]);
+    expect(universe.obsLayout.colIndex.keys()).toEqual(
+      universe.schema.layout.obs[0].dims
+    );
     expect(universe.varAnnotations.dims).toEqual([
       nVar,
-      REST.schema.schema.annotations.var.length
+      REST.schema.schema.annotations.var.columns.length
     ]);
     expect(universe.varData.isEmpty()).toBeTruthy();
   });

client/__tests__/util/stateManager/world.test.js

@@ -29,7 +29,8 @@ const defaultBigBang = () => {
   /* create crossfilter */
   const crossfilter = World.createObsDimensions(
     new Crossfilter(world.obsAnnotations),
-    world
+    world,
+    REST.schema.schema.layout.obs[0].dims
   );
 
   return {
@@ -138,7 +139,9 @@ describe("createWorldFromCurrentSelection", () => {
     expect(world.obsLayout.rowIndex.keys()).toEqual(
       new Int32Array(matchingIndices)
     );
-    expect(world.obsLayout.colIndex.keys()).toEqual(["X", "Y"]);
+    expect(world.obsLayout.colIndex.keys()).toEqual(
+      world.schema.layout.obs[0].dims
+    );
   });
 });
 
@@ -152,14 +155,18 @@ describe("createObsDimensionMap", () => {
 
     const { crossfilter } = defaultBigBang();
     const annotationNames = _.map(
-      REST.schema.schema.annotations.obs,
+      REST.schema.schema.annotations.obs.columns,
       c => c.name
     );
-    const schemaByObsName = _.keyBy(REST.schema.schema.annotations.obs, "name");
+    const obsIndexColName = REST.schema.schema.annotations.obs.index;
+    const schemaByObsName = _.keyBy(
+      REST.schema.schema.annotations.obs.columns,
+      "name"
+    );
     expect(crossfilter).toBeDefined();
     annotationNames.forEach(name => {
       const dim = crossfilter.dimensions[obsAnnoDimensionName(name)];
-      if (name === "name") {
+      if (name === obsIndexColName) {
         expect(dim).toBeUndefined();
       } else {
         const { type } = schemaByObsName[name];

client/configuration/babel/babel.dev.js

@@ -7,8 +7,8 @@ module.exports = {
   ],
   plugins: [
     "@babel/plugin-proposal-function-bind",
-    "@babel/plugin-proposal-class-properties",
     ["@babel/plugin-proposal-decorators", { legacy: true }],
+    ["@babel/plugin-proposal-class-properties", { loose: true }],
     "@babel/plugin-proposal-export-namespace-from",
     "@babel/plugin-proposal-optional-chaining",
     "@babel/plugin-proposal-nullish-coalescing-operator"

client/configuration/babel/babel.prod.js

@@ -6,8 +6,8 @@ module.exports = {
   ],
   plugins: [
     "@babel/plugin-proposal-function-bind",
-    "@babel/plugin-proposal-class-properties",
     ["@babel/plugin-proposal-decorators", { legacy: true }],
+    ["@babel/plugin-proposal-class-properties", { loose: true }],
     "@babel/plugin-proposal-export-namespace-from",
     "@babel/plugin-transform-react-constant-elements",
     "@babel/plugin-transform-runtime",

client/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cellxgene",
-  "version": "0.9.1",
+  "version": "0.10.0",
   "license": "MIT",
   "description": "cellxgene is a web application for the interactive exploration of single cell sequence data.",
   "repository": "https://github.com/chanzuckerberg/cellxgene",
@@ -9,13 +9,13 @@
     "build": "npm run clean && webpack --config configuration/webpack/webpack.config.prod.js",
     "clean": "rimraf build",
     "dev": "npm run clean && webpack --config configuration/webpack/webpack.config.dev.js",
-    "e2e": "jest --verbose false --config __tests__/e2e/e2eJestConfig.json e2e/e2e.test.js",
+    "e2e": "node node_modules/jest/bin/jest.js --verbose false --config __tests__/e2e/e2eJestConfig.json e2e/e2e.test.js",
     "lint": "eslint src",
     "smoke-test": "start-server-and-test start-server-for-test :5000 e2e",
     "start": "node server/development.js",
     "start-server-for-test": "cellxgene launch -p 5000 ../example-dataset/pbmc3k.h5ad",
-    "test": "jest",
+    "test": "node node_modules/jest/bin/jest.js",
-    "unit-test": "jest --testPathIgnorePatterns e2e"
+    "unit-test": "node node_modules/jest/bin/jest.js --testPathIgnorePatterns e2e"
   },
   "engineStrict": true,
   "engines": {
@@ -31,8 +31,8 @@
     "eslint-scope": "3.7.1"
   },
   "dependencies": {
-    "@blueprintjs/core": "^3.15.0",
+    "@blueprintjs/core": "^3.15.1",
-    "@blueprintjs/icons": "^3.3.0",
+    "@blueprintjs/icons": "^3.8.0",
     "@blueprintjs/select": "^3.8.0",
     "canvas-fit": "^1.5.0",
     "d3": "^4.10.0",
@@ -41,79 +41,77 @@
     "font-color-contrast": "^1.0.3",
     "fuzzysort": "^1.1.4",
     "gl-mat4": "^1.1.4",
-    "gl-matrix": "^2.7.1",
+    "gl-vec3": "^1.1.3",
+    "gl-matrix": "^3.0.0",
     "is-number": "^7.0.0",
     "key-pressed": "0.0.1",
     "lodash": "^4.17.4",
-    "memoize-one": "^4.0.0",
+    "memoize-one": "^5.0.4",
     "mouse-position": "^2.0.1",
     "mouse-pressed": "^1.0.0",
-    "normalize.css": "^8.0.0",
     "orbit-camera": "^1.0.0",
-    "query-string": "^6.1.0",
+    "query-string": "^6.5.0",
-    "react": "^16.6.0",
+    "react": "^16.8.6",
     "react-autocomplete": "^1.7.2",
-    "react-dom": "^16.6.0",
+    "react-dom": "^16.8.6",
-    "react-helmet": "^5.2.0",
+    "react-helmet": "^5.2.1",
-    "react-icons": "^3.2.2",
+    "react-icons": "^3.7.0",
-    "react-redux": "^5.1.0",
+    "react-redux": "^7.0.3",
     "redux": "^4.0.1",
-    "redux-devtools-extension": "^2.13.5",
     "redux-thunk": "^2.2.0",
-    "regl": "^1.3.9",
+    "regl": "^1.3.11",
     "scroll-speed": "^1.0.0",
     "urijs": "^1.19.0"
   },
   "devDependencies": {
-    "@babel/core": "^7.1.5",
+    "@babel/core": "^7.4.4",
-    "@babel/plugin-proposal-class-properties": "^7.0.0",
+    "@babel/plugin-proposal-class-properties": "^7.4.4",
-    "@babel/plugin-proposal-decorators": "^7.0.0",
+    "@babel/plugin-proposal-decorators": "^7.4.4",
-    "@babel/plugin-proposal-export-namespace-from": "^7.0.0",
+    "@babel/plugin-proposal-export-namespace-from": "^7.2.0",
-    "@babel/plugin-proposal-function-bind": "^7.0.0",
+    "@babel/plugin-proposal-function-bind": "^7.2.0",
-    "@babel/plugin-proposal-nullish-coalescing-operator": "^7.2.0",
+    "@babel/plugin-proposal-nullish-coalescing-operator": "^7.4.4",
     "@babel/plugin-proposal-optional-chaining": "^7.2.0",
-    "@babel/plugin-transform-react-constant-elements": "^7.0.0",
+    "@babel/plugin-transform-react-constant-elements": "^7.2.0",
-    "@babel/plugin-transform-runtime": "^7.1.0",
+    "@babel/plugin-transform-runtime": "^7.4.4",
-    "@babel/preset-env": "^7.1.5",
+    "@babel/preset-env": "^7.4.4",
     "@babel/preset-react": "^7.0.0",
-    "@babel/register": "^7.0.0",
+    "@babel/register": "^7.4.4",
-    "@babel/runtime": "^7.1.5",
+    "@babel/runtime": "^7.4.4",
-    "babel-core": "^7.0.0-bridge.0",
     "babel-eslint": "^10.0.1",
-    "babel-jest": "^23.6.0",
+    "babel-jest": "^24.8.0",
-    "babel-loader": "^8.0.0",
+    "babel-loader": "^8.0.6",
-    "babel-preset-modern-browsers": "^12.0.0",
+    "babel-preset-modern-browsers": "^14.0.0",
     "chalk": "^2.4.2",
     "connect-history-api-fallback": "^1.6.0",
-    "copy-webpack-plugin": "^4.6.0",
+    "copy-webpack-plugin": "^5.0.3",
-    "css-loader": "^1.0.1",
+    "css-loader": "^2.1.1",
-    "eslint": "^5.13.0",
+    "eslint": "^5.16.0",
     "eslint-config-airbnb": "^17.1.0",
-    "eslint-config-prettier": "^4.0.0",
+    "eslint-config-prettier": "^4.2.0",
     "eslint-loader": "^2.1.2",
     "eslint-plugin-filenames": "^1.3.2",
-    "eslint-plugin-import": "^2.16.0",
+    "eslint-plugin-import": "^2.17.2",
-    "eslint-plugin-jest": "^22.2.2",
+    "eslint-plugin-jest": "^22.5.1",
     "eslint-plugin-jsx-a11y": "^6.2.1",
-    "eslint-plugin-react": "^7.12.4",
+    "eslint-plugin-react": "^7.13.0",
     "express": "^4.14.0",
-    "file-loader": "^2.0.0",
+    "file-loader": "^3.0.1",
     "html-webpack-inline-source-plugin": "0.0.10",
     "html-webpack-plugin": "^3.2.0",
-    "jest": "^24.1.0",
+    "jest": "^24.8.0",
-    "jest-puppeteer": "^4.1.0",
+    "jest-puppeteer": "^4.1.1",
     "json-loader": "^0.5.4",
-    "mini-css-extract-plugin": "^0.4.1",
+    "mini-css-extract-plugin": "^0.6.0",
-    "puppeteer": "^1.15.0",
+    "puppeteer": "^1.16.0",
     "rimraf": "^2.6.3",
     "serve-favicon": "^2.3.0",
-    "start-server-and-test": "^1.7.11",
+    "start-server-and-test": "^1.9.0",
     "style-loader": "^0.23.1",
     "sw-precache-webpack-plugin": "^0.11.5",
     "url-loader": "^1.1.0",
-    "webpack": "^4.25.1",
+    "webpack": "^4.31.0",
-    "webpack-cli": "^3.1.0",
+    "webpack-cli": "^3.3.2",
-    "webpack-dev-middleware": "^3.1.3"
+    "webpack-dev-middleware": "^3.6.2"
   },
   "jest": {
     "testMatch": [
@@ -133,13 +131,18 @@
         ],
         "plugins": [
           "@babel/plugin-proposal-function-bind",
-          "@babel/plugin-proposal-class-properties",
           [
             "@babel/plugin-proposal-decorators",
             {
               "legacy": true
             }
           ],
+          [
+            "@babel/plugin-proposal-class-properties",
+            {
+              "loose": true
+            }
+          ],
           "@babel/plugin-proposal-export-namespace-from",
           "@babel/plugin-transform-react-constant-elements",
           "@babel/plugin-transform-runtime",

client/src/actions/index.js

@@ -21,24 +21,32 @@ const doInitialDataLoad = () =>
     dispatch({ type: "initial data load start" });
 
     try {
-      const requestJson = _(["config", "schema"])
+      /*
+      Step 1 - config & schema, all JSON
+      */
+      const requestJson = ["config", "schema"]
         .map(r => `${globals.API.prefix}${globals.API.version}${r}`)
-        .map(url => doJsonRequest(url))
+        .map(url => doJsonRequest(url));
-        .value();
+      const stepOneResults = await Promise.all(requestJson);
-      const requestBinary = _([
-        "annotations/obs",
-        "annotations/var?annotation-name=name",
-        "layout/obs"
-      ])
-        .map(r => `${globals.API.prefix}${globals.API.version}${r}`)
-        .map(url => doBinaryRequest(url))
-        .value();
-
-      const results = await Promise.all(_.concat(requestJson, requestBinary));
-
       /* set config defaults */
-      const config = { ...globals.configDefaults, ...results[0].config };
+      const config = { ...globals.configDefaults, ...stepOneResults[0].config };
-      const [, schema, obsAnno, varAnno, obsLayout] = [...results];
+      const schema = stepOneResults[1];
+
+      /*
+      Step 2 - dataframes, all binary.  NOTE: uses results of step 1.
+      */
+      /* only load names for var annotations, if possible*/
+      const varIndexName = schema?.schema?.annotations?.var?.index;
+      const varAnnotationsQuery = varIndexName
+        ? `?annotation-name=${varIndexName}`
+        : "";
+      const varAnnotationsURL = `annotations/var${varAnnotationsQuery}`;
+      const requestBinary = ["annotations/obs", varAnnotationsURL, "layout/obs"]
+        .map(r => `${globals.API.prefix}${globals.API.version}${r}`)
+        .map(url => doBinaryRequest(url));
+      const stepTwoResults = await Promise.all(requestBinary);
+      const [obsAnno, varAnno, obsLayout] = [...stepTwoResults];
+
       const universe = Universe.createUniverseFromResponse(
         config,
         schema,
@@ -91,6 +99,10 @@ needs expression data.
 Transparently utilizes cached data if it is already present.
 */
 async function _doRequestExpressionData(dispatch, getState, genes) {
+  const state = getState();
+  const { universe } = state;
+  const varIndexName = universe.schema.annotations.var.index;
+
   /* helper for this function only */
   const fetchData = async geneNames => {
     const res = await fetch(
@@ -100,7 +112,7 @@ async function _doRequestExpressionData(dispatch, getState, genes) {
         body: JSON.stringify({
           filter: {
             var: {
-              annotation_value: [{ name: "name", values: geneNames }]
+              annotation_value: [{ name: varIndexName, values: geneNames }]
             }
           }
         }),
@@ -123,8 +135,6 @@ async function _doRequestExpressionData(dispatch, getState, genes) {
     return Universe.convertDataFBStoObject(universe, data);
   };
 
-  const state = getState();
-  const { universe } = state;
   /* preload data already in cache */
   let expressionData = _.transform(
     genes,
@@ -241,6 +251,7 @@ const requestDifferentialExpression = (set1, set2, num_genes = 10) => async (
     */
     const state = getState();
     const { universe } = state;
+    const varIndexName = universe.schema.annotations.var.index;
 
     // Legal values are null, Array or TypedArray.  Null is initial state.
     if (!set1) set1 = [];
@@ -277,7 +288,7 @@ const requestDifferentialExpression = (set1, set2, num_genes = 10) => async (
     const data = await res.json();
     // result is [ [varIdx, ...], ... ]
     const topNGenes = _.map(data, r =>
-      universe.varAnnotations.at(r[0], "name")
+      universe.varAnnotations.at(r[0], varIndexName)
     );
 
     /*

client/src/components/categorical/category.js

@@ -27,10 +27,10 @@ class Category extends React.Component {
       const cat = categoricalSelection[metadataField];
       const categoryCount = {
         // total number of categories in this dimension
-        totalCatCount: cat.numCategories,
+        totalCatCount: cat.numCategoryValues,
         // number of selected options in this category
         selectedCatCount: _.reduce(
-          cat.categorySelected,
+          cat.categoryValueSelected,
           (res, cond) => (cond ? res + 1 : res),
           0
         )
@@ -91,7 +91,7 @@ class Category extends React.Component {
     const { categoricalSelection, metadataField } = this.props;
 
     const cat = categoricalSelection[metadataField];
-    const optTuples = sortedCategoryValues([...cat.categoryIndices]);
+    const optTuples = sortedCategoryValues([...cat.categoryValueIndices]);
     return _.map(optTuples, (tuple, i) => (
       <Value
         optTuples={optTuples}

client/src/components/categorical/value.js

@@ -46,8 +46,8 @@ class CategoryValue extends React.Component {
     if (!categoricalSelection) return null;
 
     const category = categoricalSelection[metadataField];
-    const selected = category.categorySelected[categoryIndex];
+    const selected = category.categoryValueSelected[categoryIndex];
-    const count = category.categoryCounts[categoryIndex];
+    const count = category.categoryValueCounts[categoryIndex];
     const value = category.categoryValues[categoryIndex];
     const displayString = String(
       category.categoryValues[categoryIndex]

client/src/components/continuous/continuous.js

@@ -66,7 +66,8 @@ class Continuous extends React.Component {
           ? _.map(obsAnnotations.colIndex.keys(), key => {
               const isColorField =
                 key.includes("color") || key.includes("Color");
-              if (key === "name" || isColorField) return null;
+              if (key === schema.annotations.obs.index || isColorField)
+                return null;
 
               const summary = obsAnnotations.col(key).summarize();
               const nonFiniteExtent =

client/src/components/framework/logo.js

@@ -0,0 +1,18 @@
+import React from "react";
+import * as globals from "../../globals";
+
+const Logo = props => {
+  const { size } = props;
+  return (
+    <svg width={size} height={size} viewBox={`0 0 48 48`} fill="none">
+      <rect width="48" height="48" fill="white" />
+      <rect width="48" height="48" fill={globals.logoColor} />
+      <rect x="19" y="19" width="22" height="22" fill="white" />
+      <rect x="24" y="24" width="12" height="12" fill={globals.logoColor} />
+      <rect x="7" y="19" width="7" height="22" fill="white" />
+      <rect x="19" y="7" width="22" height="7" fill="white" />
+    </svg>
+  );
+};
+
+export default Logo;

client/src/components/geneExpression/index.js

@@ -85,7 +85,8 @@ class GeneExpression extends React.Component {
     */
     const { world } = this.props;
     const { varAnnotations } = world;
-    const geneNames = varAnnotations.col("name").asArray();
+    const varIndexName = world.schema.annotations.var.index;
+    const geneNames = varAnnotations.col(varIndexName).asArray();
     if (geneNames.length > 0) {
       const placeholder = [];
       let len = geneNames.length;
@@ -107,6 +108,7 @@ class GeneExpression extends React.Component {
 
   handleClick(g) {
     const { world, dispatch, userDefinedGenes } = this.props;
+    const varIndexName = world.schema.annotations.var.index;
     const gene = g.target;
     if (userDefinedGenes.indexOf(gene) !== -1) {
       postUserErrorToast("That gene already exists");
@@ -114,7 +116,9 @@ class GeneExpression extends React.Component {
       postUserErrorToast(
         "That's too many genes, you can have at most 15 user defined genes"
       );
-    } else if (world.varAnnotations.col("name").indexOf(gene) === undefined) {
+    } else if (
+      world.varAnnotations.col(varIndexName).indexOf(gene) === undefined
+    ) {
       postUserErrorToast("That doesn't appear to be a valid gene name.");
     } else {
       dispatch({ type: "single user defined gene start" });
@@ -127,6 +131,7 @@ class GeneExpression extends React.Component {
 
   handleBulkAddClick() {
     const { world, dispatch, userDefinedGenes } = this.props;
+    const varIndexName = world.schema.annotations.var.index;
     const { bulkAdd } = this.state;
 
     /*
@@ -145,7 +150,9 @@ class GeneExpression extends React.Component {
           if (userDefinedGenes.indexOf(gene) !== -1) {
             return keepAroundErrorToast("That gene already exists");
           }
-          if (world.varAnnotations.col("name").indexOf(gene) === undefined) {
+          if (
+            world.varAnnotations.col(varIndexName).indexOf(gene) === undefined
+          ) {
             return keepAroundErrorToast(
               `${gene} doesn't appear to be a valid gene name.`
             );
@@ -168,7 +175,7 @@ class GeneExpression extends React.Component {
       userDefinedGenesLoading,
       differential
     } = this.props;
-
+    const varIndexName = world?.schema?.annotations?.var?.index;
     const { tab, bulkAdd } = this.state;
 
     return (
@@ -243,7 +250,7 @@ class GeneExpression extends React.Component {
                 itemRenderer={renderGene.bind(this)}
                 items={
                   world && world.varAnnotations
-                    ? world.varAnnotations.col("name").asArray()
+                    ? world.varAnnotations.col(varIndexName).asArray()
                     : ["No genes"]
                 }
                 popoverProps={{ minimal: true }}
@@ -322,7 +329,7 @@ class GeneExpression extends React.Component {
           <ExpressionButtons />
           {differential.diffExp
             ? _.map(differential.diffExp, (value, index) => {
-                const name = world.varAnnotations.at(value[0], "name");
+                const name = world.varAnnotations.at(value[0], varIndexName);
                 const values = world.varData.col(name);
                 if (!values) {
                   return null;

client/src/components/graph/drawPointsRegl.js

@@ -1,5 +1,6 @@
 // jshint esversion: 6
 const mat4 = require("gl-mat4");
+const vec3 = require("gl-vec3");
 
 // opacity: https://github.com/spacetx/starfish/blob/master/viz/draw/regions.js
 
@@ -38,7 +39,20 @@ export default function(regl) {
     uniforms: {
       distance: regl.prop("distance"),
       view: regl.prop("view"),
-      projection: ({viewportWidth, viewportHeight}) => mat4.perspective([], Math.PI / 2, viewportWidth / viewportHeight, 0.01, 1000)
+      projection: ({ viewportWidth, viewportHeight }) => {
+        const aspectRatio = viewportWidth / viewportHeight;
+        let m = mat4.perspective(
+          [],
+          Math.PI / 2,
+          viewportWidth / viewportHeight,
+          0.01,
+          1000
+        );
+        if (aspectRatio < 1) {
+          m = mat4.scale(m, m, vec3.fromValues(1, 1, 1 / aspectRatio));
+        }
+        return m;
+      }
     },
 
     count: regl.prop("count"),

client/src/components/graph/graph.js

@@ -3,7 +3,9 @@ import React from "react";
 import * as d3 from "d3";
 import { connect } from "react-redux";
 import mat4 from "gl-mat4";
+import vec3 from "gl-vec3";
 import _regl from "regl";
+import memoize from "memoize-one";
 import {
   Button,
   AnchorButton,
@@ -13,7 +15,9 @@ import {
   MenuItem,
   Position,
   NumericInput,
-  Icon
+  Icon,
+  RadioGroup,
+  Radio
 } from "@blueprintjs/core";
 
 import * as globals from "../../globals";
@@ -47,7 +51,8 @@ import { World } from "../../util/stateManager";
   undoDisabled: state["@@undoable/past"].length === 0,
   redoDisabled: state["@@undoable/future"].length === 0,
   selectionTool: state.graphSelection.tool,
-  currentSelection: state.graphSelection.selection
+  currentSelection: state.graphSelection.selection,
+  layoutChoice: state.layoutChoice
 }))
 class Graph extends React.Component {
   static isValidDigitKeyEvent(e) {
@@ -74,6 +79,38 @@ class Graph extends React.Component {
     return key >= 0 && key <= 9;
   }
 
+  computePointPositions = memoize((X, Y, scaleX, scaleY) => {
+    /*
+    compute webgl coordinate buffer for each point
+    */
+    const positions = new Float32Array(2 * X.length);
+    for (let i = 0, len = X.length; i < len; i += 1) {
+      positions[2 * i] = scaleX(X[i]);
+      positions[2 * i + 1] = scaleY(Y[i]);
+    }
+    return positions;
+  });
+
+  computePointColors = memoize(rgb => {
+    /*
+    compute webgl colors for each point
+    */
+    const colors = new Float32Array(3 * rgb.length);
+    for (let i = 0, len = rgb.length; i < len; i += 1) {
+      colors.set(rgb[i], 3 * i);
+    }
+    return colors;
+  });
+
+  computePointSizes = memoize((len, crossfilter) => {
+    /*
+    compute webgl dot size for each point
+    */
+    const sizes = new Float32Array(len);
+    crossfilter.fillByIsSelected(sizes, 4, 0.2);
+    return sizes;
+  });
+
   constructor(props) {
     super(props);
     this.count = 0;
@@ -81,6 +118,8 @@ class Graph extends React.Component {
     this.graphPaddingBottom = 45;
     this.graphPaddingRight = globals.leftSidebarWidth;
     this.renderCache = {
+      X: null,
+      Y: null,
       positions: null,
       colors: null,
       sizes: null
@@ -106,6 +145,12 @@ class Graph extends React.Component {
     const colorBuffer = regl.buffer();
     const sizeBuffer = regl.buffer();
 
+    // preallocate coordinate system transformation between data and gl
+    const transform = {
+      glScaleX: scaleLinear([0, 1], [-1, 1]),
+      glScaleY: scaleLinear([0, 1], [1, -1])
+    };
+
     /* first time, but this duplicates above function, should be possile to avoid this */
     const reglRender = regl.frame(() => {
       this.reglDraw(
@@ -128,7 +173,8 @@ class Graph extends React.Component {
       colorBuffer,
       sizeBuffer,
       camera,
-      reglRender
+      reglRender,
+      transform
     });
   }
 
@@ -140,19 +186,10 @@ class Graph extends React.Component {
       colorRGB,
       responsive,
       selectionTool,
-      currentSelection
+      currentSelection,
+      layoutChoice
     } = this.props;
-    const {
+    const { reglRender, mode, regl, svg } = this.state;
-      reglRender,
-      mode,
-      regl,
-      drawPoints,
-      camera,
-      pointBuffer,
-      colorBuffer,
-      sizeBuffer,
-      svg
-    } = this.state;
     let stateChanges = {};
 
     if (reglRender && this.reglRenderState === "rendering" && mode !== "zoom") {
@@ -163,53 +200,40 @@ class Graph extends React.Component {
     if (regl && world) {
       /* update the regl state */
       const { obsLayout, nObs } = world;
-      const X = obsLayout.col("X").asArray();
+      const {
-      const Y = obsLayout.col("Y").asArray();
+        drawPoints,
+        transform,
+        camera,
+        pointBuffer,
+        colorBuffer,
+        sizeBuffer
+      } = this.state;
 
-      // X/Y positions for each point - a cached value that only
+      /* coordinates for each point */
-      // changes if we have loaded entirely new cell data
+      const { glScaleX, glScaleY } = transform;
-      //
+      const X = obsLayout.col(layoutChoice.currentDimNames[0]).asArray();
-      if (!renderCache.positions || world !== prevProps.world) {
+      const Y = obsLayout.col(layoutChoice.currentDimNames[1]).asArray();
-        renderCache.positions = new Float32Array(2 * nObs);
+      const newPositions = this.computePointPositions(X, Y, glScaleX, glScaleY);
-
+      if (renderCache.positions !== newPositions) {
-        const glScaleX = scaleLinear([0, 1], [-1, 1]);
+        /* update our cache & GL if the buffer changes */
-        const glScaleY = scaleLinear([0, 1], [1, -1]);
+        renderCache.positions = newPositions;
-
+        pointBuffer({ data: newPositions, dimension: 2 });
-        const offset = [d3.mean(X) - 0.5, d3.mean(Y) - 0.5];
-
-        for (let i = 0, { positions } = renderCache; i < nObs; i += 1) {
-          positions[2 * i] = glScaleX(X[i] - offset[0]);
-          positions[2 * i + 1] = glScaleY(Y[i] - offset[1]);
-        }
-        pointBuffer({
-          data: renderCache.positions,
-          dimension: 2
-        });
-
-        stateChanges.offset = offset;
       }
 
-      // Colors for each point - a cached value that only changes when
+      /* colors for each point */
-      // the cell metadata changes.
+      const newColors = this.computePointColors(colorRGB);
-      if (!renderCache.colors || colorRGB !== prevProps.colorRGB) {
+      if (renderCache.colors !== newColors) {
-        const rgb = colorRGB;
+        /* update our cache & GL if the buffer changes */
-        if (!renderCache.colors) {
+        renderCache.colors = newColors;
-          renderCache.colors = new Float32Array(3 * rgb.length);
+        colorBuffer({ data: newColors, dimension: 3 });
-        }
-        for (let i = 0, { colors } = renderCache; i < rgb.length; i += 1) {
-          colors.set(rgb[i], 3 * i);
-        }
-        colorBuffer({ data: renderCache.colors, dimension: 3 });
       }
 
-      // Sizes for each point - updates are triggered only when selected
+      /* sizes for each point */
-      // obs change
+      const newSizes = this.computePointSizes(nObs, crossfilter);
-      if (!renderCache.sizes || crossfilter !== prevProps.crossfilter) {
+      if (renderCache.sizes !== newSizes) {
-        if (!renderCache.sizes) {
+        /* update our cache & GL if the buffer changes */
-          renderCache.sizes = new Float32Array(nObs);
+        renderCache.size = newSizes;
-        }
+        sizeBuffer({ data: newSizes, dimension: 1 });
-        crossfilter.fillByIsSelected(renderCache.sizes, 4, 0.2);
-        sizeBuffer({ data: renderCache.sizes, dimension: 1 });
       }
 
       this.count = nObs;
@@ -271,11 +295,10 @@ class Graph extends React.Component {
       mode !== prevState.mode ||
       stateChanges.svg
     ) {
-      const { tool, container, offset } = this.state;
+      const { tool, container } = this.state;
       this.selectionToolUpdate(
         stateChanges.tool ? stateChanges.tool : tool,
-        stateChanges.container ? stateChanges.container : container,
+        stateChanges.container ? stateChanges.container : container
-        stateChanges.offset ? stateChanges.offset : offset
       );
     }
 
@@ -435,7 +458,15 @@ class Graph extends React.Component {
     this.setState({ pendingClipPercentiles: null });
   };
 
-  brushToolUpdate(tool, container, offset) {
+  handleLayoutChoiceChange = e => {
+    const { dispatch } = this.props;
+    dispatch({
+      type: "set layout choice",
+      layoutChoice: e.currentTarget.value
+    });
+  };
+
+  brushToolUpdate(tool, container) {
     /*
     this is called from componentDidUpdate(), so be very careful using
     anything from this.state, which may be updated asynchronously.
@@ -449,8 +480,8 @@ class Graph extends React.Component {
         if there is a selection, make sure the brush tool matches
         */
         const screenCoords = [
-          this.mapPointToScreen(currentSelection.brushCoords.northwest, offset),
+          this.mapPointToScreen(currentSelection.brushCoords.northwest),
-          this.mapPointToScreen(currentSelection.brushCoords.southeast, offset)
+          this.mapPointToScreen(currentSelection.brushCoords.southeast)
         ];
         if (!toolCurrentSelection) {
           /* tool is not selected, so just move the brush */
@@ -477,7 +508,7 @@ class Graph extends React.Component {
     }
   }
 
-  lassoToolUpdate(tool, container, offset) {
+  lassoToolUpdate(tool, container) {
     /*
     this is called from componentDidUpdate(), so be very careful using
     anything from this.state, which may be updated asynchronously.
@@ -488,7 +519,7 @@ class Graph extends React.Component {
       if there is a current selection, make sure the lasso tool matches
       */
       const polygon = currentSelection.polygon.map(p =>
-        this.mapPointToScreen(p, offset)
+        this.mapPointToScreen(p)
       );
       tool.move(polygon);
     } else {
@@ -496,7 +527,7 @@ class Graph extends React.Component {
     }
   }
 
-  selectionToolUpdate(tool, container, offset) {
+  selectionToolUpdate(tool, container) {
     /*
     this is called from componentDidUpdate(), so be very careful using
     anything from this.state, which may be updated asynchronously.
@@ -504,10 +535,10 @@ class Graph extends React.Component {
     const { selectionTool } = this.props;
     switch (selectionTool) {
       case "brush":
-        this.brushToolUpdate(tool, container, offset);
+        this.brushToolUpdate(tool, container);
         break;
       case "lasso":
-        this.lassoToolUpdate(tool, container, offset);
+        this.lassoToolUpdate(tool, container);
         break;
       default:
         /* punt? */
@@ -564,12 +595,14 @@ class Graph extends React.Component {
     accounting for current pan/zoom camera.
     */
     const { responsive } = this.props;
-    const { regl, camera, offset } = this.state;
+    const { regl, camera, transform } = this.state;
+    const { glScaleX, glScaleY } = transform;
 
     const gl = regl._gl;
 
     // get aspect ratio
     const aspect = gl.drawingBufferWidth / gl.drawingBufferHeight;
+    const scale = aspect < 1 ? 1 / aspect : 1;
 
     // compute inverse view matrix
     const inverse = mat4.invert([], camera.view());
@@ -578,37 +611,37 @@ class Graph extends React.Component {
     const x = (2 * pin[0]) / (responsive.width - this.graphPaddingRight) - 1;
     const y = 2 * (1 - pin[1] / (responsive.height - this.graphPaddingTop)) - 1;
     const pout = [
-      x * inverse[14] * aspect + inverse[12],
+      x * inverse[14] * aspect * scale + inverse[12],
-      y * inverse[14] + inverse[13]
+      -(y * inverse[14] * scale + inverse[13])
     ];
 
-    return [(pout[0] + 1) / 2 + offset[0], (pout[1] + 1) / 2 + offset[1]];
+    const xy = [glScaleX.invert(pout[0]), glScaleY.invert(pout[1])];
+    return xy;
   }
 
-  mapPointToScreen(xyCell, offset) {
+  mapPointToScreen(xyCell) {
     /*
     Map an XY coordinate from cell/point domain to screen range.  Inverse
     of mapScreenToPoint()
     */
     const { responsive } = this.props;
-    const { regl, camera } = this.state;
+    const { regl, camera, transform } = this.state;
+    const { glScaleX, glScaleY } = transform;
 
     const gl = regl._gl;
 
     // get aspect ratio
     const aspect = gl.drawingBufferWidth / gl.drawingBufferHeight;
+    const scale = aspect < 1 ? 1 / aspect : 1;
 
     // compute inverse view matrix
-    const inverse = mat4.invert([], camera.view());
+    let inverse = mat4.invert([], camera.view());
 
     // variable names are choosen to reflect inverse of those used
     // in mapScreenToPoint().
-    const pout = [
+    const pout = [glScaleX(xyCell[0]), glScaleY(xyCell[1])];
-      (xyCell[0] - offset[0]) * 2 - 1,
+    const x = (pout[0] - inverse[12]) / aspect / scale / inverse[14];
-      (xyCell[1] - offset[1]) * 2 - 1
+    const y = (-pout[1] - inverse[13]) / scale / inverse[14];
-    ];
-    const x = (pout[0] - inverse[12]) / aspect / inverse[14];
-    const y = (pout[1] - inverse[13]) / inverse[14];
 
     const pin = [
       Math.round(((x + 1) * (responsive.width - this.graphPaddingRight)) / 2),
@@ -745,7 +778,8 @@ class Graph extends React.Component {
       redoDisabled,
       selectionTool,
       clipPercentileMin,
-      clipPercentileMax
+      clipPercentileMax,
+      layoutChoice
     } = this.props;
     const { mode, pendingClipPercentiles } = this.state;
 
@@ -888,6 +922,49 @@ class Graph extends React.Component {
                 />
               </Tooltip>
             </div>
+
+            <div
+              className="bp3-button-group"
+              style={{
+                marginLeft: 10
+              }}
+            >
+              <Popover
+                target={
+                  <Button
+                    type="button"
+                    data-testid="layout-choice"
+                    className="bp3-button bp3-icon-heatmap"
+                    style={{
+                      cursor: "pointer"
+                    }}
+                  />
+                }
+                position={Position.BOTTOM_RIGHT}
+                content={
+                  <div
+                    style={{
+                      display: "flex",
+                      justifyContent: "flex-start",
+                      alignItems: "flex-start",
+                      flexDirection: "column",
+                      padding: 10
+                    }}
+                  >
+                    <RadioGroup
+                      label="Layout Choice"
+                      onChange={this.handleLayoutChoiceChange}
+                      selectedValue={layoutChoice.current}
+                    >
+                      {layoutChoice.available.map(name => (
+                        <Radio label={name} value={name} key={name} />
+                      ))}
+                    </RadioGroup>
+                  </div>
+                }
+              />
+            </div>
+
             <div
               className="bp3-button-group"
               style={{

client/src/components/leftsidebar.js

@@ -6,6 +6,7 @@ import Continuous from "./continuous/continuous";
 import GeneExpression from "./geneExpression";
 import * as globals from "../globals";
 import DynamicScatterplot from "./scatterplot/scatterplot";
+import Logo from "./framework/logo.js";
 
 @connect(state => ({
   responsive: state.responsive,
@@ -27,7 +28,6 @@ class LeftSideBar extends React.Component {
     if cellxgene logo or tabs change, this must as well
     */
     const metadataSectionPadding = 0;
-    // scatterplotXXaccessor && scatterplotYYaccessor ? 450 : 0;
 
     return (
       <div
@@ -39,20 +39,49 @@ class LeftSideBar extends React.Component {
         }}
       >
         <p
-          data-testid="header"
           style={{
             position: "fixed",
             top: globals.cellxgeneTitleTopPadding,
             left: globals.leftSidebarWidth + globals.cellxgeneTitleLeftPadding,
-            margin: 0,
+            margin: 0
-            fontSize: globals.largestFontSize,
-            color: globals.darkerGrey,
-            width: "100%"
           }}
         >
-          cellxgene: {datasetTitle}
+          <Logo size={32} />
+          <span
+            style={{
+              fontSize: 28,
+              position: "relative",
+              top: -4,
+              fontWeight: "bold",
+              marginLeft: 5,
+              color: globals.logoColor,
+              userSelect: "none"
+            }}
+          >
+            cell<span
+              style={{
+                position: "relative",
+                top: 1,
+                fontWeight: 300,
+                fontSize: 24
+              }}
+            >
+              ×
+            </span>gene
+          </span>
+          <span
+            data-testid="header"
+            style={{
+              fontSize: 16,
+              display: "block",
+              position: "relative",
+              marginTop: 10,
+              top: -4
+            }}
+          >
+            {datasetTitle}
+          </span>
         </p>
-
         <div
           style={{
             height: responsive.height - metadataSectionPadding,

client/src/globals.js

@@ -31,6 +31,7 @@ export const darkGreen = "#448C4D";
 
 export const nonFiniteCellColor = lightGrey;
 export const defaultCellColor = "rgb(0,0,0,1)";
+export const logoColor = "black"; /* logo pink: "#E9429A" */
 
 /* typography constants */
 

client/src/reducers/categoricalSelection.js

@@ -30,15 +30,15 @@ const CategoricalSelection = (
       /*
       Set the specific category in this field to false
       */
-      const newCategorySelected = Array.from(
+      const newCategoryValueSelected = Array.from(
-        state[action.metadataField].categorySelected
+        state[action.metadataField].categoryValueSelected
       );
-      newCategorySelected[action.categoryIndex] = true;
+      newCategoryValueSelected[action.categoryIndex] = true;
       const newCategoricalSelection = {
         ...state,
         [action.metadataField]: {
           ...state[action.metadataField],
-          categorySelected: newCategorySelected
+          categoryValueSelected: newCategoryValueSelected
         }
       };
       return newCategoricalSelection;
@@ -48,15 +48,15 @@ const CategoricalSelection = (
       /*
       Set the specific category in this field to false
       */
-      const newCategorySelected = Array.from(
+      const newCategoryValueSelected = Array.from(
-        state[action.metadataField].categorySelected
+        state[action.metadataField].categoryValueSelected
       );
-      newCategorySelected[action.categoryIndex] = false;
+      newCategoryValueSelected[action.categoryIndex] = false;
       const newCategoricalSelection = {
         ...state,
         [action.metadataField]: {
           ...state[action.metadataField],
-          categorySelected: newCategorySelected
+          categoryValueSelected: newCategoryValueSelected
         }
       };
       return newCategoricalSelection;
@@ -70,8 +70,9 @@ const CategoricalSelection = (
         ...state,
         [action.metadataField]: {
           ...state[action.metadataField],
-          categorySelected: Array.from(
+          categorySelected: false,
-            state[action.metadataField].categorySelected
+          categoryValueSelected: Array.from(
+            state[action.metadataField].categoryValueSelected
           ).fill(false)
         }
       };
@@ -86,8 +87,9 @@ const CategoricalSelection = (
         ...state,
         [action.metadataField]: {
           ...state[action.metadataField],
-          categorySelected: Array.from(
+          categorySelected: true,
-            state[action.metadataField].categorySelected
+          categoryValueSelected: Array.from(
+            state[action.metadataField].categoryValueSelected
           ).fill(true)
         }
       };

client/src/reducers/controls.js

@@ -93,9 +93,10 @@ const Controls = (
     }
     case "request differential expression success": {
       const { world } = prevSharedState;
+      const varIndexName = world.schema.annotations.var.index;
       const _diffexpGenes = [];
       action.data.forEach(d => {
-        _diffexpGenes.push(world.varAnnotations.at(d[0], "name"));
+        _diffexpGenes.push(world.varAnnotations.at(d[0], varIndexName));
       });
       return {
         ...state,

client/src/reducers/crossfilter.js

@@ -20,10 +20,11 @@ const CrossfilterReducer = (
 ) => {
   switch (action.type) {
     case "initial data load complete (universe exists)": {
-      const { world } = nextSharedState;
+      const { world, layoutChoice } = nextSharedState;
       const crossfilter = World.createObsDimensions(
         new Crossfilter(world.obsAnnotations),
-        world
+        world,
+        layoutChoice.currentDimNames
       );
       return crossfilter;
     }
@@ -43,9 +44,13 @@ const CrossfilterReducer = (
     case "set clip quantiles":
     case "set World to current selection": {
       const { userDefinedGenes, diffexpGenes } = prevSharedState.controls;
-      const { world } = nextSharedState;
+      const { world, layoutChoice } = nextSharedState;
       let crossfilter = new Crossfilter(world.obsAnnotations);
-      crossfilter = World.createObsDimensions(crossfilter, world);
+      crossfilter = World.createObsDimensions(
+        crossfilter,
+        world,
+        layoutChoice.currentDimNames
+      );
       crossfilter = ControlsHelpers.createGeneDimensions(
         userDefinedGenes,
         diffexpGenes,
@@ -55,6 +60,23 @@ const CrossfilterReducer = (
       return crossfilter;
     }
 
+    case "set layout choice": {
+      /*
+      when switching layouts:
+      - delete the existing XY index
+      - add the new XY index (which implicitly selects all on it)
+      */
+      const { world, layoutChoice } = nextSharedState;
+      return state
+        .delDimension(layoutDimensionName("XY"))
+        .addDimension(
+          layoutDimensionName("XY"),
+          "spatial",
+          world.obsLayout.col(layoutChoice.currentDimNames[0]).asArray(),
+          world.obsLayout.col(layoutChoice.currentDimNames[1]).asArray()
+        );
+    }
+
     case "request user defined gene success": {
       const { world } = prevSharedState;
       const gene = action.data.genes[0];
@@ -68,8 +90,9 @@ const CrossfilterReducer = (
 
     case "request differential expression success": {
       const { world } = prevSharedState;
+      const varIndexName = world.schema.annotations.var.index;
       const genes = _.map(action.data, d =>
-        world.varAnnotations.at(d[0], "name")
+        world.varAnnotations.at(d[0], varIndexName)
       );
       const crossfilter = _.reduce(
         genes,
@@ -87,10 +110,11 @@ const CrossfilterReducer = (
 
     case "clear differential expression": {
       const { world } = prevSharedState;
+      const varIndexName = world.schema.annotations.var.index;
       const crossfilter = _.reduce(
         action.diffExp,
         (xfltr, values) => {
-          const name = world.varAnnotations.at(values[0], "name");
+          const name = world.varAnnotations.at(values[0], varIndexName);
           return xfltr.delDimension(diffexpDimensionName(name));
         },
         state
@@ -161,10 +185,12 @@ const CrossfilterReducer = (
     case "categorical metadata filter select":
     case "categorical metadata filter deselect": {
       const { categoricalSelection } = nextSharedState;
+      const { world } = prevSharedState;
       const cat = categoricalSelection[action.metadataField];
+      const col = world.obsAnnotations.col(action.metadataField);
       return state.select(obsAnnoDimensionName(action.metadataField), {
         mode: "exact",
-        values: ControlsHelpers.selectedValuesForCategory(cat)
+        values: ControlsHelpers.selectedValuesForCategory(cat, col)
       });
     }
 

client/src/reducers/graphSelection.js

@@ -7,7 +7,8 @@ const GraphSelection = (
 ) => {
   switch (action.type) {
     case "set clip quantiles":
-    case "reset World to eq Universe": {
+    case "reset World to eq Universe":
+    case "set layout choice": {
       return {
         ...state,
         selection: {

client/src/reducers/index.js

@@ -12,6 +12,7 @@ import graphSelection from "./graphSelection";
 import crossfilter from "./crossfilter";
 import colors from "./colors";
 import differential from "./differential";
+import layoutChoice from "./layoutChoice";
 import responsive from "./responsive";
 import controls from "./controls";
 import resetCache from "./resetCache";
@@ -19,31 +20,33 @@ import resetCache from "./resetCache";
 import undoableConfig from "./undoableConfig";
 
 const Reducer = undoable(
-  cascadeReducers([
+    cascadeReducers([
-    ["config", config],
+        ["config", config],
-    ["universe", universe],
+        ["universe", universe],
-    ["world", world],
+        ["world", world],
-    ["categoricalSelection", categoricalSelection],
+        ["layoutChoice", layoutChoice],
-    ["continuousSelection", continuousSelection],
+        ["categoricalSelection", categoricalSelection],
-    ["graphSelection", graphSelection],
+        ["continuousSelection", continuousSelection],
-    ["crossfilter", crossfilter],
+        ["graphSelection", graphSelection],
-    ["colors", colors],
+        ["crossfilter", crossfilter],
-    ["controls", controls],
+        ["colors", colors],
-    ["differential", differential],
+        ["controls", controls],
-    ["responsive", responsive],
+        ["differential", differential],
-    ["resetCache", resetCache]
+        ["responsive", responsive],
-  ]),
+        ["resetCache", resetCache]
-  [
+    ]),
-    "world",
+    [
-    "categoricalSelection",
+        "world",
-    "continuousSelection",
+        "categoricalSelection",
-    "graphSelection",
+        "continuousSelection",
-    "crossfilter",
+        "graphSelection",
-    "colors",
+        "crossfilter",
-    "controls",
+        "colors",
-    "differential"
+        "controls",
-  ],
+        "differential",
-  undoableConfig
+        "layoutChoice"
+    ],
+    undoableConfig
 );
 
 const store = createStore(Reducer, applyMiddleware(thunk));

client/src/reducers/layoutChoice.js

@@ -0,0 +1,50 @@
+/*
+we have a UI heuristic to pick the default layout, based on assumptions
+about commonly used names.  Preferentially, pick in the following order:
+
+  1. "umap"
+  2. "tsne"
+  3. "pca"
+  4. give up, use the first available
+*/
+function bestDefaultLayout(layouts) {
+  const preferredNames = ["umap", "tsne", "pca"];
+  const idx = preferredNames.findIndex(name => layouts.indexOf(name) !== -1);
+  if (idx !== -1) return preferredNames[idx];
+  return layouts[0];
+}
+
+const LayoutChoice = (
+  state = {
+    available: [], // all available choices
+    current: undefined, // name of the current layout, eg, 'umap'
+    currentDimNames: [] // dimension name
+  },
+  action,
+  nextSharedState
+) => {
+  switch (action.type) {
+    case "initial data load complete (universe exists)":
+    case "reset World to eq Universe": {
+      // set default to default
+      const { schema } = nextSharedState.world;
+      const available = schema.layout.obs.map(v => v.name);
+      const current = bestDefaultLayout(available);
+      const currentDimNames = schema.layout.obsByName[current].dims;
+      return { available, current, currentDimNames };
+    }
+
+    case "set layout choice": {
+      const { schema } = nextSharedState.world;
+      const current = action.layoutChoice;
+      const currentDimNames = schema.layout.obsByName[current].dims;
+      return { ...state, current, currentDimNames };
+    }
+
+    default: {
+      return state;
+    }
+  }
+};
+
+export default LayoutChoice;

client/src/reducers/undoableConfig.js

@@ -59,7 +59,7 @@ const saveOnActions = new Set([
   "categorical metadata filter select",
   "categorical metadata filter deselect",
   "categorical metadata filter all of these",
-  "categorical metadata none of these",
+  "categorical metadata filter none of these",
 
   "color by categorical metadata",
   "color by continuous metadata",
@@ -72,7 +72,9 @@ const saveOnActions = new Set([
   "store current cell selection as differential set 2",
 
   "set World to current selection",
-  "set clip quantiles"
+  "set clip quantiles",
+
+  "set layout choice"
 ]);
 
 /**
@@ -98,9 +100,15 @@ const applyPending = () => ({
   [actionKey]: "applyPending",
   [stateKey]: { fsm: null }
 });
-const skip = fsm => ({ [actionKey]: "skip", [stateKey]: { fsm } });
+const skip = (fsm, transition) => ({
+  [actionKey]: "skip",
+  [stateKey]: { fsm: transition.to !== "done" ? fsm : null }
+});
 const clear = () => ({ [actionKey]: "clear", [stateKey]: { fsm: null } });
-const save = fsm => ({ [actionKey]: "save", [stateKey]: { fsm } });
+const save = (fsm, transition) => ({
+  [actionKey]: "save",
+  [stateKey]: { fsm: transition.to !== "done" ? fsm : null }
+});
 
 /*
 Error handler for state transitions that are unexpected.  Called by

client/src/util/camera.js

@@ -5,10 +5,9 @@ const mp = require("mouse-position");
 const mb = require("mouse-pressed");
 const key = require("key-pressed");
 
-const panSpeed = 0.4;
+const panSpeed = 1.0; // changed from 0.4 to 1.0 per issue #722
 const scaleSpeed = 0.5;
 const scaleMax = 3;
-// const scaleMin = 1.15
 const scaleMin = 1.03;
 
 function attachCamera(canvas, opts) {

client/src/util/scaleLinear.js

@@ -9,8 +9,14 @@
 // this is is equivalent to d3.scaleLinear().domain([0,1]).range([-1,1])
 
 export default (domain, range) => {
-  const domainStart = domain[0];
+	const domainStart = domain[0];
-  const scale = (range[1] - range[0]) / (domain[1] - domain[0]);
+	const scale = (range[1] - range[0]) / (domain[1] - domain[0]);
-  const rangeStart = range[0];
+	const invScale = 1 / scale;
-  return value => (value - domainStart) * scale + rangeStart;
+	const rangeStart = range[0];
+	const f = value => (value - domainStart) * scale + rangeStart;
+
+	// inverter
+	f.invert = value => (value - rangeStart) * invScale + domainStart;
+
+	return f;
 };

client/src/util/stateManager/controlsHelpers.js

@@ -21,16 +21,16 @@ Remember that option values can be ANY js type, except undefined/null.
   {
     _category_name_1: {
       // map of option value to index
-      categoryIndices: Map([
+      categoryValueIndices: Map([
         catval1: index,
         ...
       ])
 
       // index->selection true/false state
-      categorySelected: [ true/false, true/false, ... ]
+      categoryValueSelected: [ true/false, true/false, ... ]
 
       // number of options
-      numCategories: number,
+      numCategoryValues: number,
 
       // isTruncated - true if the options for selection has
       // been truncated (ie, was too large to implement)
@@ -56,27 +56,31 @@ function topNCategories(summary) {
 
 export function createCategoricalSelection(maxCategoryItems, world) {
   const res = {};
+  const obsIndexName = world.schema.annotations.obs.index;
   _.forEach(world.obsAnnotations.colIndex.keys(), key => {
     const summary = world.obsAnnotations.col(key).summarize();
     if (summary.categories) {
       const isColorField = key.includes("color") || key.includes("Color");
       const isSelectableCategory =
         !isColorField &&
-        key !== "name" &&
+        key !== obsIndexName &&
         summary.categories.length < maxCategoryItems;
       if (isSelectableCategory) {
-        const [categoryValues, categoryCounts] = topNCategories(summary);
+        const [categoryValues, categoryValueCounts] = topNCategories(summary);
-        const categoryIndices = new Map(categoryValues.map((v, i) => [v, i]));
+        const categoryValueIndices = new Map(
-        const numCategories = categoryIndices.size;
+          categoryValues.map((v, i) => [v, i])
-        const categorySelected = new Array(numCategories).fill(true);
+        );
+        const numCategoryValues = categoryValueIndices.size;
+        const categoryValueSelected = new Array(numCategoryValues).fill(true);
         const isTruncated = categoryValues.length < summary.numCategories;
         res[key] = {
           categoryValues, // array: of natively typed category values
-          categoryIndices, // map: category value (native type) -> category index
+          categoryValueIndices, // map: category value (native type) -> category index
-          categorySelected, // array: t/f selection state
+          categoryValueSelected, // array: t/f selection state
-          numCategories, // number: of categories
+          numCategoryValues, // number: of values in the category
           isTruncated, // bool: true if list was truncated
-          categoryCounts // array: cardinality of each category
+          categoryValueCounts, // array: cardinality of each category,
+          categorySelected: true // bool - default state for entire category
         };
       }
     }
@@ -88,12 +92,26 @@ export function createCategoricalSelection(maxCategoryItems, world) {
 given a categoricalSelection, return the list of all category values
 where selection state is true (ie, they are selected).
 */
-export function selectedValuesForCategory(categorySelectionState) {
+export function selectedValuesForCategory(categorySelectionState, dfColumn) {
-  const selectedValues = _([...categorySelectionState.categoryIndices])
+  const {
-    .filter(tuple => categorySelectionState.categorySelected[tuple[1]])
+    categorySelected,
-    .map(tuple => tuple[0])
+    categoryValueSelected,
-    .value();
+    categoryValueIndices
-  return selectedValues;
+  } = categorySelectionState;
+  let selectedValues;
+  if (categorySelected) {
+    selectedValues = new Set(dfColumn.summarize().categories);
+  } else {
+    selectedValues = new Set();
+  }
+  categoryValueIndices.forEach((catIndex, catValue) => {
+    if (!categoryValueSelected[catIndex]) {
+      selectedValues.delete(catValue);
+    } else {
+      selectedValues.add(catValue);
+    }
+  });
+  return [...selectedValues.values()];
 }
 
 /*

client/src/util/stateManager/universe.js

@@ -104,27 +104,11 @@ function LayoutFBSToDataframe(arrayBuffer) {
     throw new Error("Unexpected layout data type returned from server");
   }
 
-  /*
-  TODO: XXX
-
-  TEMPORARY CODE AND COMMENT to support the progressive implementation
-  of multi-layout support.  For now, we search for one of the following 
-  in the layouts and use it if we find it: umap, then tsne, then pca, 
-  then whatever is first in the list.
-  */
-  let layoutIndex = 0;
-  ["umap", "tsne", "pca"].some(name => {
-    const idx = fbs.colIdx.indexOf(`${name}_0`);
-    if (idx !== -1) {
-      layoutIndex = idx;
-    }
-    return idx !== -1;
-  });
   const df = new Dataframe.Dataframe(
-    [fbs.nRows, 2],
+    [fbs.nRows, fbs.nCols],
-    [fbs.columns[layoutIndex], fbs.columns[layoutIndex + 1]],
+    fbs.columns,
     null,
-    new Dataframe.KeyIndex(["X", "Y"])
+    new Dataframe.KeyIndex(fbs.colIdx)
   );
   return df;
 }
@@ -140,7 +124,7 @@ function reconcileSchemaCategoriesWithSummary(universe) {
   cases, add a 'categories' field to the schema so it is accessible.
   */
 
-  universe.schema.annotations.obs.forEach(s => {
+  universe.schema.annotations.obs.columns.forEach(s => {
     if (
       s.type === "string" ||
       s.type === "boolean" ||
@@ -172,6 +156,9 @@ export function createUniverseFromResponse(
   universe.schema = schema;
   universe.nObs = schema.dataframe.nObs;
   universe.nVar = schema.dataframe.nVar;
+  /* add defaults, as we can't assume back-end will fully populate schema */
+  if (!schema.layout.var) schema.layout.var = [];
+  if (!schema.layout.obs) schema.layout.obs = [];
 
   /* annotations */
   universe.obsAnnotations = AnnotationsFBSToDataframe(annotationsObsResponse);
@@ -192,10 +179,16 @@ export function createUniverseFromResponse(
 
   /* Index schema for ease of use */
   universe.schema.annotations.obsByName = fromEntries(
-    universe.schema.annotations.obs.map(v => [v.name, v])
+    universe.schema.annotations.obs.columns.map(v => [v.name, v])
   );
   universe.schema.annotations.varByName = fromEntries(
-    universe.schema.annotations.var.map(v => [v.name, v])
+    universe.schema.annotations.var.columns.map(v => [v.name, v])
+  );
+  universe.schema.layout.obsByName = fromEntries(
+    universe.schema.layout.obs.map(v => [v.name, v])
+  );
+  universe.schema.layout.varByName = fromEntries(
+    universe.schema.layout.var.map(v => [v.name, v])
   );
   return universe;
 }
@@ -220,8 +213,9 @@ export function convertDataFBStoObject(universe, arrayBuffer) {
     throw new Error("Unexpected non-floating point response from server.");
   }
 
+  const varIndexName = universe.schema.annotations.var.index;
   for (let c = 0; c < colIdx.length; c += 1) {
-    const varName = universe.varAnnotations.at(colIdx[c], "name");
+    const varName = universe.varAnnotations.at(colIdx[c], varIndexName);
     result[varName] = columns[c];
   }
   return result;

client/src/util/stateManager/world.js

@@ -260,13 +260,17 @@ function deduceDimensionType(attributes, fieldName) {
   return dimensionType;
 }
 
-export function createObsDimensions(crossfilter, world) {
+export function createObsDimensions(crossfilter, world, XYdimNames) {
   /*
   create and return a crossfilter with a dimension for every obs annotation
-  for which we have a supported type, *except* 'name'
+  for which we have a supported type, *except* for the index column, indicated
+  by schema.annotations.obs.index.
   */
   const { schema, obsLayout, obsAnnotations } = world;
-  const annoList = schema.annotations.obs.filter(anno => anno.name !== "name");
+  const indexName = schema.annotations.obs.index;
+  const annoList = schema.annotations.obs.columns.filter(
+    anno => anno.name !== indexName
+  );
   crossfilter = annoList.reduce((xfltr, anno) => {
     const dimType = deduceDimensionType(anno, anno.name);
     const colData = obsAnnotations.col(anno.name).asArray();
@@ -283,8 +287,8 @@ export function createObsDimensions(crossfilter, world) {
   return crossfilter.addDimension(
     layoutDimensionName("XY"),
     "spatial",
-    obsLayout.col("X").asArray(),
+    obsLayout.col(XYdimNames[0]).asArray(),
-    obsLayout.col("Y").asArray()
+    obsLayout.col(XYdimNames[1]).asArray()
   );
 }
 

dev_docs/developer_guidelines.md

@@ -0,0 +1,91 @@
+# Developer guidelines
+
+### Requirements
+- npm
+- Python 3.6+
+- Chrome
+
+[See dev section of README](../README.md)
+
+**All instructions are expected to be run from the top level cellxgene directory unless otherwise specified.**
+
+## Server dev
+### Install
+* Build the client and put static files in place: `make build-for-server-dev`
+* Install from local files: `make install-dev`
+
+### Launch
+* `cellxgene launch [options] <datafile>`
+
+### Reloading
+If you install cellxgene using `make install-dev` the server will be restarted every time you make changes on the server code. If changes affects the client, the browser must be reloaded.
+
+### Linter
+We use `flake8` to lint code. Travis CI runs `flake8 server`.
+
+### Test
+1. Install development requirements `pip install -r server/requirements-dev.txt`
+2. Run tests `pytest server/test`
+
+### Tips
+* Install in a virtualenv
+* May need to rebuild/reinstall when you make client changes
+
+## Client dev
+### Install
+1. Install prereqs for client: `npm install --prefix client/ client`
+2. Install cellxgene server: `pip install -e .` Caveat: this will not build the production client package - you must use the [server install](#install) instructions above to serve web assets.
+
+### Launch
+To launch with hot reloading you need to launch the server and the client separately. Node's hot reloading starts the client on its own node server and auto-refreshes when changes are made.
+1. Launch server (the client relies on the REST API being available): `cellxgene launch [options] <datafile>`
+2. Launch client: in `client/` directory run `npm run start`
+3. Client will be served on localhost:3000
+
+### Build
+To build only the client: `make build-client`
+
+### Linter
+We use `eslint` to lint the code and `prettier` as our code formatter.
+
+### Test
+In `client/` directory run `npm run unit-test`
+
+### Tips
+* You can also install/launch the server side code from npm scrips (requires python3.6 with virtualenv) in `client/` directory run `npm run backend-dev`
+
+## Running tests
+Client and server tests run on Travis CI for every push, PR, and commit to master on github. End to end tests run nightly on master only. 
+
+### Server unit tests
+Install development requirements `pip install -r server/requirements-dev.txt`
+Run tests `pytest server/test`
+
+### Client unit tests
+In `client/` directory run `npm run unit-test`
+
+### End to end tests
+
+End to end tests use two env variables:
+* `JEST_ENV` - environment to run end to end tests. Default `dev`
+    * `prod` - run headless with no slowdown, chromium will not open.
+    * `dev` - opens chromimum, runs tests with minimal slowdown, close on exit.
+    * `debug` - opens chromium, runs tests with 100ms slowdown, dev tools open, chrome stays open on exit.
+* `JEST_CXG_PORT` - port that end to end tests are being run on. Default `3000` (client hosted port).
+
+On CI the end to end tests are run with `JEST_ENV` set to `prod` using the `smoke-test` npm script
+
+To run end to end tests as they will be run on CI
+1. cellxgene should be built and installed as [specified in server dev](#install)
+2. `export JEST_ENV='prod'`
+3. `export JEST_CXG_PORT='5000'`
+4. Run `npm run --prefix client/ smoke-test`
+
+Run end to end tests interactively during development
+1. cellxgene should be installed as [specified in client dev](#install-1)
+2. Follow [launch](#launch-1) instructions for client dev with dataset `example-dataset/pbmc3k`
+3. Run `npm run --prefix client/ e2e`
+4. To debug a failing test `export JEST_ENV='debug'` and re-run.
+
+
+

server/app/scanpy_engine/scanpy_engine.py

@@ -50,41 +50,61 @@ class ScanpyEngine(CXGDriver):
             "diffexp_lfc_cutoff": 0.01,
         }
 
-    def _alias_annotation_names(self, axis, name):
+    @staticmethod
-        """
+    def _create_unique_column_name(df, col_name_prefix):
-        Do all user-specified annotation aliasing.
+        """ given the columns of a dataframe, and a name prefix, return a column name which
+            does not exist in the dataframe, AND which is prefixed by `prefix`
 
-        As a *critical* side-effect, ensure the indices are simple number ranges
+            The approach is to append a numeric suffix, starting at zero and increasing by
-        (accomplished by calling pandas.DataFrame.reset_index())
+            one, until an unused name is found (eg, prefix_0, prefix_1, ...).
         """
-        if name == "name":
+        suffix = 0
-            # a noop, so skip it
+        while f"{col_name_prefix}{suffix}" in df:
-            return
+            suffix += 1
+        return f"{col_name_prefix}{suffix}"
 
-        ax_name = str(axis)
+    def _alias_annotation_names(self):
-        df_axis = getattr(self.data, ax_name)
+        """
-        if name is None:
+        The front-end relies on the existance of a unique, human-readable
-            # reset index to simple range; alias "name" to point at the
+        index for obs & var (eg, var is typically gene name, obs the cell name).
-            # previously specified index.
+        The user can specify these via the --obs-names and --var-names config.
-            df_axis.reset_index(inplace=True)
+        If they are not specified, use the existing index to create them, giving
-            df_axis.rename(inplace=True, columns={"index": "name"})
+        the resulting column a unique name (eg, "name").
-        elif name in df_axis.columns:
+
-            if name not in df_axis.columns:
+        In both cases, enforce that the result is unique, and communicate the
+        index column name to the front-end via the obs_names and var_names config
+        (which is incorporated into the schema).
+        """
+        for (ax_name, config_name) in ((Axis.OBS, "obs_names"), (Axis.VAR, "var_names")):
+            name = self.config[config_name]
+            df_axis = getattr(self.data, str(ax_name))
+            if name is None:
+                # Default: create unique names from index
+                if not df_axis.index.is_unique:
+                    raise KeyError(
+                        f"Values in {ax_name}.index must be unique. "
+                        "Please prepare data to contain unique index values, or specify an "
+                        "alternative with --{ax_name}-name."
+                    )
+                name = self._create_unique_column_name(df_axis.columns, "name_")
+                self.config[config_name] = name
+                # reset index to simple range; alias name to point at the
+                # previously specified index.
+                df_axis.rename_axis(name, inplace=True)
+                df_axis.reset_index(inplace=True)
+            elif name in df_axis.columns:
+                # User has specified alternative column for unique names, and it exists
+                if not df_axis[name].is_unique:
+                    raise KeyError(
+                        f"Values in {ax_name}.{name} must be unique. "
+                        "Please prepare data to contain unique values."
+                    )
+                df_axis.reset_index(drop=True, inplace=True)
+            else:
+                # user specified a non-existent column name
                 raise KeyError(
                     f"Annotation name {name}, specified in --{ax_name}-name does not exist."
                 )
-            if not df_axis[name].is_unique:
-                raise KeyError(
-                    f"Values in -{ax_name}-name must be unique. "
-                    "Please prepare data to contain unique values."
-                )
-            # reset index to simple range; alias user-specified annotation to "name"
-            df_axis.reset_index(drop=True, inplace=True)
-            df_axis.rename(inplace=True, columns={name: "name"})
-        else:
-            raise KeyError(
-                f"Annotation name {name}, specified in --{ax_name}_name does not exist."
-            )
 
     @staticmethod
     def _can_cast_to_float32(ann):
@@ -114,7 +134,17 @@ class ScanpyEngine(CXGDriver):
                 "nVar": self.gene_count,
                 "type": str(self.data.X.dtype),
             },
-            "annotations": {"obs": [], "var": []},
+            "annotations": {
+                "obs": {
+                    "index": self.config["obs_names"],
+                    "columns": []
+                },
+                "var": {
+                    "index": self.config["var_names"],
+                    "columns": []
+                }
+            },
+            "layout": {"obs": []}
         }
         for ax in Axis:
             curr_axis = getattr(self.data, str(ax))
@@ -138,7 +168,15 @@ class ScanpyEngine(CXGDriver):
                     raise TypeError(
                         f"Annotations of type {curr_axis[ann].dtype} are unsupported by cellxgene."
                     )
-                self.schema["annotations"][ax].append(ann_schema)
+                self.schema["annotations"][ax]["columns"].append(ann_schema)
+
+        for layout in self.config['layout']:
+            layout_schema = {
+                "name": layout,
+                "type": "float32",
+                "dims": [f"{layout}_0", f"{layout}_1"]
+            }
+            self.schema["layout"]["obs"].append(layout_schema)
 
     def _load_data(self, data):
         # as of AnnData 0.6.19, backed mode performs initial load fast, but at the
@@ -164,8 +202,11 @@ class ScanpyEngine(CXGDriver):
 
     @requires_data
     def _validate_and_initialize(self):
-        self._alias_annotation_names(Axis.OBS, self.config["obs_names"])
+        # var and obs column names must be unique
-        self._alias_annotation_names(Axis.VAR, self.config["var_names"])
+        if not self.data.obs.columns.is_unique or not self.data.var.columns.is_unique:
+            raise KeyError(f"All annotation column names must be unique.")
+
+        self._alias_annotation_names()
         self._validate_data_types()
         self.cell_count = self.data.shape[0]
         self.gene_count = self.data.shape[1]

server/cli/cli.py

@@ -5,7 +5,7 @@ from .prepare import prepare
 
 
 @click.group(name="cellxgene", context_settings=dict(max_content_width=85))
-@click.version_option(version="0.9.1", prog_name="cellxgene", message="[%(prog)s] Version %(version)s")
+@click.version_option(version="0.10.0", prog_name="cellxgene", message="[%(prog)s] Version %(version)s")
 def cli():
     pass
 

server/cli/launch.py

@@ -7,12 +7,11 @@ import warnings
 import webbrowser
 
 import click
-import psutil
 
 from server.app.app import Server
 from server.app.util.errors import ScanpyFileError
 from server.app.util.utils import custom_format_warning
-from server.utils.utils import find_available_port
+from server.utils.utils import find_available_port, is_port_available
 
 
 # anything bigger than this will generate a special message
@@ -142,7 +141,14 @@ security risk by including the --scripts flag. Make sure you trust the scripts t
         file_parts = splitext(basename(data))
         title = file_parts[0]
 
-    if not port:
+    if port:
+        if debug:
+            raise click.ClickException("--port and --debug may not be used together (try --verbose for error logging).")
+        if not is_port_available(host, int(port)):
+            raise click.ClickException(
+                f"The port selected {port} is in use, please specify an open port using the --port flag."
+            )
+    else:
         port = find_available_port(host)
 
     # Setup app
@@ -165,9 +171,6 @@ security risk by including the --scripts flag. Make sure you trust the scripts t
         click.echo(f"[cellxgene] Loading data from {basename(data)}, this may take awhile...")
     else:
         click.echo(f"[cellxgene] Loading data from {basename(data)}.")
-    # if file is larger than main memory, let the user know performance may suffer
-    if file_size > .95 * psutil.virtual_memory().total:
-        click.echo(f"[cellxgene] Warning: data file is larger than RAM - application may be very slow.")
 
     # Fix for anaconda python. matplotlib typically expects python to be installed as a framework TKAgg is usually
     # available and fixes this issue. See https://matplotlib.org/faq/virtualenv_faq.html
@@ -203,7 +206,7 @@ security risk by including the --scripts flag. Make sure you trust the scripts t
         sys.stdout = f
 
     try:
-        server.app.run(host=host, debug=debug, port=port, threaded=True)
+        server.app.run(host=host, debug=debug, port=port, threaded=True, use_debugger=False)
     except OSError as e:
         if e.errno == errno.EADDRINUSE:
             raise click.ClickException("Port is in use, please specify an open port using the --port flag.") from e

server/requirements.txt

@@ -1,4 +1,4 @@
-anndata>=0.6.15
+anndata>=0.6.20
 click>=6.7
 Flask>=1.0.2
 Flask-Caching>=1.4.0
@@ -9,8 +9,7 @@ flatbuffers>=1.10.0
 matplotlib>=2.2
 numpy>=1.15.2
 pandas>=0.23.1
-psutil>=5.6.2
 scanpy>=1.3.7
-scipy>=1.1.0,<1.3
+scipy>=1.1.0
 scikit-learn>=0.19.1,!=0.20.0
 tables>=3.5.1

server/test/schema.json

@@ -5,46 +5,61 @@
     "type": "float32"
   },
   "annotations": {
+    "obs": {
+      "index": "name_0",
+      "columns": [
+        {
+          "name": "name_0",
+          "type": "string"
+        },
+        {
+          "name": "n_genes",
+          "type": "int32"
+        },
+        {
+          "name": "percent_mito",
+          "type": "float32"
+        },
+        {
+          "name": "n_counts",
+          "type": "float32"
+        },
+        {
+          "name": "louvain",
+          "type": "categorical",
+          "categories": [
+            "CD4 T cells",
+            "CD14+ Monocytes",
+            "B cells",
+            "CD8 T cells",
+            "NK cells",
+            "FCGR3A+ Monocytes",
+            "Dendritic cells",
+            "Megakaryocytes"
+          ]
+        }
+      ]
+    },
+    "var": {
+      "index": "name_0",
+      "columns": [
+        {
+          "name": "name_0",
+          "type": "string"
+        },
+        {
+          "name": "n_cells",
+          "type": "int32"
+        }
+      ]
+    }
+  },
+  "layout": {
     "obs": [
       {
-        "name": "name",
+        "name": "umap",
-        "type": "string"
+        "type": "float32",
-      },
+        "dims": ["umap_0", "umap_1"]
-      {
-        "name": "n_genes",
-        "type": "int32"
-      },
-      {
-        "name": "percent_mito",
-        "type": "float32"
-      },
-      {
-        "name": "n_counts",
-        "type": "float32"
-      },
-      {
-        "name": "louvain",
-        "type": "categorical",
-        "categories": [
-          "CD4 T cells",
-          "CD14+ Monocytes",
-          "B cells",
-          "CD8 T cells",
-          "NK cells",
-          "FCGR3A+ Monocytes",
-          "Dendritic cells",
-          "Megakaryocytes"
-        ]
-      }
-    ],
-    "var": [
-      {
-        "name": "name",
-        "type": "string"
-      },
-      {
-        "name": "n_cells",
-        "type": "int32"
       }
     ]
   }

server/test/test_api.py

@@ -19,11 +19,12 @@ class EndPoints(unittest.TestCase):
 
     @classmethod
     def setUpClass(cls):
-        cls.ps = Popen(["cellxgene", "launch", "example-dataset/pbmc3k.h5ad", "--debug", "--port", "5005"])
+        cls.ps = Popen(["cellxgene", "launch", "example-dataset/pbmc3k.h5ad", "--verbose", "--port", "5005"])
         session = requests.Session()
         for i in range(90):
             try:
-                session.get(f"{URL_BASE}schema")
+                result = session.get(f"{URL_BASE}schema")
+                cls.schema = result.json()
             except requests.exceptions.ConnectionError:
                 time.sleep(1)
 
@@ -45,7 +46,8 @@ class EndPoints(unittest.TestCase):
         self.assertEqual(result.headers["Content-Type"], "application/json")
         result_data = result.json()
         self.assertEqual(result_data["schema"]["dataframe"]["nObs"], 2638)
-        self.assertEqual(len(result_data["schema"]["annotations"]["obs"]), 5)
+        self.assertEqual(len(result_data["schema"]["annotations"]["obs"]), 2)
+        self.assertEqual(len(result_data["schema"]["annotations"]["obs"]["columns"]), 5)
 
     def test_config(self):
         endpoint = "config"
@@ -95,7 +97,8 @@ class EndPoints(unittest.TestCase):
         self.assertIsNotNone(df['col_idx'])
         self.assertIsNone(df['row_idx'])
         self.assertEqual(len(df['columns']), df['n_cols'])
-        self.assertListEqual(df['col_idx'], ['name', 'n_genes', 'percent_mito', 'n_counts', 'louvain'])
+        obs_index_col_name = self.schema["schema"]["annotations"]["obs"]["index"]
+        self.assertListEqual(df['col_idx'], [obs_index_col_name, 'n_genes', 'percent_mito', 'n_counts', 'louvain'])
 
     def test_get_annotations_obs_keys_fbs(self):
         endpoint = "annotations/obs"
@@ -165,7 +168,8 @@ class EndPoints(unittest.TestCase):
         self.assertIsNotNone(df['col_idx'])
         self.assertIsNone(df['row_idx'])
         self.assertEqual(len(df['columns']), df['n_cols'])
-        self.assertListEqual(df['col_idx'], ['name', 'n_cells'])
+        var_index_col_name = self.schema["schema"]["annotations"]["var"]["index"]
+        self.assertListEqual(df['col_idx'], [var_index_col_name, 'n_cells'])
 
     def test_get_annotations_var_keys_fbs(self):
         endpoint = "annotations/var"
@@ -247,7 +251,8 @@ class EndPoints(unittest.TestCase):
         endpoint = f"data/var"
         url = f"{URL_BASE}{endpoint}"
         header = {"Accept": "application/octet-stream"}
-        var_filter = {"filter": {"var": {"annotation_value": [{"name": "name", "values": ["RER1"]}]}}}
+        index_col_name = self.schema["schema"]["annotations"]["var"]["index"]
+        var_filter = {"filter": {"var": {"annotation_value": [{"name": index_col_name, "values": ["RER1"]}]}}}
         result = self.session.put(url, headers=header, json=var_filter)
         self.assertEqual(result.status_code, HTTPStatus.OK)
         self.assertEqual(result.headers["Content-Type"], "application/octet-stream")

server/test/test_nan_rest.py

@@ -8,7 +8,7 @@ import decode_fbs
 
 import requests
 
-LOCAL_URL = "http://127.0.0.1:5005/"
+LOCAL_URL = "http://127.0.0.1:5006/"
 VERSION = "v0.2"
 URL_BASE = f"{LOCAL_URL}api/{VERSION}/"
 
@@ -21,7 +21,7 @@ class WithNaNs(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
         cls.ps = Popen(
-            ["cellxgene", "launch", "server/test/test_datasets/nan.h5ad", "--debug", "--port", "5005"]
+            ["cellxgene", "launch", "server/test/test_datasets/nan.h5ad", "--verbose", "--port", "5006"]
         )
         session = requests.Session()
         for i in range(90):

server/test/test_nan_scanpy_engine.py

@@ -58,14 +58,16 @@ class NaNTest(unittest.TestCase):
 
     def test_annotation(self):
         annotations = decode_fbs.decode_matrix_FBS(self.data.annotation_to_fbs_matrix("obs"))
+        obs_index_col_name = self.data.schema["annotations"]["obs"]["index"]
         self.assertEqual(
             annotations["col_idx"],
-            ["name", "n_genes", "percent_mito", "n_counts", "louvain"]
+            [obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain"]
         )
         self.assertEqual(annotations["n_rows"], 100)
         self.assertTrue(math.isnan(annotations["columns"][2][0]))
 
         annotations = decode_fbs.decode_matrix_FBS(self.data.annotation_to_fbs_matrix("var"))
-        self.assertEqual(annotations["col_idx"], ["name", "n_cells", "var_with_nans"])
+        var_index_col_name = self.data.schema["annotations"]["var"]["index"]
+        self.assertEqual(annotations["col_idx"], [var_index_col_name, "n_cells", "var_with_nans"])
         self.assertEqual(annotations["n_rows"], 100)
         self.assertTrue(math.isnan(annotations["columns"][2][0]))

server/test/test_scanpy_engine.py

@@ -31,9 +31,11 @@ class EngineTest(unittest.TestCase):
         self.assertTrue(self.data.data.X[0, 0] - -0.171_469_51 < epsilon)
 
     def test_mandatory_annotations(self):
-        self.assertIn("name", self.data.data.obs)
+        obs_index_col_name = self.data.schema["annotations"]["obs"]["index"]
+        self.assertIn(obs_index_col_name, self.data.data.obs)
         self.assertEqual(list(self.data.data.obs.index), list(range(2638)))
-        self.assertIn("name", self.data.data.var)
+        var_index_col_name = self.data.schema["annotations"]["var"]["index"]
+        self.assertIn(var_index_col_name, self.data.data.var)
         self.assertEqual(list(self.data.data.var.index), list(range(1838)))
 
     @pytest.mark.filterwarnings("ignore:Scanpy data matrix")
@@ -70,12 +72,14 @@ class EngineTest(unittest.TestCase):
         self.assertEqual(data["n_cols"], 91)
 
     def test_obs_and_var_names(self):
-        self.assertEqual(np.sum(self.data.data.var["name"].isna()), 0)
+        self.assertEqual(np.sum(self.data.data.var[self.data.schema["annotations"]["var"]["index"]].isna()), 0)
-        self.assertEqual(np.sum(self.data.data.obs["name"].isna()), 0)
+        self.assertEqual(np.sum(self.data.data.obs[self.data.schema["annotations"]["obs"]["index"]].isna()), 0)
 
     def test_schema(self):
         with open(path.join(path.dirname(__file__), "schema.json")) as fh:
             schema = json.load(fh)
+            print(schema)
+            print(self.data.schema)
             self.assertEqual(self.data.schema, schema)
 
     def test_schema_produces_error(self):
@@ -108,16 +112,18 @@ class EngineTest(unittest.TestCase):
         annotations = decode_fbs.decode_matrix_FBS(fbs)
         self.assertEqual(annotations["n_rows"], 2638)
         self.assertEqual(annotations["n_cols"], 5)
+        obs_index_col_name = self.data.schema["annotations"]["obs"]["index"]
         self.assertEqual(
             annotations["col_idx"],
-            ["name", "n_genes", "percent_mito", "n_counts", "louvain"],
+            [obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain"],
         )
 
         fbs = self.data.annotation_to_fbs_matrix("var")
         annotations = decode_fbs.decode_matrix_FBS(fbs)
         self.assertEqual(annotations['n_rows'], 1838)
         self.assertEqual(annotations['n_cols'], 2)
-        self.assertEqual(annotations["col_idx"], ["name", "n_cells"])
+        var_index_col_name = self.data.schema["annotations"]["var"]["index"]
+        self.assertEqual(annotations["col_idx"], [var_index_col_name, "n_cells"])
 
     def test_annotation_fields(self):
         fbs = self.data.annotation_to_fbs_matrix("obs", ["n_genes", "n_counts"])
@@ -125,7 +131,8 @@ class EngineTest(unittest.TestCase):
         self.assertEqual(annotations["n_rows"], 2638)
         self.assertEqual(annotations['n_cols'], 2)
 
-        fbs = self.data.annotation_to_fbs_matrix("var", ["name"])
+        var_index_col_name = self.data.schema["annotations"]["var"]["index"]
+        fbs = self.data.annotation_to_fbs_matrix("var", [var_index_col_name])
         annotations = decode_fbs.decode_matrix_FBS(fbs)
         self.assertEqual(annotations['n_rows'], 1838)
         self.assertEqual(annotations['n_cols'], 1)
@@ -163,9 +170,10 @@ class EngineTest(unittest.TestCase):
             self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
 
     def test_data_named_gene(self):
+        var_index_col_name = self.data.schema["annotations"]["var"]["index"]
         filter_ = {
             "filter": {
-                "var": {"annotation_value": [{"name": "name", "values": ["RER1"]}]}
+                "var": {"annotation_value": [{"name": var_index_col_name, "values": ["RER1"]}]}
             }
         }
         fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
@@ -176,7 +184,7 @@ class EngineTest(unittest.TestCase):
 
         filter_ = {
             "filter": {
-                "var": {"annotation_value": [{"name": "name", "values": ["SPEN", "TYMP", "PRMT2"]}]}
+                "var": {"annotation_value": [{"name": var_index_col_name, "values": ["SPEN", "TYMP", "PRMT2"]}]}
             }
         }
         fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")

server/utils/utils.py

@@ -10,10 +10,17 @@ def find_available_port(host, port=5005):
     # Takes approx 2 seconds to do a scan of 5000 ports on my laptop
     num_ports_to_try = 5000
     for port_to_try in range(port, port + num_ports_to_try):
-        with contextlib.closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s:
+        if is_port_available(host, port_to_try):
-            try:
+            return port_to_try
-                s.bind((host, port_to_try))
-                return port_to_try
-            except socket.error:
-                pass
     raise socket.error(errno.EADDRINUSE, f"No port in range {port} - {port + num_ports_to_try - 1} available.")
+
+
+def is_port_available(host, port):
+    is_available = False
+    with contextlib.closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s:
+        try:
+            s.bind((host, port))
+            is_available = True
+        except socket.error:
+            pass
+    return is_available

setup.py

@@ -8,7 +8,7 @@ with open("server/requirements.txt") as fh:
 
 setup(
     name="cellxgene",
-    version="0.9.1",
+    version="0.10.0",
     packages=find_packages(),
     url="https://github.com/chanzuckerberg/cellxgene",
     license="MIT",