release 0.10.0 (#794 )

fix incompatibility of flask reload and port searching (#793 )
* WIP * add --developer; fix incompatibility of --port and --debug * put REST tests on separate ports * PR review
2026-09-27 09:58:12 +08:00 · 2019-05-29 16:55:43 -07:00 · 2019-05-29 16:38:57 -07:00 · 2019-05-29 12:23:30 -04:00 · 2019-05-24 21:01:11 -07:00 · 2019-05-24 21:00:54 -07:00
46 changed files with 5949 additions and 3854 deletions
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.9.1
+current_version = 0.10.0

 [bumpversion:file:setup.py]
 search = version="{current_version}"
@@ -27,6 +27,5 @@ jobs:
    script: docker build .
  - name: "Smoke Tests"
    python: "3.6"
-    if: branch = master AND type = cron
    script:
      - npm run --prefix client/ smoke-test
@@ -1,6 +1,6 @@
 export const datasets = {
  pbmc3k: {
-    title: "cellxgene: pbmc3k",
+    title: "pbmc3k",
    dataframe: {
      nObs: "2638",
      nVar: "1838",
@@ -26,8 +26,8 @@ export const datasets = {
    cellsets: {
      lasso: [
        {
-          "coordinates-as-percent": { x1: 0.25, y1: 0.25, x2: 0.35, y2: 0.35 },
-          count: "26"
+          "coordinates-as-percent": { x1: 0.05, y1: 0.25, x2: 0.15, y2: 0.35 },
+          count: "104"
        }
      ],
      categorical: [
@@ -91,8 +91,8 @@ export const datasets = {
        }
      },
      lasso: {
-        "coordinates-as-percent": { x1: 0.45, y1: 0.45, x2: 0.5, y2: 0.5 },
-        count: "67"
+        "coordinates-as-percent": { x1: 0.45, y1: 0.05, x2: 0.5, y2: 0.1 },
+        count: "76"
      }
    },
    scatter: {
@@ -272,7 +272,7 @@ describe("scatter plot", async () => {

 describe("clipping", async () => {
  test("clip continuous", async () => {
-    await cxgActions.clip(data.clip.min, data.clip.max)
+    await cxgActions.clip(data.clip.min, data.clip.max);
    const histId = `histogram-${data.clip.metadata}-plot-brush`;
    const coords = await cxgActions.calcDragCoordinates(
      histId,
@@ -281,16 +281,13 @@ describe("clipping", async () => {
    await cxgActions.drag(histId, coords.start, coords.end);
    const cellCount = await cxgActions.cellSet(1);
    expect(cellCount).toBe(data.clip.count);
-    
  });

  test("clip gene", async () => {
    await utils.typeInto("gene-search", data.clip.gene);
    await page.keyboard.press("Enter");
-    await page.waitForSelector(
-      `[data-testid='histogram-${data.clip.gene}']`
-    );
-    await cxgActions.clip(data.clip.min, data.clip.max)
+    await page.waitForSelector(`[data-testid='histogram-${data.clip.gene}']`);
+    await cxgActions.clip(data.clip.min, data.clip.max);
    const histId = `histogram-${data.clip.gene}-plot-brush`;
    const coords = await cxgActions.calcDragCoordinates(
      histId,
@@ -44,11 +44,13 @@ export const puppeteerUtils = puppeteerPage => ({
  },

  async getOneElementInnerHTML(selector) {
+    await puppeteerPage.waitForSelector(selector);
    let text = await puppeteerPage.$eval(selector, el => el.innerHTML);
    return text;
  },

  async getOneElementInnerText(selector) {
+    await puppeteerPage.waitForSelector(selector);
    let text = await puppeteerPage.$eval(selector, el => el.innerText);
    return text;
  }
@@ -34,28 +34,38 @@ const aSchemaResponse = {
      type: "float32"
    },
    annotations: {
-      obs: [
-        { name: "name", type: "string" },
-        { name: "field1", type: "int32" },
-        { name: "field2", type: "float32" },
-        { name: "field3", type: "boolean" },
-        {
-          name: "field4",
-          type: "categorical",
-          categories: field4Categories
-        }
-      ],
-      var: [
-        { name: "name", type: "string" },
-        { name: "fieldA", type: "int32" },
-        { name: "fieldB", type: "float32" },
-        { name: "fieldC", type: "boolean" },
-        {
-          name: "fieldD",
-          type: "categorical",
-          categories: fieldDCategories
-        }
-      ]
+      obs: {
+        index: "name",
+        columns: [
+          { name: "name", type: "string" },
+          { name: "field1", type: "int32" },
+          { name: "field2", type: "float32" },
+          { name: "field3", type: "boolean" },
+          {
+            name: "field4",
+            type: "categorical",
+            categories: field4Categories
+          }
+        ]
+      },
+      var: {
+        index: "name",
+        columns: [
+          { name: "name", type: "string" },
+          { name: "fieldA", type: "int32" },
+          { name: "fieldB", type: "float32" },
+          { name: "fieldC", type: "boolean" },
+          {
+            name: "fieldD",
+            type: "categorical",
+            categories: fieldDCategories
+          }
+        ]
+      }
+    },
+    layout: {
+      obs: [{ name: "umap", type: "float32", dims: ["umap_0", "umap_1"] }],
+      var: []
    }
  }
 };
@@ -53,13 +53,15 @@ describe("createUniverseFromResponse", () => {

    expect(universe.obsAnnotations.dims).toEqual([
      nObs,
-      REST.schema.schema.annotations.obs.length
+      REST.schema.schema.annotations.obs.columns.length
    ]);
    expect(universe.obsLayout.dims).toEqual([nObs, 2]);
-    expect(universe.obsLayout.colIndex.keys()).toEqual(["X", "Y"]);
+    expect(universe.obsLayout.colIndex.keys()).toEqual(
+      universe.schema.layout.obs[0].dims
+    );
    expect(universe.varAnnotations.dims).toEqual([
      nVar,
-      REST.schema.schema.annotations.var.length
+      REST.schema.schema.annotations.var.columns.length
    ]);
    expect(universe.varData.isEmpty()).toBeTruthy();
  });
@@ -29,7 +29,8 @@ const defaultBigBang = () => {
  /* create crossfilter */
  const crossfilter = World.createObsDimensions(
    new Crossfilter(world.obsAnnotations),
-    world
+    world,
+    REST.schema.schema.layout.obs[0].dims
  );

  return {
@@ -138,7 +139,9 @@ describe("createWorldFromCurrentSelection", () => {
    expect(world.obsLayout.rowIndex.keys()).toEqual(
      new Int32Array(matchingIndices)
    );
-    expect(world.obsLayout.colIndex.keys()).toEqual(["X", "Y"]);
+    expect(world.obsLayout.colIndex.keys()).toEqual(
+      world.schema.layout.obs[0].dims
+    );
  });
 });

@@ -152,14 +155,18 @@ describe("createObsDimensionMap", () => {

    const { crossfilter } = defaultBigBang();
    const annotationNames = _.map(
-      REST.schema.schema.annotations.obs,
+      REST.schema.schema.annotations.obs.columns,
      c => c.name
    );
-    const schemaByObsName = _.keyBy(REST.schema.schema.annotations.obs, "name");
+    const obsIndexColName = REST.schema.schema.annotations.obs.index;
+    const schemaByObsName = _.keyBy(
+      REST.schema.schema.annotations.obs.columns,
+      "name"
+    );
    expect(crossfilter).toBeDefined();
    annotationNames.forEach(name => {
      const dim = crossfilter.dimensions[obsAnnoDimensionName(name)];
-      if (name === "name") {
+      if (name === obsIndexColName) {
        expect(dim).toBeUndefined();
      } else {
        const { type } = schemaByObsName[name];
@@ -7,8 +7,8 @@ module.exports = {
  ],
  plugins: [
    "@babel/plugin-proposal-function-bind",
-    "@babel/plugin-proposal-class-properties",
    ["@babel/plugin-proposal-decorators", { legacy: true }],
+    ["@babel/plugin-proposal-class-properties", { loose: true }],
    "@babel/plugin-proposal-export-namespace-from",
    "@babel/plugin-proposal-optional-chaining",
    "@babel/plugin-proposal-nullish-coalescing-operator"
@@ -6,8 +6,8 @@ module.exports = {
  ],
  plugins: [
    "@babel/plugin-proposal-function-bind",
-    "@babel/plugin-proposal-class-properties",
    ["@babel/plugin-proposal-decorators", { legacy: true }],
+    ["@babel/plugin-proposal-class-properties", { loose: true }],
    "@babel/plugin-proposal-export-namespace-from",
    "@babel/plugin-transform-react-constant-elements",
    "@babel/plugin-transform-runtime",
@@ -1,6 +1,6 @@
 {
  "name": "cellxgene",
-  "version": "0.9.1",
+  "version": "0.10.0",
  "license": "MIT",
  "description": "cellxgene is a web application for the interactive exploration of single cell sequence data.",
  "repository": "https://github.com/chanzuckerberg/cellxgene",
@@ -9,13 +9,13 @@
    "build": "npm run clean && webpack --config configuration/webpack/webpack.config.prod.js",
    "clean": "rimraf build",
    "dev": "npm run clean && webpack --config configuration/webpack/webpack.config.dev.js",
-    "e2e": "jest --verbose false --config __tests__/e2e/e2eJestConfig.json e2e/e2e.test.js",
+    "e2e": "node node_modules/jest/bin/jest.js --verbose false --config __tests__/e2e/e2eJestConfig.json e2e/e2e.test.js",
    "lint": "eslint src",
    "smoke-test": "start-server-and-test start-server-for-test :5000 e2e",
    "start": "node server/development.js",
    "start-server-for-test": "cellxgene launch -p 5000 ../example-dataset/pbmc3k.h5ad",
-    "test": "jest",
-    "unit-test": "jest --testPathIgnorePatterns e2e"
+    "test": "node node_modules/jest/bin/jest.js",
+    "unit-test": "node node_modules/jest/bin/jest.js --testPathIgnorePatterns e2e"
  },
  "engineStrict": true,
  "engines": {
@@ -31,8 +31,8 @@
    "eslint-scope": "3.7.1"
  },
  "dependencies": {
-    "@blueprintjs/core": "^3.15.0",
-    "@blueprintjs/icons": "^3.3.0",
+    "@blueprintjs/core": "^3.15.1",
+    "@blueprintjs/icons": "^3.8.0",
    "@blueprintjs/select": "^3.8.0",
    "canvas-fit": "^1.5.0",
    "d3": "^4.10.0",
@@ -41,79 +41,77 @@
    "font-color-contrast": "^1.0.3",
    "fuzzysort": "^1.1.4",
    "gl-mat4": "^1.1.4",
-    "gl-matrix": "^2.7.1",
+    "gl-vec3": "^1.1.3",
+    "gl-matrix": "^3.0.0",
    "is-number": "^7.0.0",
    "key-pressed": "0.0.1",
    "lodash": "^4.17.4",
-    "memoize-one": "^4.0.0",
+    "memoize-one": "^5.0.4",
    "mouse-position": "^2.0.1",
    "mouse-pressed": "^1.0.0",
-    "normalize.css": "^8.0.0",
    "orbit-camera": "^1.0.0",
-    "query-string": "^6.1.0",
-    "react": "^16.6.0",
+    "query-string": "^6.5.0",
+    "react": "^16.8.6",
    "react-autocomplete": "^1.7.2",
-    "react-dom": "^16.6.0",
-    "react-helmet": "^5.2.0",
-    "react-icons": "^3.2.2",
-    "react-redux": "^5.1.0",
+    "react-dom": "^16.8.6",
+    "react-helmet": "^5.2.1",
+    "react-icons": "^3.7.0",
+    "react-redux": "^7.0.3",
    "redux": "^4.0.1",
-    "redux-devtools-extension": "^2.13.5",
    "redux-thunk": "^2.2.0",
-    "regl": "^1.3.9",
+    "regl": "^1.3.11",
    "scroll-speed": "^1.0.0",
    "urijs": "^1.19.0"
  },
  "devDependencies": {
-    "@babel/core": "^7.1.5",
-    "@babel/plugin-proposal-class-properties": "^7.0.0",
-    "@babel/plugin-proposal-decorators": "^7.0.0",
-    "@babel/plugin-proposal-export-namespace-from": "^7.0.0",
-    "@babel/plugin-proposal-function-bind": "^7.0.0",
-    "@babel/plugin-proposal-nullish-coalescing-operator": "^7.2.0",
+    "@babel/core": "^7.4.4",
+    "@babel/plugin-proposal-class-properties": "^7.4.4",
+    "@babel/plugin-proposal-decorators": "^7.4.4",
+    "@babel/plugin-proposal-export-namespace-from": "^7.2.0",
+    "@babel/plugin-proposal-function-bind": "^7.2.0",
+    "@babel/plugin-proposal-nullish-coalescing-operator": "^7.4.4",
    "@babel/plugin-proposal-optional-chaining": "^7.2.0",
-    "@babel/plugin-transform-react-constant-elements": "^7.0.0",
-    "@babel/plugin-transform-runtime": "^7.1.0",
-    "@babel/preset-env": "^7.1.5",
+    "@babel/plugin-transform-react-constant-elements": "^7.2.0",
+    "@babel/plugin-transform-runtime": "^7.4.4",
+    "@babel/preset-env": "^7.4.4",
    "@babel/preset-react": "^7.0.0",
-    "@babel/register": "^7.0.0",
-    "@babel/runtime": "^7.1.5",
-    "babel-core": "^7.0.0-bridge.0",
+    "@babel/register": "^7.4.4",
+    "@babel/runtime": "^7.4.4",
    "babel-eslint": "^10.0.1",
-    "babel-jest": "^23.6.0",
-    "babel-loader": "^8.0.0",
-    "babel-preset-modern-browsers": "^12.0.0",
+    "babel-jest": "^24.8.0",
+    "babel-loader": "^8.0.6",
+    "babel-preset-modern-browsers": "^14.0.0",
    "chalk": "^2.4.2",
    "connect-history-api-fallback": "^1.6.0",
-    "copy-webpack-plugin": "^4.6.0",
-    "css-loader": "^1.0.1",
-    "eslint": "^5.13.0",
+    "copy-webpack-plugin": "^5.0.3",
+    "css-loader": "^2.1.1",
+    "eslint": "^5.16.0",
    "eslint-config-airbnb": "^17.1.0",
-    "eslint-config-prettier": "^4.0.0",
+    "eslint-config-prettier": "^4.2.0",
    "eslint-loader": "^2.1.2",
    "eslint-plugin-filenames": "^1.3.2",
-    "eslint-plugin-import": "^2.16.0",
-    "eslint-plugin-jest": "^22.2.2",
+    "eslint-plugin-import": "^2.17.2",
+    "eslint-plugin-jest": "^22.5.1",
    "eslint-plugin-jsx-a11y": "^6.2.1",
-    "eslint-plugin-react": "^7.12.4",
+    "eslint-plugin-react": "^7.13.0",
    "express": "^4.14.0",
-    "file-loader": "^2.0.0",
+    "file-loader": "^3.0.1",
    "html-webpack-inline-source-plugin": "0.0.10",
    "html-webpack-plugin": "^3.2.0",
-    "jest": "^24.1.0",
-    "jest-puppeteer": "^4.1.0",
+    "jest": "^24.8.0",
+    "jest-puppeteer": "^4.1.1",
    "json-loader": "^0.5.4",
-    "mini-css-extract-plugin": "^0.4.1",
-    "puppeteer": "^1.15.0",
+    "mini-css-extract-plugin": "^0.6.0",
+    "puppeteer": "^1.16.0",
    "rimraf": "^2.6.3",
    "serve-favicon": "^2.3.0",
-    "start-server-and-test": "^1.7.11",
+    "start-server-and-test": "^1.9.0",
    "style-loader": "^0.23.1",
    "sw-precache-webpack-plugin": "^0.11.5",
    "url-loader": "^1.1.0",
-    "webpack": "^4.25.1",
-    "webpack-cli": "^3.1.0",
-    "webpack-dev-middleware": "^3.1.3"
+    "webpack": "^4.31.0",
+    "webpack-cli": "^3.3.2",
+    "webpack-dev-middleware": "^3.6.2"
  },
  "jest": {
    "testMatch": [
@@ -133,13 +131,18 @@
        ],
        "plugins": [
          "@babel/plugin-proposal-function-bind",
-          "@babel/plugin-proposal-class-properties",
          [
            "@babel/plugin-proposal-decorators",
            {
              "legacy": true
            }
          ],
+          [
+            "@babel/plugin-proposal-class-properties",
+            {
+              "loose": true
+            }
+          ],
          "@babel/plugin-proposal-export-namespace-from",
          "@babel/plugin-transform-react-constant-elements",
          "@babel/plugin-transform-runtime",
@@ -21,24 +21,32 @@ const doInitialDataLoad = () =>
    dispatch({ type: "initial data load start" });

    try {
-      const requestJson = _(["config", "schema"])
+      /*
+      Step 1 - config & schema, all JSON
+      */
+      const requestJson = ["config", "schema"]
        .map(r => `${globals.API.prefix}${globals.API.version}${r}`)
-        .map(url => doJsonRequest(url))
-        .value();
-      const requestBinary = _([
-        "annotations/obs",
-        "annotations/var?annotation-name=name",
-        "layout/obs"
-      ])
-        .map(r => `${globals.API.prefix}${globals.API.version}${r}`)
-        .map(url => doBinaryRequest(url))
-        .value();
-
-      const results = await Promise.all(_.concat(requestJson, requestBinary));
-
+        .map(url => doJsonRequest(url));
+      const stepOneResults = await Promise.all(requestJson);
      /* set config defaults */
-      const config = { ...globals.configDefaults, ...results[0].config };
-      const [, schema, obsAnno, varAnno, obsLayout] = [...results];
+      const config = { ...globals.configDefaults, ...stepOneResults[0].config };
+      const schema = stepOneResults[1];
+
+      /*
+      Step 2 - dataframes, all binary.  NOTE: uses results of step 1.
+      */
+      /* only load names for var annotations, if possible*/
+      const varIndexName = schema?.schema?.annotations?.var?.index;
+      const varAnnotationsQuery = varIndexName
+        ? `?annotation-name=${varIndexName}`
+        : "";
+      const varAnnotationsURL = `annotations/var${varAnnotationsQuery}`;
+      const requestBinary = ["annotations/obs", varAnnotationsURL, "layout/obs"]
+        .map(r => `${globals.API.prefix}${globals.API.version}${r}`)
+        .map(url => doBinaryRequest(url));
+      const stepTwoResults = await Promise.all(requestBinary);
+      const [obsAnno, varAnno, obsLayout] = [...stepTwoResults];
+
      const universe = Universe.createUniverseFromResponse(
        config,
        schema,
@@ -91,6 +99,10 @@ needs expression data.
 Transparently utilizes cached data if it is already present.
 */
 async function _doRequestExpressionData(dispatch, getState, genes) {
+  const state = getState();
+  const { universe } = state;
+  const varIndexName = universe.schema.annotations.var.index;
+
  /* helper for this function only */
  const fetchData = async geneNames => {
    const res = await fetch(
@@ -100,7 +112,7 @@ async function _doRequestExpressionData(dispatch, getState, genes) {
        body: JSON.stringify({
          filter: {
            var: {
-              annotation_value: [{ name: "name", values: geneNames }]
+              annotation_value: [{ name: varIndexName, values: geneNames }]
            }
          }
        }),
@@ -123,8 +135,6 @@ async function _doRequestExpressionData(dispatch, getState, genes) {
    return Universe.convertDataFBStoObject(universe, data);
  };

-  const state = getState();
-  const { universe } = state;
  /* preload data already in cache */
  let expressionData = _.transform(
    genes,
@@ -241,6 +251,7 @@ const requestDifferentialExpression = (set1, set2, num_genes = 10) => async (
    */
    const state = getState();
    const { universe } = state;
+    const varIndexName = universe.schema.annotations.var.index;

    // Legal values are null, Array or TypedArray.  Null is initial state.
    if (!set1) set1 = [];
@@ -277,7 +288,7 @@ const requestDifferentialExpression = (set1, set2, num_genes = 10) => async (
    const data = await res.json();
    // result is [ [varIdx, ...], ... ]
    const topNGenes = _.map(data, r =>
-      universe.varAnnotations.at(r[0], "name")
+      universe.varAnnotations.at(r[0], varIndexName)
    );

    /*
@@ -27,10 +27,10 @@ class Category extends React.Component {
      const cat = categoricalSelection[metadataField];
      const categoryCount = {
        // total number of categories in this dimension
-        totalCatCount: cat.numCategories,
+        totalCatCount: cat.numCategoryValues,
        // number of selected options in this category
        selectedCatCount: _.reduce(
-          cat.categorySelected,
+          cat.categoryValueSelected,
          (res, cond) => (cond ? res + 1 : res),
          0
        )
@@ -91,7 +91,7 @@ class Category extends React.Component {
    const { categoricalSelection, metadataField } = this.props;

    const cat = categoricalSelection[metadataField];
-    const optTuples = sortedCategoryValues([...cat.categoryIndices]);
+    const optTuples = sortedCategoryValues([...cat.categoryValueIndices]);
    return _.map(optTuples, (tuple, i) => (
      <Value
        optTuples={optTuples}
@@ -46,8 +46,8 @@ class CategoryValue extends React.Component {
    if (!categoricalSelection) return null;

    const category = categoricalSelection[metadataField];
-    const selected = category.categorySelected[categoryIndex];
-    const count = category.categoryCounts[categoryIndex];
+    const selected = category.categoryValueSelected[categoryIndex];
+    const count = category.categoryValueCounts[categoryIndex];
    const value = category.categoryValues[categoryIndex];
    const displayString = String(
      category.categoryValues[categoryIndex]
@@ -66,7 +66,8 @@ class Continuous extends React.Component {
          ? _.map(obsAnnotations.colIndex.keys(), key => {
              const isColorField =
                key.includes("color") || key.includes("Color");
-              if (key === "name" || isColorField) return null;
+              if (key === schema.annotations.obs.index || isColorField)
+                return null;

              const summary = obsAnnotations.col(key).summarize();
              const nonFiniteExtent =
@@ -0,0 +1,18 @@
+import React from "react";
+import * as globals from "../../globals";
+
+const Logo = props => {
+  const { size } = props;
+  return (
+    <svg width={size} height={size} viewBox={`0 0 48 48`} fill="none">
+      <rect width="48" height="48" fill="white" />
+      <rect width="48" height="48" fill={globals.logoColor} />
+      <rect x="19" y="19" width="22" height="22" fill="white" />
+      <rect x="24" y="24" width="12" height="12" fill={globals.logoColor} />
+      <rect x="7" y="19" width="7" height="22" fill="white" />
+      <rect x="19" y="7" width="22" height="7" fill="white" />
+    </svg>
+  );
+};
+
+export default Logo;
@@ -85,7 +85,8 @@ class GeneExpression extends React.Component {
    */
    const { world } = this.props;
    const { varAnnotations } = world;
-    const geneNames = varAnnotations.col("name").asArray();
+    const varIndexName = world.schema.annotations.var.index;
+    const geneNames = varAnnotations.col(varIndexName).asArray();
    if (geneNames.length > 0) {
      const placeholder = [];
      let len = geneNames.length;
@@ -107,6 +108,7 @@ class GeneExpression extends React.Component {

  handleClick(g) {
    const { world, dispatch, userDefinedGenes } = this.props;
+    const varIndexName = world.schema.annotations.var.index;
    const gene = g.target;
    if (userDefinedGenes.indexOf(gene) !== -1) {
      postUserErrorToast("That gene already exists");
@@ -114,7 +116,9 @@ class GeneExpression extends React.Component {
      postUserErrorToast(
        "That's too many genes, you can have at most 15 user defined genes"
      );
-    } else if (world.varAnnotations.col("name").indexOf(gene) === undefined) {
+    } else if (
+      world.varAnnotations.col(varIndexName).indexOf(gene) === undefined
+    ) {
      postUserErrorToast("That doesn't appear to be a valid gene name.");
    } else {
      dispatch({ type: "single user defined gene start" });
@@ -127,6 +131,7 @@ class GeneExpression extends React.Component {

  handleBulkAddClick() {
    const { world, dispatch, userDefinedGenes } = this.props;
+    const varIndexName = world.schema.annotations.var.index;
    const { bulkAdd } = this.state;

    /*
@@ -145,7 +150,9 @@ class GeneExpression extends React.Component {
          if (userDefinedGenes.indexOf(gene) !== -1) {
            return keepAroundErrorToast("That gene already exists");
          }
-          if (world.varAnnotations.col("name").indexOf(gene) === undefined) {
+          if (
+            world.varAnnotations.col(varIndexName).indexOf(gene) === undefined
+          ) {
            return keepAroundErrorToast(
              `${gene} doesn't appear to be a valid gene name.`
            );
@@ -168,7 +175,7 @@ class GeneExpression extends React.Component {
      userDefinedGenesLoading,
      differential
    } = this.props;
-
+    const varIndexName = world?.schema?.annotations?.var?.index;
    const { tab, bulkAdd } = this.state;

    return (
@@ -243,7 +250,7 @@ class GeneExpression extends React.Component {
                itemRenderer={renderGene.bind(this)}
                items={
                  world && world.varAnnotations
-                    ? world.varAnnotations.col("name").asArray()
+                    ? world.varAnnotations.col(varIndexName).asArray()
                    : ["No genes"]
                }
                popoverProps={{ minimal: true }}
@@ -322,7 +329,7 @@ class GeneExpression extends React.Component {
          <ExpressionButtons />
          {differential.diffExp
            ? _.map(differential.diffExp, (value, index) => {
-                const name = world.varAnnotations.at(value[0], "name");
+                const name = world.varAnnotations.at(value[0], varIndexName);
                const values = world.varData.col(name);
                if (!values) {
                  return null;
@@ -1,5 +1,6 @@
 // jshint esversion: 6
 const mat4 = require("gl-mat4");
+const vec3 = require("gl-vec3");

 // opacity: https://github.com/spacetx/starfish/blob/master/viz/draw/regions.js

@@ -38,7 +39,20 @@ export default function(regl) {
    uniforms: {
      distance: regl.prop("distance"),
      view: regl.prop("view"),
-      projection: ({viewportWidth, viewportHeight}) => mat4.perspective([], Math.PI / 2, viewportWidth / viewportHeight, 0.01, 1000)
+      projection: ({ viewportWidth, viewportHeight }) => {
+        const aspectRatio = viewportWidth / viewportHeight;
+        let m = mat4.perspective(
+          [],
+          Math.PI / 2,
+          viewportWidth / viewportHeight,
+          0.01,
+          1000
+        );
+        if (aspectRatio < 1) {
+          m = mat4.scale(m, m, vec3.fromValues(1, 1, 1 / aspectRatio));
+        }
+        return m;
+      }
    },

    count: regl.prop("count"),
@@ -3,7 +3,9 @@ import React from "react";
 import * as d3 from "d3";
 import { connect } from "react-redux";
 import mat4 from "gl-mat4";
+import vec3 from "gl-vec3";
 import _regl from "regl";
+import memoize from "memoize-one";
 import {
  Button,
  AnchorButton,
@@ -13,7 +15,9 @@ import {
  MenuItem,
  Position,
  NumericInput,
-  Icon
+  Icon,
+  RadioGroup,
+  Radio
 } from "@blueprintjs/core";

 import * as globals from "../../globals";
@@ -47,7 +51,8 @@ import { World } from "../../util/stateManager";
  undoDisabled: state["@@undoable/past"].length === 0,
  redoDisabled: state["@@undoable/future"].length === 0,
  selectionTool: state.graphSelection.tool,
-  currentSelection: state.graphSelection.selection
+  currentSelection: state.graphSelection.selection,
+  layoutChoice: state.layoutChoice
 }))
 class Graph extends React.Component {
  static isValidDigitKeyEvent(e) {
@@ -74,6 +79,38 @@ class Graph extends React.Component {
    return key >= 0 && key <= 9;
  }

+  computePointPositions = memoize((X, Y, scaleX, scaleY) => {
+    /*
+    compute webgl coordinate buffer for each point
+    */
+    const positions = new Float32Array(2 * X.length);
+    for (let i = 0, len = X.length; i < len; i += 1) {
+      positions[2 * i] = scaleX(X[i]);
+      positions[2 * i + 1] = scaleY(Y[i]);
+    }
+    return positions;
+  });
+
+  computePointColors = memoize(rgb => {
+    /*
+    compute webgl colors for each point
+    */
+    const colors = new Float32Array(3 * rgb.length);
+    for (let i = 0, len = rgb.length; i < len; i += 1) {
+      colors.set(rgb[i], 3 * i);
+    }
+    return colors;
+  });
+
+  computePointSizes = memoize((len, crossfilter) => {
+    /*
+    compute webgl dot size for each point
+    */
+    const sizes = new Float32Array(len);
+    crossfilter.fillByIsSelected(sizes, 4, 0.2);
+    return sizes;
+  });
+
  constructor(props) {
    super(props);
    this.count = 0;
@@ -81,6 +118,8 @@ class Graph extends React.Component {
    this.graphPaddingBottom = 45;
    this.graphPaddingRight = globals.leftSidebarWidth;
    this.renderCache = {
+      X: null,
+      Y: null,
      positions: null,
      colors: null,
      sizes: null
@@ -106,6 +145,12 @@ class Graph extends React.Component {
    const colorBuffer = regl.buffer();
    const sizeBuffer = regl.buffer();

+    // preallocate coordinate system transformation between data and gl
+    const transform = {
+      glScaleX: scaleLinear([0, 1], [-1, 1]),
+      glScaleY: scaleLinear([0, 1], [1, -1])
+    };
+
    /* first time, but this duplicates above function, should be possile to avoid this */
    const reglRender = regl.frame(() => {
      this.reglDraw(
@@ -128,7 +173,8 @@ class Graph extends React.Component {
      colorBuffer,
      sizeBuffer,
      camera,
-      reglRender
+      reglRender,
+      transform
    });
  }

@@ -140,19 +186,10 @@ class Graph extends React.Component {
      colorRGB,
      responsive,
      selectionTool,
-      currentSelection
+      currentSelection,
+      layoutChoice
    } = this.props;
-    const {
-      reglRender,
-      mode,
-      regl,
-      drawPoints,
-      camera,
-      pointBuffer,
-      colorBuffer,
-      sizeBuffer,
-      svg
-    } = this.state;
+    const { reglRender, mode, regl, svg } = this.state;
    let stateChanges = {};

    if (reglRender && this.reglRenderState === "rendering" && mode !== "zoom") {
@@ -163,53 +200,40 @@ class Graph extends React.Component {
    if (regl && world) {
      /* update the regl state */
      const { obsLayout, nObs } = world;
-      const X = obsLayout.col("X").asArray();
-      const Y = obsLayout.col("Y").asArray();
+      const {
+        drawPoints,
+        transform,
+        camera,
+        pointBuffer,
+        colorBuffer,
+        sizeBuffer
+      } = this.state;

-      // X/Y positions for each point - a cached value that only
-      // changes if we have loaded entirely new cell data
-      //
-      if (!renderCache.positions || world !== prevProps.world) {
-        renderCache.positions = new Float32Array(2 * nObs);
-
-        const glScaleX = scaleLinear([0, 1], [-1, 1]);
-        const glScaleY = scaleLinear([0, 1], [1, -1]);
-
-        const offset = [d3.mean(X) - 0.5, d3.mean(Y) - 0.5];
-
-        for (let i = 0, { positions } = renderCache; i < nObs; i += 1) {
-          positions[2 * i] = glScaleX(X[i] - offset[0]);
-          positions[2 * i + 1] = glScaleY(Y[i] - offset[1]);
-        }
-        pointBuffer({
-          data: renderCache.positions,
-          dimension: 2
-        });
-
-        stateChanges.offset = offset;
+      /* coordinates for each point */
+      const { glScaleX, glScaleY } = transform;
+      const X = obsLayout.col(layoutChoice.currentDimNames[0]).asArray();
+      const Y = obsLayout.col(layoutChoice.currentDimNames[1]).asArray();
+      const newPositions = this.computePointPositions(X, Y, glScaleX, glScaleY);
+      if (renderCache.positions !== newPositions) {
+        /* update our cache & GL if the buffer changes */
+        renderCache.positions = newPositions;
+        pointBuffer({ data: newPositions, dimension: 2 });
      }

-      // Colors for each point - a cached value that only changes when
-      // the cell metadata changes.
-      if (!renderCache.colors || colorRGB !== prevProps.colorRGB) {
-        const rgb = colorRGB;
-        if (!renderCache.colors) {
-          renderCache.colors = new Float32Array(3 * rgb.length);
-        }
-        for (let i = 0, { colors } = renderCache; i < rgb.length; i += 1) {
-          colors.set(rgb[i], 3 * i);
-        }
-        colorBuffer({ data: renderCache.colors, dimension: 3 });
+      /* colors for each point */
+      const newColors = this.computePointColors(colorRGB);
+      if (renderCache.colors !== newColors) {
+        /* update our cache & GL if the buffer changes */
+        renderCache.colors = newColors;
+        colorBuffer({ data: newColors, dimension: 3 });
      }

-      // Sizes for each point - updates are triggered only when selected
-      // obs change
-      if (!renderCache.sizes || crossfilter !== prevProps.crossfilter) {
-        if (!renderCache.sizes) {
-          renderCache.sizes = new Float32Array(nObs);
-        }
-        crossfilter.fillByIsSelected(renderCache.sizes, 4, 0.2);
-        sizeBuffer({ data: renderCache.sizes, dimension: 1 });
+      /* sizes for each point */
+      const newSizes = this.computePointSizes(nObs, crossfilter);
+      if (renderCache.sizes !== newSizes) {
+        /* update our cache & GL if the buffer changes */
+        renderCache.size = newSizes;
+        sizeBuffer({ data: newSizes, dimension: 1 });
      }

      this.count = nObs;
@@ -271,11 +295,10 @@ class Graph extends React.Component {
      mode !== prevState.mode ||
      stateChanges.svg
    ) {
-      const { tool, container, offset } = this.state;
+      const { tool, container } = this.state;
      this.selectionToolUpdate(
        stateChanges.tool ? stateChanges.tool : tool,
-        stateChanges.container ? stateChanges.container : container,
-        stateChanges.offset ? stateChanges.offset : offset
+        stateChanges.container ? stateChanges.container : container
      );
    }

@@ -435,7 +458,15 @@ class Graph extends React.Component {
    this.setState({ pendingClipPercentiles: null });
  };

-  brushToolUpdate(tool, container, offset) {
+  handleLayoutChoiceChange = e => {
+    const { dispatch } = this.props;
+    dispatch({
+      type: "set layout choice",
+      layoutChoice: e.currentTarget.value
+    });
+  };
+
+  brushToolUpdate(tool, container) {
    /*
    this is called from componentDidUpdate(), so be very careful using
    anything from this.state, which may be updated asynchronously.
@@ -449,8 +480,8 @@ class Graph extends React.Component {
        if there is a selection, make sure the brush tool matches
        */
        const screenCoords = [
-          this.mapPointToScreen(currentSelection.brushCoords.northwest, offset),
-          this.mapPointToScreen(currentSelection.brushCoords.southeast, offset)
+          this.mapPointToScreen(currentSelection.brushCoords.northwest),
+          this.mapPointToScreen(currentSelection.brushCoords.southeast)
        ];
        if (!toolCurrentSelection) {
          /* tool is not selected, so just move the brush */
@@ -477,7 +508,7 @@ class Graph extends React.Component {
    }
  }

-  lassoToolUpdate(tool, container, offset) {
+  lassoToolUpdate(tool, container) {
    /*
    this is called from componentDidUpdate(), so be very careful using
    anything from this.state, which may be updated asynchronously.
@@ -488,7 +519,7 @@ class Graph extends React.Component {
      if there is a current selection, make sure the lasso tool matches
      */
      const polygon = currentSelection.polygon.map(p =>
-        this.mapPointToScreen(p, offset)
+        this.mapPointToScreen(p)
      );
      tool.move(polygon);
    } else {
@@ -496,7 +527,7 @@ class Graph extends React.Component {
    }
  }

-  selectionToolUpdate(tool, container, offset) {
+  selectionToolUpdate(tool, container) {
    /*
    this is called from componentDidUpdate(), so be very careful using
    anything from this.state, which may be updated asynchronously.
@@ -504,10 +535,10 @@ class Graph extends React.Component {
    const { selectionTool } = this.props;
    switch (selectionTool) {
      case "brush":
-        this.brushToolUpdate(tool, container, offset);
+        this.brushToolUpdate(tool, container);
        break;
      case "lasso":
-        this.lassoToolUpdate(tool, container, offset);
+        this.lassoToolUpdate(tool, container);
        break;
      default:
        /* punt? */
@@ -564,12 +595,14 @@ class Graph extends React.Component {
    accounting for current pan/zoom camera.
    */
    const { responsive } = this.props;
-    const { regl, camera, offset } = this.state;
+    const { regl, camera, transform } = this.state;
+    const { glScaleX, glScaleY } = transform;

    const gl = regl._gl;

    // get aspect ratio
    const aspect = gl.drawingBufferWidth / gl.drawingBufferHeight;
+    const scale = aspect < 1 ? 1 / aspect : 1;

    // compute inverse view matrix
    const inverse = mat4.invert([], camera.view());
@@ -578,37 +611,37 @@ class Graph extends React.Component {
    const x = (2 * pin[0]) / (responsive.width - this.graphPaddingRight) - 1;
    const y = 2 * (1 - pin[1] / (responsive.height - this.graphPaddingTop)) - 1;
    const pout = [
-      x * inverse[14] * aspect + inverse[12],
-      y * inverse[14] + inverse[13]
+      x * inverse[14] * aspect * scale + inverse[12],
+      -(y * inverse[14] * scale + inverse[13])
    ];

-    return [(pout[0] + 1) / 2 + offset[0], (pout[1] + 1) / 2 + offset[1]];
+    const xy = [glScaleX.invert(pout[0]), glScaleY.invert(pout[1])];
+    return xy;
  }

-  mapPointToScreen(xyCell, offset) {
+  mapPointToScreen(xyCell) {
    /*
    Map an XY coordinate from cell/point domain to screen range.  Inverse
    of mapScreenToPoint()
    */
    const { responsive } = this.props;
-    const { regl, camera } = this.state;
+    const { regl, camera, transform } = this.state;
+    const { glScaleX, glScaleY } = transform;

    const gl = regl._gl;

    // get aspect ratio
    const aspect = gl.drawingBufferWidth / gl.drawingBufferHeight;
+    const scale = aspect < 1 ? 1 / aspect : 1;

    // compute inverse view matrix
-    const inverse = mat4.invert([], camera.view());
+    let inverse = mat4.invert([], camera.view());

    // variable names are choosen to reflect inverse of those used
    // in mapScreenToPoint().
-    const pout = [
-      (xyCell[0] - offset[0]) * 2 - 1,
-      (xyCell[1] - offset[1]) * 2 - 1
-    ];
-    const x = (pout[0] - inverse[12]) / aspect / inverse[14];
-    const y = (pout[1] - inverse[13]) / inverse[14];
+    const pout = [glScaleX(xyCell[0]), glScaleY(xyCell[1])];
+    const x = (pout[0] - inverse[12]) / aspect / scale / inverse[14];
+    const y = (-pout[1] - inverse[13]) / scale / inverse[14];

    const pin = [
      Math.round(((x + 1) * (responsive.width - this.graphPaddingRight)) / 2),
@@ -745,7 +778,8 @@ class Graph extends React.Component {
      redoDisabled,
      selectionTool,
      clipPercentileMin,
-      clipPercentileMax
+      clipPercentileMax,
+      layoutChoice
    } = this.props;
    const { mode, pendingClipPercentiles } = this.state;

@@ -888,6 +922,49 @@ class Graph extends React.Component {
                />
              </Tooltip>
            </div>
+
+            <div
+              className="bp3-button-group"
+              style={{
+                marginLeft: 10
+              }}
+            >
+              <Popover
+                target={
+                  <Button
+                    type="button"
+                    data-testid="layout-choice"
+                    className="bp3-button bp3-icon-heatmap"
+                    style={{
+                      cursor: "pointer"
+                    }}
+                  />
+                }
+                position={Position.BOTTOM_RIGHT}
+                content={
+                  <div
+                    style={{
+                      display: "flex",
+                      justifyContent: "flex-start",
+                      alignItems: "flex-start",
+                      flexDirection: "column",
+                      padding: 10
+                    }}
+                  >
+                    <RadioGroup
+                      label="Layout Choice"
+                      onChange={this.handleLayoutChoiceChange}
+                      selectedValue={layoutChoice.current}
+                    >
+                      {layoutChoice.available.map(name => (
+                        <Radio label={name} value={name} key={name} />
+                      ))}
+                    </RadioGroup>
+                  </div>
+                }
+              />
+            </div>
+
            <div
              className="bp3-button-group"
              style={{
@@ -6,6 +6,7 @@ import Continuous from "./continuous/continuous";
 import GeneExpression from "./geneExpression";
 import * as globals from "../globals";
 import DynamicScatterplot from "./scatterplot/scatterplot";
+import Logo from "./framework/logo.js";

@connect(state => ({
  responsive: state.responsive,
@@ -27,7 +28,6 @@ class LeftSideBar extends React.Component {
    if cellxgene logo or tabs change, this must as well
    */
    const metadataSectionPadding = 0;
-    // scatterplotXXaccessor && scatterplotYYaccessor ? 450 : 0;

    return (
      <div
@@ -39,20 +39,49 @@ class LeftSideBar extends React.Component {
        }}
      >
        <p
-          data-testid="header"
          style={{
            position: "fixed",
            top: globals.cellxgeneTitleTopPadding,
            left: globals.leftSidebarWidth + globals.cellxgeneTitleLeftPadding,
-            margin: 0,
-            fontSize: globals.largestFontSize,
-            color: globals.darkerGrey,
-            width: "100%"
+            margin: 0
          }}
        >
-          cellxgene: {datasetTitle}
+          <Logo size={32} />
+          <span
+            style={{
+              fontSize: 28,
+              position: "relative",
+              top: -4,
+              fontWeight: "bold",
+              marginLeft: 5,
+              color: globals.logoColor,
+              userSelect: "none"
+            }}
+          >
+            cell<span
+              style={{
+                position: "relative",
+                top: 1,
+                fontWeight: 300,
+                fontSize: 24
+              }}
+            >
+              ×
+            </span>gene
+          </span>
+          <span
+            data-testid="header"
+            style={{
+              fontSize: 16,
+              display: "block",
+              position: "relative",
+              marginTop: 10,
+              top: -4
+            }}
+          >
+            {datasetTitle}
+          </span>
        </p>
-
        <div
          style={{
            height: responsive.height - metadataSectionPadding,
@@ -31,6 +31,7 @@ export const darkGreen = "#448C4D";

 export const nonFiniteCellColor = lightGrey;
 export const defaultCellColor = "rgb(0,0,0,1)";
+export const logoColor = "black"; /* logo pink: "#E9429A" */

 /* typography constants */

@@ -30,15 +30,15 @@ const CategoricalSelection = (
      /*
      Set the specific category in this field to false
      */
-      const newCategorySelected = Array.from(
-        state[action.metadataField].categorySelected
+      const newCategoryValueSelected = Array.from(
+        state[action.metadataField].categoryValueSelected
      );
-      newCategorySelected[action.categoryIndex] = true;
+      newCategoryValueSelected[action.categoryIndex] = true;
      const newCategoricalSelection = {
        ...state,
        [action.metadataField]: {
          ...state[action.metadataField],
-          categorySelected: newCategorySelected
+          categoryValueSelected: newCategoryValueSelected
        }
      };
      return newCategoricalSelection;
@@ -48,15 +48,15 @@ const CategoricalSelection = (
      /*
      Set the specific category in this field to false
      */
-      const newCategorySelected = Array.from(
-        state[action.metadataField].categorySelected
+      const newCategoryValueSelected = Array.from(
+        state[action.metadataField].categoryValueSelected
      );
-      newCategorySelected[action.categoryIndex] = false;
+      newCategoryValueSelected[action.categoryIndex] = false;
      const newCategoricalSelection = {
        ...state,
        [action.metadataField]: {
          ...state[action.metadataField],
-          categorySelected: newCategorySelected
+          categoryValueSelected: newCategoryValueSelected
        }
      };
      return newCategoricalSelection;
@@ -70,8 +70,9 @@ const CategoricalSelection = (
        ...state,
        [action.metadataField]: {
          ...state[action.metadataField],
-          categorySelected: Array.from(
-            state[action.metadataField].categorySelected
+          categorySelected: false,
+          categoryValueSelected: Array.from(
+            state[action.metadataField].categoryValueSelected
          ).fill(false)
        }
      };
@@ -86,8 +87,9 @@ const CategoricalSelection = (
        ...state,
        [action.metadataField]: {
          ...state[action.metadataField],
-          categorySelected: Array.from(
-            state[action.metadataField].categorySelected
+          categorySelected: true,
+          categoryValueSelected: Array.from(
+            state[action.metadataField].categoryValueSelected
          ).fill(true)
        }
      };
@@ -93,9 +93,10 @@ const Controls = (
    }
    case "request differential expression success": {
      const { world } = prevSharedState;
+      const varIndexName = world.schema.annotations.var.index;
      const _diffexpGenes = [];
      action.data.forEach(d => {
-        _diffexpGenes.push(world.varAnnotations.at(d[0], "name"));
+        _diffexpGenes.push(world.varAnnotations.at(d[0], varIndexName));
      });
      return {
        ...state,
@@ -20,10 +20,11 @@ const CrossfilterReducer = (
 ) => {
  switch (action.type) {
    case "initial data load complete (universe exists)": {
-      const { world } = nextSharedState;
+      const { world, layoutChoice } = nextSharedState;
      const crossfilter = World.createObsDimensions(
        new Crossfilter(world.obsAnnotations),
-        world
+        world,
+        layoutChoice.currentDimNames
      );
      return crossfilter;
    }
@@ -43,9 +44,13 @@ const CrossfilterReducer = (
    case "set clip quantiles":
    case "set World to current selection": {
      const { userDefinedGenes, diffexpGenes } = prevSharedState.controls;
-      const { world } = nextSharedState;
+      const { world, layoutChoice } = nextSharedState;
      let crossfilter = new Crossfilter(world.obsAnnotations);
-      crossfilter = World.createObsDimensions(crossfilter, world);
+      crossfilter = World.createObsDimensions(
+        crossfilter,
+        world,
+        layoutChoice.currentDimNames
+      );
      crossfilter = ControlsHelpers.createGeneDimensions(
        userDefinedGenes,
        diffexpGenes,
@@ -55,6 +60,23 @@ const CrossfilterReducer = (
      return crossfilter;
    }

+    case "set layout choice": {
+      /*
+      when switching layouts:
+      - delete the existing XY index
+      - add the new XY index (which implicitly selects all on it)
+      */
+      const { world, layoutChoice } = nextSharedState;
+      return state
+        .delDimension(layoutDimensionName("XY"))
+        .addDimension(
+          layoutDimensionName("XY"),
+          "spatial",
+          world.obsLayout.col(layoutChoice.currentDimNames[0]).asArray(),
+          world.obsLayout.col(layoutChoice.currentDimNames[1]).asArray()
+        );
+    }
+
    case "request user defined gene success": {
      const { world } = prevSharedState;
      const gene = action.data.genes[0];
@@ -68,8 +90,9 @@ const CrossfilterReducer = (

    case "request differential expression success": {
      const { world } = prevSharedState;
+      const varIndexName = world.schema.annotations.var.index;
      const genes = _.map(action.data, d =>
-        world.varAnnotations.at(d[0], "name")
+        world.varAnnotations.at(d[0], varIndexName)
      );
      const crossfilter = _.reduce(
        genes,
@@ -87,10 +110,11 @@ const CrossfilterReducer = (

    case "clear differential expression": {
      const { world } = prevSharedState;
+      const varIndexName = world.schema.annotations.var.index;
      const crossfilter = _.reduce(
        action.diffExp,
        (xfltr, values) => {
-          const name = world.varAnnotations.at(values[0], "name");
+          const name = world.varAnnotations.at(values[0], varIndexName);
          return xfltr.delDimension(diffexpDimensionName(name));
        },
        state
@@ -161,10 +185,12 @@ const CrossfilterReducer = (
    case "categorical metadata filter select":
    case "categorical metadata filter deselect": {
      const { categoricalSelection } = nextSharedState;
+      const { world } = prevSharedState;
      const cat = categoricalSelection[action.metadataField];
+      const col = world.obsAnnotations.col(action.metadataField);
      return state.select(obsAnnoDimensionName(action.metadataField), {
        mode: "exact",
-        values: ControlsHelpers.selectedValuesForCategory(cat)
+        values: ControlsHelpers.selectedValuesForCategory(cat, col)
      });
    }

@@ -7,7 +7,8 @@ const GraphSelection = (
 ) => {
  switch (action.type) {
    case "set clip quantiles":
-    case "reset World to eq Universe": {
+    case "reset World to eq Universe":
+    case "set layout choice": {
      return {
        ...state,
        selection: {
@@ -12,6 +12,7 @@ import graphSelection from "./graphSelection";
 import crossfilter from "./crossfilter";
 import colors from "./colors";
 import differential from "./differential";
+import layoutChoice from "./layoutChoice";
 import responsive from "./responsive";
 import controls from "./controls";
 import resetCache from "./resetCache";
@@ -19,31 +20,33 @@ import resetCache from "./resetCache";
 import undoableConfig from "./undoableConfig";

 const Reducer = undoable(
-  cascadeReducers([
-    ["config", config],
-    ["universe", universe],
-    ["world", world],
-    ["categoricalSelection", categoricalSelection],
-    ["continuousSelection", continuousSelection],
-    ["graphSelection", graphSelection],
-    ["crossfilter", crossfilter],
-    ["colors", colors],
-    ["controls", controls],
-    ["differential", differential],
-    ["responsive", responsive],
-    ["resetCache", resetCache]
-  ]),
-  [
-    "world",
-    "categoricalSelection",
-    "continuousSelection",
-    "graphSelection",
-    "crossfilter",
-    "colors",
-    "controls",
-    "differential"
-  ],
-  undoableConfig
+    cascadeReducers([
+        ["config", config],
+        ["universe", universe],
+        ["world", world],
+        ["layoutChoice", layoutChoice],
+        ["categoricalSelection", categoricalSelection],
+        ["continuousSelection", continuousSelection],
+        ["graphSelection", graphSelection],
+        ["crossfilter", crossfilter],
+        ["colors", colors],
+        ["controls", controls],
+        ["differential", differential],
+        ["responsive", responsive],
+        ["resetCache", resetCache]
+    ]),
+    [
+        "world",
+        "categoricalSelection",
+        "continuousSelection",
+        "graphSelection",
+        "crossfilter",
+        "colors",
+        "controls",
+        "differential",
+        "layoutChoice"
+    ],
+    undoableConfig
 );

 const store = createStore(Reducer, applyMiddleware(thunk));
@@ -0,0 +1,50 @@
+/*
+we have a UI heuristic to pick the default layout, based on assumptions
+about commonly used names.  Preferentially, pick in the following order:
+
+  1. "umap"
+  2. "tsne"
+  3. "pca"
+  4. give up, use the first available
+*/
+function bestDefaultLayout(layouts) {
+  const preferredNames = ["umap", "tsne", "pca"];
+  const idx = preferredNames.findIndex(name => layouts.indexOf(name) !== -1);
+  if (idx !== -1) return preferredNames[idx];
+  return layouts[0];
+}
+
+const LayoutChoice = (
+  state = {
+    available: [], // all available choices
+    current: undefined, // name of the current layout, eg, 'umap'
+    currentDimNames: [] // dimension name
+  },
+  action,
+  nextSharedState
+) => {
+  switch (action.type) {
+    case "initial data load complete (universe exists)":
+    case "reset World to eq Universe": {
+      // set default to default
+      const { schema } = nextSharedState.world;
+      const available = schema.layout.obs.map(v => v.name);
+      const current = bestDefaultLayout(available);
+      const currentDimNames = schema.layout.obsByName[current].dims;
+      return { available, current, currentDimNames };
+    }
+
+    case "set layout choice": {
+      const { schema } = nextSharedState.world;
+      const current = action.layoutChoice;
+      const currentDimNames = schema.layout.obsByName[current].dims;
+      return { ...state, current, currentDimNames };
+    }
+
+    default: {
+      return state;
+    }
+  }
+};
+
+export default LayoutChoice;
@@ -59,7 +59,7 @@ const saveOnActions = new Set([
  "categorical metadata filter select",
  "categorical metadata filter deselect",
  "categorical metadata filter all of these",
-  "categorical metadata none of these",
+  "categorical metadata filter none of these",

  "color by categorical metadata",
  "color by continuous metadata",
@@ -72,7 +72,9 @@ const saveOnActions = new Set([
  "store current cell selection as differential set 2",

  "set World to current selection",
-  "set clip quantiles"
+  "set clip quantiles",
+
+  "set layout choice"
 ]);

 /**
@@ -98,9 +100,15 @@ const applyPending = () => ({
  [actionKey]: "applyPending",
  [stateKey]: { fsm: null }
 });
-const skip = fsm => ({ [actionKey]: "skip", [stateKey]: { fsm } });
+const skip = (fsm, transition) => ({
+  [actionKey]: "skip",
+  [stateKey]: { fsm: transition.to !== "done" ? fsm : null }
+});
 const clear = () => ({ [actionKey]: "clear", [stateKey]: { fsm: null } });
-const save = fsm => ({ [actionKey]: "save", [stateKey]: { fsm } });
+const save = (fsm, transition) => ({
+  [actionKey]: "save",
+  [stateKey]: { fsm: transition.to !== "done" ? fsm : null }
+});

 /*
 Error handler for state transitions that are unexpected.  Called by
@@ -5,10 +5,9 @@ const mp = require("mouse-position");
 const mb = require("mouse-pressed");
 const key = require("key-pressed");

-const panSpeed = 0.4;
+const panSpeed = 1.0; // changed from 0.4 to 1.0 per issue #722
 const scaleSpeed = 0.5;
 const scaleMax = 3;
-// const scaleMin = 1.15
 const scaleMin = 1.03;

 function attachCamera(canvas, opts) {
@@ -9,8 +9,14 @@
 // this is is equivalent to d3.scaleLinear().domain([0,1]).range([-1,1])

 export default (domain, range) => {
-  const domainStart = domain[0];
-  const scale = (range[1] - range[0]) / (domain[1] - domain[0]);
-  const rangeStart = range[0];
-  return value => (value - domainStart) * scale + rangeStart;
+	const domainStart = domain[0];
+	const scale = (range[1] - range[0]) / (domain[1] - domain[0]);
+	const invScale = 1 / scale;
+	const rangeStart = range[0];
+	const f = value => (value - domainStart) * scale + rangeStart;
+
+	// inverter
+	f.invert = value => (value - rangeStart) * invScale + domainStart;
+
+	return f;
 };
@@ -21,16 +21,16 @@ Remember that option values can be ANY js type, except undefined/null.
  {
    _category_name_1: {
      // map of option value to index
-      categoryIndices: Map([
+      categoryValueIndices: Map([
        catval1: index,
        ...
      ])

      // index->selection true/false state
-      categorySelected: [ true/false, true/false, ... ]
+      categoryValueSelected: [ true/false, true/false, ... ]

      // number of options
-      numCategories: number,
+      numCategoryValues: number,

      // isTruncated - true if the options for selection has
      // been truncated (ie, was too large to implement)
@@ -56,27 +56,31 @@ function topNCategories(summary) {

 export function createCategoricalSelection(maxCategoryItems, world) {
  const res = {};
+  const obsIndexName = world.schema.annotations.obs.index;
  _.forEach(world.obsAnnotations.colIndex.keys(), key => {
    const summary = world.obsAnnotations.col(key).summarize();
    if (summary.categories) {
      const isColorField = key.includes("color") || key.includes("Color");
      const isSelectableCategory =
        !isColorField &&
-        key !== "name" &&
+        key !== obsIndexName &&
        summary.categories.length < maxCategoryItems;
      if (isSelectableCategory) {
-        const [categoryValues, categoryCounts] = topNCategories(summary);
-        const categoryIndices = new Map(categoryValues.map((v, i) => [v, i]));
-        const numCategories = categoryIndices.size;
-        const categorySelected = new Array(numCategories).fill(true);
+        const [categoryValues, categoryValueCounts] = topNCategories(summary);
+        const categoryValueIndices = new Map(
+          categoryValues.map((v, i) => [v, i])
+        );
+        const numCategoryValues = categoryValueIndices.size;
+        const categoryValueSelected = new Array(numCategoryValues).fill(true);
        const isTruncated = categoryValues.length < summary.numCategories;
        res[key] = {
          categoryValues, // array: of natively typed category values
-          categoryIndices, // map: category value (native type) -> category index
-          categorySelected, // array: t/f selection state
-          numCategories, // number: of categories
+          categoryValueIndices, // map: category value (native type) -> category index
+          categoryValueSelected, // array: t/f selection state
+          numCategoryValues, // number: of values in the category
          isTruncated, // bool: true if list was truncated
-          categoryCounts // array: cardinality of each category
+          categoryValueCounts, // array: cardinality of each category,
+          categorySelected: true // bool - default state for entire category
        };
      }
    }
@@ -88,12 +92,26 @@ export function createCategoricalSelection(maxCategoryItems, world) {
 given a categoricalSelection, return the list of all category values
 where selection state is true (ie, they are selected).
 */
-export function selectedValuesForCategory(categorySelectionState) {
-  const selectedValues = _([...categorySelectionState.categoryIndices])
-    .filter(tuple => categorySelectionState.categorySelected[tuple[1]])
-    .map(tuple => tuple[0])
-    .value();
-  return selectedValues;
+export function selectedValuesForCategory(categorySelectionState, dfColumn) {
+  const {
+    categorySelected,
+    categoryValueSelected,
+    categoryValueIndices
+  } = categorySelectionState;
+  let selectedValues;
+  if (categorySelected) {
+    selectedValues = new Set(dfColumn.summarize().categories);
+  } else {
+    selectedValues = new Set();
+  }
+  categoryValueIndices.forEach((catIndex, catValue) => {
+    if (!categoryValueSelected[catIndex]) {
+      selectedValues.delete(catValue);
+    } else {
+      selectedValues.add(catValue);
+    }
+  });
+  return [...selectedValues.values()];
 }

 /*
@@ -104,27 +104,11 @@ function LayoutFBSToDataframe(arrayBuffer) {
    throw new Error("Unexpected layout data type returned from server");
  }

-  /*
-  TODO: XXX
-
-  TEMPORARY CODE AND COMMENT to support the progressive implementation
-  of multi-layout support.  For now, we search for one of the following 
-  in the layouts and use it if we find it: umap, then tsne, then pca, 
-  then whatever is first in the list.
-  */
-  let layoutIndex = 0;
-  ["umap", "tsne", "pca"].some(name => {
-    const idx = fbs.colIdx.indexOf(`${name}_0`);
-    if (idx !== -1) {
-      layoutIndex = idx;
-    }
-    return idx !== -1;
-  });
  const df = new Dataframe.Dataframe(
-    [fbs.nRows, 2],
-    [fbs.columns[layoutIndex], fbs.columns[layoutIndex + 1]],
+    [fbs.nRows, fbs.nCols],
+    fbs.columns,
    null,
-    new Dataframe.KeyIndex(["X", "Y"])
+    new Dataframe.KeyIndex(fbs.colIdx)
  );
  return df;
 }
@@ -140,7 +124,7 @@ function reconcileSchemaCategoriesWithSummary(universe) {
  cases, add a 'categories' field to the schema so it is accessible.
  */

-  universe.schema.annotations.obs.forEach(s => {
+  universe.schema.annotations.obs.columns.forEach(s => {
    if (
      s.type === "string" ||
      s.type === "boolean" ||
@@ -172,6 +156,9 @@ export function createUniverseFromResponse(
  universe.schema = schema;
  universe.nObs = schema.dataframe.nObs;
  universe.nVar = schema.dataframe.nVar;
+  /* add defaults, as we can't assume back-end will fully populate schema */
+  if (!schema.layout.var) schema.layout.var = [];
+  if (!schema.layout.obs) schema.layout.obs = [];

  /* annotations */
  universe.obsAnnotations = AnnotationsFBSToDataframe(annotationsObsResponse);
@@ -192,10 +179,16 @@ export function createUniverseFromResponse(

  /* Index schema for ease of use */
  universe.schema.annotations.obsByName = fromEntries(
-    universe.schema.annotations.obs.map(v => [v.name, v])
+    universe.schema.annotations.obs.columns.map(v => [v.name, v])
  );
  universe.schema.annotations.varByName = fromEntries(
-    universe.schema.annotations.var.map(v => [v.name, v])
+    universe.schema.annotations.var.columns.map(v => [v.name, v])
+  );
+  universe.schema.layout.obsByName = fromEntries(
+    universe.schema.layout.obs.map(v => [v.name, v])
+  );
+  universe.schema.layout.varByName = fromEntries(
+    universe.schema.layout.var.map(v => [v.name, v])
  );
  return universe;
 }
@@ -220,8 +213,9 @@ export function convertDataFBStoObject(universe, arrayBuffer) {
    throw new Error("Unexpected non-floating point response from server.");
  }

+  const varIndexName = universe.schema.annotations.var.index;
  for (let c = 0; c < colIdx.length; c += 1) {
-    const varName = universe.varAnnotations.at(colIdx[c], "name");
+    const varName = universe.varAnnotations.at(colIdx[c], varIndexName);
    result[varName] = columns[c];
  }
  return result;
@@ -260,13 +260,17 @@ function deduceDimensionType(attributes, fieldName) {
  return dimensionType;
 }

-export function createObsDimensions(crossfilter, world) {
+export function createObsDimensions(crossfilter, world, XYdimNames) {
  /*
  create and return a crossfilter with a dimension for every obs annotation
-  for which we have a supported type, *except* 'name'
+  for which we have a supported type, *except* for the index column, indicated
+  by schema.annotations.obs.index.
  */
  const { schema, obsLayout, obsAnnotations } = world;
-  const annoList = schema.annotations.obs.filter(anno => anno.name !== "name");
+  const indexName = schema.annotations.obs.index;
+  const annoList = schema.annotations.obs.columns.filter(
+    anno => anno.name !== indexName
+  );
  crossfilter = annoList.reduce((xfltr, anno) => {
    const dimType = deduceDimensionType(anno, anno.name);
    const colData = obsAnnotations.col(anno.name).asArray();
@@ -283,8 +287,8 @@ export function createObsDimensions(crossfilter, world) {
  return crossfilter.addDimension(
    layoutDimensionName("XY"),
    "spatial",
-    obsLayout.col("X").asArray(),
-    obsLayout.col("Y").asArray()
+    obsLayout.col(XYdimNames[0]).asArray(),
+    obsLayout.col(XYdimNames[1]).asArray()
  );
 }

@@ -0,0 +1,91 @@
+# Developer guidelines
+
+### Requirements
+- npm
+- Python 3.6+
+- Chrome
+
+[See dev section of README](../README.md)
+
+**All instructions are expected to be run from the top level cellxgene directory unless otherwise specified.**
+
+## Server dev
+### Install
+* Build the client and put static files in place: `make build-for-server-dev`
+* Install from local files: `make install-dev`
+
+### Launch
+* `cellxgene launch [options] <datafile>`
+
+### Reloading
+If you install cellxgene using `make install-dev` the server will be restarted every time you make changes on the server code. If changes affects the client, the browser must be reloaded.
+
+### Linter
+We use `flake8` to lint code. Travis CI runs `flake8 server`.
+
+### Test
+1. Install development requirements `pip install -r server/requirements-dev.txt`
+2. Run tests `pytest server/test`
+
+### Tips
+* Install in a virtualenv
+* May need to rebuild/reinstall when you make client changes
+
+## Client dev
+### Install
+1. Install prereqs for client: `npm install --prefix client/ client`
+2. Install cellxgene server: `pip install -e .` Caveat: this will not build the production client package - you must use the [server install](#install) instructions above to serve web assets.
+
+### Launch
+To launch with hot reloading you need to launch the server and the client separately. Node's hot reloading starts the client on its own node server and auto-refreshes when changes are made.
+1. Launch server (the client relies on the REST API being available): `cellxgene launch [options] <datafile>`
+2. Launch client: in `client/` directory run `npm run start`
+3. Client will be served on localhost:3000
+
+### Build
+To build only the client: `make build-client`
+
+### Linter
+We use `eslint` to lint the code and `prettier` as our code formatter.
+
+### Test
+In `client/` directory run `npm run unit-test`
+
+### Tips
+* You can also install/launch the server side code from npm scrips (requires python3.6 with virtualenv) in `client/` directory run `npm run backend-dev`
+
+## Running tests
+Client and server tests run on Travis CI for every push, PR, and commit to master on github. End to end tests run nightly on master only. 
+
+### Server unit tests
+Install development requirements `pip install -r server/requirements-dev.txt`
+Run tests `pytest server/test`
+
+### Client unit tests
+In `client/` directory run `npm run unit-test`
+
+### End to end tests
+
+End to end tests use two env variables:
+* `JEST_ENV` - environment to run end to end tests. Default `dev`
+    * `prod` - run headless with no slowdown, chromium will not open.
+    * `dev` - opens chromimum, runs tests with minimal slowdown, close on exit.
+    * `debug` - opens chromium, runs tests with 100ms slowdown, dev tools open, chrome stays open on exit.
+* `JEST_CXG_PORT` - port that end to end tests are being run on. Default `3000` (client hosted port).
+
+On CI the end to end tests are run with `JEST_ENV` set to `prod` using the `smoke-test` npm script
+
+To run end to end tests as they will be run on CI
+1. cellxgene should be built and installed as [specified in server dev](#install)
+2. `export JEST_ENV='prod'`
+3. `export JEST_CXG_PORT='5000'`
+4. Run `npm run --prefix client/ smoke-test`
+
+Run end to end tests interactively during development
+1. cellxgene should be installed as [specified in client dev](#install-1)
+2. Follow [launch](#launch-1) instructions for client dev with dataset `example-dataset/pbmc3k`
+3. Run `npm run --prefix client/ e2e`
+4. To debug a failing test `export JEST_ENV='debug'` and re-run.
+
+
+
@@ -50,41 +50,61 @@ class ScanpyEngine(CXGDriver):
            "diffexp_lfc_cutoff": 0.01,
        }

-    def _alias_annotation_names(self, axis, name):
-        """
-        Do all user-specified annotation aliasing.
+    @staticmethod
+    def _create_unique_column_name(df, col_name_prefix):
+        """ given the columns of a dataframe, and a name prefix, return a column name which
+            does not exist in the dataframe, AND which is prefixed by `prefix`

-        As a *critical* side-effect, ensure the indices are simple number ranges
-        (accomplished by calling pandas.DataFrame.reset_index())
+            The approach is to append a numeric suffix, starting at zero and increasing by
+            one, until an unused name is found (eg, prefix_0, prefix_1, ...).
        """
-        if name == "name":
-            # a noop, so skip it
-            return
+        suffix = 0
+        while f"{col_name_prefix}{suffix}" in df:
+            suffix += 1
+        return f"{col_name_prefix}{suffix}"

-        ax_name = str(axis)
-        df_axis = getattr(self.data, ax_name)
-        if name is None:
-            # reset index to simple range; alias "name" to point at the
-            # previously specified index.
-            df_axis.reset_index(inplace=True)
-            df_axis.rename(inplace=True, columns={"index": "name"})
-        elif name in df_axis.columns:
-            if name not in df_axis.columns:
+    def _alias_annotation_names(self):
+        """
+        The front-end relies on the existance of a unique, human-readable
+        index for obs & var (eg, var is typically gene name, obs the cell name).
+        The user can specify these via the --obs-names and --var-names config.
+        If they are not specified, use the existing index to create them, giving
+        the resulting column a unique name (eg, "name").
+
+        In both cases, enforce that the result is unique, and communicate the
+        index column name to the front-end via the obs_names and var_names config
+        (which is incorporated into the schema).
+        """
+        for (ax_name, config_name) in ((Axis.OBS, "obs_names"), (Axis.VAR, "var_names")):
+            name = self.config[config_name]
+            df_axis = getattr(self.data, str(ax_name))
+            if name is None:
+                # Default: create unique names from index
+                if not df_axis.index.is_unique:
+                    raise KeyError(
+                        f"Values in {ax_name}.index must be unique. "
+                        "Please prepare data to contain unique index values, or specify an "
+                        "alternative with --{ax_name}-name."
+                    )
+                name = self._create_unique_column_name(df_axis.columns, "name_")
+                self.config[config_name] = name
+                # reset index to simple range; alias name to point at the
+                # previously specified index.
+                df_axis.rename_axis(name, inplace=True)
+                df_axis.reset_index(inplace=True)
+            elif name in df_axis.columns:
+                # User has specified alternative column for unique names, and it exists
+                if not df_axis[name].is_unique:
+                    raise KeyError(
+                        f"Values in {ax_name}.{name} must be unique. "
+                        "Please prepare data to contain unique values."
+                    )
+                df_axis.reset_index(drop=True, inplace=True)
+            else:
+                # user specified a non-existent column name
                raise KeyError(
                    f"Annotation name {name}, specified in --{ax_name}-name does not exist."
                )
-            if not df_axis[name].is_unique:
-                raise KeyError(
-                    f"Values in -{ax_name}-name must be unique. "
-                    "Please prepare data to contain unique values."
-                )
-            # reset index to simple range; alias user-specified annotation to "name"
-            df_axis.reset_index(drop=True, inplace=True)
-            df_axis.rename(inplace=True, columns={name: "name"})
-        else:
-            raise KeyError(
-                f"Annotation name {name}, specified in --{ax_name}_name does not exist."
-            )

    @staticmethod
    def _can_cast_to_float32(ann):
@@ -114,7 +134,17 @@ class ScanpyEngine(CXGDriver):
                "nVar": self.gene_count,
                "type": str(self.data.X.dtype),
            },
-            "annotations": {"obs": [], "var": []},
+            "annotations": {
+                "obs": {
+                    "index": self.config["obs_names"],
+                    "columns": []
+                },
+                "var": {
+                    "index": self.config["var_names"],
+                    "columns": []
+                }
+            },
+            "layout": {"obs": []}
        }
        for ax in Axis:
            curr_axis = getattr(self.data, str(ax))
@@ -138,7 +168,15 @@ class ScanpyEngine(CXGDriver):
                    raise TypeError(
                        f"Annotations of type {curr_axis[ann].dtype} are unsupported by cellxgene."
                    )
-                self.schema["annotations"][ax].append(ann_schema)
+                self.schema["annotations"][ax]["columns"].append(ann_schema)
+
+        for layout in self.config['layout']:
+            layout_schema = {
+                "name": layout,
+                "type": "float32",
+                "dims": [f"{layout}_0", f"{layout}_1"]
+            }
+            self.schema["layout"]["obs"].append(layout_schema)

    def _load_data(self, data):
        # as of AnnData 0.6.19, backed mode performs initial load fast, but at the
@@ -164,8 +202,11 @@ class ScanpyEngine(CXGDriver):

    @requires_data
    def _validate_and_initialize(self):
-        self._alias_annotation_names(Axis.OBS, self.config["obs_names"])
-        self._alias_annotation_names(Axis.VAR, self.config["var_names"])
+        # var and obs column names must be unique
+        if not self.data.obs.columns.is_unique or not self.data.var.columns.is_unique:
+            raise KeyError(f"All annotation column names must be unique.")
+
+        self._alias_annotation_names()
        self._validate_data_types()
        self.cell_count = self.data.shape[0]
        self.gene_count = self.data.shape[1]
@@ -5,7 +5,7 @@ from .prepare import prepare


@click.group(name="cellxgene", context_settings=dict(max_content_width=85))
-@click.version_option(version="0.9.1", prog_name="cellxgene", message="[%(prog)s] Version %(version)s")
+@click.version_option(version="0.10.0", prog_name="cellxgene", message="[%(prog)s] Version %(version)s")
 def cli():
    pass

@@ -7,12 +7,11 @@ import warnings
 import webbrowser

 import click
-import psutil

 from server.app.app import Server
 from server.app.util.errors import ScanpyFileError
 from server.app.util.utils import custom_format_warning
-from server.utils.utils import find_available_port
+from server.utils.utils import find_available_port, is_port_available


 # anything bigger than this will generate a special message
@@ -142,7 +141,14 @@ security risk by including the --scripts flag. Make sure you trust the scripts t
        file_parts = splitext(basename(data))
        title = file_parts[0]

-    if not port:
+    if port:
+        if debug:
+            raise click.ClickException("--port and --debug may not be used together (try --verbose for error logging).")
+        if not is_port_available(host, int(port)):
+            raise click.ClickException(
+                f"The port selected {port} is in use, please specify an open port using the --port flag."
+            )
+    else:
        port = find_available_port(host)

    # Setup app
@@ -165,9 +171,6 @@ security risk by including the --scripts flag. Make sure you trust the scripts t
        click.echo(f"[cellxgene] Loading data from {basename(data)}, this may take awhile...")
    else:
        click.echo(f"[cellxgene] Loading data from {basename(data)}.")
-    # if file is larger than main memory, let the user know performance may suffer
-    if file_size > .95 * psutil.virtual_memory().total:
-        click.echo(f"[cellxgene] Warning: data file is larger than RAM - application may be very slow.")

    # Fix for anaconda python. matplotlib typically expects python to be installed as a framework TKAgg is usually
    # available and fixes this issue. See https://matplotlib.org/faq/virtualenv_faq.html
@@ -203,7 +206,7 @@ security risk by including the --scripts flag. Make sure you trust the scripts t
        sys.stdout = f

    try:
-        server.app.run(host=host, debug=debug, port=port, threaded=True)
+        server.app.run(host=host, debug=debug, port=port, threaded=True, use_debugger=False)
    except OSError as e:
        if e.errno == errno.EADDRINUSE:
            raise click.ClickException("Port is in use, please specify an open port using the --port flag.") from e
@@ -1,4 +1,4 @@
-anndata>=0.6.15
+anndata>=0.6.20
 click>=6.7
 Flask>=1.0.2
 Flask-Caching>=1.4.0
@@ -9,8 +9,7 @@ flatbuffers>=1.10.0
 matplotlib>=2.2
 numpy>=1.15.2
 pandas>=0.23.1
-psutil>=5.6.2
 scanpy>=1.3.7
-scipy>=1.1.0,<1.3
+scipy>=1.1.0
 scikit-learn>=0.19.1,!=0.20.0
 tables>=3.5.1
@@ -5,46 +5,61 @@
    "type": "float32"
  },
  "annotations": {
+    "obs": {
+      "index": "name_0",
+      "columns": [
+        {
+          "name": "name_0",
+          "type": "string"
+        },
+        {
+          "name": "n_genes",
+          "type": "int32"
+        },
+        {
+          "name": "percent_mito",
+          "type": "float32"
+        },
+        {
+          "name": "n_counts",
+          "type": "float32"
+        },
+        {
+          "name": "louvain",
+          "type": "categorical",
+          "categories": [
+            "CD4 T cells",
+            "CD14+ Monocytes",
+            "B cells",
+            "CD8 T cells",
+            "NK cells",
+            "FCGR3A+ Monocytes",
+            "Dendritic cells",
+            "Megakaryocytes"
+          ]
+        }
+      ]
+    },
+    "var": {
+      "index": "name_0",
+      "columns": [
+        {
+          "name": "name_0",
+          "type": "string"
+        },
+        {
+          "name": "n_cells",
+          "type": "int32"
+        }
+      ]
+    }
+  },
+  "layout": {
    "obs": [
      {
-        "name": "name",
-        "type": "string"
-      },
-      {
-        "name": "n_genes",
-        "type": "int32"
-      },
-      {
-        "name": "percent_mito",
-        "type": "float32"
-      },
-      {
-        "name": "n_counts",
-        "type": "float32"
-      },
-      {
-        "name": "louvain",
-        "type": "categorical",
-        "categories": [
-          "CD4 T cells",
-          "CD14+ Monocytes",
-          "B cells",
-          "CD8 T cells",
-          "NK cells",
-          "FCGR3A+ Monocytes",
-          "Dendritic cells",
-          "Megakaryocytes"
-        ]
-      }
-    ],
-    "var": [
-      {
-        "name": "name",
-        "type": "string"
-      },
-      {
-        "name": "n_cells",
-        "type": "int32"
+        "name": "umap",
+        "type": "float32",
+        "dims": ["umap_0", "umap_1"]
      }
    ]
  }
@@ -19,11 +19,12 @@ class EndPoints(unittest.TestCase):

    @classmethod
    def setUpClass(cls):
-        cls.ps = Popen(["cellxgene", "launch", "example-dataset/pbmc3k.h5ad", "--debug", "--port", "5005"])
+        cls.ps = Popen(["cellxgene", "launch", "example-dataset/pbmc3k.h5ad", "--verbose", "--port", "5005"])
        session = requests.Session()
        for i in range(90):
            try:
-                session.get(f"{URL_BASE}schema")
+                result = session.get(f"{URL_BASE}schema")
+                cls.schema = result.json()
            except requests.exceptions.ConnectionError:
                time.sleep(1)

@@ -45,7 +46,8 @@ class EndPoints(unittest.TestCase):
        self.assertEqual(result.headers["Content-Type"], "application/json")
        result_data = result.json()
        self.assertEqual(result_data["schema"]["dataframe"]["nObs"], 2638)
-        self.assertEqual(len(result_data["schema"]["annotations"]["obs"]), 5)
+        self.assertEqual(len(result_data["schema"]["annotations"]["obs"]), 2)
+        self.assertEqual(len(result_data["schema"]["annotations"]["obs"]["columns"]), 5)

    def test_config(self):
        endpoint = "config"
@@ -95,7 +97,8 @@ class EndPoints(unittest.TestCase):
        self.assertIsNotNone(df['col_idx'])
        self.assertIsNone(df['row_idx'])
        self.assertEqual(len(df['columns']), df['n_cols'])
-        self.assertListEqual(df['col_idx'], ['name', 'n_genes', 'percent_mito', 'n_counts', 'louvain'])
+        obs_index_col_name = self.schema["schema"]["annotations"]["obs"]["index"]
+        self.assertListEqual(df['col_idx'], [obs_index_col_name, 'n_genes', 'percent_mito', 'n_counts', 'louvain'])

    def test_get_annotations_obs_keys_fbs(self):
        endpoint = "annotations/obs"
@@ -165,7 +168,8 @@ class EndPoints(unittest.TestCase):
        self.assertIsNotNone(df['col_idx'])
        self.assertIsNone(df['row_idx'])
        self.assertEqual(len(df['columns']), df['n_cols'])
-        self.assertListEqual(df['col_idx'], ['name', 'n_cells'])
+        var_index_col_name = self.schema["schema"]["annotations"]["var"]["index"]
+        self.assertListEqual(df['col_idx'], [var_index_col_name, 'n_cells'])

    def test_get_annotations_var_keys_fbs(self):
        endpoint = "annotations/var"
@@ -247,7 +251,8 @@ class EndPoints(unittest.TestCase):
        endpoint = f"data/var"
        url = f"{URL_BASE}{endpoint}"
        header = {"Accept": "application/octet-stream"}
-        var_filter = {"filter": {"var": {"annotation_value": [{"name": "name", "values": ["RER1"]}]}}}
+        index_col_name = self.schema["schema"]["annotations"]["var"]["index"]
+        var_filter = {"filter": {"var": {"annotation_value": [{"name": index_col_name, "values": ["RER1"]}]}}}
        result = self.session.put(url, headers=header, json=var_filter)
        self.assertEqual(result.status_code, HTTPStatus.OK)
        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
@@ -8,7 +8,7 @@ import decode_fbs

 import requests

-LOCAL_URL = "http://127.0.0.1:5005/"
+LOCAL_URL = "http://127.0.0.1:5006/"
 VERSION = "v0.2"
 URL_BASE = f"{LOCAL_URL}api/{VERSION}/"

@@ -21,7 +21,7 @@ class WithNaNs(unittest.TestCase):
    @classmethod
    def setUpClass(cls):
        cls.ps = Popen(
-            ["cellxgene", "launch", "server/test/test_datasets/nan.h5ad", "--debug", "--port", "5005"]
+            ["cellxgene", "launch", "server/test/test_datasets/nan.h5ad", "--verbose", "--port", "5006"]
        )
        session = requests.Session()
        for i in range(90):
@@ -58,14 +58,16 @@ class NaNTest(unittest.TestCase):

    def test_annotation(self):
        annotations = decode_fbs.decode_matrix_FBS(self.data.annotation_to_fbs_matrix("obs"))
+        obs_index_col_name = self.data.schema["annotations"]["obs"]["index"]
        self.assertEqual(
            annotations["col_idx"],
-            ["name", "n_genes", "percent_mito", "n_counts", "louvain"]
+            [obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain"]
        )
        self.assertEqual(annotations["n_rows"], 100)
        self.assertTrue(math.isnan(annotations["columns"][2][0]))

        annotations = decode_fbs.decode_matrix_FBS(self.data.annotation_to_fbs_matrix("var"))
-        self.assertEqual(annotations["col_idx"], ["name", "n_cells", "var_with_nans"])
+        var_index_col_name = self.data.schema["annotations"]["var"]["index"]
+        self.assertEqual(annotations["col_idx"], [var_index_col_name, "n_cells", "var_with_nans"])
        self.assertEqual(annotations["n_rows"], 100)
        self.assertTrue(math.isnan(annotations["columns"][2][0]))
@@ -31,9 +31,11 @@ class EngineTest(unittest.TestCase):
        self.assertTrue(self.data.data.X[0, 0] - -0.171_469_51 < epsilon)

    def test_mandatory_annotations(self):
-        self.assertIn("name", self.data.data.obs)
+        obs_index_col_name = self.data.schema["annotations"]["obs"]["index"]
+        self.assertIn(obs_index_col_name, self.data.data.obs)
        self.assertEqual(list(self.data.data.obs.index), list(range(2638)))
-        self.assertIn("name", self.data.data.var)
+        var_index_col_name = self.data.schema["annotations"]["var"]["index"]
+        self.assertIn(var_index_col_name, self.data.data.var)
        self.assertEqual(list(self.data.data.var.index), list(range(1838)))

    @pytest.mark.filterwarnings("ignore:Scanpy data matrix")
@@ -70,12 +72,14 @@ class EngineTest(unittest.TestCase):
        self.assertEqual(data["n_cols"], 91)

    def test_obs_and_var_names(self):
-        self.assertEqual(np.sum(self.data.data.var["name"].isna()), 0)
-        self.assertEqual(np.sum(self.data.data.obs["name"].isna()), 0)
+        self.assertEqual(np.sum(self.data.data.var[self.data.schema["annotations"]["var"]["index"]].isna()), 0)
+        self.assertEqual(np.sum(self.data.data.obs[self.data.schema["annotations"]["obs"]["index"]].isna()), 0)

    def test_schema(self):
        with open(path.join(path.dirname(__file__), "schema.json")) as fh:
            schema = json.load(fh)
+            print(schema)
+            print(self.data.schema)
            self.assertEqual(self.data.schema, schema)

    def test_schema_produces_error(self):
@@ -108,16 +112,18 @@ class EngineTest(unittest.TestCase):
        annotations = decode_fbs.decode_matrix_FBS(fbs)
        self.assertEqual(annotations["n_rows"], 2638)
        self.assertEqual(annotations["n_cols"], 5)
+        obs_index_col_name = self.data.schema["annotations"]["obs"]["index"]
        self.assertEqual(
            annotations["col_idx"],
-            ["name", "n_genes", "percent_mito", "n_counts", "louvain"],
+            [obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain"],
        )

        fbs = self.data.annotation_to_fbs_matrix("var")
        annotations = decode_fbs.decode_matrix_FBS(fbs)
        self.assertEqual(annotations['n_rows'], 1838)
        self.assertEqual(annotations['n_cols'], 2)
-        self.assertEqual(annotations["col_idx"], ["name", "n_cells"])
+        var_index_col_name = self.data.schema["annotations"]["var"]["index"]
+        self.assertEqual(annotations["col_idx"], [var_index_col_name, "n_cells"])

    def test_annotation_fields(self):
        fbs = self.data.annotation_to_fbs_matrix("obs", ["n_genes", "n_counts"])
@@ -125,7 +131,8 @@ class EngineTest(unittest.TestCase):
        self.assertEqual(annotations["n_rows"], 2638)
        self.assertEqual(annotations['n_cols'], 2)

-        fbs = self.data.annotation_to_fbs_matrix("var", ["name"])
+        var_index_col_name = self.data.schema["annotations"]["var"]["index"]
+        fbs = self.data.annotation_to_fbs_matrix("var", [var_index_col_name])
        annotations = decode_fbs.decode_matrix_FBS(fbs)
        self.assertEqual(annotations['n_rows'], 1838)
        self.assertEqual(annotations['n_cols'], 1)
@@ -163,9 +170,10 @@ class EngineTest(unittest.TestCase):
            self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")

    def test_data_named_gene(self):
+        var_index_col_name = self.data.schema["annotations"]["var"]["index"]
        filter_ = {
            "filter": {
-                "var": {"annotation_value": [{"name": "name", "values": ["RER1"]}]}
+                "var": {"annotation_value": [{"name": var_index_col_name, "values": ["RER1"]}]}
            }
        }
        fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
@@ -176,7 +184,7 @@ class EngineTest(unittest.TestCase):

        filter_ = {
            "filter": {
-                "var": {"annotation_value": [{"name": "name", "values": ["SPEN", "TYMP", "PRMT2"]}]}
+                "var": {"annotation_value": [{"name": var_index_col_name, "values": ["SPEN", "TYMP", "PRMT2"]}]}
            }
        }
        fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
@@ -10,10 +10,17 @@ def find_available_port(host, port=5005):
    # Takes approx 2 seconds to do a scan of 5000 ports on my laptop
    num_ports_to_try = 5000
    for port_to_try in range(port, port + num_ports_to_try):
-        with contextlib.closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s:
-            try:
-                s.bind((host, port_to_try))
-                return port_to_try
-            except socket.error:
-                pass
+        if is_port_available(host, port_to_try):
+            return port_to_try
    raise socket.error(errno.EADDRINUSE, f"No port in range {port} - {port + num_ports_to_try - 1} available.")
+
+
+def is_port_available(host, port):
+    is_available = False
+    with contextlib.closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s:
+        try:
+            s.bind((host, port))
+            is_available = True
+        except socket.error:
+            pass
+    return is_available
@@ -8,7 +8,7 @@ with open("server/requirements.txt") as fh:

 setup(
    name="cellxgene",
-    version="0.9.1",
+    version="0.10.0",
    packages=find_packages(),
    url="https://github.com/chanzuckerberg/cellxgene",
    license="MIT",
Author	SHA1	Message	Date
Bruce Martin	c6252825f3	release 0.10.0 (#794 )	2019-05-29 16:55:43 -07:00
Bruce Martin	4b96b3a635	fix incompatibility of flask reload and port searching (#793 ) * WIP * add --developer; fix incompatibility of --port and --debug * put REST tests on separate ports * PR review	2019-05-29 16:38:57 -07:00
Colin Megill	862d8feb5e	x (#792 )	2019-05-29 12:23:30 -04:00
Bruce Martin	1ef77d1596	fix misconfiguration for history management (#787 )	2019-05-24 21:01:11 -07:00
Bruce Martin	3dc45d6330	do not hard-wire column names in annotations (#785 ) * enforce column name uniqueness for obs and var * parameterize the column name containing obs and var user-readable names * use the new annotation index value from schema * update f/e unit tests * PR review suggestions * lint	2019-05-24 21:00:54 -07:00
Bruce Martin	a8c2e408d1	update to latest anndata and remove restriction on scipy (#790 )	2019-05-24 11:23:35 -07:00
Bruce Martin	e941c1a496	scaling omitted from event handlers (#789 ) * scaling omitted from event handlers * fix smoke tests	2019-05-24 07:00:04 -07:00
Colin Megill	a657eb3152	Logo (#782 ) * logo, black * fixes * remove template, move header	2019-05-23 11:47:24 -04:00
Bruce Martin	ef7c26e799	correctly handle selection of trunctated categories (#781 )	2019-05-23 08:46:22 -07:00
Bruce Martin	49af278de7	cleanup memoiziation in graph component (#783 )	2019-05-22 17:37:18 -07:00
Bruce Martin	2357d0c1b8	layout change UI (#776 ) * add layout to schema * add layout choice action and reducer * multi layout UI * update unit tests * add missing file * update test schema * fix duplicate test id * fix tabs * PR lint * fix pytest	2019-05-22 13:21:33 -07:00
Charlotte Weaver	63af79d3f8	Add developer guidelines (#769 ) * Add developer guidelines * minor formatting * PR clarifications/lint * more pr fixes * link fix * below->above * pr suggestions	2019-05-21 13:57:34 -07:00
Bruce Martin	fcc05f6a00	coordinate system fixes for embedded graph (#768 ) * change pan speed to 1 per issue #722 * correct handle scaling of graph when aspect ratio less than one * add package lock * add invert to our scale functions * correctly transform to/from gl coordinates * remove unused import * fix naming of import * update smoke tests	2019-05-20 14:22:41 -07:00
Bruce Martin	de3407d875	change scripts to support windows (#775 )	2019-05-20 11:42:57 -07:00
Charlotte Weaver	2d4e827bea	wait for element before getting text/html (#777 )	2019-05-20 11:35:53 -07:00
Bruce Martin	1fa4838863	npm (js) package dependency updates (#765 ) * JS package dependency updates * additional package updates * more package version updates * more js package updates * more JS dependency updates	2019-05-20 10:11:49 -07:00
Charlotte Weaver	ab4c74a321	remove psutil (#773 )	2019-05-18 10:53:45 -07:00
Charlotte Weaver	82d65addec	always run smoke tests (#772 )	2019-05-18 10:50:17 -07:00