exclude recent scipy versions (#770)

* add multi-layout support to back-end

* remove obsolete code

* temporary code to apply heuristic choice of default layout

* fix tests

* update python tests

* more py lint

* PR review changes

* more PR lint

* PR lint

.travis.yml

@@ -9,8 +9,8 @@ cache:
 install:
   - set -eo pipefail
   - pip install flake8
-  - make build
-  - make install
+  - make pydist
+  - make install-dist
   - pip install -r server/requirements-dev.txt
 
 jobs:

client/__tests__/e2e/data.js

@@ -108,6 +108,15 @@ export const datasets = {
           count: "24"
         }
       }
+    },
+    clip: {
+      min: "30",
+      max: "70",
+      metadata: "n_genes",
+      gene: "S100A8",
+      "coordinates-as-percent": { x1: 0.25, y1: 0.5, x2: 0.55, y2: 0.5 },
+      count: "392",
+      "gene-cell-count": "421"
     }
   }
 };

client/__tests__/e2e/e2e.test.js

@@ -182,7 +182,6 @@ describe("diffexp", async () => {
     );
   });
 });
-//
 
 describe("subset/reset", async () => {
   test("subset - cell count matches", async () => {
@@ -271,6 +270,38 @@ describe("scatter plot", async () => {
   });
 });
 
+describe("clipping", async () => {
+  test("clip continuous", async () => {
+    await cxgActions.clip(data.clip.min, data.clip.max)
+    const histId = `histogram-${data.clip.metadata}-plot-brush`;
+    const coords = await cxgActions.calcDragCoordinates(
+      histId,
+      data.clip["coordinates-as-percent"]
+    );
+    await cxgActions.drag(histId, coords.start, coords.end);
+    const cellCount = await cxgActions.cellSet(1);
+    expect(cellCount).toBe(data.clip.count);
+    
+  });
+
+  test("clip gene", async () => {
+    await utils.typeInto("gene-search", data.clip.gene);
+    await page.keyboard.press("Enter");
+    await page.waitForSelector(
+      `[data-testid='histogram-${data.clip.gene}']`
+    );
+    await cxgActions.clip(data.clip.min, data.clip.max)
+    const histId = `histogram-${data.clip.gene}-plot-brush`;
+    const coords = await cxgActions.calcDragCoordinates(
+      histId,
+      data.clip["coordinates-as-percent"]
+    );
+    await cxgActions.drag(histId, coords.start, coords.end);
+    const cellCount = await cxgActions.cellSet(1);
+    expect(cellCount).toBe(data.clip["gene-cell-count"]);
+  });
+});
+
 // interact with UI elements just that they do not break
 describe("ui elements don't error", async () => {
   test("color by", async () => {

client/__tests__/e2e/puppeteerUtils.js

@@ -16,10 +16,25 @@ export const puppeteerUtils = puppeteerPage => ({
   async typeInto(testid, text) {
     // only works for text without special characters
     await this.waitByID(testid);
+    const selector = `[data-testid='${testid}']`;
     // type ahead can be annoying if you don't pause before you type
-    await puppeteerPage.click(`[data-testid='${testid}']`);
+    await puppeteerPage.click(selector);
     await puppeteerPage.waitFor(200);
-    await puppeteerPage.type(`[data-testid='${testid}']`, text);
+    await puppeteerPage.type(selector, text);
+  },
+
+  async clearInputAndTypeInto(testid, text) {
+    await this.waitByID(testid);
+    const selector = `[data-testid='${testid}']`;
+    // only works for text without special characters
+    // type ahead can be annoying if you don't pause before you type
+    await puppeteerPage.click(selector);
+    await puppeteerPage.waitFor(200);
+    // select all
+
+    await puppeteerPage.click(selector, {clickCount: 3})
+    await puppeteerPage.keyboard.type("Backspace")
+    await puppeteerPage.type(selector, text);
   },
 
   async clickOn(testid) {
@@ -161,5 +176,13 @@ export const cellxgeneActions = puppeteerPage => ({
     await puppeteerUtils(puppeteerPage).clickOn("reset");
     // loading state never actually happens, reset is too fast
     await page.waitFor(200);
+  },
+
+  async clip(min = 0, max = 100) { 
+    await puppeteerUtils(puppeteerPage).clickOn("visualization-settings");
+    await puppeteerUtils(puppeteerPage).clearInputAndTypeInto("clip-min-input", min);
+    await puppeteerUtils(puppeteerPage).clearInputAndTypeInto("clip-max-input", max);
+    await puppeteerUtils(puppeteerPage).clickOn("clip-commit");
   }
+
 });

client/__tests__/util/range.test.js

@@ -0,0 +1,42 @@
+import { range, rangeFill } from "../../src/util/range";
+
+describe("range", () => {
+	test("no defaults", () => {
+		expect(range(0, 3, 1)).toMatchObject([0, 1, 2]);
+	});
+
+	test("range(stop)", () => {
+		expect(range(3)).toMatchObject([0, 1, 2]);
+		expect(range(0)).toMatchObject([]);
+		expect(range(1)).toMatchObject([0]);
+	});
+
+	test("range(start,stop)", () => {
+		expect(range(0, 0)).toMatchObject([]);
+		expect(range(0, 2)).toMatchObject([0, 1]);
+		expect(range(4, 8)).toMatchObject([4, 5, 6, 7]);
+	});
+
+	test("range(start, stop, step", () => {
+		expect(range(4, 0, -1)).toMatchObject([4, 3, 2, 1]);
+		expect(range(0, 4, 2)).toMatchObject([0, 2]);
+	});
+});
+
+describe("rangefill", () => {
+	test("rangeFill(arr)", () => {
+		expect(rangeFill(new Int32Array(3))).toMatchObject(
+			new Int32Array([0, 1, 2])
+		);
+	});
+	test("rangeFill(arr, start)", () => {
+		expect(rangeFill(new Int32Array(2), 1)).toMatchObject(
+			new Int32Array([1, 2])
+		);
+	});
+	test("rangeFill(arr, start, step)", () => {
+		expect(rangeFill(new Int32Array(3), 2, -1)).toMatchObject(
+			new Int32Array([2, 1, 0])
+		);
+	});
+});

client/__tests__/util/stateManager/sampleResponses.js

@@ -162,29 +162,7 @@ const aLayoutFBSResponse = (() => {
     new Float32Array(nObs).fill(Math.random()),
     new Float32Array(nObs).fill(Math.random())
   ];
-  const builder = new flatbuffers.Builder(1024);
-
-  const cols = _.map(coords, carr => {
-    const cdv = NetEncoding.Float32Array.createDataVector(builder, carr);
-    NetEncoding.Float32Array.startFloat32Array(builder);
-    NetEncoding.Float32Array.addData(builder, cdv);
-    const floatArr = NetEncoding.Float32Array.endFloat32Array(builder);
-
-    NetEncoding.Column.startColumn(builder);
-    NetEncoding.Column.addUType(builder, NetEncoding.TypedArray.Float32Array);
-    NetEncoding.Column.addU(builder, floatArr);
-    return NetEncoding.Column.endColumn(builder);
-  });
-
-  const columns = NetEncoding.Matrix.createColumnsVector(builder, cols);
-
-  NetEncoding.Matrix.startMatrix(builder);
-  NetEncoding.Matrix.addNRows(builder, nObs);
-  NetEncoding.Matrix.addNCols(builder, coords.length);
-  NetEncoding.Matrix.addColumns(builder, columns);
-  const matrix = NetEncoding.Matrix.endMatrix(builder);
-  builder.finish(matrix);
-  return builder.asUint8Array();
+  return encodeMatrix(coords, ["umap_0", "umap_1"]);
 })();
 
 const aDataObsResponse = {

client/__tests__/util/typedCrossfilter/util.test.js

@@ -1,8 +1,8 @@
 import {
-  fillRange,
   sliceByIndex,
   makeSortIndex
 } from "../../../src/util/typedCrossfilter/util";
+import { rangeFill as fillRange } from "../../../src/util/range";
 
 describe("fillRange", () => {
   test("Array", () => {

client/package-lock.json

@@ -11318,9 +11318,9 @@
       "dev": true
     },
     "puppeteer": {
-      "version": "1.12.2",
-      "resolved": "https://registry.npmjs.org/puppeteer/-/puppeteer-1.12.2.tgz",
-      "integrity": "sha512-xWSyCeD6EazGlfnQweMpM+Hs6X6PhUYhNTHKFj/axNZDq4OmrVERf70isBf7HsnFgB3zOC1+23/8+wCAZYg+Pg==",
+      "version": "1.15.0",
+      "resolved": "https://registry.npmjs.org/puppeteer/-/puppeteer-1.15.0.tgz",
+      "integrity": "sha512-D2y5kwA9SsYkNUmcBzu9WZ4V1SGHiQTmgvDZSx6sRYFsgV25IebL4V6FaHjF6MbwLK9C6f3G3pmck9qmwM8H3w==",
       "dev": true,
       "requires": {
         "debug": "^4.1.0",
@@ -11343,15 +11343,15 @@
           }
         },
         "mime": {
-          "version": "2.4.0",
-          "resolved": "https://registry.npmjs.org/mime/-/mime-2.4.0.tgz",
-          "integrity": "sha512-ikBcWwyqXQSHKtciCcctu9YfPbFYZ4+gbHEmE0Q8jzcTYQg5dHCr3g2wwAZjPoJfQVXZq6KXAjpXOTf5/cjT7w==",
+          "version": "2.4.2",
+          "resolved": "https://registry.npmjs.org/mime/-/mime-2.4.2.tgz",
+          "integrity": "sha512-zJBfZDkwRu+j3Pdd2aHsR5GfH2jIWhmL1ZzBoc+X+3JEti2hbArWcyJ+1laC1D2/U/W1a/+Cegj0/OnEU2ybjg==",
           "dev": true
         },
         "ws": {
-          "version": "6.1.4",
-          "resolved": "https://registry.npmjs.org/ws/-/ws-6.1.4.tgz",
-          "integrity": "sha512-eqZfL+NE/YQc1/ZynhojeV8q+H050oR8AZ2uIev7RU10svA9ZnJUddHcOUZTJLinZ9yEfdA2kSATS2qZK5fhJA==",
+          "version": "6.2.1",
+          "resolved": "https://registry.npmjs.org/ws/-/ws-6.2.1.tgz",
+          "integrity": "sha512-GIyAXC2cB7LjvpgMt9EKS2ldqr0MTrORaleiOno6TweZ6r3TKtoFQWay/2PceJ3RuBasOHzXNn5Lrw1X0bEjqA==",
           "dev": true,
           "requires": {
             "async-limiter": "~1.0.0"

client/package.json

@@ -104,7 +104,7 @@
     "jest-puppeteer": "^4.1.0",
     "json-loader": "^0.5.4",
     "mini-css-extract-plugin": "^0.4.1",
-    "puppeteer": "^1.12.1",
+    "puppeteer": "^1.15.0",
     "rimraf": "^2.6.3",
     "serve-favicon": "^2.3.0",
     "start-server-and-test": "^1.7.11",
@@ -142,7 +142,9 @@
           ],
           "@babel/plugin-proposal-export-namespace-from",
           "@babel/plugin-transform-react-constant-elements",
-          "@babel/plugin-transform-runtime"
+          "@babel/plugin-transform-runtime",
+          "@babel/plugin-proposal-optional-chaining",
+          "@babel/plugin-proposal-nullish-coalescing-operator"
         ]
       }
     }

client/src/components/categorical/occupancy.js

@@ -1,25 +1,16 @@
 // jshint esversion: 6
 import React from "react";
-import _ from "lodash";
 import { connect } from "react-redux";
 import * as d3 from "d3";
 
 @connect()
 class Occupancy extends React.Component {
   render() {
-    const {
-      occupancy,
-      colorScale,
-      categoricalSelection,
-      colorAccessor,
-      schema
-    } = this.props;
+    const { occupancy, colorScale, colorAccessor, schema, world } = this.props;
     const width = 100;
     const height = 11;
 
-    const categories = _.filter(schema.annotations.obs, {
-      name: colorAccessor
-    })[0].categories;
+    const categories = schema.annotations.obsByName[colorAccessor]?.categories;
 
     const x = d3
       .scaleLinear()
@@ -28,8 +19,9 @@ class Occupancy extends React.Component {
       .range([0, width]);
 
     let currentOffset = 0;
-
-    const stacks = categoricalSelection[colorAccessor].categoryValues.map(d => {
+    const dfColumn = world.obsAnnotations.col(colorAccessor);
+    const categoryValues = dfColumn.summarize().categories;
+    const stacks = categoryValues.map(d => {
       const o = occupancy.get(d);
 
       const scaledValue = x(o);

client/src/components/categorical/util.js

@@ -5,7 +5,6 @@
 // return sorted index
 
 import isNumber from "is-number";
-import _ from "lodash";
 
 const sortedCategoryValues = values => {
   /* this sort could be memoized for perf */
@@ -13,7 +12,7 @@ const sortedCategoryValues = values => {
   const strings = [];
   const ints = [];
 
-  _.forEach(values, v => {
+  values.forEach(v => {
     if (isNumber(v[0])) {
       ints.push(v);
     } else {

client/src/components/categorical/value.js

@@ -1,7 +1,6 @@
 // jshint esversion: 6
 import { connect } from "react-redux";
 import React from "react";
-import _ from "lodash";
 import Occupancy from "./occupancy";
 import { countCategoryValues2D } from "../../util/stateManager/worldUtil";
 import * as globals from "../../globals";
@@ -10,7 +9,7 @@ import * as globals from "../../globals";
   categoricalSelection: state.categoricalSelection,
   colorScale: state.colors.scale,
   colorAccessor: state.colors.colorAccessor,
-  schema: _.get(state.world, "schema", null),
+  schema: state.world?.schema,
   world: state.world
 }))
 class CategoryValue extends React.Component {
@@ -60,9 +59,7 @@ class CategoryValue extends React.Component {
     let occupancy = null;
 
     if (isColorBy && schema) {
-      categories = _.filter(schema.annotations.obs, {
-        name: colorAccessor
-      })[0].categories;
+      categories = schema.annotations.obsByName[colorAccessor]?.categories;
     }
 
     if (colorAccessor && !isColorBy && categoricalSelection[colorAccessor]) {

client/src/components/continuous/continuous.js

@@ -9,10 +9,10 @@ import * as globals from "../../globals";
 import HistogramBrush from "../brushableHistogram";
 
 @connect(state => ({
-  obsAnnotations: _.get(state.world, "obsAnnotations", null),
+  obsAnnotations: state.world?.obsAnnotations,
   colorAccessor: state.colors.colorAccessor,
   colorScale: state.colors.scale,
-  schema: _.get(state.world, "schema", null)
+  schema: state.world?.schema
 }))
 class Continuous extends React.Component {
   constructor(props) {

client/src/components/geneExpression/index.js

@@ -57,7 +57,7 @@ const filterGenes = (query, genes) =>
 
 @connect(state => {
   return {
-    obsAnnotations: _.get(state.world, "obsAnnotations", null),
+    obsAnnotations: state.world?.obsAnnotations,
     userDefinedGenes: state.controls.userDefinedGenes,
     userDefinedGenesLoading: state.controls.userDefinedGenesLoading,
     world: state.world,

client/src/components/graph/graph.js

@@ -898,6 +898,7 @@ class Graph extends React.Component {
                 target={
                   <Button
                     type="button"
+                    data-testid="visualization-settings"
                     className={`bp3-button bp3-icon-timeline-bar-chart ${activeClipClass}`}
                     style={{
                       cursor: "pointer"
@@ -929,6 +930,7 @@ class Graph extends React.Component {
                     >
                       <NumericInput
                         style={{ width: 50 }}
+                        data-testid={"clip-min-input"}
                         onValueChange={this.handleClipPercentileMinValueChange}
                         onKeyPress={this.handleClipOnKeyPress}
                         value={clipMin}
@@ -949,6 +951,7 @@ class Graph extends React.Component {
                       <span style={{ marginRight: 5, marginLeft: 5 }}> - </span>
                       <NumericInput
                         style={{ width: 50 }}
+                        data-testid={"clip-max-input"}
                         onValueChange={this.handleClipPercentileMaxValueChange}
                         onKeyPress={this.handleClipOnKeyPress}
                         value={clipMax}
@@ -968,6 +971,7 @@ class Graph extends React.Component {
                       />
                       <Button
                         type="button"
+                        data-testid="clip-commit"
                         className="bp3-button"
                         disabled={this.isClipDisabled()}
                         style={{

client/src/components/leftsidebar.js

@@ -1,5 +1,4 @@
 // jshint esversion: 6
-import _ from "lodash";
 import React from "react";
 import { connect } from "react-redux";
 import Categorical from "./categorical/categorical";
@@ -10,7 +9,7 @@ import DynamicScatterplot from "./scatterplot/scatterplot";
 
 @connect(state => ({
   responsive: state.responsive,
-  datasetTitle: _.get(state.config, "displayNames.dataset"),
+  datasetTitle: state.config?.displayNames?.dataset,
   scatterplotXXaccessor: state.controls.scatterplotXXaccessor,
   scatterplotYYaccessor: state.controls.scatterplotYYaccessor
 }))

client/src/reducers/categoricalSelection.js

@@ -1,12 +1,9 @@
-import _ from "lodash";
-
 import { ControlsHelpers } from "../util/stateManager";
 import * as globals from "../globals";
 
 function maxCategoryItems(state) {
-  return _.get(
-    state.config,
-    "parameters.max-category-items",
+  return (
+    state.config.parameters?.["max-category-items"] ??
     globals.configDefaults.parameters["max-category-items"]
   );
 }

client/src/util/dataframe/labelIndex.js

@@ -3,6 +3,8 @@ Label indexing - map a label to & from an integer offset.  See Dataframe
 for how this is used.
 **/
 
+import { rangeFill as fillRange } from "../range";
+
 /*
 Private utility functions
 */
@@ -21,14 +23,6 @@ function extent(tarr) {
   return [min, max];
 }
 
-function fillRange(arr, start = 0) {
-  const larr = arr;
-  for (let i = 0, l = larr.length; i < l; i += 1) {
-    larr[i] = i + start;
-  }
-  return larr;
-}
-
 /* eslint-disable class-methods-use-this */
 class IdentityInt32Index {
   /*

client/src/util/range.js

@@ -0,0 +1,45 @@
+/*
+Array range creation
+
+range(start, stop, step) -> Array
+	This is identical to https://docs.python.org/3/library/functions.html#func-range
+	Returns new array filled with a range of numbers.
+
+	Usage:
+
+		range(stop) - start defaults to zero, step defaults to 1
+		range(start, stop, [step]) - step defaults to 1
+
+	Examples:
+		range(3) -> [0, 1, 2]
+		range(1, 3) -> [1, 2]
+		range(1, 5, 2) -> [1, 3]
+
+
+rangeFill(array, start, step) -> array
+	Fill entire array with values, from start, by step.  Returns first array.
+	start defaults to zero, step defaults to one.
+
+*/
+
+function _doFill(arr, start, step, count) {
+	for (let idx = 0, val = start; idx < count; idx += 1, val += step) {
+		arr[idx] = val;
+	}
+	return arr;
+}
+
+export function rangeFill(arr, start = 0, step = 1) {
+	return _doFill(arr, start, step, arr.length);
+}
+
+export function range(start, stop, step) {
+	if (start === undefined) return [];
+	if (stop === undefined) {
+		stop = start;
+		start = 0;
+	}
+	step = step || 1; // catch undefind and zero
+	const len = Math.max(Math.ceil((stop - start) / step), 0);
+	return _doFill(new Array(len), start, step, len);
+}

client/src/util/stateManager/colorHelpers.js

@@ -1,12 +1,12 @@
 /*
 Helper functions for the embedded graph colors
 */
-import _ from "lodash";
 import * as d3 from "d3";
 import { interpolateRainbow, interpolateCool } from "d3-scale-chromatic";
 import * as globals from "../../globals";
 import parseRGB from "../parseRGB";
 import finiteExtent from "../finiteExtent";
+import { range } from "../range";
 
 /*
 create new colors state object.   Paramters:
@@ -37,9 +37,7 @@ function createColors(world, colorMode = null, colorAccessor = null) {
 }
 
 function createColorsByCategoricalMetadata(world, accessor) {
-  const { categories } = _.filter(world.schema.annotations.obs, {
-    name: accessor
-  })[0];
+  const { categories } = world.schema.annotations.obsByName[accessor];
 
   const scale = d3
     .scaleSequential(interpolateRainbow)
@@ -67,7 +65,7 @@ function createColorsByContinuousMetadata(world, accessor) {
   const scale = d3
     .scaleQuantile()
     .domain([min, max])
-    .range(_.range(colorBins - 1, -1, -1));
+    .range(range(colorBins - 1, -1, -1));
 
   /* pre-create colors - much faster than doing it for each obs */
   const colors = new Array(colorBins);
@@ -97,7 +95,7 @@ function createColorsByExpression(world, accessor) {
   const scale = d3
     .scaleQuantile()
     .domain([min, max])
-    .range(_.range(colorBins - 1, -1, -1));
+    .range(range(colorBins - 1, -1, -1));
 
   /* pre-create colors - much faster than doing it for each obs */
   const colors = new Array(colorBins);

client/src/util/stateManager/controlsHelpers.js

@@ -5,7 +5,7 @@ Helper functions for the controls reducer
 import _ from "lodash";
 
 import * as globals from "../../globals";
-import { fillRange } from "../typedCrossfilter/util";
+import { rangeFill as fillRange } from "../range";
 import {
   userDefinedDimensionName,
   diffexpDimensionName

client/src/util/stateManager/universe.js

@@ -78,6 +78,7 @@ function AnnotationsFBSToDataframe(arrayBuffer) {
   The application has strong assumptions that all scalar data will be
   stored as a float32 or float64 (regardless of underlying data types).
   For example, clipping of value ranges (eg, user-selected percentiles)
+  depends on the ability to use NaN in any numeric type.
 
   All float data from the server is left as is.  All non-float is promoted
   to an appropriate float.
@@ -98,13 +99,30 @@ function AnnotationsFBSToDataframe(arrayBuffer) {
 
 function LayoutFBSToDataframe(arrayBuffer) {
   const fbs = decodeMatrixFBS(arrayBuffer, true);
-  if (fbs.columns.length !== 2 || !fbs.columns.every(isFpTypedArray)) {
+  if (fbs.columns.length < 2 || !fbs.columns.every(isFpTypedArray)) {
     // We have strong assumptions about the shape & type of layout data.
     throw new Error("Unexpected layout data type returned from server");
   }
+
+  /*
+  TODO: XXX
+
+  TEMPORARY CODE AND COMMENT to support the progressive implementation
+  of multi-layout support.  For now, we search for one of the following 
+  in the layouts and use it if we find it: umap, then tsne, then pca, 
+  then whatever is first in the list.
+  */
+  let layoutIndex = 0;
+  ["umap", "tsne", "pca"].some(name => {
+    const idx = fbs.colIdx.indexOf(`${name}_0`);
+    if (idx !== -1) {
+      layoutIndex = idx;
+    }
+    return idx !== -1;
+  });
   const df = new Dataframe.Dataframe(
-    [fbs.nRows, fbs.nCols],
-    fbs.columns,
+    [fbs.nRows, 2],
+    [fbs.columns[layoutIndex], fbs.columns[layoutIndex + 1]],
     null,
     new Dataframe.KeyIndex(["X", "Y"])
   );
@@ -122,15 +140,15 @@ function reconcileSchemaCategoriesWithSummary(universe) {
   cases, add a 'categories' field to the schema so it is accessible.
   */
 
-  _.forEach(universe.schema.annotations.obs, s => {
+  universe.schema.annotations.obs.forEach(s => {
     if (
       s.type === "string" ||
       s.type === "boolean" ||
       s.type === "categorical"
     ) {
       const categories = _.union(
-        _.get(s, "categories", []),
-        _.get(universe.obsAnnotations.col(s.name).summarize(), "categories", [])
+        s.categories ?? [],
+        universe.obsAnnotations.col(s.name).summarize().categories ?? []
       );
       s.categories = categories;
     }

client/src/util/typedCrossfilter/util.js

@@ -1,22 +1,12 @@
 // jshint esversion: 6
 
 import { sortIndex } from "./sort";
+import { rangeFill as fillRange } from "../range";
 
 /*
     Utility functions, private to this module.
 */
 
-// fill an array or typedarray with a sequential range of numbers,
-// starting with `start`
-//
-export function fillRange(arr, start = 0) {
-  const larr = arr;
-  for (let i = 0, len = larr.length; i < len; i += 1) {
-    larr[i] = i + start;
-  }
-  return larr;
-}
-
 // slice out of one array into another, using an index array
 //
 export function sliceByIndex(src, index) {

dev_docs/release_process.md

@@ -43,24 +43,53 @@ Follow these steps to create a release.
 8. Publish to pypi by performing the following steps (assumes you that you have registered for pypi, 
 and that you have write access to the cellxgene pypi package):
     - Build the distribution and upload to test pypi `make release-stage-2`
-    - [optional] Test the test installation in a fresh virtual environment using `make install-release-test`
+    - Test the test installation in a fresh virtual environment using `make install-release-test`
     - Upload the package to real pypi using `make release-stage-final`
-    - [optional] Test the installation in a fresh virtual environment using
+    - Test the installation in a fresh virtual environment using
       `pip install cellxgene`
-    - **Troubleshooting**:
-        - Fails to upload to test.pypi: pypi doesn't allow you to reupload a release with the same version number, 
-        if you accidentally burned a release number you want to use on prod, you have a couple options. 
-            1) OPTION 1: Create distribution `make pydist`; test release locally `pip install dist/<release tarball>`;
-             then upload to prod `make release-stage-final`. 
-            2) OPTION 2: (DANGER) release directly to prod: `make release-burned`.  
-            3) OPTION 3: If the release was burned on prod as well run from Step 3 again with option
-            PART=patch until you get to an unburned version.
-        - The release doesn't install or fails your tests when you install it: Delete it from pypi - Go to pypi.org, sign in, 
-        go to the cellxgene package, click manage, then in the options drop down, click delete and 
-        follow the instructions. You will not be able to use that release number again. If it is a minor bug 
-        and not a major regression, you can just release a patch. 
-    
 
 The optional steps are for testing purposes, and are recommended
 for publishing any major releases, and any releases that significantly
 change the packaging (e.g. new bundled files, new dependencies, etc.)
+
+## Troubleshooting
+### Fails to upload to test.pypi
+ 
+ _PyPi doesn't allow you to reupload a release with the same version number_ 
+If you accidentally burned a release number you want to use on prod, you have a few options:
+1) OPTION 1: Create distribution `make pydist`; test release locally `pip install dist/<release tarball>`;
+ then upload to prod `make release-stage-final`. 
+2) OPTION 2: (DANGER) release directly to prod: `make release-directly-to-prod`.  
+3) OPTION 3: If the release was burned on prod as well run from Step 3 again with option
+PART=patch until you get to an unburned version.
+    
+### The release doesn't install or fails your tests when you install it
+ 
+Delete it from pypi - Go to pypi.org -> sign in -> go to the cellxgene package -> click manage -> then in the options drop down click delete -> follow the instructions. You will not be able to use that release number again. If it is a minor bug and not a major regression, you can just release a patch. 
+### If you need to run stage final on a different computer than stage 2
+ If you run stage final without running stage 2 first, the dist will not have been build on the computer running stage final. The solution is to run `make release-directly-to-prod`. This both builds the distribution files and then releases directly to prod pypi.org. 
+
+## Stage Details
+### Stage 1 - `make release-stage-1`
+    1. Pip installs requirements-dev
+    2. Bumps version by [PART]
+    3. Deletes build directory, client/build, dist and cellxgene.egg-info
+    4. Creates the package-lock.json
+
+### Stage 2 - `make release-stage-2`
+    1. Pip installs requirements-dev
+    2. Builds client and server
+    3. Creates distribution release (sdist)
+    4. Uploads to test.pypi.org
+
+### Stage final - `make release-stage-final`
+    ** Does not build distribution **
+    1. Uploads to pypi.org
+    
+### (DANGER) Release directly to prod `make release-directly-to-prod`
+    ** builds distribution and uploads directly to prod **
+    Only use this if you are directed to by the troubleshooting guide
+    1. Pip installs requirements-dev
+    2. Builds client and server
+    3. Creates distribution release (sdist)
+    4. Uploads to pypi.org

docs/faq.md

@@ -38,7 +38,7 @@ Currently this is not supported directly, but you should be able to do this your
 
 - `.obs` and `.var` annotations are use to extract metadata for filtering
 - `.X` is used to display expression (histograms, scatterplot & colorscale) and to compute differential expression
-- `.obsm` is used for layout
+- `.obsm` is used for layout. If an embedding has more than two components, the first two will be used for visualization.
 
 #### I have a BIG dataset - how can I make cellxgene run as fast as possible?
 

makefile

@@ -76,7 +76,7 @@ release-stage-final: twine-prod
 
 # DANGER: releases directly to prod
 # use this if you accidently burned a test release version number,
-release-burned : dev-env pydist twine-prod
+release-directly-to-prod : dev-env pydist twine-prod
 	@echo "Dist built and uploaded to pypi.org"
 	@echo "Test the install:"
 	@echo "    make install-release"
@@ -114,14 +114,18 @@ install-dev : uninstall
 
 # install from test.pypi to test your release
 install-release-test : uninstall
-	pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple cellxgene
+	pip install --no-cache-dir --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple cellxgene
 	@echo "Installed cellxgene from test.pypi.org, now run and smoke test"
 
 # install from pypi to test your release
 install-release : uninstall
-	pip install cellxgene
+	pip install --no-cache-dir cellxgene
 	@echo "Installed cellxgene from pypi.org"
 
+# install from dist
+install-dist : uninstall
+	pip install dist/cellxgene*.tar.gz
+
 uninstall :
 	pip uninstall -y cellxgene || :
 

server/app/scanpy_engine/scanpy_engine.py

@@ -1,12 +1,13 @@
 import warnings
 
 import numpy as np
+import pandas
 from pandas.core.dtypes.dtypes import CategoricalDtype
-import scanpy as sc
+import anndata
 from scipy import sparse
 
 from server.app.driver.driver import CXGDriver
-from server.app.util.constants import Axis, DEFAULT_TOP_N
+from server.app.util.constants import Axis, DEFAULT_TOP_N, MAX_LAYOUTS
 from server.app.util.errors import (
     FilterError,
     JSONEncodingValueError,
@@ -41,7 +42,7 @@ class ScanpyEngine(CXGDriver):
     @staticmethod
     def _get_default_config():
         return {
-            "layout": "umap",
+            "layout": [],
             "diffexp": "ttest",
             "max_category_items": 100,
             "obs_names": None,
@@ -140,11 +141,10 @@ class ScanpyEngine(CXGDriver):
                 self.schema["annotations"][ax].append(ann_schema)
 
     def _load_data(self, data):
-        # Based on benchmarking, cache=True has no impact on perf.
-        # Note: as of current scanpy/anndata release, setting backed='r' will
-        # result in an error.  https://github.com/theislab/anndata/issues/79
+        # as of AnnData 0.6.19, backed mode performs initial load fast, but at the
+        # cost of significantly slower access to X data.
         try:
-            self.data = sc.read(data, cache=True)
+            self.data = anndata.read_h5ad(data)
         except ValueError:
             raise ScanpyFileError(
                 "File must be in the .h5ad format. Please read "
@@ -167,13 +167,62 @@ class ScanpyEngine(CXGDriver):
         self._alias_annotation_names(Axis.OBS, self.config["obs_names"])
         self._alias_annotation_names(Axis.VAR, self.config["var_names"])
         self._validate_data_types()
-        self._validate_data_calculations()
         self.cell_count = self.data.shape[0]
         self.gene_count = self.data.shape[1]
+        self._default_and_validate_layouts()
         self._create_schema()
 
+    @requires_data
+    def _default_and_validate_layouts(self):
+        """ function:
+            a) generate list of default layouts, if not already user specified
+            b) validate layouts are legal.  remove/warn on any that are not
+            c) cap total list of layouts at global const MAX_LAYOUTS
+        """
+        layouts = self.config['layout']
+        # handle default
+        if layouts is None or len(layouts) == 0:
+            # load default layouts from the data.
+            layouts = [key[2:] for key in self.data.obsm_keys() if type(key) == str and key.startswith("X_")]
+            if len(layouts) == 0:
+                raise PrepareError(f"Unable to find any precomputed layouts within the dataset.")
+
+        # remove invalid layouts
+        valid_layouts = []
+        obsm_keys = self.data.obsm_keys()
+        for layout in layouts:
+            layout_name = f"X_{layout}"
+            if layout_name not in obsm_keys:
+                warnings.warn(f"Ignoring unknown layout name: {layout}.")
+            elif not self._is_valid_layout(self.data.obsm[layout_name]):
+                warnings.warn(f"Ignoring layout due to malformed shape or data type: {layout}")
+            else:
+                valid_layouts.append(layout)
+
+        if len(valid_layouts) == 0:
+            raise PrepareError(f"No valid layout data.")
+
+        # cap layouts to MAX_LAYOUTS
+        self.config['layout'] = valid_layouts[0:MAX_LAYOUTS]
+
+    @requires_data
+    def _is_valid_layout(self, arr):
+        """ return True if this layout data is a valid array for front-end presentation:
+            * ndarray, with shape (n_obs, >= 2), dtype float/int/uint
+            * contains only finite values
+        """
+        is_valid = type(arr) == np.ndarray and arr.dtype.kind in "fiu"
+        is_valid = is_valid and arr.shape[0] == self.data.n_obs and arr.shape[1] >= 2
+        is_valid = is_valid and np.all(np.isfinite(arr))
+        return is_valid
+
     @requires_data
     def _validate_data_types(self):
+        if sparse.isspmatrix(self.data.X) and not sparse.isspmatrix_csc(self.data.X):
+            warnings.warn(
+                f"Scanpy data matrix is sparse, but not a CSC (columnar) matrix.  "
+                f"Performance may be improved by using CSC."
+            )
         if self.data.X.dtype != "float32":
             warnings.warn(
                 f"Scanpy data matrix is in {self.data.X.dtype} format not float32. "
@@ -204,20 +253,6 @@ class ScanpyEngine(CXGDriver):
                             f"annotations with more than 500 categories in the UI"
                         )
 
-    @requires_data
-    def _validate_data_calculations(self):
-        layout_key = f"X_{self.config['layout']}"
-        try:
-            assert layout_key in self.data.obsm_keys()
-        except AssertionError:
-            raise PrepareError(
-                f"Cannot find a field with coordinates for the {self.config['layout']} layout requested. A different"
-                f" layout may have been computed. The requested layout must be pre-calculated and saved "
-                f"back in the h5ad file. You can run "
-                f"`cellxgene prepare --layout {self.config['layout']} <datafile>` "
-                f"to solve this problem. "
-            )
-
     @staticmethod
     def _annotation_filter_to_mask(filter, d_axis, count):
         mask = np.ones((count,), dtype=bool)
@@ -304,7 +339,7 @@ class ScanpyEngine(CXGDriver):
         if sparse.issparse(X):  # use tuned getcol/hstack for performance
             indices = np.nonzero(var_mask)[0]
             cols = [X.getcol(i) for i in indices]
-            return sparse.hstack(cols)
+            return sparse.hstack(cols, format="csc")
         else:   # else, just use standard slicing, which is fine for dense arrays
             return X[:, var_mask]
 
@@ -368,15 +403,18 @@ class ScanpyEngine(CXGDriver):
         * only returns Matrix in columnar layout
         """
         try:
-            full_embedding = self.data.obsm[f"X_{self.config['layout']}"]
-            if full_embedding.shape[1] > 2:
-                warnings.warn(f"Warning: found {full_embedding.shape[1]} \
-                components of embedding. Using the first two for layout display.")
-            df_layout = full_embedding[:, :2]
+            layout_data = []
+            for layout in self.config["layout"]:
+                full_embedding = self.data.obsm[f"X_{layout}"]
+                embedding = full_embedding[:, :2]
+                normalized_layout = (embedding - embedding.min()) / (embedding.max() - embedding.min())
+                normalized_layout = normalized_layout.astype(dtype=np.float32)
+                layout_data.append(pandas.DataFrame(normalized_layout, columns=[f"{layout}_0", f"{layout}_1"]))
+
         except ValueError as e:
             raise PrepareError(
                 f"Layout has not been calculated using {self.config['layout']}, "
                 f"please prepare your datafile and relaunch cellxgene") from e
 
-        normalized_layout = (df_layout - df_layout.min()) / (df_layout.max() - df_layout.min())
-        return encode_matrix_fbs(normalized_layout.astype(dtype=np.float32), col_idx=None, row_idx=None)
+        df = pandas.concat(layout_data, axis=1, copy=False)
+        return encode_matrix_fbs(df, col_idx=df.columns, row_idx=None)

server/app/util/constants.py

@@ -31,3 +31,5 @@ JSON_NaN_to_num_warning_msg = (
     "JSON encoding failure - please verify all data are finite values (no NaN or Infinities)"
 )
 REACTIVE_LIMIT = 1_000_000
+
+MAX_LAYOUTS = 30

server/cli/launch.py

@@ -1,16 +1,22 @@
+import errno
 import logging
 from os import devnull
-from os.path import splitext, basename
+from os.path import splitext, basename, getsize
 import sys
 import warnings
 import webbrowser
 
 import click
+import psutil
 
 from server.app.app import Server
 from server.app.util.errors import ScanpyFileError
 from server.app.util.utils import custom_format_warning
-from server.utils.constants import MODES
+from server.utils.utils import find_available_port
+
+
+# anything bigger than this will generate a special message
+BIG_FILE_SIZE_THRESHOLD = 100 * 2**20  # 100MB
 
 
 @click.command()
@@ -18,10 +24,10 @@ from server.utils.constants import MODES
 @click.option(
     "--layout",
     "-l",
-    type=click.Choice(MODES),
-    default="umap",
+    default=[],
+    multiple=True,
     show_default=True,
-    help="Method for layout."
+    help="Layout name, eg, 'umap'."
 )
 @click.option(
     "--diffexp",
@@ -50,7 +56,8 @@ from server.utils.constants import MODES
     show_default=True,
     help="Open the web browser after launch.",
 )
-@click.option("--port", "-p", help="Port to run server on.", metavar="", default=5005, show_default=True)
+@click.option("--port", "-p", help="Port to run server on, if not specified cellxgene will find an available port.",
+              metavar="", show_default=True)
 @click.option("--obs-names", default=None, metavar="", help="Name of annotation field to use for observations.")
 @click.option("--var-names", default=None, metavar="", help="Name of annotation to use for variables.")
 @click.option("--host", default="127.0.0.1", help="Host IP address")
@@ -135,6 +142,9 @@ security risk by including the --scripts flag. Make sure you trust the scripts t
         file_parts = splitext(basename(data))
         title = file_parts[0]
 
+    if not port:
+        port = find_available_port(host)
+
     # Setup app
     cellxgene_url = f"http://{host}:{port}"
 
@@ -148,7 +158,16 @@ security risk by including the --scripts flag. Make sure you trust the scripts t
         log = logging.getLogger("werkzeug")
         log.setLevel(logging.ERROR)
 
-    click.echo(f"[cellxgene] Loading data from {basename(data)}, this may take awhile...")
+    file_size = getsize(data)
+
+    # if a big file, let the user know it may take a while to load.
+    if file_size > BIG_FILE_SIZE_THRESHOLD:
+        click.echo(f"[cellxgene] Loading data from {basename(data)}, this may take awhile...")
+    else:
+        click.echo(f"[cellxgene] Loading data from {basename(data)}.")
+    # if file is larger than main memory, let the user know performance may suffer
+    if file_size > .95 * psutil.virtual_memory().total:
+        click.echo(f"[cellxgene] Warning: data file is larger than RAM - application may be very slow.")
 
     # Fix for anaconda python. matplotlib typically expects python to be installed as a framework TKAgg is usually
     # available and fixes this issue. See https://matplotlib.org/faq/virtualenv_faq.html
@@ -183,4 +202,9 @@ security risk by including the --scripts flag. Make sure you trust the scripts t
         f = open(devnull, "w")
         sys.stdout = f
 
-    server.app.run(host=host, debug=debug, port=port, threaded=True)
+    try:
+        server.app.run(host=host, debug=debug, port=port, threaded=True)
+    except OSError as e:
+        if e.errno == errno.EADDRINUSE:
+            raise click.ClickException("Port is in use, please specify an open port using the --port flag.") from e
+        raise

server/requirements.txt

@@ -1,3 +1,4 @@
+anndata>=0.6.15
 click>=6.7
 Flask>=1.0.2
 Flask-Caching>=1.4.0
@@ -8,7 +9,8 @@ flatbuffers>=1.10.0
 matplotlib>=2.2
 numpy>=1.15.2
 pandas>=0.23.1
+psutil>=5.6.2
 scanpy>=1.3.7
-scipy>=1.1.0
+scipy>=1.1.0,<1.3
 scikit-learn>=0.19.1,!=0.20.0
 tables>=3.5.1

server/test/test_api.py

@@ -19,7 +19,7 @@ class EndPoints(unittest.TestCase):
 
     @classmethod
     def setUpClass(cls):
-        cls.ps = Popen(["cellxgene", "launch", "example-dataset/pbmc3k.h5ad", "--debug"])
+        cls.ps = Popen(["cellxgene", "launch", "example-dataset/pbmc3k.h5ad", "--debug", "--port", "5005"])
         session = requests.Session()
         for i in range(90):
             try:
@@ -67,9 +67,11 @@ class EndPoints(unittest.TestCase):
         self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
         df = decode_fbs.decode_matrix_FBS(result.content)
         self.assertEqual(df['n_rows'], 2638)
-        self.assertEqual(df['n_cols'], 2)
+        self.assertEqual(df['n_cols'], 8)
         self.assertIsNotNone(df['columns'])
-        self.assertIsNone(df['col_idx'])
+        self.assertListEqual(df['col_idx'], [
+            'pca_0', 'pca_1', 'tsne_0', 'tsne_1', 'umap_0', 'umap_1', 'draw_graph_fr_0', 'draw_graph_fr_1'
+        ])
         self.assertIsNone(df['row_idx'])
         self.assertEqual(len(df['columns']), df['n_cols'])
 

server/test/test_nan_rest.py

@@ -21,7 +21,7 @@ class WithNaNs(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
         cls.ps = Popen(
-            ["cellxgene", "launch", "server/test/test_datasets/nan.h5ad", "--debug"]
+            ["cellxgene", "launch", "server/test/test_datasets/nan.h5ad", "--debug", "--port", "5005"]
         )
         session = requests.Session()
         for i in range(90):

server/test/test_nan_scanpy_engine.py

@@ -12,7 +12,7 @@ from server.app.util.errors import FilterError
 class NaNTest(unittest.TestCase):
     def setUp(self):
         self.args = {
-            "layout": "umap",
+            "layout": ["umap"],
             "diffexp": "ttest",
             "max_category_items": 100,
             "obs_names": None,

server/test/test_scanpy_engine.py

@@ -15,7 +15,7 @@ from server.app.util.errors import FilterError
 class EngineTest(unittest.TestCase):
     def setUp(self):
         args = {
-            "layout": "umap",
+            "layout": ["umap"],
             "diffexp": "ttest",
             "max_category_items": 100,
             "obs_names": None,

server/test/test_scanpy_engine_data_load.py

@@ -15,7 +15,7 @@ class DataLoadEngineTest(unittest.TestCase):
 
     def test_delayed_load_args(self):
         args = {
-            "layout": "tsne",
+            "layout": ["tsne"],
             "diffexp": "ttest",
             "max_category_items": 1000,
             "obs_names": "foo",

server/utils/utils.py

@@ -0,0 +1,19 @@
+import contextlib
+import errno
+import socket
+
+
+def find_available_port(host, port=5005):
+    """
+    Helper method to find open port on host. Tries 5000 ports incremented from the specified port
+    """
+    # Takes approx 2 seconds to do a scan of 5000 ports on my laptop
+    num_ports_to_try = 5000
+    for port_to_try in range(port, port + num_ports_to_try):
+        with contextlib.closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s:
+            try:
+                s.bind((host, port_to_try))
+                return port_to_try
+            except socket.error:
+                pass
+    raise socket.error(errno.EADDRINUSE, f"No port in range {port} - {port + num_ports_to_try - 1} available.")