Performance work, plus fix #405 (#406)

* remove memoization

* update to flash 1.0.2; turn on threading

* stand-alone helper routines for array slicing

* fix issue #405

* reset diffexp state when world changes

* performance work in dimension creation; fix world slicing bug

* update tests to match new state mgmt api

* update flask

* do not make dimensions for useless annotations

* update test to match optimizations

client/__tests__/util/stateManager/world.test.js

@@ -77,7 +77,8 @@ describe("createWorldFromEntireUniverse", () => {
 
         varDataCache: expect.any(Object),
 
-        worldObsIndex: null // indicating full universe
+        obsIndex: null, // null indicating full universe
+        obsBackIndex: null
       })
     );
   });
@@ -120,16 +121,21 @@ describe("createWorldFromCurrentSelection", () => {
       nObs: universeIndices.length,
       obsAnnotations: _.map(universeIndices, i => universe.obsAnnotations[i]),
       obsLayout: {
-        X: _.map(universeIndices, i => universe.obsLayout.X[i]),
+        X: new Float32Array(
-        Y: _.map(universeIndices, i => universe.obsLayout.Y[i])
+          _.map(universeIndices, i => universe.obsLayout.X[i])
+        ),
+        Y: new Float32Array(
+          _.map(universeIndices, i => universe.obsLayout.Y[i])
+        )
       },
-      worldObsIndex: _.transform(
+      obsBackIndex: _.transform(
         universeIndices,
         (result, univIdx, worldIdx) => {
           result[univIdx] = worldIdx;
         },
-        new Array(universe.nObs).fill(-1)
+        new Uint32Array(universe.nObs).fill(-1)
-      )
+      ),
+      obsIndex: new Uint32Array(universeIndices)
     };
 
     expect(world).toMatchObject(
@@ -141,9 +147,13 @@ describe("createWorldFromCurrentSelection", () => {
         obsAnnotations: expected.obsAnnotations,
         varAnnotations: universe.varAnnotations,
         obsLayout: expected.obsLayout,
-        summary: expect.any(Object) /* we could do better! */,
+        summary: {
+          obs: expect.any(Object) /* we could do better! */,
+          var: expect.any(Object) /* we could do better! */
+        },
         varDataCache: expect.any(Object),
-        worldObsIndex: expected.worldObsIndex
+        obsIndex: expected.obsIndex,
+        obsBackIndex: expected.obsBackIndex
       })
     );
   });
@@ -163,11 +173,15 @@ describe("createObsDimensionMap", () => {
     expect(dimensionMap).toBeDefined();
     REST.annotationsObs.names.forEach(name => {
       const dim = dimensionMap[obsAnnoDimensionName(name)];
-      const { type } = schemaByObsName[name];
+      if (name === "name") {
-      if (type === "string" || type === "boolean" || type === "categorical") {
+        expect(dim).toBeUndefined();
-        expect(dim).toBeInstanceOf(Crossfilter.EnumDimension);
       } else {
-        expect(dim).toBeInstanceOf(Crossfilter.ScalarDimension);
+        const { type } = schemaByObsName[name];
+        if (type === "string" || type === "boolean" || type === "categorical") {
+          expect(dim).toBeInstanceOf(Crossfilter.EnumDimension);
+        } else {
+          expect(dim).toBeInstanceOf(Crossfilter.ScalarDimension);
+        }
       }
     });
     expect(dimensionMap[layoutDimensionName("X")]).toBeInstanceOf(
@@ -205,6 +219,7 @@ describe("subsetVarData", () => {
       world,
       crossfilter
     );
+    expect(newWorld.obsIndex).toMatchObject(new Uint32Array([0, 2]));
 
     /* expect a subset */
     const result = World.subsetVarData(newWorld, universe, sourceVarData);

client/__tests__/util/typedCrossfilter/util.test.js

@@ -0,0 +1,115 @@
+import {
+  fillRange,
+  sliceByIndex,
+  makeSortIndex,
+  lowerBound,
+  lowerBoundIndirect,
+  upperBoundIndirect
+} from "../../../src/util/typedCrossfilter/util";
+
+describe("fillRange", () => {
+  test("Array", () => {
+    expect(fillRange(new Array(6))).toMatchObject([0, 1, 2, 3, 4, 5]);
+    expect(fillRange(new Array(4), 1)).toMatchObject([1, 2, 3, 4]);
+    expect(fillRange([])).toMatchObject([]);
+  });
+
+  test("Uint32Array", () => {
+    expect(fillRange(new Uint32Array(6))).toMatchObject(
+      new Uint32Array([0, 1, 2, 3, 4, 5])
+    );
+    expect(fillRange(new Array(4), 1)).toMatchObject(
+      new Uint32Array([1, 2, 3, 4])
+    );
+    expect(fillRange([])).toMatchObject(new Uint32Array([]));
+  });
+});
+
+describe("sliceByIndex", () => {
+  test("Array", () => {
+    expect(sliceByIndex([0, 1, 2, 3, 4], [0, 1, 2])).toMatchObject([0, 1, 2]);
+    expect(sliceByIndex([0, 1, 2, 3, 4], [2, 1, 0])).toMatchObject([2, 1, 0]);
+    expect(sliceByIndex([0, 1, 2, 3, 4], [])).toMatchObject([]);
+    expect(sliceByIndex([], [])).toMatchObject([]);
+  });
+
+  test("Uint32Array", () => {
+    expect(
+      sliceByIndex([0, 1, 2, 3, 4], new Uint32Array([0, 1, 2]))
+    ).toMatchObject([0, 1, 2]);
+    expect(
+      sliceByIndex([0, 1, 2, 3, 4], new Uint32Array([2, 1, 0]))
+    ).toMatchObject([2, 1, 0]);
+    expect(sliceByIndex([0, 1, 2, 3, 4], new Uint32Array([]))).toMatchObject(
+      []
+    );
+    expect(sliceByIndex([], new Uint32Array([]))).toMatchObject([]);
+
+    expect(
+      sliceByIndex(new Uint32Array([0, 1, 2, 3, 4]), new Uint32Array([0, 1, 2]))
+    ).toMatchObject(new Uint32Array([0, 1, 2]));
+    expect(
+      sliceByIndex(new Uint32Array([0, 1, 2, 3, 4]), new Uint32Array([2, 1, 0]))
+    ).toMatchObject(new Uint32Array([2, 1, 0]));
+    expect(
+      sliceByIndex(
+        new Uint32Array([0, 1, 2, 3, 4]),
+        new Uint32Array(new Uint32Array([]))
+      )
+    ).toMatchObject(new Uint32Array([]));
+    expect(
+      sliceByIndex(new Uint32Array([]), new Uint32Array([]))
+    ).toMatchObject(new Uint32Array([]));
+  });
+
+  test("Float32Array", () => {
+    expect(
+      sliceByIndex(new Float32Array([0, 1, 2, 3, 4]), [0, 1, 2])
+    ).toMatchObject(new Float32Array([0, 1, 2]));
+    expect(
+      sliceByIndex(new Float32Array([0, 1, 2, 3, 4]), [2, 1, 0])
+    ).toMatchObject(new Float32Array([2, 1, 0]));
+    expect(sliceByIndex(new Float32Array([0, 1, 2, 3, 4]), [])).toMatchObject(
+      new Float32Array([])
+    );
+    expect(sliceByIndex(new Float32Array([]), [])).toMatchObject(
+      new Float32Array([])
+    );
+  });
+});
+
+describe("makeSortIndex", () => {
+  test("Array", () => {
+    expect(makeSortIndex([3, 2, 1, 0])).toMatchObject(
+      new Uint32Array([3, 2, 1, 0])
+    );
+    expect(makeSortIndex([3, 2, 1, 0, 4])).toMatchObject(
+      new Uint32Array([3, 2, 1, 0, 4])
+    );
+    expect(makeSortIndex([])).toMatchObject(new Uint32Array([]));
+  });
+
+  test("Float32Array", () => {
+    expect(makeSortIndex(new Float32Array([3, 2, 1, 0]))).toMatchObject(
+      new Uint32Array([3, 2, 1, 0])
+    );
+    expect(makeSortIndex(new Float32Array([3, 2, 1, 0, 4]))).toMatchObject(
+      new Uint32Array([3, 2, 1, 0, 4])
+    );
+    expect(makeSortIndex(new Float32Array([]))).toMatchObject(
+      new Uint32Array([])
+    );
+  });
+
+  test("Int32Array", () => {
+    expect(makeSortIndex(new Int32Array([3, 2, 1, 0]))).toMatchObject(
+      new Uint32Array([3, 2, 1, 0])
+    );
+    expect(makeSortIndex(new Int32Array([3, 2, 1, 0, 4]))).toMatchObject(
+      new Uint32Array([3, 2, 1, 0, 4])
+    );
+    expect(makeSortIndex(new Int32Array([]))).toMatchObject(
+      new Uint32Array([])
+    );
+  });
+});

client/src/actions/index.js

@@ -149,16 +149,13 @@ function requestSingleGeneExpressionCountsForColoringPOST(gene) {
   return async (dispatch, getState) => {
     dispatch({ type: "get single gene expression for coloring started" });
     try {
-      const expressionData = await _doRequestExpressionData(
+      await _doRequestExpressionData(dispatch, getState, [gene]);
-        dispatch,
+      const { world } = getState().controls;
-        getState,
-        [gene]
-      );
       dispatch({
         type: "color by expression",
         gene,
         data: {
-          [gene]: expressionData[gene]
+          [gene]: kvCache.get(world.varDataCache, gene)
         }
       });
     } catch (error) {
@@ -173,16 +170,15 @@ function requestSingleGeneExpressionCountsForColoringPOST(gene) {
 const requestUserDefinedGene = gene => async (dispatch, getState) => {
   dispatch({ type: "request user defined gene started" });
   try {
-    const data = await await _doRequestExpressionData(dispatch, getState, [
+    await await _doRequestExpressionData(dispatch, getState, [gene]);
-      gene
+    const { world } = getState().controls;
-    ]);
 
     /* then send the success case action through */
     return dispatch({
       type: "request user defined gene success",
       data: {
         genes: [gene],
-        expression: data[gene]
+        expression: kvCache.get(world.varDataCache, gene)
       }
     });
   } catch (error) {

client/src/reducers/differential.js

@@ -51,6 +51,7 @@ const Differential = (
     case "set World to current selection":
       return {
         ...state,
+        diffExp: null,
         celllist1: null,
         celllist2: null
       };

client/src/util/stateManager/world.js

@@ -4,6 +4,7 @@ import _ from "lodash";
 import * as kvCache from "./keyvalcache";
 import summarizeAnnotations from "./summarizeAnnotations";
 import { layoutDimensionName, obsAnnoDimensionName } from "../nameCreators";
+import { sliceByIndex } from "../typedCrossfilter/util";
 
 /*
 World is a subset of universe.   Most code should use world, and should
@@ -56,7 +57,8 @@ function templateWorld() {
   return {
     // map from universe obsIndex to world offset.
     // Undefined / null indicates identity mapping.
-    worldObsIndex: null,
+    obsIndex: null,
+    obsBackIndex: null,
 
     /* schema/version related */
     api: null,
@@ -90,7 +92,12 @@ export function createWorldFromEntireUniverse(universe) {
 
   // map from the universe obsIndex to our world offset.
   // undefined/null indicates identity map.
-  world.worldObsIndex = null;
+  // In other words   obsBackIndex[universeIdx] -> worldIdx
+  world.obsBackIndex = null;
+  // Map to the universe index for each element in world.
+  // Null indicates identity map (aka world === universe)
+  // In other wrods  obsIndex[worldIdx]  ->  universeIdx
+  world.obsIndex = null;
 
   /*
   public interface follows
@@ -130,42 +137,40 @@ export function createWorldFromCurrentSelection(universe, world, crossfilter) {
   const newWorld = templateWorld();
 
   /* these don't change as only OBS are selected in our current implementation */
-  newWorld.api = world.api;
+  newWorld.api = universe.api;
-  newWorld.nVar = world.nVar;
+  newWorld.nVar = universe.nVar;
-  newWorld.schema = world.schema;
+  newWorld.schema = universe.schema;
-  newWorld.varAnnotations = world.varAnnotations;
+  newWorld.varAnnotations = universe.varAnnotations;
 
-  /*
+  /* build index maps and back maps based upon current selection state */
-  Subset world from universe based upon world's current selection.  Only those
+  const obsBackIndex = new Uint32Array(universe.nObs);
-  fields which are subset by observation selection/filtering need to be updated.
+  obsBackIndex.fill(-1); // default - aka unused
-  */
+  const notSelected = obsBackIndex[0];
-  const numSelected = crossfilter.countFiltered();
+  let nObs = 0;
-
+  for (let i = 0; i < universe.nObs; i += 1) {
-  /*
-  Create a world which is based upon current selection
-  */
-  newWorld.nObs = numSelected;
-  newWorld.obsAnnotations = new Array(numSelected);
-  newWorld.obsLayout = {
-    X: new Array(numSelected),
-    Y: new Array(numSelected)
-  };
-  newWorld.worldObsIndex = new Array(universe.nObs);
-
-  for (let i = 0, sel = 0; i < world.nObs; i += 1) {
     if (crossfilter.isElementFiltered(i)) {
-      newWorld.obsAnnotations[sel] = world.obsAnnotations[i];
+      obsBackIndex[i] = nObs;
-      newWorld.obsLayout.X[sel] = world.obsLayout.X[i];
+      nObs += 1;
-      newWorld.obsLayout.Y[sel] = world.obsLayout.Y[i];
+    }
-      sel += 1;
+  }
+  const obsIndex = new Uint32Array(nObs);
+  for (let i = 0; i < universe.nObs; i += 1) {
+    const worldIdx = obsBackIndex[i];
+    if (worldIdx !== notSelected) {
+      obsIndex[worldIdx] = i;
     }
   }
 
-  // build index to our world offset
+  newWorld.nObs = nObs;
-  newWorld.worldObsIndex.fill(-1); // default - aka unused
+  newWorld.obsIndex = obsIndex;
-  for (let i = 0; i < newWorld.nObs; i += 1) {
+  newWorld.obsBackIndex = obsBackIndex;
-    newWorld.worldObsIndex[newWorld.obsAnnotations[i].__index__] = i;
+
-  }
+  /* now slice */
+  newWorld.obsAnnotations = sliceByIndex(universe.obsAnnotations, obsIndex);
+  newWorld.obsLayout = {
+    X: sliceByIndex(universe.obsLayout.X, obsIndex),
+    Y: sliceByIndex(universe.obsLayout.Y, obsIndex)
+  };
 
   /* derived data & summaries */
   newWorld.summary = summarizeAnnotations(
@@ -226,14 +231,7 @@ export function createVarDimension(
   crossfilter,
   geneName
 ) {
-  const { worldObsIndex } = world;
+  return crossfilter.dimension(_worldVarDataCache[geneName], Float32Array);
-  const varData = _worldVarDataCache[geneName];
-  const worldIndex = worldObsIndex ? idx => worldObsIndex[idx] : idx => idx;
-
-  return crossfilter.dimension(
-    r => varData[worldIndex(r.__index__)],
-    Float32Array
-  );
 }
 
 export function createObsDimensionMap(crossfilter, world) {
@@ -241,32 +239,32 @@ export function createObsDimensionMap(crossfilter, world) {
   create and return a crossfilter dimension for every obs annotation
   for which we have a supported type.
   */
-  const { schema, obsLayout, worldObsIndex } = world;
+  const { schema, obsLayout } = world;
 
-  const dimensionMap = _.transform(
+  // Create a crossfilter dimension for all obs annotations *except* 'name'
-    schema.annotations.obs,
+  const dimensionMap = _(schema.annotations.obs)
-    (result, anno) => {
+    .filter(anno => anno.name !== "name")
+    .transform((result, anno) => {
       const dimType = deduceDimensionType(anno, anno.name);
+      // XXX if dimtype is a scalar, we may be able to do better?
       if (dimType) {
         result[obsAnnoDimensionName(anno.name)] = crossfilter.dimension(
           r => r[anno.name],
           dimType
         );
       } // else ignore the annotation
-    },
+    }, {})
-    {}
+    .value();
-  );
 
   /*
   Add crossfilter dimensions allowing filtering on layout
   */
-  const worldIndex = worldObsIndex ? idx => worldObsIndex[idx] : idx => idx;
   dimensionMap[layoutDimensionName("X")] = crossfilter.dimension(
-    r => obsLayout.X[worldIndex(r.__index__)],
+    obsLayout.X,
     Float32Array
   );
   dimensionMap[layoutDimensionName("Y")] = crossfilter.dimension(
-    r => obsLayout.Y[worldIndex(r.__index__)],
+    obsLayout.Y,
     Float32Array
   );
 
@@ -282,10 +280,5 @@ export function subsetVarData(world, universe, varData) {
   if (worldEqUniverse(world, universe)) {
     return varData;
   }
-
+  return sliceByIndex(varData, world.obsIndex);
-  const newVarData = new Float32Array(world.nObs);
-  for (let i = 0; i < world.nObs; i += 1) {
-    newVarData[i] = varData[world.obsAnnotations[i].__index__];
-  }
-  return newVarData;
 }

client/src/util/typedCrossfilter/index.js

@@ -31,7 +31,7 @@ more complex API.  In a few cases, elements of that API were incorporated.
 import PositiveIntervals from "./positiveIntervals";
 import BitArray from "./bitArray";
 import {
-  fillRange,
+  makeSortIndex,
   lowerBound,
   lowerBoundIndirect,
   upperBoundIndirect
@@ -126,16 +126,31 @@ class ScalarDimension {
     // current selection filter, expressed as PostiveIntervals.
     this.currentFilter = [];
 
-    // Create value array
+    // Two modes - caller can provide a pre-created value array,
-    const array = this._createValueArray(
+    // or a map function which will create it.
-      value,
+    let array;
-      new ValueArrayType(this.crossfilter.data.length)
+    if (value instanceof ValueArrayType) {
-    );
+      if (value.length !== this.crossfilter.data.length) {
+        throw new RangeError(
+          "ScalarDimension values length must equal crossfilter data record count"
+        );
+      }
+      array = value;
+    } else if (value instanceof Function) {
+      // Create value array
+      array = this._createValueArray(
+        value,
+        new ValueArrayType(this.crossfilter.data.length)
+      );
+    } else {
+      throw new NotImplementedError(
+        "dimension value must be function or value array type"
+      );
+    }
     this.value = array;
 
     // create sort index
-    this.index = fillRange(new Uint32Array(this.crossfilter.data.length));
+    this.index = makeSortIndex(array);
-    this.index.sort((a, b) => array[a] - array[b]);
 
     // groups, if any
     this.groups = [];

client/src/util/typedCrossfilter/util.js

@@ -16,6 +16,25 @@ export function fillRange(arr, start = 0) {
   return larr;
 }
 
+// slice out of one array into another, using an index array
+//
+export function sliceByIndex(src, index) {
+  if (index === undefined || index === null) {
+    return src;
+  }
+  const dst = new src.constructor(index.length);
+  for (let i = 0; i < index.length; i += 1) {
+    dst[i] = src[index[i]];
+  }
+  return dst;
+}
+
+export function makeSortIndex(src) {
+  const index = fillRange(new Uint32Array(src.length));
+  index.sort((a, b) => src[a] - src[b]);
+  return index;
+}
+
 // Search for `value` in the sorted array `arr`, in the range [first, last).
 // Return the first (left most) index where arr[index] >= value.
 //

server/app/scanpy_engine/scanpy_engine.py

@@ -5,7 +5,6 @@ from pandas import DataFrame
 import scanpy.api as sc
 from scipy import stats, sparse
 
-from server.app.app import cache
 from server.app.driver.driver import CXGDriver
 from server.app.util.constants import Axis, DEFAULT_TOP_N, DiffExpMode
 from server.app.util.utils import FilterError, InteractiveError, PrepareError
@@ -96,11 +95,10 @@ class ScanpyEngine(CXGDriver):
 
     @staticmethod
     def _load_data(data):
-        # See https://scanpy.readthedocs.io/en/latest/api/scanpy.api.read.html
+        # Based on benchmarking, cache=True has no impact on perf.
-        # Based upon this advice, setting cache=True parameter
         # Note: as of current scanpy/anndata release, setting backed='r' will
-        # result in an error.
+        # result in an error.  https://github.com/theislab/anndata/issues/79
-        return sc.read(data, cache=True)
+        return sc.read(data, cache=False)
 
     @staticmethod
     def _top_sort(values, sort_order, top_n=None):
@@ -251,7 +249,6 @@ class ScanpyEngine(CXGDriver):
 
         return data
 
-    @cache.memoize()
     def annotation(self, filter, axis, fields=None):
         """
          Gets annotation value for each observation
@@ -274,7 +271,6 @@ class ScanpyEngine(CXGDriver):
         }
         return result
 
-    @cache.memoize()
     def data_frame(self, filter, axis):
         """
         Retrieves data for each variable for observations in data frame
@@ -366,7 +362,6 @@ class ScanpyEngine(CXGDriver):
         # Results need to be returned in var index order
         return sorted(result, key=lambda gene: gene[0])
 
-    @cache.memoize()
     def layout(self, filter, interactive_limit=None):
         """
         Computes a n-d layout for cells through dimensionality reduction.

server/cli/launch.py

@@ -96,4 +96,4 @@ def launch(data, layout, diffexp, title, verbose, debug, obs_names, var_names,
 
     click.echo('[cellxgene] Type CTRL-C at any time to exit.')
 
-    app.run(host=host, debug=debug, port=port)
+    app.run(host=host, debug=debug, port=port, threaded=True)

server/requirements.txt

@@ -1,6 +1,6 @@
 anndata>=0.6.12
 click==6.7
-Flask==0.12.4
+Flask>=1.0.2
 Flask-Caching==1.4.0
 Flask-Compress==1.4.0
 Flask-Cors==3.0.6