Performance work, plus fix #405 (#406)

* remove memoization

* update to flash 1.0.2; turn on threading

* stand-alone helper routines for array slicing

* fix issue #405

* reset diffexp state when world changes

* performance work in dimension creation; fix world slicing bug

* update tests to match new state mgmt api

* update flask

* do not make dimensions for useless annotations

* update test to match optimizations

client/src/actions/index.js

@@ -149,16 +149,13 @@ function requestSingleGeneExpressionCountsForColoringPOST(gene) {
   return async (dispatch, getState) => {
     dispatch({ type: "get single gene expression for coloring started" });
     try {
-      const expressionData = await _doRequestExpressionData(
-        dispatch,
-        getState,
-        [gene]
-      );
+      await _doRequestExpressionData(dispatch, getState, [gene]);
+      const { world } = getState().controls;
       dispatch({
         type: "color by expression",
         gene,
         data: {
-          [gene]: expressionData[gene]
+          [gene]: kvCache.get(world.varDataCache, gene)
         }
       });
     } catch (error) {
@@ -173,16 +170,15 @@ function requestSingleGeneExpressionCountsForColoringPOST(gene) {
 const requestUserDefinedGene = gene => async (dispatch, getState) => {
   dispatch({ type: "request user defined gene started" });
   try {
-    const data = await await _doRequestExpressionData(dispatch, getState, [
-      gene
-    ]);
+    await await _doRequestExpressionData(dispatch, getState, [gene]);
+    const { world } = getState().controls;
 
     /* then send the success case action through */
     return dispatch({
       type: "request user defined gene success",
       data: {
         genes: [gene],
-        expression: data[gene]
+        expression: kvCache.get(world.varDataCache, gene)
       }
     });
   } catch (error) {

client/src/reducers/differential.js

@@ -51,6 +51,7 @@ const Differential = (
     case "set World to current selection":
       return {
         ...state,
+        diffExp: null,
         celllist1: null,
         celllist2: null
       };

client/src/util/stateManager/world.js

@@ -4,6 +4,7 @@ import _ from "lodash";
 import * as kvCache from "./keyvalcache";
 import summarizeAnnotations from "./summarizeAnnotations";
 import { layoutDimensionName, obsAnnoDimensionName } from "../nameCreators";
+import { sliceByIndex } from "../typedCrossfilter/util";
 
 /*
 World is a subset of universe.   Most code should use world, and should
@@ -56,7 +57,8 @@ function templateWorld() {
   return {
     // map from universe obsIndex to world offset.
     // Undefined / null indicates identity mapping.
-    worldObsIndex: null,
+    obsIndex: null,
+    obsBackIndex: null,
 
     /* schema/version related */
     api: null,
@@ -90,7 +92,12 @@ export function createWorldFromEntireUniverse(universe) {
 
   // map from the universe obsIndex to our world offset.
   // undefined/null indicates identity map.
-  world.worldObsIndex = null;
+  // In other words   obsBackIndex[universeIdx] -> worldIdx
+  world.obsBackIndex = null;
+  // Map to the universe index for each element in world.
+  // Null indicates identity map (aka world === universe)
+  // In other wrods  obsIndex[worldIdx]  ->  universeIdx
+  world.obsIndex = null;
 
   /*
   public interface follows
@@ -130,42 +137,40 @@ export function createWorldFromCurrentSelection(universe, world, crossfilter) {
   const newWorld = templateWorld();
 
   /* these don't change as only OBS are selected in our current implementation */
-  newWorld.api = world.api;
-  newWorld.nVar = world.nVar;
-  newWorld.schema = world.schema;
-  newWorld.varAnnotations = world.varAnnotations;
+  newWorld.api = universe.api;
+  newWorld.nVar = universe.nVar;
+  newWorld.schema = universe.schema;
+  newWorld.varAnnotations = universe.varAnnotations;
 
-  /*
-  Subset world from universe based upon world's current selection.  Only those
-  fields which are subset by observation selection/filtering need to be updated.
-  */
-  const numSelected = crossfilter.countFiltered();
-
-  /*
-  Create a world which is based upon current selection
-  */
-  newWorld.nObs = numSelected;
-  newWorld.obsAnnotations = new Array(numSelected);
-  newWorld.obsLayout = {
-    X: new Array(numSelected),
-    Y: new Array(numSelected)
-  };
-  newWorld.worldObsIndex = new Array(universe.nObs);
-
-  for (let i = 0, sel = 0; i < world.nObs; i += 1) {
+  /* build index maps and back maps based upon current selection state */
+  const obsBackIndex = new Uint32Array(universe.nObs);
+  obsBackIndex.fill(-1); // default - aka unused
+  const notSelected = obsBackIndex[0];
+  let nObs = 0;
+  for (let i = 0; i < universe.nObs; i += 1) {
     if (crossfilter.isElementFiltered(i)) {
-      newWorld.obsAnnotations[sel] = world.obsAnnotations[i];
-      newWorld.obsLayout.X[sel] = world.obsLayout.X[i];
-      newWorld.obsLayout.Y[sel] = world.obsLayout.Y[i];
-      sel += 1;
+      obsBackIndex[i] = nObs;
+      nObs += 1;
+    }
+  }
+  const obsIndex = new Uint32Array(nObs);
+  for (let i = 0; i < universe.nObs; i += 1) {
+    const worldIdx = obsBackIndex[i];
+    if (worldIdx !== notSelected) {
+      obsIndex[worldIdx] = i;
     }
   }
 
-  // build index to our world offset
-  newWorld.worldObsIndex.fill(-1); // default - aka unused
-  for (let i = 0; i < newWorld.nObs; i += 1) {
-    newWorld.worldObsIndex[newWorld.obsAnnotations[i].__index__] = i;
-  }
+  newWorld.nObs = nObs;
+  newWorld.obsIndex = obsIndex;
+  newWorld.obsBackIndex = obsBackIndex;
+
+  /* now slice */
+  newWorld.obsAnnotations = sliceByIndex(universe.obsAnnotations, obsIndex);
+  newWorld.obsLayout = {
+    X: sliceByIndex(universe.obsLayout.X, obsIndex),
+    Y: sliceByIndex(universe.obsLayout.Y, obsIndex)
+  };
 
   /* derived data & summaries */
   newWorld.summary = summarizeAnnotations(
@@ -226,14 +231,7 @@ export function createVarDimension(
   crossfilter,
   geneName
 ) {
-  const { worldObsIndex } = world;
-  const varData = _worldVarDataCache[geneName];
-  const worldIndex = worldObsIndex ? idx => worldObsIndex[idx] : idx => idx;
-
-  return crossfilter.dimension(
-    r => varData[worldIndex(r.__index__)],
-    Float32Array
-  );
+  return crossfilter.dimension(_worldVarDataCache[geneName], Float32Array);
 }
 
 export function createObsDimensionMap(crossfilter, world) {
@@ -241,32 +239,32 @@ export function createObsDimensionMap(crossfilter, world) {
   create and return a crossfilter dimension for every obs annotation
   for which we have a supported type.
   */
-  const { schema, obsLayout, worldObsIndex } = world;
+  const { schema, obsLayout } = world;
 
-  const dimensionMap = _.transform(
-    schema.annotations.obs,
-    (result, anno) => {
+  // Create a crossfilter dimension for all obs annotations *except* 'name'
+  const dimensionMap = _(schema.annotations.obs)
+    .filter(anno => anno.name !== "name")
+    .transform((result, anno) => {
       const dimType = deduceDimensionType(anno, anno.name);
+      // XXX if dimtype is a scalar, we may be able to do better?
       if (dimType) {
         result[obsAnnoDimensionName(anno.name)] = crossfilter.dimension(
           r => r[anno.name],
           dimType
         );
       } // else ignore the annotation
-    },
-    {}
-  );
+    }, {})
+    .value();
 
   /*
   Add crossfilter dimensions allowing filtering on layout
   */
-  const worldIndex = worldObsIndex ? idx => worldObsIndex[idx] : idx => idx;
   dimensionMap[layoutDimensionName("X")] = crossfilter.dimension(
-    r => obsLayout.X[worldIndex(r.__index__)],
+    obsLayout.X,
     Float32Array
   );
   dimensionMap[layoutDimensionName("Y")] = crossfilter.dimension(
-    r => obsLayout.Y[worldIndex(r.__index__)],
+    obsLayout.Y,
     Float32Array
   );
 
@@ -282,10 +280,5 @@ export function subsetVarData(world, universe, varData) {
   if (worldEqUniverse(world, universe)) {
     return varData;
   }
-
-  const newVarData = new Float32Array(world.nObs);
-  for (let i = 0; i < world.nObs; i += 1) {
-    newVarData[i] = varData[world.obsAnnotations[i].__index__];
-  }
-  return newVarData;
+  return sliceByIndex(varData, world.obsIndex);
 }

client/src/util/typedCrossfilter/index.js

@@ -31,7 +31,7 @@ more complex API.  In a few cases, elements of that API were incorporated.
 import PositiveIntervals from "./positiveIntervals";
 import BitArray from "./bitArray";
 import {
-  fillRange,
+  makeSortIndex,
   lowerBound,
   lowerBoundIndirect,
   upperBoundIndirect
@@ -126,16 +126,31 @@ class ScalarDimension {
     // current selection filter, expressed as PostiveIntervals.
     this.currentFilter = [];
 
-    // Create value array
-    const array = this._createValueArray(
-      value,
-      new ValueArrayType(this.crossfilter.data.length)
-    );
+    // Two modes - caller can provide a pre-created value array,
+    // or a map function which will create it.
+    let array;
+    if (value instanceof ValueArrayType) {
+      if (value.length !== this.crossfilter.data.length) {
+        throw new RangeError(
+          "ScalarDimension values length must equal crossfilter data record count"
+        );
+      }
+      array = value;
+    } else if (value instanceof Function) {
+      // Create value array
+      array = this._createValueArray(
+        value,
+        new ValueArrayType(this.crossfilter.data.length)
+      );
+    } else {
+      throw new NotImplementedError(
+        "dimension value must be function or value array type"
+      );
+    }
     this.value = array;
 
     // create sort index
-    this.index = fillRange(new Uint32Array(this.crossfilter.data.length));
-    this.index.sort((a, b) => array[a] - array[b]);
+    this.index = makeSortIndex(array);
 
     // groups, if any
     this.groups = [];

client/src/util/typedCrossfilter/util.js

@@ -16,6 +16,25 @@ export function fillRange(arr, start = 0) {
   return larr;
 }
 
+// slice out of one array into another, using an index array
+//
+export function sliceByIndex(src, index) {
+  if (index === undefined || index === null) {
+    return src;
+  }
+  const dst = new src.constructor(index.length);
+  for (let i = 0; i < index.length; i += 1) {
+    dst[i] = src[index[i]];
+  }
+  return dst;
+}
+
+export function makeSortIndex(src) {
+  const index = fillRange(new Uint32Array(src.length));
+  index.sort((a, b) => src[a] - src[b]);
+  return index;
+}
+
 // Search for `value` in the sorted array `arr`, in the range [first, last).
 // Return the first (left most) index where arr[index] >= value.
 //