Dataframe (#576)

* initial dataframe commit

* initial dataframe port of core app

* rename variables for clarity

* remove unused import

* comment out unused code

* fix array handling bug in crossfilter dimension creation

* allow creation of empty dataframes

* handle non-existent columns

* handle non-existent columns

* revise tests for new dataframe

* comments for clarity

* comments for clarity

* generate bulk add placeholder with real gene names

* fix bug in gene name adding

* more dataframe unit tests

* fix bug - subset from current world, not universe

* put cut and pasted code into a single function

* improve caching of crossfilter

* remove cascading update bug from graph

* more performance work

* improve state handling for scatterplot

* performance optimization of critical path

* add column summarization

* dataframe utils

* add callOnceLazy

* fix tests

* minor updates found during review

* fix misspelling

* remove RESTv02 from function names

* comment cleanup

* cut/icut col parameter defaults to null

* break up large test

* improve tests and comments on dataframe at/has functions

client/src/util/stateManager/summarizeAnnotations.js

@@ -53,13 +53,15 @@ Example:
 NOTE: will not summarize the required 'name' annotation, as that is
 specified as unique per element.
 */
-function _summarizeAnnotations(_schema, annotations) {
+function _summarizeAnnotations(_schema, df) {
   const summary = _(_schema) // lodash wrapping: https://lodash.com/docs/4.17.11#lodash
-    .filter(v => v.name !== "name")
+    .filter(v => v.name !== "name") // don't summarize name
     .keyBy("name")
     .mapValues(anno => {
       const { name, type } = anno;
       const continuous = type === "int32" || type === "float32";
+      const numRows = df.length;
+      const col = df.col(name) ? df.col(name).asArray() : null;
 
       if (continuous) {
         let min;
@@ -67,22 +69,24 @@ function _summarizeAnnotations(_schema, annotations) {
         let nan = 0;
         let pinf = 0;
         let ninf = 0;
-        for (let r = 0; r < annotations.length; r += 1) {
-          const val = Number(annotations[r][name]);
-          if (Number.isFinite(val)) {
-            if (min === undefined) {
-              min = val;
-              max = val;
+        if (col) {
+          for (let r = 0; r < numRows; r += 1) {
+            const val = Number(col[r]);
+            if (Number.isFinite(val)) {
+              if (min === undefined) {
+                min = val;
+                max = val;
+              } else {
+                min = val < min ? val : min;
+                max = val > max ? val : max;
+              }
+            } else if (Number.isNaN(val)) {
+              nan += 1;
+            } else if (val > 0) {
+              pinf += 1;
             } else {
-              min = val < min ? val : min;
-              max = val > max ? val : max;
+              ninf += 1;
             }
-          } else if (Number.isNaN(val)) {
-            nan += 1;
-          } else if (val > 0) {
-            pinf += 1;
-          } else {
-            ninf += 1;
           }
         }
         return {
@@ -93,11 +97,13 @@ function _summarizeAnnotations(_schema, annotations) {
 
       /* else categorical */
       const categoryCounts = new Map();
-      for (let r = 0; r < annotations.length; r += 1) {
-        const val = annotations[r][name];
-        let curCount = categoryCounts.get(val);
-        if (curCount === undefined) curCount = 0;
-        categoryCounts.set(val, curCount + 1);
+      if (col) {
+        for (let r = 0; r < numRows; r += 1) {
+          const val = col[r];
+          let curCount = categoryCounts.get(val);
+          if (curCount === undefined) curCount = 0;
+          categoryCounts.set(val, curCount + 1);
+        }
       }
       return {
         categorical: true,

client/src/util/stateManager/universe.js

@@ -5,6 +5,7 @@ import _ from "lodash";
 import * as kvCache from "./keyvalcache";
 import summarizeAnnotations from "./summarizeAnnotations";
 import decodeMatrixFBS from "./matrix";
+import * as Dataframe from "../dataframe";
 
 /*
 Private helper function - create and return a template Universe
@@ -27,13 +28,10 @@ function templateUniverse() {
     /*
     Annotations
     */
-    obsAnnotations: [] /* all obs annotations, by obs index */,
-    varAnnotations: [] /* all var annotations, by var index */,
-    obsNameToIndexMap: {} /* reverse map 'name' to index */,
-    varNameToIndexMap: {} /* reverse map 'name' to index */,
-    summary: null /* derived data summaries XXX: consider exploding in place */,
-
-    obsLayout: { X: [], Y: [] } /* xy layout */,
+    obsAnnotations: null,
+    varAnnotations: null,
+    obsLayout: null,
+    summary: null /* derived data summaries. XXX: consider exploding in place */,
 
     /*
     Cache of var data (expression), by var annotation name.   Data can be
@@ -61,9 +59,8 @@ function finalize(universe) {
   /* A bit of sanity checking! */
   const { nObs, nVar } = universe;
   if (
+    nObs !== universe.obsLayout.length ||
     nObs !== universe.obsAnnotations.length ||
-    nObs !== universe.obsLayout.X.length ||
-    nObs !== universe.obsLayout.Y.length ||
     nVar !== universe.varAnnotations.length
   ) {
     throw new Error("Universe dimensionality mismatch - failed to load");
@@ -73,61 +70,33 @@ function finalize(universe) {
   //  - layout has supported number of dimensions
   //  - ...
 
-  /*
-  Create all derived (convenience) data structures.
-  */
-  universe.obsNameToIndexMap = _.transform(
-    universe.obsAnnotations,
-    (acc, value, idx) => {
-      acc[value.name] = idx;
-    },
-    {}
-  );
-  universe.varNameToIndexMap = _.transform(
-    universe.varAnnotations,
-    (acc, value, idx) => {
-      acc[value.name] = idx;
-    },
-    {}
-  );
   universe.finalized = true;
   return universe;
 }
 
-function RESTv02AnotationsFBSResponseToInternal(arrayBuffer) {
+function AnnotationsFBSToDataframe(arrayBuffer) {
   /*
-  Convert a Matrix FBS to our internal format -- row-major array of
-  observations/cells, stored as an object.  Each obs has a key for each
-  annotation, plus __index__ containing its obsIndex.
-
-  Example:
-  [
-    { __index__: 0, tissue_type: "lung", sex: "F", ... },
-    ...
-  ]
-
-  XXX TODO: we could make use of the columns in building crossfilter
-  dimensions (they have to be recreated).  Future optimization.
+  Convert a Matrix FBS to a Dataframe.
   */
   const fbs = decodeMatrixFBS(arrayBuffer);
-  const keys = fbs.colIdx;
-  const result = Array(fbs.nRows);
-  for (let row = 0; row < fbs.nRows; row += 1) {
-    const rec = { __index__: row };
-    for (let col = 0; col < fbs.nCols; col += 1) {
-      rec[keys[col]] = fbs.columns[col][row];
-    }
-    result[row] = rec;
-  }
-  return result;
+  const df = new Dataframe.Dataframe(
+    [fbs.nRows, fbs.nCols],
+    fbs.columns,
+    null,
+    new Dataframe.KeyIndex(fbs.colIdx)
+  );
+  return df;
 }
 
-function RESTv02LayoutFBSResponseToInternal(arrayBuffer) {
+function LayoutFBSToDataframe(arrayBuffer) {
   const fbs = decodeMatrixFBS(arrayBuffer, true);
-  return {
-    X: fbs.columns[0],
-    Y: fbs.columns[1]
-  };
+  const df = new Dataframe.Dataframe(
+    [fbs.nRows, fbs.nCols],
+    fbs.columns,
+    null,
+    new Dataframe.KeyIndex(["X", "Y"])
+  );
+  return df;
 }
 
 function reconcileSchemaCategoriesWithSummary(universe) {
@@ -156,7 +125,7 @@ function reconcileSchemaCategoriesWithSummary(universe) {
   });
 }
 
-export function createUniverseFromRestV02Response(
+export function createUniverseFromResponse(
   configResponse,
   schemaResponse,
   annotationsObsResponse,
@@ -178,15 +147,10 @@ export function createUniverseFromRestV02Response(
   universe.nVar = schema.dataframe.nVar;
 
   /* annotations */
-  universe.obsAnnotations = RESTv02AnotationsFBSResponseToInternal(
-    annotationsObsResponse
-  );
-  universe.varAnnotations = RESTv02AnotationsFBSResponseToInternal(
-    annotationsVarResponse
-  );
-
+  universe.obsAnnotations = AnnotationsFBSToDataframe(annotationsObsResponse);
+  universe.varAnnotations = AnnotationsFBSToDataframe(annotationsVarResponse);
   /* layout */
-  universe.obsLayout = RESTv02LayoutFBSResponseToInternal(layoutFBSResponse);
+  universe.obsLayout = LayoutFBSToDataframe(layoutFBSResponse);
 
   universe.summary = summarizeAnnotations(
     universe.schema,
@@ -214,8 +178,8 @@ export function convertDataFBStoObject(universe, arrayBuffer) {
   const result = {};
 
   for (let c = 0; c < colIdx.length; c += 1) {
-    const gene = universe.varAnnotations[colIdx[c]].name;
-    result[gene] = columns[c];
+    const varName = universe.varAnnotations.at(colIdx[c], "name");
+    result[varName] = columns[c];
   }
   return result;
 }

client/src/util/stateManager/world.js

@@ -8,6 +8,7 @@ import Crossfilter from "../typedCrossfilter";
 import { sliceByIndex } from "../typedCrossfilter/util";
 
 /*
+
 World is a subset of universe.   Most code should use world, and should
 (generally) not use Universe.   World contains any per-obs or per-var data
 that must be consistent acorss the app when we view/manipulate subsets
@@ -16,37 +17,32 @@ of Universe.
 Private API indicated by leading underscore in key name (eg, _foo).  Anything else
 is public.
 
-World contains several public keys, obsAnnotations, and obsLayout, which are
-arrays contianing information about an OBS in the same order/offset.  In
-other words, world.obsAnnotations[0] and world.obsLayout.X[0] refer to the same
-obs/cell.
+Notable keys in the world object:
+
+* nObs, nVar: dimensions
+
+* schema: data schema from the server
 
 * obsAnnotations:
 
-  obsAnnotations will return an array of objects.  Each object contains all annotation
-  values for a given observation/cell, keyed by annotation name, PLUS a key
-  '__cellId__', containing a REST API ID for this obs/cell (referred to as the
-  obsIndex in the REST 0.2 spec or cellIndex in the 0.1 spec.
+  Dataframe containing obs annotations.  Columns are indexed by annotation
+  name (eg, 'tissue type'), and rows are indexed by the REST API obsIndex
+  (ie, the offset into the underlying server-side dataframe).
 
-  Example:  [ { __cellId__: 99, cluster: 'blue', numReads: 93933 } ]
-
-  NOTE: world.obsAnnotation should be identical to the old state.cells value,
-  EXCEPT that
-    * __cellIndex__ renamed to __index__
-    * __x__ and __y__ are now in world.obsLayout
-    * __color__ and __colorRBG__ should be moved to controls reducer
+  This indexing means that you can access data by _either_ the server's
+  obxIndex, or the offset into the client-side column array .  Be careful
+  to know which you want and are using.
 
 * obsLayout:
 
-  obsLayout will return an object containing two arrays, containing X and Y
-  coordinates respectively.
+  A dataframe containing the X/Y layout for all obs.  Columns are named
+  'X' and 'Y', and rows are indexed in the same way as obsAnnotation.
 
-  Example: { X: [ 0.33, 0.23, ... ], Y: [ 0.8, 0.777, ... ]}
+* summary: summary of each obsAnnotation column (eg, numeric extent for
+  continuous data, category counts for categorical metadata)
 
-* crossfilter - a crossfilter object across world.obsAnnotations
-
-* dimensionMap - an object mapping annotation names to dimensions on
-  the crossfilter
+* varDataCache: expression columns, in a kvCache.   TODO: maybe move to a
+  Dataframe in the future.
 
 */
 
@@ -56,11 +52,6 @@ const VarDataCacheTTLMs = 1000; // min cache time in MS
 
 function templateWorld() {
   return {
-    // map from universe obsIndex to world offset.
-    // Undefined / null indicates identity mapping.
-    obsIndex: null,
-    obsBackIndex: null,
-
     /* schema/version related */
     api: null,
     schema: null,
@@ -71,7 +62,7 @@ function templateWorld() {
     obsAnnotations: null,
     varAnnotations: null,
 
-    /* layout of graph */
+    /* layout of graph. Dataframe. */
     obsLayout: null,
 
     /* derived data summaries XXX: consider exploding in place */
@@ -91,15 +82,6 @@ export function createWorldFromEntireUniverse(universe) {
 
   const world = templateWorld();
 
-  // map from the universe obsIndex to our world offset.
-  // undefined/null indicates identity map.
-  // In other words   obsBackIndex[universeIdx] -> worldIdx
-  world.obsBackIndex = null;
-  // Map to the universe index for each element in world.
-  // Null indicates identity map (aka world === universe)
-  // In other wrods  obsIndex[worldIdx]  ->  universeIdx
-  world.obsIndex = null;
-
   /*
   public interface follows
   */
@@ -143,35 +125,11 @@ export function createWorldFromCurrentSelection(universe, world, crossfilter) {
   newWorld.schema = universe.schema;
   newWorld.varAnnotations = universe.varAnnotations;
 
-  /* build index maps and back maps based upon current selection state */
-  const obsBackIndex = new Uint32Array(universe.nObs);
-  obsBackIndex.fill(-1); // default - aka unused
-  const notSelected = obsBackIndex[0];
-  let nObs = 0;
-  for (let i = 0; i < universe.nObs; i += 1) {
-    if (crossfilter.isElementFiltered(i)) {
-      obsBackIndex[i] = nObs;
-      nObs += 1;
-    }
-  }
-  const obsIndex = new Uint32Array(nObs);
-  for (let i = 0; i < universe.nObs; i += 1) {
-    const worldIdx = obsBackIndex[i];
-    if (worldIdx !== notSelected) {
-      obsIndex[worldIdx] = i;
-    }
-  }
-
-  newWorld.nObs = nObs;
-  newWorld.obsIndex = obsIndex;
-  newWorld.obsBackIndex = obsBackIndex;
-
-  /* now slice */
-  newWorld.obsAnnotations = sliceByIndex(universe.obsAnnotations, obsIndex);
-  newWorld.obsLayout = {
-    X: sliceByIndex(universe.obsLayout.X, obsIndex),
-    Y: sliceByIndex(universe.obsLayout.Y, obsIndex)
-  };
+  /* now subset/cut obs */
+  const mask = crossfilter.allFilteredMask();
+  newWorld.obsAnnotations = world.obsAnnotations.icutByMask(mask);
+  newWorld.obsLayout = world.obsLayout.icutByMask(mask);
+  newWorld.nObs = newWorld.obsAnnotations.dims[0];
 
   /* derived data & summaries */
   newWorld.summary = summarizeAnnotations(
@@ -245,22 +203,23 @@ export function createObsDimensionMap(crossfilter, world) {
   create and return a crossfilter dimension for every obs annotation
   for which we have a supported type.
   */
-  const { schema, obsLayout } = world;
+  const { schema, obsLayout, obsAnnotations } = world;
 
   // Create a crossfilter dimension for all obs annotations *except* 'name'
   const dimensionMap = _(schema.annotations.obs)
     .filter(anno => anno.name !== "name")
     .transform((result, anno) => {
       const dimType = deduceDimensionType(anno, anno.name);
+      const colData = obsAnnotations.col(anno.name).asArray();
       if (dimType === "enum") {
         result[obsAnnoDimensionName(anno.name)] = crossfilter.dimension(
           Crossfilter.EnumDimension,
-          r => r[anno.name]
+          colData
         );
-      } else {
+      } else if (dimType) {
         result[obsAnnoDimensionName(anno.name)] = crossfilter.dimension(
           Crossfilter.ScalarDimension,
-          r => r[anno.name],
+          colData,
           dimType
         );
       } // else ignore the annotation
@@ -272,8 +231,8 @@ export function createObsDimensionMap(crossfilter, world) {
   */
   dimensionMap[layoutDimensionName("XY")] = crossfilter.dimension(
     Crossfilter.SpatialDimension,
-    obsLayout.X,
-    obsLayout.Y
+    obsLayout.col("X").asArray(),
+    obsLayout.col("Y").asArray()
   );
 
   return dimensionMap;
@@ -288,5 +247,23 @@ export function subsetVarData(world, universe, varData) {
   if (worldEqUniverse(world, universe)) {
     return varData;
   }
-  return sliceByIndex(varData, world.obsIndex);
+  return sliceByIndex(varData, world.obsAnnotations.rowIndex.keys());
+}
+
+export function getSelectedByIndex(crossfilter) {
+  /*
+  return array of obsIndex, containing all selected obs/cells.
+  */
+  const selected = crossfilter.allFilteredMask(); // array of bool-ish
+  const keys = crossfilter.data.rowIndex.keys(); // row keys, aka universe rowIndex
+
+  const set = new Int32Array(selected.length);
+  let numElems = 0;
+  for (let i = 0, l = selected.length; i < l; i += 1) {
+    if (selected[i]) {
+      set[numElems] = keys[i];
+      numElems += 1;
+    }
+  }
+  return new Int32Array(set.buffer, 0, numElems);
 }

client/src/util/stateManager/worldUtil.js

@@ -18,13 +18,23 @@ Map {
    ...
 }
 
+Parameters are:
+- dim1: dimension 1 name/label
+- dim2: dimension 2 name/label
+- df: dataframe containing dim1 and dim2 on the column axis
+
 */
-function _countCategoryValues2D(dim1, dim2, rows) {
+function _countCategoryValues2D(dim1, dim2, df) {
   const dimMap = new Map();
-  for (let r = 0; r < rows.length; r += 1) {
-    const row = rows[r];
-    const val1 = row[dim1];
-    const val2 = row[dim2];
+  const col1 = df.col(dim1) ? df.col(dim1).asArray() : null;
+  const col2 = df.col(dim2) ? df.col(dim2).asArray() : null;
+  if (!col1 || !col2) {
+    return dimMap;
+  }
+
+  for (let r = 0, l = df.length; r < l; r += 1) {
+    const val1 = col1[r];
+    const val2 = col2[r];
     let d2Map = dimMap.get(val1);
     if (d2Map === undefined) {
       d2Map = new Map();