Dataframe, part deux - add varData and summarize() (#608)

* initial dataframe commit

* initial dataframe port of core app

* rename variables for clarity

* remove unused import

* comment out unused code

* fix array handling bug in crossfilter dimension creation

* allow creation of empty dataframes

* handle non-existent columns

* handle non-existent columns

* revise tests for new dataframe

* comments for clarity

* comments for clarity

* generate bulk add placeholder with real gene names

* fix bug in gene name adding

* more dataframe unit tests

* fix bug - subset from current world, not universe

* put cut and pasted code into a single function

* improve caching of crossfilter

* remove cascading update bug from graph

* more performance work

* improve state handling for scatterplot

* performance optimization of critical path

* add column summarization

* dataframe utils

* add callOnceLazy

* fix tests

* minor updates found during review

* fix misspelling

* remove RESTv02 from function names

* comment cleanup

* cut/icut col parameter defaults to null

* break up large test

* improve tests and comments on dataframe at/has functions

* add Dataframe withCol/dropCol

* expression varData now stored in a dataframe

* dead code cleanup

* use dataframe.summarize()

* test cases for Dataframe.col.summarize

* update test cases for new dataframe summarize

* improve naming

* use new hasCol API

* add comments

* add more Dataframe.withCol tests

* add ability to specify row index in cut operation

* retire subsetVarData function

* correctly handle expression subsetting

* lint and improve comments

* rename cut to subset

* changes based on PR review

client/src/util/dataframe/labelIndex.js

@@ -57,13 +57,13 @@ class IdentityInt32Index {
     return i;
   }
 
-  getMaxOffset() {
+  size() {
     return this.maxOffset;
   }
 
-  cut(labelArray) {
+  __promote(labelArray) {
     /*
-    if density of resulting integer
+    time/space decision - based on the resulting density
     */
     const [minLabel, maxLabel] = extent(labelArray);
     const labelSpaceSize = maxLabel - minLabel + 1;
@@ -74,6 +74,26 @@ class IdentityInt32Index {
     }
     return new DenseInt32Index(labelArray, [minLabel, maxLabel]);
   }
+
+  subsetLabels(labelArray) {
+    return this.__promote(labelArray);
+  }
+
+  withLabel(label) {
+    if (label === this.maxOffset) {
+      return new IdentityInt32Index(label + 1);
+    }
+    return this.__promote([...this.keys(), label]);
+  }
+
+  dropLabel(label) {
+    if (label === this.maxOffset - 1) {
+      return new IdentityInt32Index(label);
+    }
+    const labelArray = [...this.keys()];
+    labelArray.splice(labelArray.indexOf(label), 1);
+    return this.__promote(labelArray);
+  }
 }
 /* eslint-enable class-methods-use-this */
 
@@ -121,11 +141,11 @@ class DenseInt32Index {
     return this.rindex;
   }
 
-  getMaxOffset() {
+  size() {
     return this.rindex.length;
   }
 
-  cut(labelArray) {
+  __promote(labelArray) {
     /*
     time/space decision - if we are going to use less than 10% of the
     dense index space, switch to a KeyIndex (which is slower, but uses
@@ -140,6 +160,20 @@ class DenseInt32Index {
     }
     return new DenseInt32Index(labelArray, [minLabel, maxLabel]);
   }
+
+  subsetLabels(labelArray) {
+    return this.__promote(labelArray);
+  }
+
+  withLabel(label) {
+    return this.__promote([...this.keys(), label]);
+  }
+
+  dropLabel(label) {
+    const labelArray = [...this.keys()];
+    labelArray.splice(labelArray.indexOf(label), 1);
+    return this.__promote(labelArray);
+  }
 }
 /* eslint-enable class-methods-use-this */
 
@@ -151,6 +185,9 @@ class KeyIndex {
   */
   constructor(labels) {
     const index = new Map();
+    if (labels === undefined) {
+      labels = [];
+    }
     const rindex = labels;
     labels.forEach((v, i) => {
       index.set(v, i);
@@ -175,14 +212,33 @@ class KeyIndex {
     return this.rindex;
   }
 
-  getMaxOffset() {
+  size() {
     return this.rindex.length;
   }
 
-  cut(labelArray) {
+  subsetLabels(labelArray) {
+    return new KeyIndex(labelArray);
+  }
+
+  withLabel(label) {
+    return new KeyIndex([...this.rindex, label]);
+  }
+
+  dropLabel(label) {
+    const idx = this.rindex.indexOf(label);
+    const labelArray = [...this.rindex];
+    labelArray.splice(idx, 1);
     return new KeyIndex(labelArray);
   }
 }
 /* eslint-enable class-methods-use-this */
 
-export { DenseInt32Index, IdentityInt32Index, KeyIndex };
+function isLabelIndex(i) {
+  return (
+    i instanceof IdentityInt32Index ||
+    i instanceof DenseInt32Index ||
+    i instanceof KeyIndex
+  );
+}
+
+export { DenseInt32Index, IdentityInt32Index, KeyIndex, isLabelIndex };