Redux refactor (#1571)

* refactor categorical controls state

* lint

* fix race condition in tests

* fix typo

* add missing update on subset

* remove obsolete code

* update jest and puppeteer major version; update all minors

* update when label changes

* remove lint from tests; increase timeouts in e2e tests

* initial refactoring to new async annomatrix

* refine error handling

* fix bad merge

* add continuous legend

* lint

* fix memoization in color table creators

* partial implementation of user defined annotations

* add new annotations action creator file

* first pass at user annotations

* additional user annotation bug fixes

* user annotation auto-save

* unit test cleanup

* lint

* refactor into multiple files

* cleanup

* add column GC

* fix several bugs in user annotations

* remove debug code

* no anonymous functions

* undo redo cleanup

* file cleanup

* scatterplot

* performance

* cleanup

* remove old code

* render in parallel with load

* fix race condition

* simply graph rendering

* render throttle DRY

* fix category label order

* fix typo in e2e test setup

* re-fix the e2e test setup

* be more tolerant of races

* anno matrix unit tests

* temp disable reembedding

* pilot port continuous histo to react-async

* name change

* lint

* fix repaint bug

* typo fix

* update snap to match new ids

* world/universe name cleanup

* move annoMatrix to src dir

* use private underscore naming convention

* fix corner case in all selected

* name cleanup

* add layout control

* init edge case

* lint

* port scatterplot

* fix label indexing bug and improve tests

* port category to react-async

* fix user annotation labelling while subset

* select all of prev layout on layout switch

* fix race with crossfilter update

* prettier lint

* fix misleading comment

* fix url composition in loader

* first pass at crossfilter tests

* lint

* lint

* fix typo

* improved error handling for network errors

* fix memoization bug

* add memo

* refactor for performnce

* add missing single-value handling in select exact parser

* small bugs discovered by tests

* lint

* additional crossfilter unit tests

* remove extraneous comment

* add support for automatic category determination

* lint

* fix render bug in category

* take advantage of schema categories guarantee

* lint

* do not clear history when resetting

* enhanced annomatrix gc

* lint

* finish renaming to follow conventions; fix clone race bug

* lint

* add priority based loading to improve initial data load UX

* crossfilter cache perf

* perf tuning

* remove timers

* documentation

* PR review changes

* PR review changes

* more PR review edits

* improve clarity of comment

* more PR review fixes

* port centroidLabels to use react-async

* remove dead code

* pr review updates

* oops, remove logging

client/src/util/dataframe/dataframe.js

@@ -557,12 +557,14 @@ class Dataframe {
     const dims = [...this.dims];
 
     /* subset columns */
-    let { __columns, colIndex } = this;
+    let { __columns, colIndex, __columnsAccessor } = this;
     if (newColIndex) {
       const colOffsets = this.colIndex.getOffsets(newColIndex.labels());
       __columns = new Array(colOffsets.length);
+      __columnsAccessor = new Array(colOffsets.length);
       for (let i = 0, l = colOffsets.length; i < l; i += 1) {
         __columns[i] = this.__columns[colOffsets[i]];
+        __columnsAccessor[i] = this.__columnsAccessor[colOffsets[i]];
       }
       colIndex = newColIndex;
       dims[1] = colOffsets.length;
@@ -580,10 +582,17 @@ class Dataframe {
       });
       rowIndex = newRowIndex;
       dims[0] = rowOffsets.length;
+      __columnsAccessor = []; // force a recompile
     }
 
     if (dims[0] === 0 || dims[1] === 0) return Dataframe.empty();
-    return new Dataframe(dims, __columns, rowIndex, colIndex);
+    return new Dataframe(
+      dims,
+      __columns,
+      rowIndex,
+      colIndex,
+      __columnsAccessor
+    );
   }
 
   subset(rowLabels, colLabels = null, withRowIndex = null) {
@@ -792,7 +801,9 @@ class Dataframe {
 
     callback MUST not modify the column, but instead return a mutated copy.
     */
-    const columns = this.__columns.map(callback);
+    const columns = this.__columns.map((colData, colIdx) =>
+      callback(colData, colIdx, this)
+    );
     const columnsAccessor = columns.map((c, idx) =>
       this.__columns[idx] === c ? this.__columnsAccessor[idx] : undefined
     );

client/src/util/dataframe/labelIndex.js

@@ -98,6 +98,23 @@ class IdentityInt32Index {
     return this.subset(offsets);
   }
 
+  /* identity index - labels are offsets */
+  isubsetMask(mask) {
+    let count = 0;
+    if (mask.length !== this.maxOffset) {
+      throw new RangeError("mask has invalid length for index");
+    }
+    let labels = new Int32Array(mask.length);
+    for (let i = 0, l = mask.length; i < l; i += 1) {
+      if (mask[i]) {
+        labels[count] = i;
+        count += 1;
+      }
+    }
+    labels = labels.slice(0, count);
+    return this.subset(labels);
+  }
+
   withLabel(label) {
     if (label === this.maxOffset) {
       return new IdentityInt32Index(label + 1);
@@ -201,7 +218,6 @@ class DenseInt32Index {
       if (offset === undefined || offset === -1)
         throw new RangeError(`unknown label: ${label}`);
     }
-
     return this.__promote(labels);
   }
 
@@ -216,7 +232,22 @@ class DenseInt32Index {
         throw new RangeError(`out of bounds offset: ${offset}`);
       labels[i] = rindex[offset];
     }
+    return this.__promote(labels);
+  }
 
+  isubsetMask(mask) {
+    const { rindex } = this;
+    if (mask.length !== rindex.length)
+      throw new RangeError("mask has invalid length for index");
+    let count = 0;
+    let labels = new Int32Array(mask.length);
+    for (let i = 0, l = mask.length; i < l; i += 1) {
+      if (mask[i]) {
+        labels[count] = rindex[i];
+        count += 1;
+      }
+    }
+    labels = labels.slice(0, count);
     return this.__promote(labels);
   }
 
@@ -314,6 +345,22 @@ class KeyIndex {
     return new KeyIndex(labels);
   }
 
+  isubsetMask(mask) {
+    const { rindex } = this;
+    if (mask.length !== rindex.length)
+      throw new RangeError("mask has invalid length for index");
+    let labels = new Array(mask.length);
+    let count = 0;
+    for (let i = 0, l = mask.length; i < l; i += 1) {
+      if (mask[i]) {
+        labels[count] = rindex[i];
+        count += 1;
+      }
+    }
+    labels = labels.slice(0, count);
+    return new KeyIndex(labels);
+  }
+
   withLabel(label) {
     return new KeyIndex([...this.rindex, label]);
   }