Clip continuous values based on percentile cutoffs (#672)

* Add numeric inputs for percentiles

* Define initial values for percentile cutoffs in world reducer

* add percentil to crossfilter dimensions

* worldEqUniverse now handles cloned worlds

* add Dataframe.mapColumns

* Wire up handlers for percentile inputs

* World reducer and stateManager know about continuousPercentileMin/Max

* Create world as universe clone (not pointer) to avoid clobbering vals

* Define basic actions for setting continuousPercentileMin/Max

* Under the hood, deal with percentiles between 0 and 1

* Move percentile inputs to visualization settings menu

* Fix padding for undo/redo buttons

* Trigger world rebuild from percentile actions

* BROKEN - pseudocode for clamping dataframe by percentiles upon world rebuild

* fix error handling on clip quantiles; start world clipping implementation

* more unclipped reorg

* rename crossfilter.percentile to quantile

* simplify schema access

* update continuous legend when scale changes

* update color cache when clip changes

* clip obs annotations and var data when clip quantile changes

* use own fromEntries

* fix tests

* stable non-finite float sort/search

* clarify comments

* fix syntax typo

* use new stand-alone clip

* clip expresssion data

* add select tests for non-finite scalars

* basic styles

* clip UI now requires explicit commit

* reset enable/disable accounts for clip percentiles

* better error messages

* fix bug in undo interaction with programatic min brush selection

* small refactoring

* support clipping of int data

* do not perform unnecessary summarizations

* improve caching of dataframe compiled columns

* add percentile precompute to Dataframe.summarize

* use Dataframe.summarize for clip percentiles

* remove obsolete quantile code from corssfilter

* histogram scale and label Y axis, add unclipped X range labels

* layout tweaks

* scatterplot now updates when clip changes

* improve comments

* remove debugging comment

* rework clip number entry validation for usability

* ui tweaks to histogram colors and layout

* enable undo/redo for clip user action

* refine UI on clip value entry

* api cleanup

* update confusing comment

* clarify purpose of isValidDigitKeyEvent

* fix misleading comment

* apply appropriate button-group classes; do not mix span and div

* variable name and comment changes suggested in PR review

* rename sort to sortArray; remove unused and dead code path

* naming changes suggested in PR review

* code review improvements for clarity

* more small changes from PR review

* lint fixes for PR review

* fix spelling error

* clarify that function performs in-place modification of world

* add comment to clarify intent of range operation

* fix bad indents in comments

* clean up __columnsAccessor comments and code

* improve comments around clipPredicate

* field name consistency

* improve comment on quantiles params

client/src/util/dataframe/dataframe.js

@@ -1,6 +1,6 @@
 import { IdentityInt32Index, isLabelIndex } from "./labelIndex";
 // weird cross-dependency that we should clean up someday...
-import { sort } from "../typedCrossfilter/sort";
+import { sortArray } from "../typedCrossfilter/sort";
 import { isTypedArray, isArrayOrTypedArray, callOnceLazy } from "./util";
 import { summarizeContinuous, summarizeCategorical } from "./summarize";
 
@@ -63,7 +63,13 @@ class Dataframe {
   Constructors & factories
   **/
 
-  constructor(dims, columnarData, rowIndex = null, colIndex = null) {
+  constructor(
+    dims,
+    columnarData,
+    rowIndex = null,
+    colIndex = null,
+    __columnsAccessor = [] // private interface
+  ) {
     /*
     The base constructor is relatively hard to use - as an alternative,
     see factory methods and clone/slice, below.
@@ -74,6 +80,9 @@ class Dataframe {
         or TypedArray of length nRows.
       * rowIndex/colIndex - null (create default index using offsets as key),
         or a caller-provided index.
+      * __columnsAccessor - private interface, do not specify.  Used internally
+        to improve caching of column accessors when possible (eg, clone(), 
+        dropCol(), withCol()).
     All columns and indices must have appropriate dimensionality.
     */
     const [nRows, nCols] = dims;
@@ -94,7 +103,7 @@ class Dataframe {
     this.rowIndex = rowIndex;
     this.colIndex = colIndex;
 
-    this.__compile();
+    this.__compile(__columnsAccessor);
   }
 
   static __errorChecks(dims, columnarData, rowIndex, colIndex) {
@@ -135,97 +144,107 @@ class Dataframe {
     }
   }
 
-  __compile() {
+  static __compileColumn(column, getOffset, getLabel) {
+    /*
+      Each column accessor is a function which will lookup data by
+      index (ie, is equivalent to dataframe.get(row, col), where 'col'
+      is fixed.
+
+      In addition, each column accessor has several functions:
+
+      asArray() -- return the entire column as a native Array or TypedArray.
+        Crucially, this native array only supports label indexing.
+        Example:
+          const arr = df.col('a').asArray();
+
+      has(rlabel) -- return boolean indicating of the row label
+        is contained within the column.  Example:
+          const isInColumn = df.col('a').includes(99)
+        For the default offset indexing, this is identical to:
+          const isInColumn = (99 > 0) && (99 < df.nRows);
+
+      ihas(roffset) -- same as has(), but accepts a row offset
+        instead of a row label.
+
+      indexOf(value) -- return the label (not offset) of the first instance of
+        'value' in the column.  If you want the offset, just use the builtin JS
+        indexOf() function, available on both Array and TypedArray.
+
+      iget(offset) -- return the value at 'offset'
+
+    */
+    const { length } = column;
+
+    /* get value by row label */
+    const get = function get(rlabel) {
+      return column[getOffset(rlabel)];
+    };
+
+    /* get value by row offset */
+    const iget = function iget(roffset) {
+      return column[roffset];
+    };
+
+    /* full column array access */
+    const asArray = function asArray() {
+      return column;
+    };
+
+    /* test for row label inclusion in column */
+    const has = function has(rlabel) {
+      const offset = getOffset(rlabel);
+      return offset >= 0 && offset < length;
+    };
+
+    const ihas = function ihas(offset) {
+      return offset >= 0 && offset < length;
+    };
+
+    /*
+    return first label (index) at which the value is found in this column,
+    or undefined if not found.
+
+    NOTE: not found return is DIFFERENT than the default Array.indexOf as
+    -1 is a plausible Dataframe row/col label.
+    */
+    const indexOf = function indexOf(value) {
+      const offset = column.indexOf(value);
+      if (offset === -1) {
+        return undefined;
+      }
+      return getLabel(offset);
+    };
+
+    /*
+    Summarize the column data. Lazy eval;
+    */
+    const summarize = callOnceLazy(() =>
+      isTypedArray(column)
+        ? summarizeContinuous(column)
+        : summarizeCategorical(column)
+    );
+
+    get.summarize = summarize;
+    get.asArray = asArray;
+    get.has = has;
+    get.ihas = ihas;
+    get.indexOf = indexOf;
+    get.iget = iget;
+    return get;
+  }
+
+  __compile(accessors) {
     /*
     Compile data accessors for each column.
 
-    Each column accessor is a function which will lookup data by
-    index (ie, is equivalent to dataframe.get(row, col), where 'col'
-    is fixed.
-
-    In addition, each column accessor has several functions:
-
-    asArray() -- return the entire column as a native Array or TypedArray.
-      Crucially, this native array only supports label indexing.
-      Example:
-        const arr = df.col('a').asArray();
-
-    has(rlabel) -- return boolean indicating of the row label
-      is contained within the column.  Example:
-        const isInColumn = df.col('a').includes(99)
-      For the default offset indexing, this is identical to:
-        const isInColumn = (99 > 0) && (99 < df.nRows);
-
-    ihas(roffset) -- same as has(), but accepts a row offset
-      instead of a row label.
-
-    indexOf(value) -- return the label (not offset) of the first instance of
-      'value' in the column.  If you want the offset, just use the builtin JS
-      indexOf() function, available on both Array and TypedArray.
-
-    iget(offset) -- return the value at 'offset'
-
+    Use an existing accessor if provided, else compile a new one.
     */
     const { getOffset, getLabel } = this.rowIndex;
-    this.__columnsAccessor = this.__columns.map(column => {
-      const { length } = column;
-
-      /* get value by row label */
-      const get = function get(rlabel) {
-        return column[getOffset(rlabel)];
-      };
-
-      /* get value by row offset */
-      const iget = function iget(roffset) {
-        return column[roffset];
-      };
-
-      /* full column array access */
-      const asArray = function asArray() {
-        return column;
-      };
-
-      /* test for row label inclusion in column */
-      const has = function has(rlabel) {
-        const offset = getOffset(rlabel);
-        return offset >= 0 && offset < length;
-      };
-
-      const ihas = function ihas(offset) {
-        return offset >= 0 && offset < length;
-      };
-
-      /*
-      return first label (index) at which the value is found in this column,
-      or undefined if not found.
-
-      NOTE: not found return is DIFFERENT than the default Array.indexOf as
-      -1 is a plausible Dataframe row/col label.
-      */
-      const indexOf = function indexOf(value) {
-        const offset = column.indexOf(value);
-        if (offset === -1) {
-          return undefined;
-        }
-        return getLabel(offset);
-      };
-
-      /*
-      Summarize the column data. Lazy eval;
-      */
-      const summarize = callOnceLazy(() =>
-        isTypedArray(column)
-          ? summarizeContinuous(column)
-          : summarizeCategorical(column)
-      );
-
-      get.summarize = summarize;
-      get.asArray = asArray;
-      get.has = has;
-      get.ihas = ihas;
-      get.indexOf = indexOf;
-      get.iget = iget;
-      return get;
+    this.__columnsAccessor = this.__columns.map((column, idx) => {
+      if (accessors[idx]) {
+        return accessors[idx];
+      }
+      return Dataframe.__compileColumn(column, getOffset, getLabel);
     });
   }
 
@@ -237,7 +256,8 @@ class Dataframe {
       this.dims,
       [...this.__columns],
       this.rowIndex,
-      this.colIndex
+      this.colIndex,
+      [...this.__columnsAccessor]
     );
   }
 
@@ -273,7 +293,14 @@ class Dataframe {
     const columns = [...this.__columns];
     columns.push(colData);
     const colIndex = this.colIndex.withLabel(label);
-    return new this.constructor(dims, columns, rowIndex, colIndex);
+    const columnsAccessor = [...this.__columnsAccessor];
+    return new this.constructor(
+      dims,
+      columns,
+      rowIndex,
+      colIndex,
+      columnsAccessor
+    );
   }
 
   dropCol(label) {
@@ -287,7 +314,15 @@ class Dataframe {
     const columns = [...this.__columns];
     columns.splice(coffset, 1);
     const colIndex = this.colIndex.dropLabel(label);
-    return new this.constructor(dims, columns, this.rowIndex, colIndex);
+    const columnsAccessor = [...this.__columnsAccessor];
+    columnsAccessor.splice(coffset, 1);
+    return new this.constructor(
+      dims,
+      columns,
+      this.rowIndex,
+      colIndex,
+      columnsAccessor
+    );
   }
 
   static empty(rowIndex = null, colIndex = null) {
@@ -316,7 +351,7 @@ class Dataframe {
       if (!offsets) {
         return [null, null];
       }
-      const sortedOffsets = sort(offsets);
+      const sortedOffsets = sortArray(offsets);
       const sortedLabels = new Array(sortedOffsets.length);
       for (let i = 0, l = sortedOffsets.length; i < l; i += 1) {
         sortedLabels[i] = index.getLabel(sortedOffsets[i]);
@@ -543,14 +578,34 @@ class Dataframe {
   /****
   Functional (map/reduce/etc) data access
 
-  XXX: not yet implemented, as there is no clear use case.   Can easily
+  TODO: most are not yet implemented, as there is no clear use case.   Can easily
   add these as useful.
   ****/
 
+  mapColumns(callback) {
+    /*
+    map all columns in the dataframe, returning a new dataframe comprised of the
+    return values, with the same index as the original dataframe.
+
+    callback MUST not modify the column, but instead return a mutated copy.
+    */
+    const columns = this.__columns.map(callback);
+    const columnsAccessor = columns.map((c, idx) =>
+      this.__columns[idx] === c ? this.__columnsAccessor[idx] : undefined
+    );
+    return new this.constructor(
+      this.dims,
+      columns,
+      this.rowIndex,
+      this.colIndex,
+      columnsAccessor
+    );
+  }
+
   /*
   Map & reduce of column or row
 
-  XXX TODO remainder of map/reduce functions:  mapCol, mapRow, reduceRow, ...
+  TODO remainder of map/reduce functions:  mapCol, mapRow, reduceRow, ...
   */
   /* comment out until we have a use for this
 

client/src/util/dataframe/summarize.js

@@ -1,32 +1,56 @@
 /*
 Private dataframe support functions
+
+TODO / XXX: for scalar/continuous data, this uses a naive method
+of computing quantiles.  Would be good to switch from sort to 
+partition at some point.
 */
 
+import quantile from "../quantile";
+import { sortArray } from "../typedCrossfilter/sort";
+
+// [ 0, 0.01, 0.02, ..., 1.0]
+const centileNames = new Array(101).fill(0).map((v, idx) => idx / 100);
+
 export function summarizeContinuous(col) {
   let min;
   let max;
   let nan = 0;
   let pinf = 0;
   let ninf = 0;
+  let percentiles;
   if (col) {
-    for (let r = 0, l = col.length; r < l; r += 1) {
-      const val = Number(col[r]);
-      if (Number.isFinite(val)) {
-        if (min === undefined) {
-          min = val;
-          max = val;
-        } else {
-          min = val < min ? val : min;
-          max = val > max ? val : max;
-        }
-      } else if (Number.isNaN(val)) {
-        nan += 1;
-      } else if (val > 0) {
-        pinf += 1;
-      } else {
-        ninf += 1;
+    // -Inf < finite < Inf < NaN
+    const sortedCol = sortArray(new col.constructor(col));
+
+    // count non-finites, which are at each end of sorted data
+    for (let i = sortedCol.length - 1; i >= 0; i -= 1) {
+      if (!Number.isNaN(sortedCol[i])) {
+        nan = sortedCol.length - i - 1;
+        break;
       }
     }
+    for (let i = 0, l = sortedCol.length; i < l; i += 1) {
+      if (sortedCol[i] !== Number.NEGATIVE_INFINITY) {
+        ninf = i;
+        break;
+      }
+    }
+    for (let i = sortedCol.length - nan - 1; i >= 0; i -= 1) {
+      if (sortedCol[i] !== Number.POSITIVE_INFINITY) {
+        pinf = sortedCol.length - i - nan - 1;
+        break;
+      }
+    }
+
+    // compute percentiles on finite data ONLY
+    const sortedColFiniteOnly = sortedCol.slice(
+      ninf,
+      sortedCol.length - nan - pinf
+    );
+    percentiles = quantile(centileNames, sortedColFiniteOnly, true);
+    min = percentiles[0];
+    max = percentiles[100];
   }
   return {
     categorical: false,
@@ -34,7 +58,8 @@ export function summarizeContinuous(col) {
     max,
     nan,
     pinf,
-    ninf
+    ninf,
+    percentiles
   };
 }
 

client/src/util/dataframe/util.js

@@ -2,16 +2,7 @@
 Private utility code for dataframe
 */
 
-export function isTypedArray(x) {
-  return (
-    ArrayBuffer.isView(x) &&
-    Object.prototype.toString.call(x) !== "[object DataView]"
-  );
-}
-
-export function isArrayOrTypedArray(x) {
-  return Array.isArray(x) || isTypedArray(x);
-}
+export { isTypedArray, isArrayOrTypedArray } from "../typeHelpers";
 
 export function callOnceLazy(f) {
   let value;