Clip continuous values based on percentile cutoffs (#672)

* Add numeric inputs for percentiles

* Define initial values for percentile cutoffs in world reducer

* add percentil to crossfilter dimensions

* worldEqUniverse now handles cloned worlds

* add Dataframe.mapColumns

* Wire up handlers for percentile inputs

* World reducer and stateManager know about continuousPercentileMin/Max

* Create world as universe clone (not pointer) to avoid clobbering vals

* Define basic actions for setting continuousPercentileMin/Max

* Under the hood, deal with percentiles between 0 and 1

* Move percentile inputs to visualization settings menu

* Fix padding for undo/redo buttons

* Trigger world rebuild from percentile actions

* BROKEN - pseudocode for clamping dataframe by percentiles upon world rebuild

* fix error handling on clip quantiles; start world clipping implementation

* more unclipped reorg

* rename crossfilter.percentile to quantile

* simplify schema access

* update continuous legend when scale changes

* update color cache when clip changes

* clip obs annotations and var data when clip quantile changes

* use own fromEntries

* fix tests

* stable non-finite float sort/search

* clarify comments

* fix syntax typo

* use new stand-alone clip

* clip expresssion data

* add select tests for non-finite scalars

* basic styles

* clip UI now requires explicit commit

* reset enable/disable accounts for clip percentiles

* better error messages

* fix bug in undo interaction with programatic min brush selection

* small refactoring

* support clipping of int data

* do not perform unnecessary summarizations

* improve caching of dataframe compiled columns

* add percentile precompute to Dataframe.summarize

* use Dataframe.summarize for clip percentiles

* remove obsolete quantile code from corssfilter

* histogram scale and label Y axis, add unclipped X range labels

* layout tweaks

* scatterplot now updates when clip changes

* improve comments

* remove debugging comment

* rework clip number entry validation for usability

* ui tweaks to histogram colors and layout

* enable undo/redo for clip user action

* refine UI on clip value entry

* api cleanup

* update confusing comment

* clarify purpose of isValidDigitKeyEvent

* fix misleading comment

* apply appropriate button-group classes; do not mix span and div

* variable name and comment changes suggested in PR review

* rename sort to sortArray; remove unused and dead code path

* naming changes suggested in PR review

* code review improvements for clarity

* more small changes from PR review

* lint fixes for PR review

* fix spelling error

* clarify that function performs in-place modification of world

* add comment to clarify intent of range operation

* fix bad indents in comments

* clean up __columnsAccessor comments and code

* improve comments around clipPredicate

* field name consistency

* improve comment on quantiles params

client/src/util/stateManager/universe.js

@@ -4,6 +4,8 @@ import _ from "lodash";
 
 import decodeMatrixFBS from "./matrix";
 import * as Dataframe from "../dataframe";
+import fromEntries from "../fromEntries";
+import { isFpTypedArray } from "../typeHelpers";
 
 /*
 Private helper function - create and return a template Universe
@@ -37,14 +39,57 @@ These functions are used exclusively by the actions and reducers to
 build an internal POJO for use by the rendering components.
 */
 
+function promoteTypedArray(o) {
+  /*
+  Decide what internal data type to use for the data returned from 
+  the server.
+
+  TODO - future optimization: not all int32/uint32 data series require
+  promotion to float64.  We COULD simply look at the data to decide. 
+  */
+  if (isFpTypedArray(o) || Array.isArray(o)) return o;
+
+  let TyepdArrayCtor;
+  switch (o.constructor) {
+    case Int8Array:
+    case Uint8Array:
+    case Uint8ClampedArray:
+    case Int16Array:
+    case Uint16Array:
+      TyepdArrayCtor = Float32Array;
+      break;
+
+    case Int32Array:
+    case Uint32Array:
+      TyepdArrayCtor = Float64Array;
+      break;
+
+    default:
+      throw new Error("Unexpected data type returned from server.");
+  }
+  if (o.constructor === TyepdArrayCtor) return o;
+  return new TyepdArrayCtor(o);
+}
+
 function AnnotationsFBSToDataframe(arrayBuffer) {
   /*
   Convert a Matrix FBS to a Dataframe.
+
+  The application has strong assumptions that all scalar data will be
+  stored as a float32 or float64 (regardless of underlying data types).
+  For example, clipping of value ranges (eg, user-selected percentiles)
+
+  All float data from the server is left as is.  All non-float is promoted
+  to an appropriate float.
   */
-  const fbs = decodeMatrixFBS(arrayBuffer);
+  const fbs = decodeMatrixFBS(arrayBuffer, true); // leave in place
+  const columns = fbs.columns.map(c => {
+    if (isFpTypedArray(c) || Array.isArray(c)) return c;
+    return promoteTypedArray(c);
+  });
   const df = new Dataframe.Dataframe(
     [fbs.nRows, fbs.nCols],
-    fbs.columns,
+    columns,
     null,
     new Dataframe.KeyIndex(fbs.colIdx)
   );
@@ -53,6 +98,10 @@ function AnnotationsFBSToDataframe(arrayBuffer) {
 
 function LayoutFBSToDataframe(arrayBuffer) {
   const fbs = decodeMatrixFBS(arrayBuffer, true);
+  if (fbs.columns.length !== 2 || !fbs.columns.every(isFpTypedArray)) {
+    // We have strong assumptions about the shape & type of layout data.
+    throw new Error("Unexpected layout data type returned from server");
+  }
   const df = new Dataframe.Dataframe(
     [fbs.nRows, fbs.nCols],
     fbs.columns,
@@ -122,6 +171,14 @@ export function createUniverseFromResponse(
   }
 
   reconcileSchemaCategoriesWithSummary(universe);
+
+  /* Index schema for ease of use */
+  universe.schema.annotations.obsByName = fromEntries(
+    universe.schema.annotations.obs.map(v => [v.name, v])
+  );
+  universe.schema.annotations.varByName = fromEntries(
+    universe.schema.annotations.var.map(v => [v.name, v])
+  );
   return universe;
 }
 
@@ -140,6 +197,11 @@ export function convertDataFBStoObject(universe, arrayBuffer) {
   const { colIdx, columns } = fbs;
   const result = {};
 
+  if (!columns.every(isFpTypedArray)) {
+    // We have strong assumptions that all var data is float
+    throw new Error("Unexpected non-floating point response from server.");
+  }
+
   for (let c = 0; c < colIdx.length; c += 1) {
     const varName = universe.varAnnotations.at(colIdx[c], "name");
     result[varName] = columns[c];

client/src/util/stateManager/world.js

@@ -1,7 +1,14 @@
 // jshint esversion: 6
 
-import { layoutDimensionName, obsAnnoDimensionName } from "../nameCreators";
+import clip from "../clip";
+import {
+  layoutDimensionName,
+  obsAnnoDimensionName,
+  diffexpDimensionName,
+  userDefinedDimensionName
+} from "../nameCreators";
 import * as Dataframe from "../dataframe";
+import ImmutableTypedCrossfilter from "../typedCrossfilter/crossfilter";
 
 /*
 
@@ -19,6 +26,8 @@ Notable keys in the world object:
 
 * schema: data schema from the server
 
+* clipQuantiles: the quantiles used to clip all data in world.
+
 * obsAnnotations:
 
   Dataframe containing obs annotations.  Columns are indexed by annotation
@@ -37,78 +46,192 @@ Notable keys in the world object:
 * varData: a cache of expression columns, stored in a Dataframe.  Cache
   managed by controls reducer.
 
+* unclipped: will contain unclipped variants of all potentiall clipped
+  dataframes (obsAnnotations, varData).
+
 */
 
 function templateWorld() {
+  const obsAnnotations = Dataframe.Dataframe.empty();
+  const varAnnotations = Dataframe.Dataframe.empty();
+  const obsLayout = Dataframe.Dataframe.empty();
+  const varData = Dataframe.Dataframe.empty(null, new Dataframe.KeyIndex());
   return {
     /* schema/version related */
     schema: null,
     nObs: 0,
     nVar: 0,
+    clipQuantiles: { min: 0, max: 1 },
 
     /* annotations */
-    obsAnnotations: Dataframe.Dataframe.empty(),
-    varAnnotations: Dataframe.Dataframe.empty(),
+    obsAnnotations,
+    varAnnotations,
 
     /* layout of graph. Dataframe. */
-    obsLayout: Dataframe.Dataframe.empty(),
+    obsLayout,
 
-    /*
-    Var data columns - subset of all data (may be empty)
-    */
-    varData: Dataframe.Dataframe.empty(null, new Dataframe.KeyIndex())
+    /* Var data columns - subset of all data (may be empty) */
+    varData,
+
+    /* unclipped dataframes - subset, but not value clipped */
+    unclipped: {
+      obsAnnotations,
+      varData
+    }
   };
 }
 
+function clipDataframe(
+  df,
+  lowerQuantile,
+  upperQuantile,
+  quantileF,
+  clipPredicate = () => true,
+  value = Number.NaN
+) {
+  /*
+  For all columns in the dataframe, clip all values above or below specified 
+  quantiles to `value` if clipPredicate returns True for that column (if it
+  returns false, skip the column entirely).
+
+  Returns a clipped copy - does not mutate original.
+
+  clipPredicate must have signature: (dataframe, colIndex, colLabel) => boolean
+  True signifies that the column should be clipped; false indicates that the
+  column should be left intact/unchanged.
+
+  quantileF must have signature:  (label, qval) => number
+  */
+  if (lowerQuantile < 0) lowerQuantile = 0;
+  if (upperQuantile > 1) upperQuantile = 1;
+  if (lowerQuantile === 0 && upperQuantile === 1) return df;
+
+  const keys = df.colIndex.keys();
+  return df.mapColumns((col, colIdx) => {
+    const colLabel = keys[colIdx];
+    if (!clipPredicate(df, colIdx, colLabel)) return col;
+
+    const colMin = quantileF(colLabel, lowerQuantile);
+    const colMax = quantileF(colLabel, upperQuantile);
+    const newCol = clip(col.slice(), colMin, colMax, value);
+    return newCol;
+  });
+}
+
+/*
+Create World with contents eq entire universe.   Commonly used to initialize World.
+If clipQuantiles
+*/
 export function createWorldFromEntireUniverse(universe) {
   const world = templateWorld();
 
-  /*
-  public interface follows
-  */
-
   /* Schema related */
   world.schema = universe.schema;
   world.nObs = universe.nObs;
   world.nVar = universe.nVar;
+  world.clipQuantiles = { min: 0, max: 1 };
 
-  /* annotation dataframes */
-  world.obsAnnotations = universe.obsAnnotations;
-  world.varAnnotations = universe.varAnnotations;
+  /* dataframes: annotations and layout */
+  world.obsAnnotations = universe.obsAnnotations.clone();
+  world.varAnnotations = universe.varAnnotations.clone();
+  world.obsLayout = universe.obsLayout.clone();
 
-  /* layout and display characteristics dataframe */
-  world.obsLayout = universe.obsLayout;
-
-  /*
-  Var data columns - subset of all
-  */
+  /* Var dataframe - contains a subset of all var columns */
   world.varData = universe.varData.clone();
 
+  /* save unclipped copies of potentially clipped dataframes */
+  world.unclipped = {
+    obsAnnotations: world.obsAnnotations.clone(),
+    varData: world.varData.clone()
+  };
+
   return world;
 }
 
-export function createWorldFromCurrentSelection(universe, world, crossfilter) {
-  const newWorld = templateWorld();
+/*
+clip dataframes based on quantiles.
 
-  /* these don't change as only OBS are selected in our current implementation */
-  newWorld.nVar = universe.nVar;
-  newWorld.schema = universe.schema;
-  newWorld.varAnnotations = universe.varAnnotations;
+This is an in-place operation on the world object provided as an argument.
+The values in world.unclipped are clipped and assigned to world.obsAnnotations
+and world.varData.
+*/
+function setClippedDataframes(world) {
+  const { schema } = world;
+  const isContinuousObsAnnotation = (df, idx, label) =>
+    deduceDimensionType(schema.annotations.obsByName[label], label) !== "enum";
+  const obsQuantile = (label, q) =>
+    world.unclipped.obsAnnotations.col(label).summarize().percentiles[100 * q];
+  world.obsAnnotations = clipDataframe(
+    world.unclipped.obsAnnotations,
+    world.clipQuantiles.min,
+    world.clipQuantiles.max,
+    obsQuantile,
+    isContinuousObsAnnotation
+  );
 
-  /* now subset/cut obs */
+  const varDataQuantile = (label, q) =>
+    world.unclipped.varData.col(label).summarize().percentiles[100 * q];
+  world.varData = clipDataframe(
+    world.unclipped.varData,
+    world.clipQuantiles.min,
+    world.clipQuantiles.max,
+    varDataQuantile,
+    () => true
+  );
+}
+
+/*
+Subset the current world based upon the current selection, maintaining any existing
+clip.  Returns new world.  Parameters:
+  * unvierse
+  * world - the current world
+  * crossfilter - the selection state
+*/
+export function createWorldBySelection(universe, world, crossfilter) {
+  const newWorld = { ...world, obsLayout: null, unclipped: {}, varData: null };
+
+  /* subset unclipped dataframes based upon current selection */
   const mask = crossfilter.allSelectedMask();
-  newWorld.obsAnnotations = world.obsAnnotations.isubsetMask(mask);
   newWorld.obsLayout = world.obsLayout.isubsetMask(mask);
-  newWorld.nObs = newWorld.obsAnnotations.dims[0];
-
-  /*
-  Var data columns - subset of all
-  */
-  if (world.varData.isEmpty()) {
-    newWorld.varData = world.varData.clone();
+  newWorld.unclipped.obsAnnotations = world.unclipped.obsAnnotations.isubsetMask(
+    mask
+  );
+  if (world.unclipped.varData.isEmpty()) {
+    newWorld.unclipped.varData = world.unclipped.varData.clone();
   } else {
-    newWorld.varData = world.varData.isubsetMask(mask);
+    newWorld.unclipped.varData = world.unclipped.varData.isubsetMask(mask);
   }
+  /* subsetting changings dimension size */
+  newWorld.nObs = newWorld.unclipped.obsAnnotations.dims[0];
+
+  /* and now clip */
+  setClippedDataframes(newWorld);
+  return newWorld;
+}
+
+/*
+Change clip quantiles on the current world, returning a new world.
+Parameters:
+  * universe
+  * world - current world
+  * clipQuantiles - new clip
+*/
+export function createWorldWithNewClip(
+  universe,
+  world,
+  crossfilter,
+  clipQuantiles
+) {
+  const newWorld = { ...world, obsAnnotation: null, varData: null };
+  newWorld.clipQuantiles = clipQuantiles;
+  newWorld.obsLayout = world.obsLayout.clone();
+  newWorld.unclipped = {
+    obsAnnotations: world.unclipped.obsAnnotations.clone(),
+    varData: world.unclipped.varData.clone()
+  };
+
+  /* and now clip */
+  setClippedDataframes(newWorld);
   return newWorld;
 }
 
@@ -166,7 +289,10 @@ export function createObsDimensions(crossfilter, world) {
 }
 
 export function worldEqUniverse(world, universe) {
-  return world.obsAnnotations === universe.obsAnnotations;
+  return (
+    world.obsAnnotations === universe.obsAnnotations ||
+    world.obsAnnotations.rowIndex === universe.obsAnnotations.rowIndex
+  );
 }
 
 export function getSelectedByIndex(crossfilter) {