Clip continuous values based on percentile cutoffs (#672)

* Add numeric inputs for percentiles

* Define initial values for percentile cutoffs in world reducer

* add percentil to crossfilter dimensions

* worldEqUniverse now handles cloned worlds

* add Dataframe.mapColumns

* Wire up handlers for percentile inputs

* World reducer and stateManager know about continuousPercentileMin/Max

* Create world as universe clone (not pointer) to avoid clobbering vals

* Define basic actions for setting continuousPercentileMin/Max

* Under the hood, deal with percentiles between 0 and 1

* Move percentile inputs to visualization settings menu

* Fix padding for undo/redo buttons

* Trigger world rebuild from percentile actions

* BROKEN - pseudocode for clamping dataframe by percentiles upon world rebuild

* fix error handling on clip quantiles; start world clipping implementation

* more unclipped reorg

* rename crossfilter.percentile to quantile

* simplify schema access

* update continuous legend when scale changes

* update color cache when clip changes

* clip obs annotations and var data when clip quantile changes

* use own fromEntries

* fix tests

* stable non-finite float sort/search

* clarify comments

* fix syntax typo

* use new stand-alone clip

* clip expresssion data

* add select tests for non-finite scalars

* basic styles

* clip UI now requires explicit commit

* reset enable/disable accounts for clip percentiles

* better error messages

* fix bug in undo interaction with programatic min brush selection

* small refactoring

* support clipping of int data

* do not perform unnecessary summarizations

* improve caching of dataframe compiled columns

* add percentile precompute to Dataframe.summarize

* use Dataframe.summarize for clip percentiles

* remove obsolete quantile code from corssfilter

* histogram scale and label Y axis, add unclipped X range labels

* layout tweaks

* scatterplot now updates when clip changes

* improve comments

* remove debugging comment

* rework clip number entry validation for usability

* ui tweaks to histogram colors and layout

* enable undo/redo for clip user action

* refine UI on clip value entry

* api cleanup

* update confusing comment

* clarify purpose of isValidDigitKeyEvent

* fix misleading comment

* apply appropriate button-group classes; do not mix span and div

* variable name and comment changes suggested in PR review

* rename sort to sortArray; remove unused and dead code path

* naming changes suggested in PR review

* code review improvements for clarity

* more small changes from PR review

* lint fixes for PR review

* fix spelling error

* clarify that function performs in-place modification of world

* add comment to clarify intent of range operation

* fix bad indents in comments

* clean up __columnsAccessor comments and code

* improve comments around clipPredicate

* field name consistency

* improve comment on quantiles params

client/src/reducers/colors.js

@@ -27,6 +27,7 @@ const ColorsReducer = (
       };
     }
 
+    case "set clip quantiles":
     case "set World to current selection": {
       const { colorMode, colorAccessor } = state;
       const { world } = nextSharedState;

client/src/reducers/continuousSelection.js

@@ -17,6 +17,14 @@ const ContinuousSelection = (state = {}, action) => {
         [name]: action.range
       };
     }
+    case "continuous metadata histogram cancel": {
+      const name = makeContinuousDimensionName(
+        action.continuousNamespace,
+        action.selection
+      );
+      const { [name]: deletedField, ...newState } = state;
+      return newState;
+    }
     default: {
       return state;
     }

client/src/reducers/crossfilter.js

@@ -40,6 +40,7 @@ const CrossfilterReducer = (
       return crossfilter;
     }
 
+    case "set clip quantiles":
     case "set World to current selection": {
       const { userDefinedGenes, diffexpGenes } = prevSharedState.controls;
       const { world } = nextSharedState;

client/src/reducers/resetCache.js

@@ -1,7 +1,10 @@
 /*
-Reducer which caches derived state to be used in a reset
-*/
+Reducer which caches derived state to be used in a reset or other 
+recomputation.
 
+Currently this only caches the baseline (full universe) world & crossfilter, 
+for use in a Reset.
+*/
 const ResetCacheReducer = (
   state = {
     world: null,

client/src/reducers/undoable.js

@@ -51,6 +51,7 @@ history state processing.  The undoable action object contents, by key:
     filter state are entirely at the discretion of the action filter.
 
 */
+import fromEntries from "../util/fromEntries";
 
 const historyKeyPrefix = "@@undoable/";
 const pastKey = `${historyKeyPrefix}past`;
@@ -283,18 +284,4 @@ function push(arr, val, limit = undefined) {
   return narr;
 }
 
-function fromEntries(arr) {
-  /*
-  Similar to Object.fromEntries, but only handles array.
-  This could be replaced with the standard fucnction once it
-  is widely available.   As of 3/20/2019, it has not yet
-  been released in the Chrome stable channel.
-  */
-  const obj = {};
-  for (let i = 0, l = arr.length; i < l; i += 1) {
-    obj[arr[i][0]] = arr[i][1];
-  }
-  return obj;
-}
-
 export default Undoable;

client/src/reducers/undoableConfig.js

@@ -71,7 +71,8 @@ const saveOnActions = new Set([
   "store current cell selection as differential set 1",
   "store current cell selection as differential set 2",
 
-  "set World to current selection"
+  "set World to current selection",
+  "set clip quantiles"
 ]);
 
 /**
@@ -107,8 +108,8 @@ StateMachine when it doesn't know what to do.
 
 Signature:  (fsm, event, from) => undoableAction
 */
-const onFsmError = (fsm, name, from) => {
-  console.error("FSM error - unexpected history state", fsm, name, from);
+const onFsmError = (fsm, event, from) => {
+  console.error(`FSM error [event: "${event}", state: "${from}"]`, fsm);
   // In production, try to recover gracefully if we have unexpected state
   return clear(fsm);
 };

client/src/reducers/universe.js

@@ -1,5 +1,3 @@
-import _ from "lodash";
-
 import { ControlsHelpers } from "../util/stateManager";
 
 const Universe = (state = null, action, nextSharedState, prevSharedState) => {
@@ -12,9 +10,10 @@ const Universe = (state = null, action, nextSharedState, prevSharedState) => {
     case "expression load success": {
       let { varData } = state;
 
-      // Load new expression data into the varData dataframes, if
+      // Lazy load new expression data into the varData dataframe, if
       // not already present.
-      _.forEach(action.expressionData, (val, key) => {
+      //
+      Object.entries(action.expressionData).forEach(([key, val]) => {
         // If not already in universe.varData, save entire expression column
         if (!varData.hasCol(key)) {
           varData = varData.withCol(key, val);
@@ -22,14 +21,15 @@ const Universe = (state = null, action, nextSharedState, prevSharedState) => {
       });
 
       // Prune size of varData "cache" if getting out of hand....
+      //
       const { userDefinedGenes, diffexpGenes } = prevSharedState;
-      const allTheGenesWeNeed = _.uniq(
-        [].concat(
+      const allTheGenesWeNeed = [
+        ...new Set(
           userDefinedGenes,
           diffexpGenes,
           Object.keys(action.expressionData)
         )
-      );
+      ];
       varData = ControlsHelpers.pruneVarDataCache(varData, allTheGenesWeNeed);
 
       return {

client/src/reducers/world.js

@@ -1,6 +1,6 @@
-import _ from "lodash";
-
 import { World, ControlsHelpers } from "../util/stateManager";
+import clip from "../util/clip";
+import quantile from "../util/quantile";
 
 const WorldReducer = (
   state = null,
@@ -21,7 +21,7 @@ const WorldReducer = (
 
     case "set World to current selection": {
       /* Set viewable world to be the currently selected data */
-      const world = World.createWorldFromCurrentSelection(
+      const world = World.createWorldBySelection(
         action.universe,
         action.world,
         action.crossfilter
@@ -29,16 +29,27 @@ const WorldReducer = (
       return world;
     }
 
+    case "set clip quantiles": {
+      const world = World.createWorldWithNewClip(
+        prevSharedState.universe,
+        state,
+        prevSharedState.crossfilter,
+        action.clipQuantiles
+      );
+      return world;
+    }
+
     case "expression load success": {
       const { universe } = nextSharedState;
       const universeVarData = universe.varData;
-      let worldVarData = state.varData;
+      let unclippedVarData = state.unclipped.varData;
 
-      // Load new expression data into the varData dataframes, if
+      // Lazy load new expression data into the unclipped varData dataframe, if
       // not already present.
-      _.forEach(action.expressionData, (val, key) => {
+      //
+      Object.entries(action.expressionData).forEach(([key, val]) => {
         // If not already in world.varData, save sliced expression column
-        if (!worldVarData.hasCol(key)) {
+        if (!unclippedVarData.hasCol(key)) {
           // Slice if world !== universe, else just use whole column.
           // Use the obsAnnotation index as the cut key, as we keep
           // all world dataframes in sync.
@@ -51,7 +62,7 @@ const WorldReducer = (
           }
 
           // Now build world's varData dataframe
-          worldVarData = worldVarData.withCol(
+          unclippedVarData = unclippedVarData.withCol(
             key,
             worldValSlice,
             state.obsAnnotations.rowIndex
@@ -59,23 +70,55 @@ const WorldReducer = (
         }
       });
 
-      // Prune size of varData "cache" if getting out of hand....
+      // Prune size of varData unclipped dataframe if getting out of hand....
+      //
       const { userDefinedGenes, diffexpGenes } = prevSharedState;
-      const allTheGenesWeNeed = _.uniq(
-        [].concat(
+      const allTheGenesWeNeed = [
+        ...new Set(
           userDefinedGenes,
           diffexpGenes,
           Object.keys(action.expressionData)
         )
-      );
-      worldVarData = ControlsHelpers.pruneVarDataCache(
-        worldVarData,
+      ];
+      unclippedVarData = ControlsHelpers.pruneVarDataCache(
+        unclippedVarData,
         allTheGenesWeNeed
       );
 
+      // at this point, we have the unclipped data in unclippedVarData.
+      // Now create clipped.
+      //   - Drop columns no longer needed
+      //   - Add new columns
+      //
+      let clippedVarData = state.varData;
+      const keysToDrop = clippedVarData.colIndex
+        .keys()
+        .filter(k => !unclippedVarData.hasCol(k));
+      const keysToAdd = unclippedVarData.colIndex
+        .keys()
+        .filter(k => !clippedVarData.hasCol(k));
+      keysToDrop.forEach(k => {
+        clippedVarData = clippedVarData.dropCol(k);
+      });
+      keysToAdd.forEach(k => {
+        const data = unclippedVarData.col(k).asArray();
+        const q = [state.clipQuantiles.min, state.clipQuantiles.max];
+        const [qMinVal, qMaxVal] = quantile(q, data);
+        const clippedData = clip(data, qMinVal, qMaxVal, Number.NaN);
+        clippedVarData = clippedVarData.withCol(
+          k,
+          clippedData,
+          state.obsAnnotations.rowIndex
+        );
+      });
+
       return {
         ...state,
-        varData: worldVarData
+        varData: clippedVarData,
+        unclipped: {
+          ...state.unclipped,
+          varData: unclippedVarData
+        }
       };
     }