Clip continuous values based on percentile cutoffs (#672)

* Add numeric inputs for percentiles

* Define initial values for percentile cutoffs in world reducer

* add percentil to crossfilter dimensions

* worldEqUniverse now handles cloned worlds

* add Dataframe.mapColumns

* Wire up handlers for percentile inputs

* World reducer and stateManager know about continuousPercentileMin/Max

* Create world as universe clone (not pointer) to avoid clobbering vals

* Define basic actions for setting continuousPercentileMin/Max

* Under the hood, deal with percentiles between 0 and 1

* Move percentile inputs to visualization settings menu

* Fix padding for undo/redo buttons

* Trigger world rebuild from percentile actions

* BROKEN - pseudocode for clamping dataframe by percentiles upon world rebuild

* fix error handling on clip quantiles; start world clipping implementation

* more unclipped reorg

* rename crossfilter.percentile to quantile

* simplify schema access

* update continuous legend when scale changes

* update color cache when clip changes

* clip obs annotations and var data when clip quantile changes

* use own fromEntries

* fix tests

* stable non-finite float sort/search

* clarify comments

* fix syntax typo

* use new stand-alone clip

* clip expresssion data

* add select tests for non-finite scalars

* basic styles

* clip UI now requires explicit commit

* reset enable/disable accounts for clip percentiles

* better error messages

* fix bug in undo interaction with programatic min brush selection

* small refactoring

* support clipping of int data

* do not perform unnecessary summarizations

* improve caching of dataframe compiled columns

* add percentile precompute to Dataframe.summarize

* use Dataframe.summarize for clip percentiles

* remove obsolete quantile code from corssfilter

* histogram scale and label Y axis, add unclipped X range labels

* layout tweaks

* scatterplot now updates when clip changes

* improve comments

* remove debugging comment

* rework clip number entry validation for usability

* ui tweaks to histogram colors and layout

* enable undo/redo for clip user action

* refine UI on clip value entry

* api cleanup

* update confusing comment

* clarify purpose of isValidDigitKeyEvent

* fix misleading comment

* apply appropriate button-group classes; do not mix span and div

* variable name and comment changes suggested in PR review

* rename sort to sortArray; remove unused and dead code path

* naming changes suggested in PR review

* code review improvements for clarity

* more small changes from PR review

* lint fixes for PR review

* fix spelling error

* clarify that function performs in-place modification of world

* add comment to clarify intent of range operation

* fix bad indents in comments

* clean up __columnsAccessor comments and code

* improve comments around clipPredicate

* field name consistency

* improve comment on quantiles params

client/src/reducers/world.js

@@ -1,6 +1,6 @@
-import _ from "lodash";
-
 import { World, ControlsHelpers } from "../util/stateManager";
+import clip from "../util/clip";
+import quantile from "../util/quantile";
 
 const WorldReducer = (
   state = null,
@@ -21,7 +21,7 @@ const WorldReducer = (
 
     case "set World to current selection": {
       /* Set viewable world to be the currently selected data */
-      const world = World.createWorldFromCurrentSelection(
+      const world = World.createWorldBySelection(
         action.universe,
         action.world,
         action.crossfilter
@@ -29,16 +29,27 @@ const WorldReducer = (
       return world;
     }
 
+    case "set clip quantiles": {
+      const world = World.createWorldWithNewClip(
+        prevSharedState.universe,
+        state,
+        prevSharedState.crossfilter,
+        action.clipQuantiles
+      );
+      return world;
+    }
+
     case "expression load success": {
       const { universe } = nextSharedState;
       const universeVarData = universe.varData;
-      let worldVarData = state.varData;
+      let unclippedVarData = state.unclipped.varData;
 
-      // Load new expression data into the varData dataframes, if
+      // Lazy load new expression data into the unclipped varData dataframe, if
       // not already present.
-      _.forEach(action.expressionData, (val, key) => {
+      //
+      Object.entries(action.expressionData).forEach(([key, val]) => {
         // If not already in world.varData, save sliced expression column
-        if (!worldVarData.hasCol(key)) {
+        if (!unclippedVarData.hasCol(key)) {
           // Slice if world !== universe, else just use whole column.
           // Use the obsAnnotation index as the cut key, as we keep
           // all world dataframes in sync.
@@ -51,7 +62,7 @@ const WorldReducer = (
           }
 
           // Now build world's varData dataframe
-          worldVarData = worldVarData.withCol(
+          unclippedVarData = unclippedVarData.withCol(
             key,
             worldValSlice,
             state.obsAnnotations.rowIndex
@@ -59,23 +70,55 @@ const WorldReducer = (
         }
       });
 
-      // Prune size of varData "cache" if getting out of hand....
+      // Prune size of varData unclipped dataframe if getting out of hand....
+      //
       const { userDefinedGenes, diffexpGenes } = prevSharedState;
-      const allTheGenesWeNeed = _.uniq(
-        [].concat(
+      const allTheGenesWeNeed = [
+        ...new Set(
           userDefinedGenes,
           diffexpGenes,
           Object.keys(action.expressionData)
         )
-      );
-      worldVarData = ControlsHelpers.pruneVarDataCache(
-        worldVarData,
+      ];
+      unclippedVarData = ControlsHelpers.pruneVarDataCache(
+        unclippedVarData,
         allTheGenesWeNeed
       );
 
+      // at this point, we have the unclipped data in unclippedVarData.
+      // Now create clipped.
+      //   - Drop columns no longer needed
+      //   - Add new columns
+      //
+      let clippedVarData = state.varData;
+      const keysToDrop = clippedVarData.colIndex
+        .keys()
+        .filter(k => !unclippedVarData.hasCol(k));
+      const keysToAdd = unclippedVarData.colIndex
+        .keys()
+        .filter(k => !clippedVarData.hasCol(k));
+      keysToDrop.forEach(k => {
+        clippedVarData = clippedVarData.dropCol(k);
+      });
+      keysToAdd.forEach(k => {
+        const data = unclippedVarData.col(k).asArray();
+        const q = [state.clipQuantiles.min, state.clipQuantiles.max];
+        const [qMinVal, qMaxVal] = quantile(q, data);
+        const clippedData = clip(data, qMinVal, qMaxVal, Number.NaN);
+        clippedVarData = clippedVarData.withCol(
+          k,
+          clippedData,
+          state.obsAnnotations.rowIndex
+        );
+      });
+
       return {
         ...state,
-        varData: worldVarData
+        varData: clippedVarData,
+        unclipped: {
+          ...state.unclipped,
+          varData: unclippedVarData
+        }
       };
     }