Clip continuous values based on percentile cutoffs (#672)

* Add numeric inputs for percentiles

* Define initial values for percentile cutoffs in world reducer

* add percentil to crossfilter dimensions

* worldEqUniverse now handles cloned worlds

* add Dataframe.mapColumns

* Wire up handlers for percentile inputs

* World reducer and stateManager know about continuousPercentileMin/Max

* Create world as universe clone (not pointer) to avoid clobbering vals

* Define basic actions for setting continuousPercentileMin/Max

* Under the hood, deal with percentiles between 0 and 1

* Move percentile inputs to visualization settings menu

* Fix padding for undo/redo buttons

* Trigger world rebuild from percentile actions

* BROKEN - pseudocode for clamping dataframe by percentiles upon world rebuild

* fix error handling on clip quantiles; start world clipping implementation

* more unclipped reorg

* rename crossfilter.percentile to quantile

* simplify schema access

* update continuous legend when scale changes

* update color cache when clip changes

* clip obs annotations and var data when clip quantile changes

* use own fromEntries

* fix tests

* stable non-finite float sort/search

* clarify comments

* fix syntax typo

* use new stand-alone clip

* clip expresssion data

* add select tests for non-finite scalars

* basic styles

* clip UI now requires explicit commit

* reset enable/disable accounts for clip percentiles

* better error messages

* fix bug in undo interaction with programatic min brush selection

* small refactoring

* support clipping of int data

* do not perform unnecessary summarizations

* improve caching of dataframe compiled columns

* add percentile precompute to Dataframe.summarize

* use Dataframe.summarize for clip percentiles

* remove obsolete quantile code from corssfilter

* histogram scale and label Y axis, add unclipped X range labels

* layout tweaks

* scatterplot now updates when clip changes

* improve comments

* remove debugging comment

* rework clip number entry validation for usability

* ui tweaks to histogram colors and layout

* enable undo/redo for clip user action

* refine UI on clip value entry

* api cleanup

* update confusing comment

* clarify purpose of isValidDigitKeyEvent

* fix misleading comment

* apply appropriate button-group classes; do not mix span and div

* variable name and comment changes suggested in PR review

* rename sort to sortArray; remove unused and dead code path

* naming changes suggested in PR review

* code review improvements for clarity

* more small changes from PR review

* lint fixes for PR review

* fix spelling error

* clarify that function performs in-place modification of world

* add comment to clarify intent of range operation

* fix bad indents in comments

* clean up __columnsAccessor comments and code

* improve comments around clipPredicate

* field name consistency

* improve comment on quantiles params

client/__tests__/util/dataframe/dataframe.test.js

@@ -532,6 +532,42 @@ describe("dataframe factories", () => {
       expect(df.rowIndex.keys()).toEqual(dfA.rowIndex.keys());
     });
   });
+
+  describe("mapColumns", () => {
+    test("identity", () => {
+      const dfA = Dataframe.Dataframe.create(
+        [3, 3],
+        [
+          new Array(3).fill(0),
+          new Int16Array(3).fill(99),
+          new Float64Array(3).fill(1.1)
+        ]
+      );
+      const dfB = dfA.mapColumns((col, idx) => {
+        expect(dfA.icol(idx).asArray()).toBe(col);
+        return col;
+      });
+      expect(dfA).not.toBe(dfB);
+      expect(dfA.dims).toEqual(dfB.dims);
+      for (let c = 0; c < dfA.dims[1]; c += 1) {
+        expect(dfA.icol(c).asArray()).toBe(dfB.icol(c).asArray());
+      }
+    });
+
+    test("transform", () => {
+      const dfA = Dataframe.Dataframe.create(
+        [3, 3],
+        [new Array(3).fill(0), new Array(3).fill(0), new Array(3).fill(0)]
+      );
+      const dfB = dfA.mapColumns(() => {
+        return new Array(3).fill(1);
+      });
+      expect(dfA).not.toBe(dfB);
+      expect(dfB.iat(0, 0)).toEqual(1);
+      expect(dfB.iat(0, 1)).toEqual(1);
+      expect(dfB.iat(0, 2)).toEqual(1);
+    });
+  });
 });
 
 describe("dataframe col", () => {

client/__tests__/util/quantile.test.js

@@ -0,0 +1,29 @@
+import quantile from "../../src/util/quantile";
+
+describe("quantile", () => {
+	test("single q", () => {
+		const arr = new Float32Array([9, 3, 5, 6, 0]);
+		expect(quantile([1.0], arr)).toMatchObject([9]);
+		expect(quantile([0.9], arr)).toMatchObject([9]);
+		expect(quantile([0.8], arr)).toMatchObject([9]);
+		expect(quantile([0.7], arr)).toMatchObject([6]);
+		expect(quantile([0.6], arr)).toMatchObject([6]);
+		expect(quantile([0.5], arr)).toMatchObject([5]);
+		expect(quantile([0.4], arr)).toMatchObject([5]);
+		expect(quantile([0.3], arr)).toMatchObject([3]);
+		expect(quantile([0.2], arr)).toMatchObject([3]);
+		expect(quantile([0.1], arr)).toMatchObject([0]);
+		expect(quantile([0], arr)).toMatchObject([0]);
+	});
+
+	test("multi q", () => {
+		const arr = new Float32Array([9, 3, 5, 6, 0]);
+		expect(quantile([0, 0.25, 0.5, 0.75, 1.0], arr)).toMatchObject([
+			0,
+			3,
+			5,
+			6,
+			9
+		]);
+	});
+});

client/__tests__/util/stateManager/world.test.js

@@ -58,10 +58,15 @@ describe("createWorldFromEntireUniverse", () => {
         nObs: universe.nObs,
         nVar: universe.nVar,
         schema: universe.schema,
-        obsAnnotations: universe.obsAnnotations,
+        obsAnnotations: expect.any(Dataframe.Dataframe),
-        varAnnotations: universe.varAnnotations,
+        varAnnotations: expect.any(Dataframe.Dataframe),
-        obsLayout: universe.obsLayout,
+        obsLayout: expect.any(Dataframe.Dataframe),
-        varData: expect.any(Dataframe.Dataframe)
+        varData: expect.any(Dataframe.Dataframe),
+        clipQuantiles: { min: 0, max: 1 },
+        unclipped: {
+          obsAnnotations: expect.any(Dataframe.Dataframe),
+          varData: expect.any(Dataframe.Dataframe)
+        }
       })
     );
   });
@@ -76,7 +81,7 @@ describe("createWorldFromCurrentSelection", () => {
     } = defaultBigBang();
 
     /* mock a selection */
-    let crossfilter = originalCrossfilter
+    const crossfilter = originalCrossfilter
       .select(obsAnnoDimensionName("field1"), { mode: "range", lo: 0, hi: 5 })
       .select(obsAnnoDimensionName("field3"), {
         mode: "exact",
@@ -84,7 +89,7 @@ describe("createWorldFromCurrentSelection", () => {
       });
 
     /* create the world from the selection */
-    const world = World.createWorldFromCurrentSelection(
+    const world = World.createWorldBySelection(
       universe,
       originalWorld,
       crossfilter
@@ -112,10 +117,15 @@ describe("createWorldFromCurrentSelection", () => {
         nObs: matchingIndices.length,
         nVar: universe.nVar,
         schema: universe.schema,
+        clipQuantiles: { min: 0, max: 1 },
         obsAnnotations: expect.any(Dataframe.Dataframe),
-        varAnnotations: universe.varAnnotations,
+        varAnnotations: expect.any(Dataframe.Dataframe),
         obsLayout: expect.any(Dataframe.Dataframe),
-        varData: expect.any(Dataframe.Dataframe)
+        varData: expect.any(Dataframe.Dataframe),
+        unclipped: {
+          obsAnnotations: expect.any(Dataframe.Dataframe),
+          varData: expect.any(Dataframe.Dataframe)
+        }
       })
     );
 

client/__tests__/util/typedCrossfilter/crossfilter.test.js

@@ -11,7 +11,8 @@ const someData = [
     tip: 100,
     type: "tab",
     productIDs: ["001"],
-    coords: [0, 0]
+    coords: [0, 0],
+    nonFinite: 0.0
   },
   {
     date: "2011-11-14T16:20:19Z",
@@ -20,7 +21,8 @@ const someData = [
     tip: 100,
     type: "tab",
     productIDs: ["001", "005"],
-    coords: [0.4, 0.4]
+    coords: [0.4, 0.4],
+    nonFinite: Number.NaN
   },
   {
     date: "2011-11-14T16:28:54Z",
@@ -29,7 +31,8 @@ const someData = [
     tip: 200,
     type: "visa",
     productIDs: ["004", "005"],
-    coords: [0.3, 0.1]
+    coords: [0.3, 0.1],
+    nonFinite: Number.POSITIVE_INFINITY
   },
   {
     date: "2011-11-14T16:30:43Z",
@@ -38,7 +41,8 @@ const someData = [
     tip: 0,
     type: "tab",
     productIDs: ["001", "002"],
-    coords: [0.392, 0.1]
+    coords: [0.392, 0.1],
+    nonFinite: Number.NEGATIVE_INFINITY
   },
   {
     date: "2011-11-14T16:48:46Z",
@@ -47,7 +51,8 @@ const someData = [
     tip: 0,
     type: "tab",
     productIDs: ["005"],
-    coords: [0.7, 0.0482]
+    coords: [0.7, 0.0482],
+    nonFinite: 1.0
   },
   {
     date: "2011-11-14T16:53:41Z",
@@ -56,7 +61,8 @@ const someData = [
     tip: 0,
     type: "tab",
     productIDs: ["001", "004", "005"],
-    coords: [0.9999, 1.0]
+    coords: [0.9999, 1.0],
+    nonFinite: Number.NaN
   },
   {
     date: "2011-11-14T16:54:06Z",
@@ -65,7 +71,8 @@ const someData = [
     tip: 0,
     type: "cash",
     productIDs: ["001", "002", "003", "004", "005"],
-    coords: [0.384, 0.6938]
+    coords: [0.384, 0.6938],
+    nonFinite: 99.0
   },
   {
     date: "2011-11-14T16:58:03Z",
@@ -74,7 +81,8 @@ const someData = [
     tip: 0,
     type: "tab",
     productIDs: ["001"],
-    coords: [0.4822, 0.482]
+    coords: [0.4822, 0.482],
+    nonFinite: Number.NaN
   },
   {
     date: "2011-11-14T17:07:21Z",
@@ -83,7 +91,8 @@ const someData = [
     tip: 0,
     type: "tab",
     productIDs: ["004", "005"],
-    coords: [0.2234, 0]
+    coords: [0.2234, 0],
+    nonFinite: Number.NaN
   },
   {
     date: "2011-11-14T17:22:59Z",
@@ -92,7 +101,8 @@ const someData = [
     tip: 0,
     type: "tab",
     productIDs: ["001", "002", "004", "005"],
-    coords: [0.382, 0.38485]
+    coords: [0.382, 0.38485],
+    nonFinite: -1
   },
   {
     date: "2011-11-14T17:25:45Z",
@@ -101,7 +111,8 @@ const someData = [
     tip: 0,
     type: "cash",
     productIDs: ["002"],
-    coords: [0.998, 0.8472]
+    coords: [0.998, 0.8472],
+    nonFinite: 0.0
   },
   {
     date: "2011-11-14T17:29:52Z",
@@ -110,7 +121,8 @@ const someData = [
     tip: 100,
     type: "visa",
     productIDs: ["004"],
-    coords: [0.8273, 0.3384]
+    coords: [0.8273, 0.3384],
+    nonFinite: 0.0
   }
 ];
 
@@ -327,4 +339,96 @@ describe("ImmutableTypedCrossfilter", () => {
       ).toEqual(_.filter(someData, d => polygonContains(polygon, d.coords)));
     });
   });
+
+  describe("non-finite scalars", () => {
+    let p;
+    beforeEach(() => {
+      p = payments
+        .addDimension("quantity", "scalar", (i, d) => d[i].quantity, Int32Array)
+        .addDimension(
+          "nonFinite",
+          "scalar",
+          (i, d) => d[i].nonFinite,
+          Float32Array
+        )
+        .select("quantity", { mode: "all" });
+    });
+
+    test("all or none", () => {
+      expect(p.select("nonFinite", { mode: "all" }).countSelected()).toEqual(
+        someData.length
+      );
+      expect(p.select("nonFinite", { mode: "none" }).countSelected()).toEqual(
+        0
+      );
+    });
+
+    test("exact", () => {
+      expect(
+        p.select("nonFinite", { mode: "exact", values: [0] }).countSelected()
+      ).toEqual(3);
+      expect(
+        p.select("nonFinite", { mode: "exact", values: [1] }).countSelected()
+      ).toEqual(1);
+      expect(
+        p
+          .select("nonFinite", {
+            mode: "exact",
+            values: [Number.POSITIVE_INFINITY]
+          })
+          .countSelected()
+      ).toEqual(1);
+      expect(
+        p
+          .select("nonFinite", {
+            mode: "exact",
+            values: [Number.NEGATIVE_INFINITY]
+          })
+          .countSelected()
+      ).toEqual(1);
+      expect(
+        p
+          .select("nonFinite", { mode: "exact", values: [Number.NaN] })
+          .countSelected()
+      ).toEqual(4);
+      expect(
+        p
+          .select("nonFinite", {
+            mode: "exact",
+            values: [Number.POSITIVE_INFINITY, 0, 1, 99]
+          })
+          .countSelected()
+      ).toEqual(6);
+    });
+
+    test("range", () => {
+      expect(
+        p
+          .select("nonFinite", {
+            mode: "range",
+            lo: 0,
+            hi: Number.POSITIVE_INFINITY
+          })
+          .countSelected()
+      ).toEqual(5);
+      expect(
+        p
+          .select("nonFinite", {
+            mode: "range",
+            lo: 0,
+            hi: Number.NaN
+          })
+          .countSelected()
+      ).toEqual(6);
+      expect(
+        p
+          .select("nonFinite", {
+            mode: "range",
+            lo: Number.NEGATIVE_INFINITY,
+            hi: Number.POSITIVE_INFINITY
+          })
+          .countSelected()
+      ).toEqual(7);
+    });
+  });
 });

client/__tests__/util/typedCrossfilter/sort.test.js

@@ -1,4 +1,22 @@
-import { sort, sortIndex } from "../../../src/util/typedCrossfilter/sort";
+import {
+  sortArray,
+  sortIndex,
+  lowerBound,
+  upperBound,
+  lowerBoundIndirect,
+  upperBoundIndirect
+} from "../../../src/util/typedCrossfilter/sort";
+
+/*
+Sort tests should keep in mind that there are separate code
+paths for:
+  - small vs. large arrays (insertionsort only)
+  - float-only typed arrays vs. other array types (non-finite handling)
+  - indexed vs. direct sort
+*/
+
+const pInf = Number.POSITIVE_INFINITY;
+const nInf = Number.NEGATIVE_INFINITY;
 
 function fillRange(arr, start = 0) {
   const larr = arr;
@@ -15,42 +33,223 @@ function fillRand(arr) {
   return arr;
 }
 
-describe("sort", () => {
+describe("sortArray", () => {
-  [Array, Float32Array, Uint32Array, Int32Array, Float64Array].map(Type =>
+  describe("JS vals", () => {
-    test(Type.name, () => {
+    [
-      expect(sort(Type.from([6, 5, 4, 3, 2, 1, 0]))).toMatchObject(
+      [true, false],
-        Type.from([0, 1, 2, 3, 4, 5, 6])
+      ["a", "b", "0", "1"],
-      );
+      [0, "a", true, null, undefined, 3.1415],
-      expect(sort(Type.from([6, 5, 4, 3, 2, 1]))).toMatchObject(
+      fillRand(new Array(1000)),
-        Type.from([1, 2, 3, 4, 5, 6])
+      ["a", NaN, null, pInf]
-      );
+    ].map((val, idx) =>
+      test(`JS vals ${idx}`, () => {
+        expect(sortArray(val)).toMatchObject(val.sort());
+      })
+    );
+  });
 
-      const source = fillRand(new Type(1000));
+  describe("finite numbers", () => {
-      expect(sort(Type.from(source))).toMatchObject(Type.from(source).sort());
+    [Array, Float32Array, Uint32Array, Int32Array, Float64Array].map(Type =>
-    })
+      test(Type.name, () => {
-  );
+        expect(sortArray(Type.from([6, 5, 4, 3, 2, 1, 0]))).toMatchObject(
+          Type.from([0, 1, 2, 3, 4, 5, 6])
+        );
+
+        expect(sortArray(Type.from([6, 5, 4, 3, 2, 1]))).toMatchObject(
+          Type.from([1, 2, 3, 4, 5, 6])
+        );
+
+        const source = fillRand(new Type(1000));
+        expect(sortArray(Type.from(source))).toMatchObject(
+          Type.from(source).sort()
+        );
+      })
+    );
+  });
+
+  describe("non-finite numbers", () => {
+    test("inifinity", () => {
+      expect(sortArray(new Float32Array([pInf, nInf, 0, 1, 2]))).toMatchObject(
+        new Float32Array([nInf, 0, 1, 2, pInf])
+      );
+      expect(
+        sortArray(new Float32Array([pInf, nInf, pInf, nInf]))
+      ).toMatchObject(new Float32Array([nInf, nInf, pInf, pInf]));
+      expect(
+        sortArray(new Float32Array([pInf, nInf, pInf, nInf, pInf]))
+      ).toMatchObject(new Float32Array([nInf, nInf, pInf, pInf, pInf]));
+      expect(
+        sortArray(
+          new Float32Array(100).fill(Infinity, 0, 50).fill(-Infinity, 50, 100)
+        )
+      ).toMatchObject(
+        new Float32Array(100).fill(-Infinity, 0, 50).fill(Infinity, 50, 100)
+      );
+    });
+    test("NaN", () => {
+      expect(sortArray(new Float64Array([NaN, 2, 1, 0]))).toMatchObject(
+        new Float64Array([0, 1, 2, NaN])
+      );
+      expect(sortArray(new Float32Array([NaN, 2, 1, 0]))).toMatchObject(
+        new Float32Array([0, 1, 2, NaN])
+      );
+      expect(sortArray(new Float32Array([NaN, 2, NaN, 1, 0]))).toMatchObject(
+        new Float32Array([0, 1, 2, NaN, NaN])
+      );
+      expect(sortArray(new Float32Array([NaN, 2, 1, NaN, 0]))).toMatchObject(
+        new Float32Array([0, 1, 2, NaN, NaN])
+      );
+      expect(
+        sortArray(fillRange(new Float32Array(100)).fill(NaN, 0, 10))
+      ).toMatchObject(fillRange(new Float32Array(100), 10).fill(NaN, 90, 100));
+    });
+
+    test("mixed numbers", () => {
+      expect(
+        sortArray(new Float32Array([NaN, pInf, nInf, NaN, NaN]))
+      ).toMatchObject(new Float32Array([nInf, pInf, NaN, NaN, NaN]));
+      expect(
+        sortArray(new Float32Array([NaN, pInf, nInf, NaN, 1, NaN, 2]))
+      ).toMatchObject(new Float32Array([nInf, 1, 2, pInf, NaN, NaN, NaN]));
+      expect(
+        sortArray(new Float32Array([NaN, pInf, nInf, 0, 1, NaN, 2]))
+      ).toMatchObject(new Float32Array([nInf, 0, 1, 2, pInf, NaN, NaN]));
+      expect(
+        sortArray(
+          fillRange(new Float32Array(100))
+            .fill(NaN, 0, 10)
+            .fill(Infinity, 10, 20)
+        )
+      ).toMatchObject(
+        fillRange(new Float32Array(100), 20)
+          .fill(Infinity, 80, 90)
+          .fill(NaN, 90, 100)
+      );
+    });
+  });
 });
 
 describe("sortIndex", () => {
-  [Array, Float32Array, Uint32Array, Int32Array, Float64Array].map(Type =>
+  describe("finite numbers", () => {
-    test(Type.name, () => {
+    [Array, Float32Array, Uint32Array, Int32Array, Float64Array].map(Type =>
-      const source1 = Type.from([6, 5, 4, 3, 2, 1, 0]);
+      test(Type.name, () => {
+        const source1 = Type.from([6, 5, 4, 3, 2, 1, 0]);
+        const index1 = fillRange(new Uint32Array(source1.length));
+        expect(sortIndex(index1, source1)).toMatchObject(
+          index1.sort((a, b) => source1[a] - source1[b])
+        );
+
+        const source2 = Type.from([6, 5, 4, 3, 2, 1]);
+        const index2 = fillRange(new Uint32Array(source2.length));
+        expect(sortIndex(index2, source2)).toMatchObject(
+          index2.sort((a, b) => source1[a] - source1[b])
+        );
+
+        const source3 = fillRand(new Type(1000));
+        const index3 = fillRange(new Uint32Array(source3.length));
+        expect(sortIndex(index3, source3)).toMatchObject(
+          index3.sort((a, b) => source1[a] - source1[b])
+        );
+      })
+    );
+  });
+
+  describe("non-finite numbers", () => {
+    test("mixed numbers", () => {
+      const source1 = new Float32Array([NaN, pInf, nInf, NaN, 1, NaN, 2]);
       const index1 = fillRange(new Uint32Array(source1.length));
       expect(sortIndex(index1, source1)).toMatchObject(
-        index1.sort((a, b) => source1[a] - source1[b])
+        new Uint32Array([2, 4, 6, 1, 0, 3, 5])
       );
 
-      const source2 = Type.from([6, 5, 4, 3, 2, 1]);
+      const source2 = new Float32Array([NaN, pInf, nInf, 0, 1, NaN, 2]);
       const index2 = fillRange(new Uint32Array(source2.length));
       expect(sortIndex(index2, source2)).toMatchObject(
-        index2.sort((a, b) => source1[a] - source1[b])
+        new Uint32Array([2, 3, 4, 6, 1, 0, 5])
       );
+    });
+  });
+});
 
-      const source3 = fillRand(new Type(1000));
+describe("lowerBound", () => {
-      const index3 = fillRange(new Uint32Array(source3.length));
+  test("non-float path", () => {
-      expect(sortIndex(index3, source3)).toMatchObject(
+    expect(lowerBound([], 0, 0, 0)).toEqual(0);
-        index3.sort((a, b) => source1[a] - source1[b])
+
-      );
+    expect(lowerBound([0, 1, 2, 3], -1, 0, 4)).toEqual(0);
-    })
+    expect(lowerBound([0, 1, 2, 3], 0, 0, 4)).toEqual(0);
-  );
+    expect(lowerBound([0, 1, 2, 3], 1, 0, 4)).toEqual(1);
+    expect(lowerBound([0, 1, 2, 3], 3, 0, 4)).toEqual(3);
+    expect(lowerBound([0, 1, 2, 3], 4, 0, 4)).toEqual(4);
+
+    expect(lowerBound([0, 1, 2, 3, 4], -1, 0, 5)).toEqual(0);
+    expect(lowerBound([0, 1, 2, 3, 4], 0, 0, 5)).toEqual(0);
+    expect(lowerBound([0, 1, 2, 3, 4], 2, 0, 5)).toEqual(2);
+    expect(lowerBound([0, 1, 2, 3, 4], 4, 0, 5)).toEqual(4);
+    expect(lowerBound([0, 1, 2, 3, 4], 5, 0, 5)).toEqual(5);
+
+    expect(lowerBound([0, 2, 4, 6, 8], 5, 0, 5)).toEqual(3);
+    expect(lowerBound([0, 2, 2, 2, 8], 5, 0, 5)).toEqual(4);
+
+    expect(lowerBound([0, 1, 2, 3, 4, 5, 6, 7, 8], 3, 2, 4)).toEqual(3);
+    expect(lowerBound([0, 1, 2, 3, 4, 5, 6, 7, 8], 99, 2, 4)).toEqual(4);
+  });
+
+  test("float path, finites", () => {
+    expect(lowerBound(new Float32Array([0, 1, 2, 3]), 1, 0, 4)).toEqual(1);
+
+    expect(lowerBound(new Float32Array([]), 0, 0, 0)).toEqual(0);
+
+    expect(lowerBound(new Float32Array([0, 1, 2, 3]), -1, 0, 4)).toEqual(0);
+    expect(lowerBound(new Float32Array([0, 1, 2, 3]), 0, 0, 4)).toEqual(0);
+    expect(lowerBound(new Float32Array([0, 1, 2, 3]), 1, 0, 4)).toEqual(1);
+    expect(lowerBound(new Float32Array([0, 1, 2, 3]), 3, 0, 4)).toEqual(3);
+    expect(lowerBound(new Float32Array([0, 1, 2, 3]), 4, 0, 4)).toEqual(4);
+
+    expect(lowerBound(new Float32Array([0, 1, 2, 3, 4]), -1, 0, 5)).toEqual(0);
+    expect(lowerBound(new Float32Array([0, 1, 2, 3, 4]), 0, 0, 5)).toEqual(0);
+    expect(lowerBound(new Float32Array([0, 1, 2, 3, 4]), 2, 0, 5)).toEqual(2);
+    expect(lowerBound(new Float32Array([0, 1, 2, 3, 4]), 4, 0, 5)).toEqual(4);
+    expect(lowerBound(new Float32Array([0, 1, 2, 3, 4]), 5, 0, 5)).toEqual(5);
+
+    expect(lowerBound(new Float32Array([0, 2, 4, 6, 8]), 5, 0, 5)).toEqual(3);
+    expect(lowerBound(new Float32Array([0, 2, 2, 2, 8]), 5, 0, 5)).toEqual(4);
+
+    expect(
+      lowerBound(new Float32Array([0, 1, 2, 3, 4, 5, 6, 7, 8]), 3, 2, 4)
+    ).toEqual(3);
+    expect(
+      lowerBound(new Float32Array([0, 1, 2, 3, 4, 5, 6, 7, 8]), 99, 2, 4)
+    ).toEqual(4);
+  });
+
+  test("float path, non-finite", () => {
+    expect(
+      lowerBound(
+        new Float32Array([-Infinity, 0, 1, Infinity, NaN]),
+        -Infinity,
+        0,
+        5
+      )
+    ).toEqual(0);
+    expect(
+      lowerBound(new Float32Array([-Infinity, 0, 1, Infinity, NaN]), 0, 0, 5)
+    ).toEqual(1);
+    expect(
+      lowerBound(new Float32Array([-Infinity, 0, 1, Infinity, NaN]), 1, 0, 5)
+    ).toEqual(2);
+    expect(
+      lowerBound(new Float32Array([-Infinity, 0, 1, Infinity, NaN]), 2, 0, 5)
+    ).toEqual(3);
+    expect(
+      lowerBound(
+        new Float32Array([-Infinity, 0, 1, Infinity, NaN]),
+        Infinity,
+        0,
+        5
+      )
+    ).toEqual(3);
+    expect(
+      lowerBound(new Float32Array([-Infinity, 0, 1, Infinity, NaN]), NaN, 0, 5)
+    ).toEqual(4);
+  });
 });

client/__tests__/util/typedCrossfilter/util.test.js

@@ -1,10 +1,7 @@
 import {
   fillRange,
   sliceByIndex,
-  makeSortIndex,
+  makeSortIndex
-  lowerBound,
-  lowerBoundIndirect,
-  upperBoundIndirect
 } from "../../../src/util/typedCrossfilter/util";
 
 describe("fillRange", () => {

client/package-lock.json

@@ -1049,12 +1049,12 @@
       }
     },
     "@blueprintjs/core": {
-      "version": "3.8.0",
+      "version": "3.15.0",
-      "resolved": "https://registry.npmjs.org/@blueprintjs/core/-/core-3.8.0.tgz",
+      "resolved": "https://registry.npmjs.org/@blueprintjs/core/-/core-3.15.0.tgz",
-      "integrity": "sha512-maw63uME+spdyDUOJH47r+YRc8kDeux8kQz4poiAQKk4eJavc160/aU7w5k4sV29mVputp7P22qNY1oBNXjtDQ==",
+      "integrity": "sha512-znXO0UaWBJO7Nm2qEvZzURCDuogsJ9DJlIw3N2XnIi+k/c8mQ1xlvtg0TT2pS88lMG9HUEKky3b0UQaSXBw4LA==",
       "requires": {
-        "@blueprintjs/icons": "^3.3.0",
+        "@blueprintjs/icons": "^3.7.0",
-        "@types/dom4": "^2.0.0",
+        "@types/dom4": "^2.0.1",
         "classnames": "^2.2",
         "dom4": "^2.0.1",
         "normalize.css": "^8.0.0",
@@ -1063,6 +1063,17 @@
         "react-transition-group": "^2.2.1",
         "resize-observer-polyfill": "^1.5.0",
         "tslib": "^1.9.0"
+      },
+      "dependencies": {
+        "@blueprintjs/icons": {
+          "version": "3.8.0",
+          "resolved": "https://registry.npmjs.org/@blueprintjs/icons/-/icons-3.8.0.tgz",
+          "integrity": "sha512-yHaRQ3vfV9Gf3foZ4ONtxddz+u5ufkHqHj8Ia5VhPbFgG4el+cPdmsGGIIM72rgKS1KQa5Ay+ggjpByUlXvrKg==",
+          "requires": {
+            "classnames": "^2.2",
+            "tslib": "^1.9.0"
+          }
+        }
       }
     },
     "@blueprintjs/icons": {
@@ -5524,7 +5535,8 @@
         "ansi-regex": {
           "version": "2.1.1",
           "bundled": true,
-          "dev": true
+          "dev": true,
+          "optional": true
         },
         "aproba": {
           "version": "1.2.0",
@@ -5545,12 +5557,14 @@
         "balanced-match": {
           "version": "1.0.0",
           "bundled": true,
-          "dev": true
+          "dev": true,
+          "optional": true
         },
         "brace-expansion": {
           "version": "1.1.11",
           "bundled": true,
           "dev": true,
+          "optional": true,
           "requires": {
             "balanced-match": "^1.0.0",
             "concat-map": "0.0.1"
@@ -5565,17 +5579,20 @@
         "code-point-at": {
           "version": "1.1.0",
           "bundled": true,
-          "dev": true
+          "dev": true,
+          "optional": true
         },
         "concat-map": {
           "version": "0.0.1",
           "bundled": true,
-          "dev": true
+          "dev": true,
+          "optional": true
         },
         "console-control-strings": {
           "version": "1.1.0",
           "bundled": true,
-          "dev": true
+          "dev": true,
+          "optional": true
         },
         "core-util-is": {
           "version": "1.0.2",
@@ -5692,7 +5709,8 @@
         "inherits": {
           "version": "2.0.3",
           "bundled": true,
-          "dev": true
+          "dev": true,
+          "optional": true
         },
         "ini": {
           "version": "1.3.5",
@@ -5704,6 +5722,7 @@
           "version": "1.0.0",
           "bundled": true,
           "dev": true,
+          "optional": true,
           "requires": {
             "number-is-nan": "^1.0.0"
           }
@@ -5718,6 +5737,7 @@
           "version": "3.0.4",
           "bundled": true,
           "dev": true,
+          "optional": true,
           "requires": {
             "brace-expansion": "^1.1.7"
           }
@@ -5725,12 +5745,14 @@
         "minimist": {
           "version": "0.0.8",
           "bundled": true,
-          "dev": true
+          "dev": true,
+          "optional": true
         },
         "minipass": {
           "version": "2.2.4",
           "bundled": true,
           "dev": true,
+          "optional": true,
           "requires": {
             "safe-buffer": "^5.1.1",
             "yallist": "^3.0.0"
@@ -5749,6 +5771,7 @@
           "version": "0.5.1",
           "bundled": true,
           "dev": true,
+          "optional": true,
           "requires": {
             "minimist": "0.0.8"
           }
@@ -5829,7 +5852,8 @@
         "number-is-nan": {
           "version": "1.0.1",
           "bundled": true,
-          "dev": true
+          "dev": true,
+          "optional": true
         },
         "object-assign": {
           "version": "4.1.1",
@@ -5841,6 +5865,7 @@
           "version": "1.4.0",
           "bundled": true,
           "dev": true,
+          "optional": true,
           "requires": {
             "wrappy": "1"
           }
@@ -5926,7 +5951,8 @@
         "safe-buffer": {
           "version": "5.1.1",
           "bundled": true,
-          "dev": true
+          "dev": true,
+          "optional": true
         },
         "safer-buffer": {
           "version": "2.1.2",
@@ -5962,6 +5988,7 @@
           "version": "1.0.2",
           "bundled": true,
           "dev": true,
+          "optional": true,
           "requires": {
             "code-point-at": "^1.0.0",
             "is-fullwidth-code-point": "^1.0.0",
@@ -5981,6 +6008,7 @@
           "version": "3.0.1",
           "bundled": true,
           "dev": true,
+          "optional": true,
           "requires": {
             "ansi-regex": "^2.0.0"
           }
@@ -6024,12 +6052,14 @@
         "wrappy": {
           "version": "1.0.2",
           "bundled": true,
-          "dev": true
+          "dev": true,
+          "optional": true
         },
         "yallist": {
           "version": "3.0.2",
           "bundled": true,
-          "dev": true
+          "dev": true,
+          "optional": true
         }
       }
     },

client/package.json

@@ -31,7 +31,7 @@
     "eslint-scope": "3.7.1"
   },
   "dependencies": {
-    "@blueprintjs/core": "^3.8.0",
+    "@blueprintjs/core": "^3.15.0",
     "@blueprintjs/icons": "^3.3.0",
     "@blueprintjs/select": "^3.8.0",
     "canvas-fit": "^1.5.0",

client/src/components/brushableHistogram/index.js

@@ -5,14 +5,12 @@ https://bl.ocks.org/SpaceActuary/2f004899ea1b2bd78d6f1dbb2febf771
 */
 // jshint esversion: 6
 import React from "react";
-import _ from "lodash";
 import { Button, ButtonGroup, Tooltip } from "@blueprintjs/core";
 import { connect } from "react-redux";
 import * as d3 from "d3";
 import memoize from "memoize-one";
 import * as globals from "../../globals";
 import actions from "../../actions";
-import finiteExtent from "../../util/finiteExtent";
 import { makeContinuousDimensionName } from "../../util/nameCreators";
 
 @connect(state => ({
@@ -21,53 +19,52 @@ import { makeContinuousDimensionName } from "../../util/nameCreators";
   scatterplotYYaccessor: state.controls.scatterplotYYaccessor,
   continuousSelection: state.continuousSelection,
   differential: state.differential,
-  colorAccessor: state.colors.colorAccessor,
+  colorAccessor: state.colors.colorAccessor
-  obsAnnotations: _.get(state.world, "obsAnnotations", null)
 }))
 class HistogramBrush extends React.Component {
-  calcHistogramCache = memoize((obsAnnotations, field, rangeMin, rangeMax) => {
+  static getColumn(world, field, clipped = true) {
-    const { world } = this.props;
+    /*
-    const histogramCache = {};
+    Return the underlying Dataframe column for our field.   By default, 
+    returns the clipped column.   If clipped===false, will return the 
+    unclipped column.
+    */
+    const obsAnnotations = clipped
+      ? world.obsAnnotations
+      : world.unclipped.obsAnnotations;
+    const varData = clipped ? world.varData : world.unclipped.varData;
+    if (obsAnnotations.hasCol(field)) {
+      return obsAnnotations.col(field);
+    }
+    return varData.col(field);
+  }
 
+  calcHistogramCache = memoize((world, field) => {
+    /*
+     recalculate expensive stuff, notably bins, summaries, etc.
+    */
+    const histogramCache = {};
+    const col = HistogramBrush.getColumn(world, field);
+    const values = col.asArray();
+    const summary = col.summarize();
+    const { min: domainMin, max: domainMax } = summary;
+    histogramCache.x = d3
+      .scaleLinear()
+      .domain([domainMin, domainMax])
+      .range([0, this.width - this.marginRight]);
+
+    histogramCache.bins = d3
+      .histogram()
+      .domain(histogramCache.x.domain())
+      .thresholds(40)(values);
+
+    const yMax = histogramCache.bins
+      .map(b => b.length)
+      .reduce((a, b) => Math.max(a, b));
     histogramCache.y = d3
       .scaleLinear()
+      .domain([0, yMax])
       .range([this.height - this.marginBottom, 0]);
 
-    if (obsAnnotations.hasCol(field)) {
-      // recalculate expensive stuff
-      const allValuesForContinuousFieldAsArray = obsAnnotations
-        .col(field)
-        .asArray();
-
-      histogramCache.x = d3
-        .scaleLinear()
-        .domain([rangeMin, rangeMax])
-        .range([0, this.width]);
-
-      histogramCache.bins = d3
-        .histogram()
-        .domain(histogramCache.x.domain())
-        .thresholds(40)(allValuesForContinuousFieldAsArray);
-
-      histogramCache.numValues = allValuesForContinuousFieldAsArray.length;
-    } else if (world.varData.hasCol(field)) {
-      const varValues = world.varData.col(field).asArray();
-
-      histogramCache.x = d3
-        .scaleLinear()
-        .domain(
-          finiteExtent(varValues)
-        ) /* replace this if we have ranges for genes back from server like we do for annotations on cells */
-        .range([0, this.width]);
-
-      histogramCache.bins = d3
-        .histogram()
-        .domain(histogramCache.x.domain())
-        .thresholds(40)(varValues);
-
-      histogramCache.numValues = varValues.length;
-    }
-
     return histogramCache;
   });
 
@@ -76,22 +73,23 @@ class HistogramBrush extends React.Component {
 
     this.width = 340;
     this.height = 100;
-    this.marginBottom = 20;
+    this.marginBottom = 20; // space for X axis & labels
+    this.marginRight = 40; // space for Y axis & labels
   }
 
   componentDidMount() {
     const { field } = this.props;
-    const { x, y, bins, numValues, svgRef } = this._histogram;
+    const { x, y, bins, svgRef } = this._histogram;
 
-    this.renderAxesBrushBins(x, y, bins, numValues, svgRef, field);
+    this.renderAxesBrushBins(x, y, bins, svgRef, field);
   }
 
   componentDidUpdate(prevProps) {
-    const { field, obsAnnotations, continuousSelection } = this.props;
+    const { field, world, continuousSelection } = this.props;
-    const { x, y, bins, numValues, svgRef } = this._histogram;
+    const { x, y, bins, svgRef } = this._histogram;
 
-    if (obsAnnotations !== prevProps.obsAnnotations) {
+    if (world !== prevProps.world) {
-      this.renderAxesBrushBins(x, y, bins, numValues, svgRef, field);
+      this.renderAxesBrushBins(x, y, bins, svgRef, field);
     }
 
     /*
@@ -172,7 +170,6 @@ class HistogramBrush extends React.Component {
   onBrushEnd(selection, x) {
     return () => {
       const { dispatch, field, isObs, isUserDefined, isDiffExp } = this.props;
-      const { brushXselection } = this.state;
       const minAllowedBrushSize = 10;
       const smallAmountToAvoidInfiniteLoop = 0.1;
 
@@ -197,11 +194,6 @@ class HistogramBrush extends React.Component {
             smallAmountToAvoidInfiniteLoop; //
 
           _range = [x(d3.event.selection[0]), x(procedurallyResizedBrushWidth)];
-
-          d3.event.target.move(brushXselection, [
-            d3.event.selection[0],
-            procedurallyResizedBrushWidth
-          ]);
         }
 
         dispatch({
@@ -229,23 +221,16 @@ class HistogramBrush extends React.Component {
   }
 
   drawHistogram(svgRef) {
-    const { obsAnnotations, field, ranges } = this.props;
+    const { field, world } = this.props;
-    const histogramCache = this.calcHistogramCache(
+    const histogramCache = this.calcHistogramCache(world, field);
-      obsAnnotations,
+    const { x, y, bins } = histogramCache;
-      field,
+    this._histogram = { x, y, bins, svgRef };
-      ranges.min,
-      ranges.max
-    );
-
-    const { x, y, bins, numValues } = histogramCache;
-
-    this._histogram = { x, y, bins, numValues, svgRef };
   }
 
   handleColorAction() {
-    const { obsAnnotations, dispatch, field, world, ranges } = this.props;
+    const { dispatch, field, world, ranges } = this.props;
 
-    if (obsAnnotations.hasCol(field)) {
+    if (world.obsAnnotations.hasCol(field)) {
       dispatch({
         type: "color by continuous metadata",
         colorAccessor: field,
@@ -307,29 +292,29 @@ class HistogramBrush extends React.Component {
     };
   }
 
-  renderAxesBrushBins(x, y, bins, numValues, svgRef, field) {
+  renderAxesBrushBins(x, y, bins, svgRef, field) {
+    const svg = d3.select(svgRef);
+
     /* Remove everything */
-    d3.select(svgRef)
+    svg.selectAll("*").remove();
-      .selectAll("*")
-      .remove();
 
     /* BINS */
-    d3.select(svgRef)
+    svg
       .insert("g", "*")
       .attr("fill", "#bbb")
       .selectAll("rect")
       .data(bins)
       .enter()
       .append("rect")
-      .attr("class", "bar")
       .attr("x", d => x(d.x0) + 1)
-      .attr("y", d => y(d.length / numValues))
+      .attr("y", d => y(d.length))
       .attr("width", d => Math.abs(x(d.x1) - x(d.x0) - 1))
-      .attr("height", d => y(0) - y(d.length / numValues));
+      .attr("height", d => y(0) - y(d.length));
 
     /* BRUSH */
     const brushX = d3
       .brushX()
+      .extent([[0, 0], [this.width - this.marginRight, this.height]])
       /*
       emit start so that the Undoable history can save an undo point
       upon drag start, and ignore the subsequent intermediate drag events.
@@ -344,24 +329,24 @@ class HistogramBrush extends React.Component {
       .attr("data-testid", `${svgRef.dataset.testid}-brush`)
       .call(brushX);
 
-    /* AXIS */
+    /* X AXIS */
-    d3.select(svgRef)
+    svg
       .append("g")
       .attr("class", "axis axis--x")
       .attr("transform", `translate(0,${this.height - this.marginBottom})`)
       .call(d3.axisBottom(x).ticks(5));
 
-    d3.select(svgRef)
+    /* Y AXIS */
-      .selectAll(".axis--x text")
+    svg
-      .style("fill", "rgb(80,80,80)");
+      .append("g")
+      .attr("class", "axis axis--y")
+      .attr("transform", `translate(${this.width - this.marginRight},0)`)
+      .call(d3.axisRight(y).ticks(3));
 
-    d3.select(svgRef)
+    /* axis style */
-      .selectAll(".axis--x path")
+    svg.selectAll(".axis text").style("fill", "rgb(80,80,80)");
-      .style("stroke", "rgb(230,230,230)");
+    svg.selectAll(".axis path").style("stroke", "rgb(230,230,230)");
-
+    svg.selectAll(".axis line").style("stroke", "rgb(230,230,230)");
-    d3.select(svgRef)
-      .selectAll(".axis--x line")
-      .style("stroke", "rgb(230,230,230)");
 
     this.setState({ brushX, brushXselection });
   }
@@ -369,20 +354,29 @@ class HistogramBrush extends React.Component {
   render() {
     const {
       field,
+      world,
       colorAccessor,
       isUserDefined,
       isDiffExp,
       logFoldChange,
-      pval,
       pvalAdj,
       scatterplotXXaccessor,
       scatterplotYYaccessor,
       zebra
     } = this.props;
-    const field_for_id = field.replace(/\s/g, "_");
+    const fieldForId = field.replace(/\s/g, "_");
+    const {
+      min: unclippedRangeMin,
+      max: unclippedRangeMax
+    } = HistogramBrush.getColumn(world, field, false).summarize();
+    const unclippedRangeMinColor =
+      world.clipQuantiles.min === 0 ? "#bbb" : globals.blue;
+    const unclippedRangeMaxColor =
+      world.clipQuantiles.max === 1 ? "#bbb" : globals.blue;
+
     return (
       <div
-        id={`histogram_${field_for_id}`}
+        id={`histogram_${fieldForId}`}
         data-testid={`histogram-${field}`}
         data-testclass={
           isDiffExp
@@ -396,7 +390,13 @@ class HistogramBrush extends React.Component {
           backgroundColor: zebra ? globals.lightestGrey : "white"
         }}
       >
-        <div style={{ display: "flex", justifyContent: "flex-end" }}>
+        <div
+          style={{
+            display: "flex",
+            justifyContent: "flex-end",
+            paddingBottom: "8px"
+          }}
+        >
           {isDiffExp || isUserDefined ? (
             <span>
               <span
@@ -450,7 +450,7 @@ class HistogramBrush extends React.Component {
         <svg
           width={this.width}
           height={this.height}
-          id={`histogram_${field_for_id}_svg`}
+          id={`histogram_${fieldForId}_svg`}
           data-testclass="histogram-plot"
           data-testid={`histogram-${field}-plot`}
           ref={svgRef => {
@@ -460,15 +460,21 @@ class HistogramBrush extends React.Component {
         <div
           style={{
             display: "flex",
-            justifyContent: "center"
+            justifyContent: "space-between"
           }}
         >
+          <span style={{ color: unclippedRangeMinColor }}>
+            min {unclippedRangeMin.toPrecision(4)}
+          </span>
           <span
             data-testclass="brushable-histogram-field-name"
             style={{ fontStyle: "italic" }}
           >
             {field}
           </span>
+          <span style={{ color: unclippedRangeMaxColor }}>
+            max {unclippedRangeMax.toPrecision(4)}
+          </span>
         </div>
 
         {isDiffExp ? (

client/src/components/continuous/continuous.js

@@ -64,18 +64,15 @@ class Continuous extends React.Component {
         ) : null}
         {obsAnnotations
           ? _.map(obsAnnotations.colIndex.keys(), key => {
-              const summary = obsAnnotations.col(key).summarize();
               const isColorField =
                 key.includes("color") || key.includes("Color");
+              if (key === "name" || isColorField) return null;
+
+              const summary = obsAnnotations.col(key).summarize();
               const nonFiniteExtent =
                 summary.min === undefined || summary.max === undefined;
-              zebra += 1;
+              if (!summary.categorical && !nonFiniteExtent) {
-              if (
+                zebra += 1;
-                !summary.categorical &&
-                key !== "name" &&
-                !isColorField &&
-                !nonFiniteExtent
-              ) {
                 return (
                   <HistogramBrush
                     key={key}

client/src/components/continuousLegend/index.js

@@ -112,6 +112,7 @@ class ContinuousLegend extends React.Component {
     const { colorAccessor, responsive, colorScale } = this.props;
     if (
       prevProps.colorAccessor !== colorAccessor ||
+      prevProps.colorScale !== colorScale ||
       prevProps.responsive.height !== responsive.height ||
       prevProps.responsive.width !== responsive.width
     ) {

client/src/components/geneExpression/index.js

@@ -22,7 +22,6 @@ import {
   keepAroundErrorToast
 } from "../framework/toasters";
 import ExpressionButtons from "./expressionButtons";
-import finiteExtent from "../../util/finiteExtent";
 
 const renderGene = (fuzzySortResult, { handleClick, modifiers, query }) => {
   if (!modifiers.matchesPredicate) {

client/src/components/graph/graph.js

@@ -1,6 +1,5 @@
 // jshint esversion: 6
 import React from "react";
-import _ from "lodash";
 import * as d3 from "d3";
 import { connect } from "react-redux";
 import mat4 from "gl-mat4";
@@ -12,7 +11,9 @@ import {
   Popover,
   Menu,
   MenuItem,
-  Position
+  Position,
+  NumericInput,
+  Icon
 } from "@blueprintjs/core";
 
 import * as globals from "../../globals";
@@ -29,6 +30,8 @@ import { World } from "../../util/stateManager";
   world: state.world,
   universe: state.universe,
   crossfilter: state.crossfilter,
+  clipPercentileMin: Math.round(100 * (state.world?.clipQuantiles?.min ?? 0)),
+  clipPercentileMax: Math.round(100 * (state.world?.clipQuantiles?.max ?? 1)),
   responsive: state.responsive,
   colorRGB: state.colors.rgb,
   opacityForDeselectedCells: state.controls.opacityForDeselectedCells,
@@ -47,6 +50,30 @@ import { World } from "../../util/stateManager";
   currentSelection: state.graphSelection.selection
 }))
 class Graph extends React.Component {
+  static isValidDigitKeyEvent(e) {
+    /*
+    Return true if this event is necessary to enter a percent number input.
+    Return false if not.
+
+    Returns true for events with keys: backspace, control, alt, meta, [0-9],
+    or events that don't have a key.
+    */
+    if (e.key === null) return true;
+    if (e.ctrlKey || e.altKey || e.metaKey) return true;
+
+    // concept borrowed from blueprint's numericInputUtils:
+    // keys that print a single character when pressed have a `key` name of
+    // length 1. every other key has a longer `key` name (e.g. "Backspace",
+    // "ArrowUp", "Shift"). since none of those keys can print a character
+    // to the field--and since they may have important native behaviors
+    // beyond printing a character--we don't want to disable their effects.
+    const isSingleCharKey = e.key.length === 1;
+    if (!isSingleCharKey) return true;
+
+    const key = e.key.charCodeAt(0) - 48; /* "0" */
+    return key >= 0 && key <= 9;
+  }
+
   constructor(props) {
     super(props);
     this.count = 0;
@@ -62,7 +89,8 @@ class Graph extends React.Component {
       svg: null,
       tool: null,
       container: null,
-      mode: "select"
+      mode: "select",
+      pendingClipPercentiles: null
     };
   }
 
@@ -265,6 +293,7 @@ class Graph extends React.Component {
       * there are no userDefinedGenes or diffexpGenes displayed
       * scatterplot is not displayed
       * nothing in cellset1 or cellset2
+      * clip percentiles are [0,100]
     */
     const {
       crossfilter,
@@ -276,7 +305,9 @@ class Graph extends React.Component {
       scatterplotXXaccessor,
       scatterplotYYaccessor,
       celllist1,
-      celllist2
+      celllist2,
+      clipPercentileMin,
+      clipPercentileMax
     } = this.props;
 
     if (!crossfilter || !world || !universe) {
@@ -294,7 +325,9 @@ class Graph extends React.Component {
       nothingColoredBy &&
       noGenes &&
       scatterNotDpl &&
-      nothingInCellsets
+      nothingInCellsets &&
+      clipPercentileMax === 100 &&
+      clipPercentileMin === 0
     );
   };
 
@@ -306,6 +339,102 @@ class Graph extends React.Component {
     dispatch(actions.resetInterface());
   };
 
+  isClipDisabled = () => {
+    /*
+    return true if clip button should be disabled.
+    */
+    const { pendingClipPercentiles } = this.state;
+    const clipPercentileMin = pendingClipPercentiles?.clipPercentileMin;
+    const clipPercentileMax = pendingClipPercentiles?.clipPercentileMax;
+
+    const { world } = this.props;
+    const currentClipMin = 100 * world?.clipQuantiles?.min;
+    const currentClipMax = 100 * world?.clipQuantiles?.max;
+
+    // if you change this test, be careful with logic around
+    // comparisons between undefined / NaN handling.
+    const isDisabled =
+      !(clipPercentileMin < clipPercentileMax) ||
+      (clipPercentileMin === currentClipMin &&
+        clipPercentileMax === currentClipMax);
+
+    return isDisabled;
+  };
+
+  handleClipOnKeyPress = e => {
+    /* 
+    allow only numbers, plus other critical keys which 
+    may be required to make a number 
+    */
+    if (!Graph.isValidDigitKeyEvent(e)) {
+      e.preventDefault();
+    }
+  };
+
+  handleClipPercentileMinValueChange = v => {
+    /*
+    Ignore anything that isn't a legit number
+    */
+    if (!Number.isFinite(v)) return;
+
+    const { pendingClipPercentiles } = this.state;
+    const clipPercentileMax = pendingClipPercentiles?.clipPercentileMax;
+
+    /*
+    clamp to [0, currentClipPercentileMax]
+    */
+    if (v <= 0) v = 0;
+    if (v > 100) v = 100;
+    const clipPercentileMin = Math.round(v); // paranoia
+    this.setState({
+      pendingClipPercentiles: { clipPercentileMin, clipPercentileMax }
+    });
+  };
+
+  handleClipPercentileMaxValueChange = v => {
+    /*
+    Ignore anything that isn't a legit number
+    */
+    if (!Number.isFinite(v)) return;
+
+    const { pendingClipPercentiles } = this.state;
+    const clipPercentileMin = pendingClipPercentiles?.clipPercentileMin;
+
+    /*
+    clamp to [0, 100]
+    */
+    if (v < 0) v = 0;
+    if (v > 100) v = 100;
+    const clipPercentileMax = Math.round(v); // paranoia
+
+    this.setState({
+      pendingClipPercentiles: { clipPercentileMin, clipPercentileMax }
+    });
+  };
+
+  handleClipCommit = () => {
+    const { dispatch } = this.props;
+    const { pendingClipPercentiles } = this.state;
+    const { clipPercentileMin, clipPercentileMax } = pendingClipPercentiles;
+    const min = clipPercentileMin / 100;
+    const max = clipPercentileMax / 100;
+    dispatch({
+      type: "set clip quantiles",
+      clipQuantiles: { min, max }
+    });
+  };
+
+  handleClipOpening = () => {
+    const { clipPercentileMin, clipPercentileMax } = this.props;
+    this.setState({
+      pendingClipPercentiles: { clipPercentileMin, clipPercentileMax }
+    });
+  };
+
+  handleClipClosing = () => {
+    this.setState({ pendingClipPercentiles: null });
+  };
+
   brushToolUpdate(tool, container, offset) {
     /*
     this is called from componentDidUpdate(), so be very careful using
@@ -614,9 +743,20 @@ class Graph extends React.Component {
       libraryVersions,
       undoDisabled,
       redoDisabled,
-      selectionTool
+      selectionTool,
+      clipPercentileMin,
+      clipPercentileMax
     } = this.props;
-    const { mode } = this.state;
+    const { mode, pendingClipPercentiles } = this.state;
+
+    const clipMin =
+      pendingClipPercentiles?.clipPercentileMin ?? clipPercentileMin;
+    const clipMax =
+      pendingClipPercentiles?.clipPercentileMax ?? clipPercentileMax;
+    const activeClipClass =
+      clipPercentileMin > 0 || clipPercentileMax < 100
+        ? " bp3-intent-warning"
+        : "";
 
     // constants used to create selection tool button
     let selectionTooltip;
@@ -684,66 +824,175 @@ class Graph extends React.Component {
                 reset
               </AnchorButton>
             </Tooltip>
-            <div>
+            <div className="bp3-button-group">
-              <div className="bp3-button-group">
+              <Tooltip content={selectionTooltip} position="left">
-                <Tooltip content={selectionTooltip} position="left">
+                <Button
-                  <Button
+                  type="button"
-                    type="button"
+                  data-testid="mode-lasso"
-                    data-testid="mode-lasso"
+                  className={`bp3-button ${selectionButtonClass}`}
-                    className={`bp3-button ${selectionButtonClass}`}
+                  active={mode === "select"}
-                    active={mode === "select"}
+                  onClick={() => {
-                    onClick={() => {
+                    this.setState({ mode: "select" });
-                      this.setState({ mode: "select" });
+                  }}
-                    }}
+                  style={{
-                    style={{
+                    cursor: "pointer"
-                      cursor: "pointer"
+                  }}
-                    }}
+                />
-                  />
+              </Tooltip>
-                </Tooltip>
+              <Tooltip content="Pan and zoom" position="left">
-                <Tooltip content="Pan and zoom" position="left">
+                <Button
-                  <Button
+                  type="button"
-                    type="button"
+                  data-testid="mode-pan-zoom"
-                    data-testid="mode-pan-zoom"
+                  className="bp3-button bp3-icon-zoom-in"
-                    className="bp3-button bp3-icon-zoom-in"
+                  active={mode === "zoom"}
-                    active={mode === "zoom"}
+                  onClick={() => {
-                    onClick={() => {
+                    this.restartReglLoop();
-                      this.restartReglLoop();
+                    this.setState({ mode: "zoom" });
-                      this.setState({ mode: "zoom" });
+                  }}
-                    }}
+                  style={{
-                    style={{
+                    cursor: "pointer"
-                      cursor: "pointer"
+                  }}
-                    }}
+                />
-                  />
+              </Tooltip>
-                </Tooltip>
-                <Tooltip content="Undo" position="left">
-                  <AnchorButton
-                    type="button"
-                    className="bp3-button bp3-icon-undo"
-                    disabled={undoDisabled}
-                    onClick={() => {
-                      dispatch({ type: "@@undoable/undo" });
-                    }}
-                    style={{
-                      cursor: "pointer"
-                    }}
-                  />
-                </Tooltip>
-                <Tooltip content="Redo" position="left">
-                  <AnchorButton
-                    type="button"
-                    className="bp3-button bp3-icon-redo"
-                    disabled={redoDisabled}
-                    onClick={() => {
-                      dispatch({ type: "@@undoable/redo" });
-                    }}
-                    style={{
-                      cursor: "pointer"
-                    }}
-                  />
-                </Tooltip>
-              </div>
             </div>
-            <div style={{ marginLeft: 10 }}>
+            <div
+              className="bp3-button-group"
+              style={{
+                marginLeft: 10
+              }}
+            >
+              <Tooltip content="Undo" position="left">
+                <AnchorButton
+                  type="button"
+                  className="bp3-button bp3-icon-undo"
+                  disabled={undoDisabled}
+                  onClick={() => {
+                    dispatch({ type: "@@undoable/undo" });
+                  }}
+                  style={{
+                    cursor: "pointer"
+                  }}
+                />
+              </Tooltip>
+              <Tooltip content="Redo" position="left">
+                <AnchorButton
+                  type="button"
+                  className="bp3-button bp3-icon-redo"
+                  disabled={redoDisabled}
+                  onClick={() => {
+                    dispatch({ type: "@@undoable/redo" });
+                  }}
+                  style={{
+                    cursor: "pointer"
+                  }}
+                />
+              </Tooltip>
+            </div>
+            <div
+              className="bp3-button-group"
+              style={{
+                marginLeft: 10
+              }}
+            >
+              <Tooltip content="Visualization settings" position="left">
+                <Popover
+                  target={
+                    <Button
+                      type="button"
+                      className={`bp3-button bp3-icon-timeline-bar-chart ${activeClipClass}`}
+                      style={{
+                        cursor: "pointer"
+                      }}
+                    />
+                  }
+                  onOpening={this.handleClipOpening}
+                  onClosing={this.handleClipClosing}
+                  content={
+                    <div
+                      style={{
+                        display: "flex",
+                        justifyContent: "flex-start",
+                        alignItems: "flex-start",
+                        flexDirection: "column",
+                        padding: 10
+                      }}
+                    >
+                      <div>Clip all continuous values to percentile range</div>
+                      <div
+                        style={{
+                          display: "flex",
+                          justifyContent: "space-between",
+                          alignItems: "center",
+                          paddingTop: 5,
+                          paddingBottom: 5
+                        }}
+                      >
+                        <NumericInput
+                          style={{ width: 50 }}
+                          onValueChange={
+                            this.handleClipPercentileMinValueChange
+                          }
+                          onKeyPress={this.handleClipOnKeyPress}
+                          value={clipMin}
+                          min={0}
+                          max={100}
+                          fill={false}
+                          minorStepSize={null}
+                          rightElement={
+                            <div style={{ padding: "4px 2px" }}>
+                              <Icon
+                                icon="percentage"
+                                intent="primary"
+                                iconSize={14}
+                              />
+                            </div>
+                          }
+                        />
+                        <span style={{ marginRight: 5, marginLeft: 5 }}>
+                          {" "}
+                          -{" "}
+                        </span>
+                        <NumericInput
+                          style={{ width: 50 }}
+                          onValueChange={
+                            this.handleClipPercentileMaxValueChange
+                          }
+                          onKeyPress={this.handleClipOnKeyPress}
+                          value={clipMax}
+                          min={0}
+                          max={100}
+                          fill={false}
+                          minorStepSize={null}
+                          rightElement={
+                            <div style={{ padding: "4px 2px" }}>
+                              <Icon
+                                icon="percentage"
+                                intent="primary"
+                                iconSize={14}
+                              />
+                            </div>
+                          }
+                        />
+                        <span style={{ marginRight: 5, marginLeft: 5 }}> </span>
+                        <Button
+                          type="button"
+                          className="bp3-button"
+                          disabled={this.isClipDisabled()}
+                          style={{
+                            cursor: "pointer"
+                          }}
+                          onClick={this.handleClipCommit}
+                        >
+                          Clip
+                        </Button>
+                      </div>
+                    </div>
+                  }
+                />
+              </Tooltip>
+            </div>
+
+            <div style={{ marginLeft: 10 }} className="bp3-button-group">
               <Popover
                 content={
                   <Menu>
@@ -786,7 +1035,7 @@ class Graph extends React.Component {
               >
                 <Button
                   type="button"
-                  className="bp3-button bp3-icon-cog"
+                  className="bp3-button bp3-icon-info-sign"
                   style={{
                     cursor: "pointer"
                   }}

client/src/components/scatterplot/scatterplot.js

@@ -145,7 +145,8 @@ class Scatterplot extends React.Component {
 
     if (
       scatterplotXXaccessor !== prevProps.scatterplotXXaccessor || // was CLU now FTH1 etc
-      scatterplotYYaccessor !== prevProps.scatterplotYYaccessor // was CLU now FTH1 etc
+      scatterplotYYaccessor !== prevProps.scatterplotYYaccessor || // was CLU now FTH1 etc
+      world !== prevProps.world // shape or clip of world changed
     ) {
       const scales = Scatterplot.setupScales(expressionX, expressionY);
       this.drawAxesSVG(scales.xScale, scales.yScale, svg);
@@ -263,9 +264,15 @@ class Scatterplot extends React.Component {
 
     // the axes are much cleaner and easier now. No need to rotate and orient
     // the axis, just call axisBottom, axisLeft etc.
-    const xAxis = d3.axisBottom().scale(xScale);
+    const xAxis = d3
+      .axisBottom()
+      .ticks(7)
+      .scale(xScale);
 
-    const yAxis = d3.axisLeft().scale(yScale);
+    const yAxis = d3
+      .axisLeft()
+      .ticks(7)
+      .scale(yScale);
 
     // adding axes is also simpler now, just translate x-axis to (0,height)
     // and it's alread defined to be a bottom axis.

client/src/reducers/colors.js

@@ -27,6 +27,7 @@ const ColorsReducer = (
       };
     }
 
+    case "set clip quantiles":
     case "set World to current selection": {
       const { colorMode, colorAccessor } = state;
       const { world } = nextSharedState;

client/src/reducers/continuousSelection.js

@@ -17,6 +17,14 @@ const ContinuousSelection = (state = {}, action) => {
         [name]: action.range
       };
     }
+    case "continuous metadata histogram cancel": {
+      const name = makeContinuousDimensionName(
+        action.continuousNamespace,
+        action.selection
+      );
+      const { [name]: deletedField, ...newState } = state;
+      return newState;
+    }
     default: {
       return state;
     }

client/src/reducers/crossfilter.js

@@ -40,6 +40,7 @@ const CrossfilterReducer = (
       return crossfilter;
     }
 
+    case "set clip quantiles":
     case "set World to current selection": {
       const { userDefinedGenes, diffexpGenes } = prevSharedState.controls;
       const { world } = nextSharedState;

client/src/reducers/resetCache.js

@@ -1,7 +1,10 @@
 /*
-Reducer which caches derived state to be used in a reset
+Reducer which caches derived state to be used in a reset or other 
-*/
+recomputation.
 
+Currently this only caches the baseline (full universe) world & crossfilter, 
+for use in a Reset.
+*/
 const ResetCacheReducer = (
   state = {
     world: null,

client/src/reducers/undoable.js

@@ -51,6 +51,7 @@ history state processing.  The undoable action object contents, by key:
     filter state are entirely at the discretion of the action filter.
 
 */
+import fromEntries from "../util/fromEntries";
 
 const historyKeyPrefix = "@@undoable/";
 const pastKey = `${historyKeyPrefix}past`;
@@ -283,18 +284,4 @@ function push(arr, val, limit = undefined) {
   return narr;
 }
 
-function fromEntries(arr) {
-  /*
-  Similar to Object.fromEntries, but only handles array.
-  This could be replaced with the standard fucnction once it
-  is widely available.   As of 3/20/2019, it has not yet
-  been released in the Chrome stable channel.
-  */
-  const obj = {};
-  for (let i = 0, l = arr.length; i < l; i += 1) {
-    obj[arr[i][0]] = arr[i][1];
-  }
-  return obj;
-}
-
 export default Undoable;

client/src/reducers/undoableConfig.js

@@ -71,7 +71,8 @@ const saveOnActions = new Set([
   "store current cell selection as differential set 1",
   "store current cell selection as differential set 2",
 
-  "set World to current selection"
+  "set World to current selection",
+  "set clip quantiles"
 ]);
 
 /**
@@ -107,8 +108,8 @@ StateMachine when it doesn't know what to do.
 
 Signature:  (fsm, event, from) => undoableAction
 */
-const onFsmError = (fsm, name, from) => {
+const onFsmError = (fsm, event, from) => {
-  console.error("FSM error - unexpected history state", fsm, name, from);
+  console.error(`FSM error [event: "${event}", state: "${from}"]`, fsm);
   // In production, try to recover gracefully if we have unexpected state
   return clear(fsm);
 };

client/src/reducers/universe.js

@@ -1,5 +1,3 @@
-import _ from "lodash";
-
 import { ControlsHelpers } from "../util/stateManager";
 
 const Universe = (state = null, action, nextSharedState, prevSharedState) => {
@@ -12,9 +10,10 @@ const Universe = (state = null, action, nextSharedState, prevSharedState) => {
     case "expression load success": {
       let { varData } = state;
 
-      // Load new expression data into the varData dataframes, if
+      // Lazy load new expression data into the varData dataframe, if
       // not already present.
-      _.forEach(action.expressionData, (val, key) => {
+      //
+      Object.entries(action.expressionData).forEach(([key, val]) => {
         // If not already in universe.varData, save entire expression column
         if (!varData.hasCol(key)) {
           varData = varData.withCol(key, val);
@@ -22,14 +21,15 @@ const Universe = (state = null, action, nextSharedState, prevSharedState) => {
       });
 
       // Prune size of varData "cache" if getting out of hand....
+      //
       const { userDefinedGenes, diffexpGenes } = prevSharedState;
-      const allTheGenesWeNeed = _.uniq(
+      const allTheGenesWeNeed = [
-        [].concat(
+        ...new Set(
           userDefinedGenes,
           diffexpGenes,
           Object.keys(action.expressionData)
         )
-      );
+      ];
       varData = ControlsHelpers.pruneVarDataCache(varData, allTheGenesWeNeed);
 
       return {

client/src/reducers/world.js

@@ -1,6 +1,6 @@
-import _ from "lodash";
-
 import { World, ControlsHelpers } from "../util/stateManager";
+import clip from "../util/clip";
+import quantile from "../util/quantile";
 
 const WorldReducer = (
   state = null,
@@ -21,7 +21,7 @@ const WorldReducer = (
 
     case "set World to current selection": {
       /* Set viewable world to be the currently selected data */
-      const world = World.createWorldFromCurrentSelection(
+      const world = World.createWorldBySelection(
         action.universe,
         action.world,
         action.crossfilter
@@ -29,16 +29,27 @@ const WorldReducer = (
       return world;
     }
 
+    case "set clip quantiles": {
+      const world = World.createWorldWithNewClip(
+        prevSharedState.universe,
+        state,
+        prevSharedState.crossfilter,
+        action.clipQuantiles
+      );
+      return world;
+    }
+
     case "expression load success": {
       const { universe } = nextSharedState;
       const universeVarData = universe.varData;
-      let worldVarData = state.varData;
+      let unclippedVarData = state.unclipped.varData;
 
-      // Load new expression data into the varData dataframes, if
+      // Lazy load new expression data into the unclipped varData dataframe, if
       // not already present.
-      _.forEach(action.expressionData, (val, key) => {
+      //
+      Object.entries(action.expressionData).forEach(([key, val]) => {
         // If not already in world.varData, save sliced expression column
-        if (!worldVarData.hasCol(key)) {
+        if (!unclippedVarData.hasCol(key)) {
           // Slice if world !== universe, else just use whole column.
           // Use the obsAnnotation index as the cut key, as we keep
           // all world dataframes in sync.
@@ -51,7 +62,7 @@ const WorldReducer = (
           }
 
           // Now build world's varData dataframe
-          worldVarData = worldVarData.withCol(
+          unclippedVarData = unclippedVarData.withCol(
             key,
             worldValSlice,
             state.obsAnnotations.rowIndex
@@ -59,23 +70,55 @@ const WorldReducer = (
         }
       });
 
-      // Prune size of varData "cache" if getting out of hand....
+      // Prune size of varData unclipped dataframe if getting out of hand....
+      //
       const { userDefinedGenes, diffexpGenes } = prevSharedState;
-      const allTheGenesWeNeed = _.uniq(
+      const allTheGenesWeNeed = [
-        [].concat(
+        ...new Set(
           userDefinedGenes,
           diffexpGenes,
           Object.keys(action.expressionData)
         )
-      );
+      ];
-      worldVarData = ControlsHelpers.pruneVarDataCache(
+      unclippedVarData = ControlsHelpers.pruneVarDataCache(
-        worldVarData,
+        unclippedVarData,
         allTheGenesWeNeed
       );
 
+      // at this point, we have the unclipped data in unclippedVarData.
+      // Now create clipped.
+      //   - Drop columns no longer needed
+      //   - Add new columns
+      //
+      let clippedVarData = state.varData;
+      const keysToDrop = clippedVarData.colIndex
+        .keys()
+        .filter(k => !unclippedVarData.hasCol(k));
+      const keysToAdd = unclippedVarData.colIndex
+        .keys()
+        .filter(k => !clippedVarData.hasCol(k));
+      keysToDrop.forEach(k => {
+        clippedVarData = clippedVarData.dropCol(k);
+      });
+      keysToAdd.forEach(k => {
+        const data = unclippedVarData.col(k).asArray();
+        const q = [state.clipQuantiles.min, state.clipQuantiles.max];
+        const [qMinVal, qMaxVal] = quantile(q, data);
+        const clippedData = clip(data, qMinVal, qMaxVal, Number.NaN);
+        clippedVarData = clippedVarData.withCol(
+          k,
+          clippedData,
+          state.obsAnnotations.rowIndex
+        );
+      });
+
       return {
         ...state,
-        varData: worldVarData
+        varData: clippedVarData,
+        unclipped: {
+          ...state.unclipped,
+          varData: unclippedVarData
+        }
       };
     }
 

client/src/util/clip.js

@@ -0,0 +1,25 @@
+/*
+clip - clip all values in a Array or TypedArray, IN PLACE.
+
+Values in array are clipped if less than `lower` or greater than `upper`.
+
+If `setTo` is undefined, values less than `lower` will be set to `lower`,
+and values greater than `upper` will be set to `upper`.  
+
+If `setTo` is not undefined, values outside the [lower, upper] range will be set to 
+`setTo`.
+
+*/
+export default function clip(arr, lower, upper, setTo) {
+	const lowerSet = setTo === undefined ? lower : setTo;
+	const upperSet = setTo === undefined ? upper : setTo;
+	for (let i = 0, l = arr.length; i < l; i += 1) {
+		const v = arr[i];
+		if (v < lower) {
+			arr[i] = lowerSet;
+		} else if (v > upper) {
+			arr[i] = upperSet;
+		}
+	}
+	return arr;
+}

client/src/util/dataframe/dataframe.js

@@ -1,6 +1,6 @@
 import { IdentityInt32Index, isLabelIndex } from "./labelIndex";
 // weird cross-dependency that we should clean up someday...
-import { sort } from "../typedCrossfilter/sort";
+import { sortArray } from "../typedCrossfilter/sort";
 import { isTypedArray, isArrayOrTypedArray, callOnceLazy } from "./util";
 import { summarizeContinuous, summarizeCategorical } from "./summarize";
 
@@ -63,7 +63,13 @@ class Dataframe {
   Constructors & factories
   **/
 
-  constructor(dims, columnarData, rowIndex = null, colIndex = null) {
+  constructor(
+    dims,
+    columnarData,
+    rowIndex = null,
+    colIndex = null,
+    __columnsAccessor = [] // private interface
+  ) {
     /*
     The base constructor is relatively hard to use - as an alternative,
     see factory methods and clone/slice, below.
@@ -74,6 +80,9 @@ class Dataframe {
         or TypedArray of length nRows.
       * rowIndex/colIndex - null (create default index using offsets as key),
         or a caller-provided index.
+      * __columnsAccessor - private interface, do not specify.  Used internally
+        to improve caching of column accessors when possible (eg, clone(), 
+        dropCol(), withCol()).
     All columns and indices must have appropriate dimensionality.
     */
     const [nRows, nCols] = dims;
@@ -94,7 +103,7 @@ class Dataframe {
     this.rowIndex = rowIndex;
     this.colIndex = colIndex;
 
-    this.__compile();
+    this.__compile(__columnsAccessor);
   }
 
   static __errorChecks(dims, columnarData, rowIndex, colIndex) {
@@ -135,97 +144,107 @@ class Dataframe {
     }
   }
 
-  __compile() {
+  static __compileColumn(column, getOffset, getLabel) {
+    /*
+      Each column accessor is a function which will lookup data by
+      index (ie, is equivalent to dataframe.get(row, col), where 'col'
+      is fixed.
+
+      In addition, each column accessor has several functions:
+
+      asArray() -- return the entire column as a native Array or TypedArray.
+        Crucially, this native array only supports label indexing.
+        Example:
+          const arr = df.col('a').asArray();
+
+      has(rlabel) -- return boolean indicating of the row label
+        is contained within the column.  Example:
+          const isInColumn = df.col('a').includes(99)
+        For the default offset indexing, this is identical to:
+          const isInColumn = (99 > 0) && (99 < df.nRows);
+
+      ihas(roffset) -- same as has(), but accepts a row offset
+        instead of a row label.
+
+      indexOf(value) -- return the label (not offset) of the first instance of
+        'value' in the column.  If you want the offset, just use the builtin JS
+        indexOf() function, available on both Array and TypedArray.
+
+      iget(offset) -- return the value at 'offset'
+
+    */
+    const { length } = column;
+
+    /* get value by row label */
+    const get = function get(rlabel) {
+      return column[getOffset(rlabel)];
+    };
+
+    /* get value by row offset */
+    const iget = function iget(roffset) {
+      return column[roffset];
+    };
+
+    /* full column array access */
+    const asArray = function asArray() {
+      return column;
+    };
+
+    /* test for row label inclusion in column */
+    const has = function has(rlabel) {
+      const offset = getOffset(rlabel);
+      return offset >= 0 && offset < length;
+    };
+
+    const ihas = function ihas(offset) {
+      return offset >= 0 && offset < length;
+    };
+
+    /*
+    return first label (index) at which the value is found in this column,
+    or undefined if not found.
+
+    NOTE: not found return is DIFFERENT than the default Array.indexOf as
+    -1 is a plausible Dataframe row/col label.
+    */
+    const indexOf = function indexOf(value) {
+      const offset = column.indexOf(value);
+      if (offset === -1) {
+        return undefined;
+      }
+      return getLabel(offset);
+    };
+
+    /*
+    Summarize the column data. Lazy eval;
+    */
+    const summarize = callOnceLazy(() =>
+      isTypedArray(column)
+        ? summarizeContinuous(column)
+        : summarizeCategorical(column)
+    );
+
+    get.summarize = summarize;
+    get.asArray = asArray;
+    get.has = has;
+    get.ihas = ihas;
+    get.indexOf = indexOf;
+    get.iget = iget;
+    return get;
+  }
+
+  __compile(accessors) {
     /*
     Compile data accessors for each column.
 
-    Each column accessor is a function which will lookup data by
+    Use an existing accessor if provided, else compile a new one.
-    index (ie, is equivalent to dataframe.get(row, col), where 'col'
-    is fixed.
-
-    In addition, each column accessor has several functions:
-
-    asArray() -- return the entire column as a native Array or TypedArray.
-      Crucially, this native array only supports label indexing.
-      Example:
-        const arr = df.col('a').asArray();
-
-    has(rlabel) -- return boolean indicating of the row label
-      is contained within the column.  Example:
-        const isInColumn = df.col('a').includes(99)
-      For the default offset indexing, this is identical to:
-        const isInColumn = (99 > 0) && (99 < df.nRows);
-
-    ihas(roffset) -- same as has(), but accepts a row offset
-      instead of a row label.
-
-    indexOf(value) -- return the label (not offset) of the first instance of
-      'value' in the column.  If you want the offset, just use the builtin JS
-      indexOf() function, available on both Array and TypedArray.
-
-    iget(offset) -- return the value at 'offset'
-
     */
     const { getOffset, getLabel } = this.rowIndex;
-    this.__columnsAccessor = this.__columns.map(column => {
+    this.__columnsAccessor = this.__columns.map((column, idx) => {
-      const { length } = column;
+      if (accessors[idx]) {
-
+        return accessors[idx];
-      /* get value by row label */
+      }
-      const get = function get(rlabel) {
+      return Dataframe.__compileColumn(column, getOffset, getLabel);
-        return column[getOffset(rlabel)];
-      };
-
-      /* get value by row offset */
-      const iget = function iget(roffset) {
-        return column[roffset];
-      };
-
-      /* full column array access */
-      const asArray = function asArray() {
-        return column;
-      };
-
-      /* test for row label inclusion in column */
-      const has = function has(rlabel) {
-        const offset = getOffset(rlabel);
-        return offset >= 0 && offset < length;
-      };
-
-      const ihas = function ihas(offset) {
-        return offset >= 0 && offset < length;
-      };
-
-      /*
-      return first label (index) at which the value is found in this column,
-      or undefined if not found.
-
-      NOTE: not found return is DIFFERENT than the default Array.indexOf as
-      -1 is a plausible Dataframe row/col label.
-      */
-      const indexOf = function indexOf(value) {
-        const offset = column.indexOf(value);
-        if (offset === -1) {
-          return undefined;
-        }
-        return getLabel(offset);
-      };
-
-      /*
-      Summarize the column data. Lazy eval;
-      */
-      const summarize = callOnceLazy(() =>
-        isTypedArray(column)
-          ? summarizeContinuous(column)
-          : summarizeCategorical(column)
-      );
-
-      get.summarize = summarize;
-      get.asArray = asArray;
-      get.has = has;
-      get.ihas = ihas;
-      get.indexOf = indexOf;
-      get.iget = iget;
-      return get;
     });
   }
 
@@ -237,7 +256,8 @@ class Dataframe {
       this.dims,
       [...this.__columns],
       this.rowIndex,
-      this.colIndex
+      this.colIndex,
+      [...this.__columnsAccessor]
     );
   }
 
@@ -273,7 +293,14 @@ class Dataframe {
     const columns = [...this.__columns];
     columns.push(colData);
     const colIndex = this.colIndex.withLabel(label);
-    return new this.constructor(dims, columns, rowIndex, colIndex);
+    const columnsAccessor = [...this.__columnsAccessor];
+    return new this.constructor(
+      dims,
+      columns,
+      rowIndex,
+      colIndex,
+      columnsAccessor
+    );
   }
 
   dropCol(label) {
@@ -287,7 +314,15 @@ class Dataframe {
     const columns = [...this.__columns];
     columns.splice(coffset, 1);
     const colIndex = this.colIndex.dropLabel(label);
-    return new this.constructor(dims, columns, this.rowIndex, colIndex);
+    const columnsAccessor = [...this.__columnsAccessor];
+    columnsAccessor.splice(coffset, 1);
+    return new this.constructor(
+      dims,
+      columns,
+      this.rowIndex,
+      colIndex,
+      columnsAccessor
+    );
   }
 
   static empty(rowIndex = null, colIndex = null) {
@@ -316,7 +351,7 @@ class Dataframe {
       if (!offsets) {
         return [null, null];
       }
-      const sortedOffsets = sort(offsets);
+      const sortedOffsets = sortArray(offsets);
       const sortedLabels = new Array(sortedOffsets.length);
       for (let i = 0, l = sortedOffsets.length; i < l; i += 1) {
         sortedLabels[i] = index.getLabel(sortedOffsets[i]);
@@ -543,14 +578,34 @@ class Dataframe {
   /****
   Functional (map/reduce/etc) data access
 
-  XXX: not yet implemented, as there is no clear use case.   Can easily
+  TODO: most are not yet implemented, as there is no clear use case.   Can easily
   add these as useful.
   ****/
 
+  mapColumns(callback) {
+    /*
+    map all columns in the dataframe, returning a new dataframe comprised of the
+    return values, with the same index as the original dataframe.
+
+    callback MUST not modify the column, but instead return a mutated copy.
+    */
+    const columns = this.__columns.map(callback);
+    const columnsAccessor = columns.map((c, idx) =>
+      this.__columns[idx] === c ? this.__columnsAccessor[idx] : undefined
+    );
+    return new this.constructor(
+      this.dims,
+      columns,
+      this.rowIndex,
+      this.colIndex,
+      columnsAccessor
+    );
+  }
+
   /*
   Map & reduce of column or row
 
-  XXX TODO remainder of map/reduce functions:  mapCol, mapRow, reduceRow, ...
+  TODO remainder of map/reduce functions:  mapCol, mapRow, reduceRow, ...
   */
   /* comment out until we have a use for this
 

client/src/util/dataframe/summarize.js

@@ -1,32 +1,56 @@
 /*
 Private dataframe support functions
+
+TODO / XXX: for scalar/continuous data, this uses a naive method
+of computing quantiles.  Would be good to switch from sort to 
+partition at some point.
 */
 
+import quantile from "../quantile";
+import { sortArray } from "../typedCrossfilter/sort";
+
+// [ 0, 0.01, 0.02, ..., 1.0]
+const centileNames = new Array(101).fill(0).map((v, idx) => idx / 100);
+
 export function summarizeContinuous(col) {
   let min;
   let max;
   let nan = 0;
   let pinf = 0;
   let ninf = 0;
+  let percentiles;
   if (col) {
-    for (let r = 0, l = col.length; r < l; r += 1) {
+    // -Inf < finite < Inf < NaN
-      const val = Number(col[r]);
+    const sortedCol = sortArray(new col.constructor(col));
-      if (Number.isFinite(val)) {
+
-        if (min === undefined) {
+    // count non-finites, which are at each end of sorted data
-          min = val;
+    for (let i = sortedCol.length - 1; i >= 0; i -= 1) {
-          max = val;
+      if (!Number.isNaN(sortedCol[i])) {
-        } else {
+        nan = sortedCol.length - i - 1;
-          min = val < min ? val : min;
+        break;
-          max = val > max ? val : max;
-        }
-      } else if (Number.isNaN(val)) {
-        nan += 1;
-      } else if (val > 0) {
-        pinf += 1;
-      } else {
-        ninf += 1;
       }
     }
+    for (let i = 0, l = sortedCol.length; i < l; i += 1) {
+      if (sortedCol[i] !== Number.NEGATIVE_INFINITY) {
+        ninf = i;
+        break;
+      }
+    }
+    for (let i = sortedCol.length - nan - 1; i >= 0; i -= 1) {
+      if (sortedCol[i] !== Number.POSITIVE_INFINITY) {
+        pinf = sortedCol.length - i - nan - 1;
+        break;
+      }
+    }
+
+    // compute percentiles on finite data ONLY
+    const sortedColFiniteOnly = sortedCol.slice(
+      ninf,
+      sortedCol.length - nan - pinf
+    );
+    percentiles = quantile(centileNames, sortedColFiniteOnly, true);
+    min = percentiles[0];
+    max = percentiles[100];
   }
   return {
     categorical: false,
@@ -34,7 +58,8 @@ export function summarizeContinuous(col) {
     max,
     nan,
     pinf,
-    ninf
+    ninf,
+    percentiles
   };
 }
 

client/src/util/dataframe/util.js

@@ -2,16 +2,7 @@
 Private utility code for dataframe
 */
 
-export function isTypedArray(x) {
+export { isTypedArray, isArrayOrTypedArray } from "../typeHelpers";
-  return (
-    ArrayBuffer.isView(x) &&
-    Object.prototype.toString.call(x) !== "[object DataView]"
-  );
-}
-
-export function isArrayOrTypedArray(x) {
-  return Array.isArray(x) || isTypedArray(x);
-}
 
 export function callOnceLazy(f) {
   let value;

client/src/util/fromEntries.js

@@ -0,0 +1,13 @@
+export default function fromEntries(arr) {
+	/*
+	Similar to Object.fromEntries, but only handles array.
+	This could be replaced with the standard fucnction once it
+	is widely available.   As of 3/20/2019, it has not yet
+	been released in the Chrome stable channel.
+	*/
+	const obj = {};
+	for (let i = 0, l = arr.length; i < l; i += 1) {
+		obj[arr[i][0]] = arr[i][1];
+	}
+	return obj;
+}

client/src/util/quantile.js

@@ -0,0 +1,29 @@
+/*
+quantiles - calculate quantiles for the typed array.
+
+Currently interpolates to 'lower' value.
+
+Arguments:
+
+	* quantArr - array of quantiles to compute, where values:  0 <= value <= 1.0
+	* tarr - a typed array
+	* sorted - option bool.  If false (default), will assume array is not sorted.
+	 	If true, will assume it is sorted.
+
+*/
+
+import { sortArray } from "./typedCrossfilter/sort";
+
+export default function quantile(quantArr, tarr, sorted = false) {
+	/*
+	start with the naive (sort) implementation.  Later, use a faster partition
+	*/
+	const arr = sorted ? tarr : sortArray(new tarr.constructor(tarr)); // copy
+	const len = arr.length;
+	return quantArr.map(q => {
+		if (q === 1) {
+			return arr[len - 1];
+		}
+		return arr[Math.floor(q * len)];
+	});
+}

client/src/util/stateManager/universe.js

@@ -4,6 +4,8 @@ import _ from "lodash";
 
 import decodeMatrixFBS from "./matrix";
 import * as Dataframe from "../dataframe";
+import fromEntries from "../fromEntries";
+import { isFpTypedArray } from "../typeHelpers";
 
 /*
 Private helper function - create and return a template Universe
@@ -37,14 +39,57 @@ These functions are used exclusively by the actions and reducers to
 build an internal POJO for use by the rendering components.
 */
 
+function promoteTypedArray(o) {
+  /*
+  Decide what internal data type to use for the data returned from 
+  the server.
+
+  TODO - future optimization: not all int32/uint32 data series require
+  promotion to float64.  We COULD simply look at the data to decide. 
+  */
+  if (isFpTypedArray(o) || Array.isArray(o)) return o;
+
+  let TyepdArrayCtor;
+  switch (o.constructor) {
+    case Int8Array:
+    case Uint8Array:
+    case Uint8ClampedArray:
+    case Int16Array:
+    case Uint16Array:
+      TyepdArrayCtor = Float32Array;
+      break;
+
+    case Int32Array:
+    case Uint32Array:
+      TyepdArrayCtor = Float64Array;
+      break;
+
+    default:
+      throw new Error("Unexpected data type returned from server.");
+  }
+  if (o.constructor === TyepdArrayCtor) return o;
+  return new TyepdArrayCtor(o);
+}
+
 function AnnotationsFBSToDataframe(arrayBuffer) {
   /*
   Convert a Matrix FBS to a Dataframe.
+
+  The application has strong assumptions that all scalar data will be
+  stored as a float32 or float64 (regardless of underlying data types).
+  For example, clipping of value ranges (eg, user-selected percentiles)
+
+  All float data from the server is left as is.  All non-float is promoted
+  to an appropriate float.
   */
-  const fbs = decodeMatrixFBS(arrayBuffer);
+  const fbs = decodeMatrixFBS(arrayBuffer, true); // leave in place
+  const columns = fbs.columns.map(c => {
+    if (isFpTypedArray(c) || Array.isArray(c)) return c;
+    return promoteTypedArray(c);
+  });
   const df = new Dataframe.Dataframe(
     [fbs.nRows, fbs.nCols],
-    fbs.columns,
+    columns,
     null,
     new Dataframe.KeyIndex(fbs.colIdx)
   );
@@ -53,6 +98,10 @@ function AnnotationsFBSToDataframe(arrayBuffer) {
 
 function LayoutFBSToDataframe(arrayBuffer) {
   const fbs = decodeMatrixFBS(arrayBuffer, true);
+  if (fbs.columns.length !== 2 || !fbs.columns.every(isFpTypedArray)) {
+    // We have strong assumptions about the shape & type of layout data.
+    throw new Error("Unexpected layout data type returned from server");
+  }
   const df = new Dataframe.Dataframe(
     [fbs.nRows, fbs.nCols],
     fbs.columns,
@@ -122,6 +171,14 @@ export function createUniverseFromResponse(
   }
 
   reconcileSchemaCategoriesWithSummary(universe);
+
+  /* Index schema for ease of use */
+  universe.schema.annotations.obsByName = fromEntries(
+    universe.schema.annotations.obs.map(v => [v.name, v])
+  );
+  universe.schema.annotations.varByName = fromEntries(
+    universe.schema.annotations.var.map(v => [v.name, v])
+  );
   return universe;
 }
 
@@ -140,6 +197,11 @@ export function convertDataFBStoObject(universe, arrayBuffer) {
   const { colIdx, columns } = fbs;
   const result = {};
 
+  if (!columns.every(isFpTypedArray)) {
+    // We have strong assumptions that all var data is float
+    throw new Error("Unexpected non-floating point response from server.");
+  }
+
   for (let c = 0; c < colIdx.length; c += 1) {
     const varName = universe.varAnnotations.at(colIdx[c], "name");
     result[varName] = columns[c];

client/src/util/stateManager/world.js

@@ -1,7 +1,14 @@
 // jshint esversion: 6
 
-import { layoutDimensionName, obsAnnoDimensionName } from "../nameCreators";
+import clip from "../clip";
+import {
+  layoutDimensionName,
+  obsAnnoDimensionName,
+  diffexpDimensionName,
+  userDefinedDimensionName
+} from "../nameCreators";
 import * as Dataframe from "../dataframe";
+import ImmutableTypedCrossfilter from "../typedCrossfilter/crossfilter";
 
 /*
 
@@ -19,6 +26,8 @@ Notable keys in the world object:
 
 * schema: data schema from the server
 
+* clipQuantiles: the quantiles used to clip all data in world.
+
 * obsAnnotations:
 
   Dataframe containing obs annotations.  Columns are indexed by annotation
@@ -37,78 +46,192 @@ Notable keys in the world object:
 * varData: a cache of expression columns, stored in a Dataframe.  Cache
   managed by controls reducer.
 
+* unclipped: will contain unclipped variants of all potentiall clipped
+  dataframes (obsAnnotations, varData).
+
 */
 
 function templateWorld() {
+  const obsAnnotations = Dataframe.Dataframe.empty();
+  const varAnnotations = Dataframe.Dataframe.empty();
+  const obsLayout = Dataframe.Dataframe.empty();
+  const varData = Dataframe.Dataframe.empty(null, new Dataframe.KeyIndex());
   return {
     /* schema/version related */
     schema: null,
     nObs: 0,
     nVar: 0,
+    clipQuantiles: { min: 0, max: 1 },
 
     /* annotations */
-    obsAnnotations: Dataframe.Dataframe.empty(),
+    obsAnnotations,
-    varAnnotations: Dataframe.Dataframe.empty(),
+    varAnnotations,
 
     /* layout of graph. Dataframe. */
-    obsLayout: Dataframe.Dataframe.empty(),
+    obsLayout,
 
-    /*
+    /* Var data columns - subset of all data (may be empty) */
-    Var data columns - subset of all data (may be empty)
+    varData,
-    */
+
-    varData: Dataframe.Dataframe.empty(null, new Dataframe.KeyIndex())
+    /* unclipped dataframes - subset, but not value clipped */
+    unclipped: {
+      obsAnnotations,
+      varData
+    }
   };
 }
 
+function clipDataframe(
+  df,
+  lowerQuantile,
+  upperQuantile,
+  quantileF,
+  clipPredicate = () => true,
+  value = Number.NaN
+) {
+  /*
+  For all columns in the dataframe, clip all values above or below specified 
+  quantiles to `value` if clipPredicate returns True for that column (if it
+  returns false, skip the column entirely).
+
+  Returns a clipped copy - does not mutate original.
+
+  clipPredicate must have signature: (dataframe, colIndex, colLabel) => boolean
+  True signifies that the column should be clipped; false indicates that the
+  column should be left intact/unchanged.
+
+  quantileF must have signature:  (label, qval) => number
+  */
+  if (lowerQuantile < 0) lowerQuantile = 0;
+  if (upperQuantile > 1) upperQuantile = 1;
+  if (lowerQuantile === 0 && upperQuantile === 1) return df;
+
+  const keys = df.colIndex.keys();
+  return df.mapColumns((col, colIdx) => {
+    const colLabel = keys[colIdx];
+    if (!clipPredicate(df, colIdx, colLabel)) return col;
+
+    const colMin = quantileF(colLabel, lowerQuantile);
+    const colMax = quantileF(colLabel, upperQuantile);
+    const newCol = clip(col.slice(), colMin, colMax, value);
+    return newCol;
+  });
+}
+
+/*
+Create World with contents eq entire universe.   Commonly used to initialize World.
+If clipQuantiles
+*/
 export function createWorldFromEntireUniverse(universe) {
   const world = templateWorld();
 
-  /*
-  public interface follows
-  */
-
   /* Schema related */
   world.schema = universe.schema;
   world.nObs = universe.nObs;
   world.nVar = universe.nVar;
+  world.clipQuantiles = { min: 0, max: 1 };
 
-  /* annotation dataframes */
+  /* dataframes: annotations and layout */
-  world.obsAnnotations = universe.obsAnnotations;
+  world.obsAnnotations = universe.obsAnnotations.clone();
-  world.varAnnotations = universe.varAnnotations;
+  world.varAnnotations = universe.varAnnotations.clone();
+  world.obsLayout = universe.obsLayout.clone();
 
-  /* layout and display characteristics dataframe */
+  /* Var dataframe - contains a subset of all var columns */
-  world.obsLayout = universe.obsLayout;
-
-  /*
-  Var data columns - subset of all
-  */
   world.varData = universe.varData.clone();
 
+  /* save unclipped copies of potentially clipped dataframes */
+  world.unclipped = {
+    obsAnnotations: world.obsAnnotations.clone(),
+    varData: world.varData.clone()
+  };
+
   return world;
 }
 
-export function createWorldFromCurrentSelection(universe, world, crossfilter) {
+/*
-  const newWorld = templateWorld();
+clip dataframes based on quantiles.
 
-  /* these don't change as only OBS are selected in our current implementation */
+This is an in-place operation on the world object provided as an argument.
-  newWorld.nVar = universe.nVar;
+The values in world.unclipped are clipped and assigned to world.obsAnnotations
-  newWorld.schema = universe.schema;
+and world.varData.
-  newWorld.varAnnotations = universe.varAnnotations;
+*/
+function setClippedDataframes(world) {
+  const { schema } = world;
+  const isContinuousObsAnnotation = (df, idx, label) =>
+    deduceDimensionType(schema.annotations.obsByName[label], label) !== "enum";
+  const obsQuantile = (label, q) =>
+    world.unclipped.obsAnnotations.col(label).summarize().percentiles[100 * q];
+  world.obsAnnotations = clipDataframe(
+    world.unclipped.obsAnnotations,
+    world.clipQuantiles.min,
+    world.clipQuantiles.max,
+    obsQuantile,
+    isContinuousObsAnnotation
+  );
 
-  /* now subset/cut obs */
+  const varDataQuantile = (label, q) =>
+    world.unclipped.varData.col(label).summarize().percentiles[100 * q];
+  world.varData = clipDataframe(
+    world.unclipped.varData,
+    world.clipQuantiles.min,
+    world.clipQuantiles.max,
+    varDataQuantile,
+    () => true
+  );
+}
+
+/*
+Subset the current world based upon the current selection, maintaining any existing
+clip.  Returns new world.  Parameters:
+  * unvierse
+  * world - the current world
+  * crossfilter - the selection state
+*/
+export function createWorldBySelection(universe, world, crossfilter) {
+  const newWorld = { ...world, obsLayout: null, unclipped: {}, varData: null };
+
+  /* subset unclipped dataframes based upon current selection */
   const mask = crossfilter.allSelectedMask();
-  newWorld.obsAnnotations = world.obsAnnotations.isubsetMask(mask);
   newWorld.obsLayout = world.obsLayout.isubsetMask(mask);
-  newWorld.nObs = newWorld.obsAnnotations.dims[0];
+  newWorld.unclipped.obsAnnotations = world.unclipped.obsAnnotations.isubsetMask(
-
+    mask
-  /*
+  );
-  Var data columns - subset of all
+  if (world.unclipped.varData.isEmpty()) {
-  */
+    newWorld.unclipped.varData = world.unclipped.varData.clone();
-  if (world.varData.isEmpty()) {
-    newWorld.varData = world.varData.clone();
   } else {
-    newWorld.varData = world.varData.isubsetMask(mask);
+    newWorld.unclipped.varData = world.unclipped.varData.isubsetMask(mask);
   }
+  /* subsetting changings dimension size */
+  newWorld.nObs = newWorld.unclipped.obsAnnotations.dims[0];
+
+  /* and now clip */
+  setClippedDataframes(newWorld);
+  return newWorld;
+}
+
+/*
+Change clip quantiles on the current world, returning a new world.
+Parameters:
+  * universe
+  * world - current world
+  * clipQuantiles - new clip
+*/
+export function createWorldWithNewClip(
+  universe,
+  world,
+  crossfilter,
+  clipQuantiles
+) {
+  const newWorld = { ...world, obsAnnotation: null, varData: null };
+  newWorld.clipQuantiles = clipQuantiles;
+  newWorld.obsLayout = world.obsLayout.clone();
+  newWorld.unclipped = {
+    obsAnnotations: world.unclipped.obsAnnotations.clone(),
+    varData: world.unclipped.varData.clone()
+  };
+
+  /* and now clip */
+  setClippedDataframes(newWorld);
   return newWorld;
 }
 
@@ -166,7 +289,10 @@ export function createObsDimensions(crossfilter, world) {
 }
 
 export function worldEqUniverse(world, universe) {
-  return world.obsAnnotations === universe.obsAnnotations;
+  return (
+    world.obsAnnotations === universe.obsAnnotations ||
+    world.obsAnnotations.rowIndex === universe.obsAnnotations.rowIndex
+  );
 }
 
 export function getSelectedByIndex(crossfilter) {

client/src/util/typeHelpers.js

@@ -0,0 +1,29 @@
+/*
+Various type and schema related helper functions.
+*/
+
+/*
+Utility function to test for a typed array
+*/
+export function isTypedArray(x) {
+	return (
+		ArrayBuffer.isView(x) &&
+		Object.prototype.toString.call(x) !== "[object DataView]"
+	);
+}
+
+/*
+Test for float typed array, ie, Float32TypedArray or Float64TypedArray
+*/
+export function isFpTypedArray(x) {
+	let constructor;
+	const isFloatArray =
+		x &&
+		({ constructor } = x) &&
+		(constructor === Float32Array || constructor === Float64Array);
+	return isFloatArray;
+}
+
+export function isArrayOrTypedArray(x) {
+	return Array.isArray(x) || isTypedArray(x);
+}

client/src/util/typedCrossfilter/crossfilter.js

@@ -2,13 +2,13 @@ import { polygonContains } from "d3";
 
 import PositiveIntervals from "./positiveIntervals";
 import BitArray from "./bitArray";
-import { sort } from "./sort";
 import {
-  makeSortIndex,
+  sortArray,
   lowerBound,
   lowerBoundIndirect,
   upperBoundIndirect
-} from "./util";
+} from "./sort";
+import { makeSortIndex } from "./util";
 
 class NotImplementedError extends Error {
   constructor(...params) {
@@ -61,6 +61,10 @@ export default class ImmutableTypedCrossfilter {
     return Object.keys(this.dimensions);
   }
 
+  hasDimension(name) {
+    return !!this.dimensions[name];
+  }
+
   addDimension(name, type, ...rest) {
     /*
     Add a new dimension to this crossfilter, of type DimensionType.
@@ -284,15 +288,15 @@ class _ImmutableBaseDimension {
     this.name = name;
   }
 
-  /* eslint-disable class-methods-use-this */
   select(spec) {
     const { mode } = spec;
     if (mode === undefined) {
       throw new Error("select spec does not contain 'mode'");
     }
-    throw new Error(`select mode ${mode} not implemented`);
+    throw new Error(
+      `select mode ${mode} not implemented by dimension ${this.name}`
+    );
   }
-  /* eslint-enable class-methods-use-this */
 }
 
 class ImmutableScalarDimension extends _ImmutableBaseDimension {
@@ -414,7 +418,7 @@ class ImmutableEnumDimension extends ImmutableScalarDimension {
     for (let i = 0; i < len; i += 1) {
       s.add(mapf(i, data));
     }
-    const enumIndex = sort(Array.from(s));
+    const enumIndex = sortArray(Array.from(s));
     this.enumIndex = enumIndex;
 
     // create dimension value array

client/src/util/typedCrossfilter/sort.js

@@ -1,5 +1,29 @@
-const SmallArray = 32;
+import { isTypedArray, isFpTypedArray } from "../typeHelpers";
 
+/* eslint no-bitwise: "off" */
+
+/*
+ ** fast sort and search, with separate code paths for floats (NaN ordering),
+ ** indirect and direct search/sort.
+ */
+
+/*
+Comparators for float sort.   -Infinity < finite < Infinity < NaN
+*/
+function lt(a, b) {
+  if (Number.isNaN(b)) return !Number.isNaN(a);
+  return a < b;
+}
+
+function gt(a, b) {
+  if (Number.isNaN(a)) return !Number.isNaN(b);
+  return a > b;
+}
+
+/*
+insertion sort, used for small arrays (controlled by SMALL_ARRAY constant)
+*/
+const SMALL_ARRAY = 32;
 function insertionsort(a, lo, hi) {
   for (let i = lo + 1; i < hi + 1; i += 1) {
     const x = a[i];
@@ -12,6 +36,18 @@ function insertionsort(a, lo, hi) {
   return a;
 }
 
+function insertionsortFloats(a, lo, hi) {
+  for (let i = lo + 1; i < hi + 1; i += 1) {
+    const x = a[i];
+    let j;
+    for (j = i; j > lo && gt(a[j - 1], x); j -= 1) {
+      a[j] = a[j - 1];
+    }
+    a[j] = x;
+  }
+  return a;
+}
+
 function insertionsortIndirect(a, s, lo, hi) {
   for (let i = lo + 1; i < hi + 1; i += 1) {
     const x = a[i];
@@ -25,8 +61,24 @@ function insertionsortIndirect(a, s, lo, hi) {
   return a;
 }
 
+function insertionsortFloatsIndirect(a, s, lo, hi) {
+  for (let i = lo + 1; i < hi + 1; i += 1) {
+    const x = a[i];
+    const t = s[x];
+    let j;
+    for (j = i; j > lo && gt(s[a[j - 1]], t); j -= 1) {
+      a[j] = a[j - 1];
+    }
+    a[j] = x;
+  }
+  return a;
+}
+
+/*
+Quicksort - used for larger arrays
+*/
 function quicksort(a, lo, hi) {
-  if (hi - lo < SmallArray) {
+  if (hi - lo < SMALL_ARRAY) {
     return insertionsort(a, lo, hi);
   }
   if (lo < hi) {
@@ -55,8 +107,38 @@ function quicksort(a, lo, hi) {
   return a;
 }
 
+function quicksortFloats(a, lo, hi) {
+  if (hi - lo < SMALL_ARRAY) {
+    return insertionsortFloats(a, lo, hi);
+  }
+  if (lo < hi) {
+    // partition
+    const mid = Math.floor((lo + hi) / 2);
+    const p = a[mid];
+    let i = lo - 1;
+    let j = hi + 1;
+    while (i < j) {
+      do {
+        i += 1;
+      } while (lt(a[i], p));
+      do {
+        j -= 1;
+      } while (gt(a[j], p));
+      if (i < j) {
+        const tmp = a[i];
+        a[i] = a[j];
+        a[j] = tmp;
+      }
+    }
+    // sort
+    quicksortFloats(a, lo, j);
+    quicksortFloats(a, j + 1, hi);
+  }
+  return a;
+}
+
 function quicksortIndirect(a, s, lo, hi) {
-  if (hi - lo < SmallArray) {
+  if (hi - lo < SMALL_ARRAY) {
     return insertionsortIndirect(a, s, lo, hi);
   }
   if (lo < hi) {
@@ -86,15 +168,248 @@ function quicksortIndirect(a, s, lo, hi) {
   return a;
 }
 
-// Convenience wrappers
+function quicksortFloatsIndirect(a, s, lo, hi) {
-export function sort(arr, comparator = undefined) {
+  if (hi - lo < SMALL_ARRAY) {
-  if (comparator !== undefined) {
+    return insertionsortFloatsIndirect(a, s, lo, hi);
-    // XXX for now
-    return arr.sort(arr, comparator);
   }
-  return quicksort(arr, 0, arr.length - 1);
+  if (lo < hi) {
+    // partition
+    const mid = Math.floor((lo + hi) / 2);
+    const p = a[mid];
+    const t = s[p];
+    let i = lo - 1;
+    let j = hi + 1;
+    while (i < j) {
+      do {
+        i += 1;
+      } while (lt(s[a[i]], t));
+      do {
+        j -= 1;
+      } while (gt(s[a[j]], t));
+      if (i < j) {
+        const tmp = a[i];
+        a[i] = a[j];
+        a[j] = tmp;
+      }
+    }
+    // sort
+    quicksortFloatsIndirect(a, s, lo, j);
+    quicksortFloatsIndirect(a, s, j + 1, hi);
+  }
+  return a;
+}
+
+/*
+Convenience wrappers, handling optimization paths and default
+handlers for NaN comparisons.  Sorts in place.
+*/
+export function sortArray(arr) {
+  if (Array.isArray(arr)) {
+    return quicksort(arr, 0, arr.length - 1);
+  }
+  if (isTypedArray(arr)) {
+    if (isFpTypedArray(arr)) {
+      return quicksortFloats(arr, 0, arr.length - 1);
+    }
+    return quicksort(arr, 0, arr.length - 1);
+  }
+  /* else unsupported */
+  throw new Error("sortArray received unsupported object type");
 }
 
 export function sortIndex(index, source) {
+  if (isFpTypedArray(source))
+    return quicksortFloatsIndirect(index, source, 0, index.length - 1);
   return quicksortIndirect(index, source, 0, index.length - 1);
 }
+
+// Search for `value` in the sorted array `arr`, in the range [first, last).
+// Return the first (left most) index where arr[index] >= value.
+//
+// In other words, return array index I where:
+//  arr[i] < value for all tarr[lo:I]
+//  arr[i] >= value for all tarr[I:last]
+//
+// The same semantics/behavior as:
+//    C++: lower_bound()
+//    Python: bisect.bisect_left()
+//
+function lowerBoundNonFloat(valueArray, value, first, last) {
+  let lfirst = first;
+  let llast = last;
+  // this is just a binary search
+  while (lfirst < llast) {
+    const middle = (lfirst + llast) >>> 1;
+    if (valueArray[middle] < value) {
+      lfirst = middle + 1;
+    } else {
+      llast = middle;
+    }
+  }
+  return lfirst;
+}
+
+// lowerBound, but with NaN handling
+//
+// If the underlying array is a Float32Array or Float64Array, will enforce
+// the ordering -Infinity < finite < Infinity < NaN.
+//
+function lowerBoundFloat(valueArray, value, first, last) {
+  let lfirst = first;
+  let llast = last;
+  // this is just a binary search
+  while (lfirst < llast) {
+    const middle = (lfirst + llast) >>> 1;
+    if (lt(valueArray[middle], value)) {
+      lfirst = middle + 1;
+    } else {
+      llast = middle;
+    }
+  }
+  return lfirst;
+}
+
+export function lowerBound(valueArray, value, first, last) {
+  if (isFpTypedArray(valueArray)) {
+    return lowerBoundFloat(valueArray, value, first, last);
+  }
+  return lowerBoundNonFloat(valueArray, value, first, last);
+}
+
+// Inlined performance optimization - used to indirect through a sort map.
+//
+function lowerBoundNonFloatIndirect(
+  valueArray,
+  indexArray,
+  value,
+  first,
+  last
+) {
+  let lfirst = first;
+  let llast = last;
+  // this is just a binary search
+  while (lfirst < llast) {
+    const middle = (lfirst + llast) >>> 1;
+    if (valueArray[indexArray[middle]] < value) {
+      lfirst = middle + 1;
+    } else {
+      llast = middle;
+    }
+  }
+  return lfirst;
+}
+
+function lowerBoundFloatIndirect(valueArray, indexArray, value, first, last) {
+  let lfirst = first;
+  let llast = last;
+  // this is just a binary search
+  while (lfirst < llast) {
+    const middle = (lfirst + llast) >>> 1;
+    if (lt(valueArray[indexArray[middle]], value)) {
+      lfirst = middle + 1;
+    } else {
+      llast = middle;
+    }
+  }
+  return lfirst;
+}
+
+export function lowerBoundIndirect(valueArray, indexArray, value, first, last) {
+  if (isFpTypedArray(valueArray)) {
+    return lowerBoundFloatIndirect(valueArray, indexArray, value, first, last);
+  }
+  return lowerBoundNonFloatIndirect(valueArray, indexArray, value, first, last);
+}
+
+// Search for `value in the sorted array `arr`, in the range [first, last).
+// Return the first value where arr[index] > value.
+//
+// In other words, return array index I, where:
+//  arr[i] <= value for all tarr[lo:I]
+//  arr[i] > value for all tarr[I:last]
+//
+// The same semantics/behavior as:
+//    C++: upper_bound()
+//    Python: bisect.bisect_right()
+//
+function upperBoundNonFloat(valueArray, value, first, last) {
+  let lfirst = first;
+  let llast = last;
+  // this is just a binary search
+  while (lfirst < llast) {
+    const middle = (lfirst + llast) >>> 1;
+    if (valueArray[middle] > value) {
+      llast = middle;
+    } else {
+      lfirst = middle + 1;
+    }
+  }
+  return lfirst;
+}
+
+function upperBoundFloat(valueArray, value, first, last) {
+  let lfirst = first;
+  let llast = last;
+  // this is just a binary search
+  while (lfirst < llast) {
+    const middle = (lfirst + llast) >>> 1;
+    if (gt(valueArray[middle], value)) {
+      llast = middle;
+    } else {
+      lfirst = middle + 1;
+    }
+  }
+  return lfirst;
+}
+
+export function upperBound(valueArray, value, first, last) {
+  if (isFpTypedArray(valueArray)) {
+    return upperBoundFloat(valueArray, value, first, last);
+  }
+  return upperBoundNonFloat(valueArray, value, first, last);
+}
+
+// Inline performance optimization
+//
+function upperBoundNonFloatIndirect(
+  valueArray,
+  indexArray,
+  value,
+  first,
+  last
+) {
+  let lfirst = first;
+  let llast = last;
+  // this is just a binary search
+  while (lfirst < llast) {
+    const middle = (lfirst + llast) >>> 1;
+    if (valueArray[indexArray[middle]] > value) {
+      llast = middle;
+    } else {
+      lfirst = middle + 1;
+    }
+  }
+  return lfirst;
+}
+
+function upperBoundFloatIndirect(valueArray, indexArray, value, first, last) {
+  let lfirst = first;
+  let llast = last;
+  // this is just a binary search
+  while (lfirst < llast) {
+    const middle = (lfirst + llast) >>> 1;
+    if (gt(valueArray[indexArray[middle]], value)) {
+      llast = middle;
+    } else {
+      lfirst = middle + 1;
+    }
+  }
+  return lfirst;
+}
+
+export function upperBoundIndirect(valueArray, indexArray, value, first, last) {
+  if (isFpTypedArray(valueArray)) {
+    return upperBoundFloatIndirect(valueArray, indexArray, value, first, last);
+  }
+  return upperBoundNonFloatIndirect(valueArray, indexArray, value, first, last);
+}

client/src/util/typedCrossfilter/util.js

@@ -1,5 +1,4 @@
 // jshint esversion: 6
-/* eslint no-bitwise: "off" */
 
 import { sortIndex } from "./sort";
 
@@ -36,93 +35,3 @@ export function makeSortIndex(src) {
   sortIndex(index, src);
   return index;
 }
-
-// Search for `value` in the sorted array `arr`, in the range [first, last).
-// Return the first (left most) index where arr[index] >= value.
-//
-// In other words, return array index I where:
-//  arr[i] < value for all tarr[lo:I]
-//  arr[i] >= value for all tarr[I:last]
-//
-// The same semantics/behavior as:
-//    C++: lower_bound()
-//    Python: bisect.bisect_left()
-//
-// XXX: it is likely that there would be minimal performance hit from creating
-// a factory version of lowerBound that takes an accessor (rather than having
-// a special-cased version for lining the indirection).
-//
-export function lowerBound(valueArray, value, first, last) {
-  let lfirst = first;
-  let llast = last;
-  // this is just a binary search
-  while (lfirst < llast) {
-    const middle = (lfirst + llast) >>> 1;
-    if (valueArray[middle] < value) {
-      lfirst = middle + 1;
-    } else {
-      llast = middle;
-    }
-  }
-  return lfirst;
-}
-
-// Inlined performance optimization - used to indirect through a sort map.
-//
-export function lowerBoundIndirect(valueArray, indexArray, value, first, last) {
-  let lfirst = first;
-  let llast = last;
-  // this is just a binary search
-  while (lfirst < llast) {
-    const middle = (lfirst + llast) >>> 1;
-    if (valueArray[indexArray[middle]] < value) {
-      lfirst = middle + 1;
-    } else {
-      llast = middle;
-    }
-  }
-  return lfirst;
-}
-
-// Search for `value in the sorted array `arr`, in the range [first, last).
-// Return the first value where arr[index] > value.
-//
-// In other words, return array index I, where:
-//  arr[i] <= value for all tarr[lo:I]
-//  arr[i] > value for all tarr[I:last]
-//
-// The same semantics/behavior as:
-//    C++: upper_bound()
-//    Python: bisect.bisect_right()
-//
-export function upperBound(valueArray, value, first, last) {
-  let lfirst = first;
-  let llast = last;
-  // this is just a binary search
-  while (lfirst < llast) {
-    const middle = (lfirst + llast) >>> 1;
-    if (valueArray[middle] > value) {
-      llast = middle;
-    } else {
-      lfirst = middle + 1;
-    }
-  }
-  return lfirst;
-}
-
-// Inline performance optimization
-//
-export function upperBoundIndirect(valueArray, indexArray, value, first, last) {
-  let lfirst = first;
-  let llast = last;
-  // this is just a binary search
-  while (lfirst < llast) {
-    const middle = (lfirst + llast) >>> 1;
-    if (valueArray[indexArray[middle]] > value) {
-      llast = middle;
-    } else {
-      lfirst = middle + 1;
-    }
-  }
-  return lfirst;
-}