From 76446ca7d446e227e7ecccea35cadc8850402e12 Mon Sep 17 00:00:00 2001
From: bkmartinjr <bruce@chanzuckerberg.com>
Date: Tue, 10 Jul 2018 13:52:22 -0700
Subject: [PATCH 1/6] start at groups implementation for typedCrossfilter

---
 src/util/typedCrossfilter/bitArray.js |  44 +++++++--
 src/util/typedCrossfilter/index.js    | 131 ++++++++++++++++++++++++++
 2 files changed, 167 insertions(+), 8 deletions(-)

diff --git a/src/util/typedCrossfilter/bitArray.js b/src/util/typedCrossfilter/bitArray.js
index 49258aca..c022f4e2 100644
--- a/src/util/typedCrossfilter/bitArray.js
+++ b/src/util/typedCrossfilter/bitArray.js
@@ -116,7 +116,7 @@ class BitArray {
     // all selection tests assume unallocated dimensions are zero valued.
     this.deselectAll(dim);
     const col = dim >>> 5;
-    this.bitmask[col] &= ~(1 << (dim % 32));
+    this.bitmask[col] &= ~(1 << dim % 32);
     this.dimensionCount--;
   }
 
@@ -129,7 +129,35 @@ class BitArray {
 
     for (let w = 0; w < width; w++) {
       const bitmask = this.bitmask[w];
-      if (!bitmask || bitarray[w * length + index] !== bitmask) return false;
+
+      // Wha??  XXXX
+      // if (!bitmask || bitarray[w * length + index] !== bitmask) return false;
+      if (bitmask && bitarray[w * length + index] !== bitmask) return false;
+    }
+    return true;
+  }
+
+  // return true if this index is selected in ALL dimensions IGNORING dim
+  //
+  isSelectedIgnoringDim(index, dim) {
+    const ignoreOffset = dim >>> 5;
+    const ignoreMask = ~(1 << dim % 32);
+
+    const width = this.width;
+    const length = this.length;
+    const bitarray = this.bitarray;
+
+    for (let w = 0; w < width; w++) {
+      const bitmask = this.bitmask[w];
+      if (w === ignoreOffset) {
+        if (
+          bitmask &&
+          (bitarray[w * length + index] & ignoreMask) !== (bitmask & ignoreMask)
+        )
+          return false;
+      } else {
+        if (bitmask && bitarray[w * length + index] !== bitmask) return false;
+      }
     }
     return true;
   }
@@ -139,7 +167,7 @@ class BitArray {
   selectOne(dim, index) {
     const col = dim >>> 5;
     const before = this.bitarray[col * this.length + index];
-    const after = before | (1 << (dim % 32));
+    const after = before | (1 << dim % 32);
     this.bitarray[col * this.length + index] = after;
   }
 
@@ -148,7 +176,7 @@ class BitArray {
   deselectOne(dim, index) {
     const col = dim >>> 5;
     const before = this.bitarray[col * this.length + index];
-    const after = before & ~(1 << (dim % 32));
+    const after = before & ~(1 << dim % 32);
     this.bitarray[col * this.length + index] = after;
   }
 
@@ -158,7 +186,7 @@ class BitArray {
     let col = dim >> 5;
     const bitmask = this.bitmask[col];
     const bitarray = this.bitarray;
-    const one = 1 << (dim % 32);
+    const one = 1 << dim % 32;
     for (let i = col * this.length, len = i + this.length; i < len; i++) {
       bitarray[i] |= one;
     }
@@ -170,7 +198,7 @@ class BitArray {
     let col = dim >> 5;
     const bitmask = this.bitmask[col];
     const bitarray = this.bitarray;
-    const zero = ~(1 << (dim % 32));
+    const zero = ~(1 << dim % 32);
     for (let i = col * this.length, len = i + this.length; i < len; i++) {
       bitarray[i] &= zero;
     }
@@ -184,7 +212,7 @@ class BitArray {
     const first = range[0];
     const last = range[1];
     const bitarray = this.bitarray;
-    const one = 1 << (dim % 32);
+    const one = 1 << dim % 32;
     const offset = col * this.length;
     for (let i = first; i < last; i++) {
       bitarray[offset + indirect[i]] |= one;
@@ -198,7 +226,7 @@ class BitArray {
     const first = range[0];
     const last = range[1];
     const bitarray = this.bitarray;
-    const zero = ~(1 << (dim % 32));
+    const zero = ~(1 << dim % 32);
     const offset = col * this.length;
     for (let i = first; i < last; i++) {
       bitarray[offset + indirect[i]] &= zero;
diff --git a/src/util/typedCrossfilter/index.js b/src/util/typedCrossfilter/index.js
index 48cc0425..da8cfc5c 100644
--- a/src/util/typedCrossfilter/index.js
+++ b/src/util/typedCrossfilter/index.js
@@ -120,6 +120,9 @@ class ScalarDimension {
     // create sort index
     this.index = Util.fillRange(new Uint32Array(this.crossfilter.data.length));
     this.index.sort((a, b) => array[a] - array[b]);
+
+    // groups, if any
+    this.groups = [];
   }
 
   _createValueArray(value, array) {
@@ -134,12 +137,15 @@ class ScalarDimension {
 
   dispose() {
     this.crossfilter._freeDimension(this._id);
+    return this;
   }
 
   id() {
     return this._id;
   }
 
+  // Argument is an array of intervals indicating records newly selected/filtered
+  //
   _updateFilters(newFilter) {
     newFilter = PositiveIntervals.canonicalize(newFilter);
 
@@ -323,6 +329,16 @@ class ScalarDimension {
 
     return ret;
   }
+
+  group(groupValue) {
+    const grp = new ScalarGroup(groupValue, this.value.constructor, this);
+    this.groups.push(grp);
+    return grp;
+  }
+
+  _freeGroup(group) {
+    this.groups = this.groups.filter(e => e !== group);
+  }
 }
 
 // Ordered enumeration - supports any sortable enumerable type, eg,
@@ -379,6 +395,121 @@ class EnumDimension extends ScalarDimension {
   }
 }
 
+// Groups!  Map/reduce
+//
+// XXX: potential optimizations not implemented
+//  - groupValue may be identity - could skip creating separate group map
+//  - only works for scalars so far
+//
+class ScalarGroup {
+  constructor(groupValue, groupValueType, dimension) {
+    // parent dimension
+    this.dimension = dimension;
+
+    // groupValue is optional.  Defaults to identity.  Used to perform
+    // initial map operation.
+    //
+    if (groupValue === undefined) {
+      this.mapValue = dimension.value;
+    } else {
+      const data = dimension.value;
+      const len = data.length;
+      const array = new groupValueType(dimension.value.length);
+      for (let i = 0; i < len; i++) {
+        array[i] = groupValue(data[i]);
+      }
+      this.mapValue = array;
+    }
+
+    this.groups = [];
+
+    // group index is mapping from data record index to group index
+    this.groupIndex = new Uint32Array(dimension.crossfilter.data.length);
+
+    // default to counting
+    this.reduceCount();
+
+    // Creates this.groups
+    this._reduce();
+  }
+
+  // Update the group reduction incrementally
+  _updateReduce(adds, dels) {
+    // XXX
+    throw new Error("unimplemented");
+  }
+
+  // Reduce the entire data set, creating both the group index and the
+  // groups data.
+  //
+  _reduce() {
+    const dimension = this.dimension;
+    const data = dimension.crossfilter.data;
+
+    // Create groups
+    const groupNames = new Set(this.mapValue);
+    this.groups = [];
+    const groupIndexByName = {};
+    groupNames.forEach(name => {
+      this.groups.push({ key: name, value: this.reduceInitial() });
+      groupIndexByName[name] = this.groups.length - 1;
+    });
+
+    // Create groupIndex - index map between data record index and group index
+    for (let i = 0, len = this.mapValue.length; i < len; i++) {
+      // this.groupIndex[index[i]] = groupIndexByName[this.mapValue[i]];
+      this.groupIndex[i] = groupIndexByName[this.mapValue[i]];
+    }
+
+    // reduce all filtered records, IGNORING the current dimension's filter
+    const selection = dimension.crossfilter.selection;
+    for (let i = 0, len = data.length; i < len; i++) {
+      if (selection.isSelectedIgnoringDim(i, dimension.id())) {
+        const group = this.groups[this.groupIndex[i]];
+        group.value = this.reduceAdd(group.value, data[i]);
+      }
+    }
+  }
+
+  dispose() {
+    this.dimension._freeGroup(this);
+    return this;
+  }
+
+  // return number of distinct values in the group, independent of any filters.
+  //
+  size() {
+    return this.groups.length;
+  }
+
+  // Set the reduce functions and return the grouping.
+  //
+  reduce(add, remove, initial) {
+    this.reduceAdd = add;
+    this.reduceRemove = remove;
+    this.reduceInitial = initial;
+    this._reduce();
+    return this;
+  }
+
+  // set the reduce functions to count records.
+  reduceCount() {
+    return this.reduce((p, v) => p + 1, (p, v) => p - 1, () => 0);
+  }
+
+  // set the reduce functions to sum records using specified value accessor.
+  //
+  reduceSum(value) {
+    return this.reduce((p, v) => p + value(v), (p, v) => p - value(v), () => 0);
+  }
+
+  all() {
+    const res = [...this.groups];
+    res.sort((a, b) => (a.key < b.key ? -1 : a.key > b.key ? 1 : 0));
+    return res;
+  }
+}
+
 // Wrapper for backwards compat with crossfilter.
 //
 function crossfilter(data) {

From 5400df25caed7dad595c234c516093250a8be790 Mon Sep 17 00:00:00 2001
From: bkmartinjr <bruce@chanzuckerberg.com>
Date: Wed, 1 Aug 2018 16:32:31 -0700
Subject: [PATCH 2/6] finish group() implementation

---
 src/util/typedCrossfilter/bitArray.js |   5 +-
 src/util/typedCrossfilter/index.js    | 151 ++++++++++++++++++++------
 2 files changed, 120 insertions(+), 36 deletions(-)

diff --git a/src/util/typedCrossfilter/bitArray.js b/src/util/typedCrossfilter/bitArray.js
index c022f4e2..667f359c 100644
--- a/src/util/typedCrossfilter/bitArray.js
+++ b/src/util/typedCrossfilter/bitArray.js
@@ -129,10 +129,7 @@ class BitArray {
 
     for (let w = 0; w < width; w++) {
       const bitmask = this.bitmask[w];
-
-      // Wha??  XXXX
-      // if (!bitmask || bitarray[w * length + index] !== bitmask) return false;
-      if (bitmask && bitarray[w * length + index] !== bitmask) return false;
+      if (!bitmask || bitarray[w * length + index] !== bitmask) return false;
     }
     return true;
   }
diff --git a/src/util/typedCrossfilter/index.js b/src/util/typedCrossfilter/index.js
index da8cfc5c..aae260b9 100644
--- a/src/util/typedCrossfilter/index.js
+++ b/src/util/typedCrossfilter/index.js
@@ -149,35 +149,49 @@ class ScalarDimension {
   _updateFilters(newFilter) {
     newFilter = PositiveIntervals.canonicalize(newFilter);
 
+    // XXX removed optimization for now
     // special case optimization - select all/none can bypass
     // more complex work and just clobber everything.
     //
-    if (newFilter.length === 0) {
-      this.crossfilter.selection.deselectAll(this._id);
-    } else if (
-      newFilter.length === 1 &&
-      newFilter[0][0] === 0 &&
-      newFilter[0][1] == this.index.length
-    ) {
-      this.crossfilter.selection.selectAll(this._id);
-    } else {
-      const adds = PositiveIntervals.difference(newFilter, this.currentFilter);
-      const dels = PositiveIntervals.difference(this.currentFilter, newFilter);
-      dels.forEach(interval =>
-        this.crossfilter.selection.deselectIndirectFromRange(
-          this._id,
-          this.index,
-          interval
-        )
-      );
-      adds.forEach(interval =>
-        this.crossfilter.selection.selectIndirectFromRange(
-          this._id,
-          this.index,
-          interval
-        )
-      );
-    }
+    // if (newFilter.length === 0) {
+    //   this.crossfilter.selection.deselectAll(this._id);
+    // } else if (
+    //   newFilter.length === 1 &&
+    //   newFilter[0][0] === 0 &&
+    //   newFilter[0][1] == this.index.length
+    // ) {
+    //   this.crossfilter.selection.selectAll(this._id);
+    // } else {
+
+    const adds = PositiveIntervals.difference(newFilter, this.currentFilter);
+    const dels = PositiveIntervals.difference(this.currentFilter, newFilter);
+
+    this.crossfilter.filters.forEach(f =>
+      f.dim.groups.forEach(grp => grp._updateReduceDel(this, dels))
+    );
+
+    dels.forEach(interval =>
+      this.crossfilter.selection.deselectIndirectFromRange(
+        this._id,
+        this.index,
+        interval
+      )
+    );
+
+    adds.forEach(interval =>
+      this.crossfilter.selection.selectIndirectFromRange(
+        this._id,
+        this.index,
+        interval
+      )
+    );
+
+    this.crossfilter.filters.forEach(f =>
+      f.dim.groups.forEach(grp => grp._updateReduceAdd(this, adds))
+    );
+
+    // XXX removed optimization
+    // }
 
     this.currentFilter = newFilter;
   }
@@ -393,13 +407,19 @@ class EnumDimension extends ScalarDimension {
       )
     );
   }
+
+  group(groupValue) {
+    const grp = new EnumGroup(groupValue, this.value.constructor, this);
+    this.groups.push(grp);
+    return grp;
+  }
 }
 
 // Groups!  Map/reduce
 //
 // XXX: potential optimizations not implemented
 //  - groupValue may be identity - could skip creating separate group map
-//  - only works for scalars so far
+//  - only works for scalars so far (no enum)
 //
 class ScalarGroup {
   constructor(groupValue, groupValueType, dimension) {
@@ -433,10 +453,60 @@ class ScalarGroup {
     this._reduce();
   }
 
-  // Update the group reduction incrementally
-  _updateReduce(adds, dels) {
-    // XXX
-    throw new Error("unimplemented");
+  // Update the group reduction incrementally.  Called when *any* dimension filter
+  // changes.  Guaranteed to be called AFTER the crossfilter is updated.
+  //
+  // Arguments:
+  //  * dim: the dimension that is changing
+  //  * intv: interval list of newly selected values on `dim` (adds)
+  //
+  _updateReduceAdd(dim, intv) {
+    // console.log("_updateReduceAdd", dim, intv);
+
+    // ignore updates to self, as we don't reduce inclusive of our filter
+    if (dim === this.dimension || intv.length === 0) return;
+
+    // Each item in the range was just added to `dim`.  It was NOT previously
+    // selected - reduceAdd if it is now selected.
+    const selection = this.dimension.crossfilter.selection;
+    const data = this.dimension.crossfilter.data;
+    intv.forEach(rng => {
+      for (let r = rng[0]; r < rng[1]; r++) {
+        const i = dim.index[r];
+        if (selection.isSelectedIgnoringDim(i, this.dimension.id())) {
+          const group = this.groups[this.groupIndex[i]];
+          group.value = this.reduceAdd(group.value, data[i]);
+        }
+      }
+    });
+  }
+
+  // Update the group reduction incrementally.  Called when *any* dimension filter
+  // changes.   Guaranteed to be called BEFORE the crossfilter is updated.
+  //
+  // Arguments:
+  //  * dim: the dimension that is changing
+  //  * intv: interval list of previously selected values on `dim` (dels)
+  //
+  _updateReduceDel(dim, intv) {
+    // console.log("_updateReduceDel", dim, intv);
+
+    // ignore updates to self, as we don't reduce inclusive of our filter
+    if (dim === this.dimension || intv.length === 0) return;
+
+    // Each item in the range will be remved from `dim`.  reduceRemove if it
+    // is currently selected.
+    const selection = this.dimension.crossfilter.selection;
+    const data = this.dimension.crossfilter.data;
+    intv.forEach(rng => {
+      for (let r = rng[0]; r < rng[1]; r++) {
+        const i = dim.index[r];
+        if (selection.isSelectedIgnoringDim(i, this.dimension.id())) {
+          const group = this.groups[this.groupIndex[i]];
+          group.value = this.reduceRemove(group.value, data[i]);
+        }
+      }
+    });
   }
 
   // Reduce the entire data set, creating both the group index and the
@@ -457,7 +527,6 @@ class ScalarGroup {
 
     // Create groupIndex - index map between data record index and group index
     for (let i = 0, len = this.mapValue.length; i < len; i++) {
-      // this.groupIndex[index[i]] = groupIndexByName[this.mapValue[i]];
       this.groupIndex[i] = groupIndexByName[this.mapValue[i]];
     }
 
@@ -510,6 +579,24 @@ class ScalarGroup {
   }
 }
 
+class EnumGroup extends ScalarGroup {
+  constructor(groupValue, groupValueType, dimension) {
+    super(groupValue, groupValueType, dimension);
+  }
+
+  all() {
+    const res = [];
+    this.groups.forEach(e =>
+      res.push({
+        key: this.dimension.enumIndex[e.key],
+        value: e.value
+      })
+    );
+    res.sort((a, b) => (a.key < b.key ? -1 : a.key > b.key ? 1 : 0));
+    return res;
+  }
+}
+
 // Wrapper for backwards compat with crossfilter.
 //
 function crossfilter(data) {

From a39e0c79a6bc30ac1e4b1c10b51963d8634967d5 Mon Sep 17 00:00:00 2001
From: bkmartinjr <bruce@chanzuckerberg.com>
Date: Sat, 4 Aug 2018 13:03:43 -0700
Subject: [PATCH 3/6] add Jest configuration for node-based testing

---
 client/package.json | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/client/package.json b/client/package.json
index 1d458a9d..cffd9aff 100644
--- a/client/package.json
+++ b/client/package.json
@@ -4,10 +4,8 @@
   "license": "MIT",
   "repository": "https://github.com/chanzuckerberg/cellxgene",
   "scripts": {
-    "build":
-      "npm run clean && webpack --config configuration/webpack/webpack.config.prod.js",
-    "dev":
-      "npm run clean && webpack --config configuration/webpack/webpack.config.dev.js",
+    "build": "npm run clean && webpack --config configuration/webpack/webpack.config.prod.js",
+    "dev": "npm run clean && webpack --config configuration/webpack/webpack.config.dev.js",
     "clean": "rimraf build",
     "start": "node server/development.js",
     "lint": "eslint src",
@@ -117,6 +115,7 @@
   "jest": {
     "testMatch": [
       "**/__tests__/**/?(*.)(spec|test).js?(x)"
-    ]
+    ],
+    "testURL": "http://localhost/"
   }
 }

From f839653c00c0e42f94bb4b1e12a2167f9f73a7a1 Mon Sep 17 00:00:00 2001
From: bkmartinjr <bruce@chanzuckerberg.com>
Date: Sat, 4 Aug 2018 13:04:09 -0700
Subject: [PATCH 4/6] add Jest tests for new crossfilter group functions

---
 .../__tests__/util/typedCrossfilter.test.js   | 186 ++++++++++++++++++
 1 file changed, 186 insertions(+)

diff --git a/client/__tests__/util/typedCrossfilter.test.js b/client/__tests__/util/typedCrossfilter.test.js
index 00e68495..1ebbcaf1 100644
--- a/client/__tests__/util/typedCrossfilter.test.js
+++ b/client/__tests__/util/typedCrossfilter.test.js
@@ -101,6 +101,33 @@ const someData = [
   }
 ];
 
+function groupReduce(data, valueMap, valueReduce, valueInit) {
+  return _
+    .reduce(
+      data,
+      (acc, value) => {
+        const k = valueMap(value);
+        let r = _.find(acc, o => o.key === k);
+        if (!r) {
+          r = { key: k, value: valueInit() };
+          acc.push(r);
+        }
+        r.value = valueReduce(r.value, value);
+        return acc;
+      },
+      []
+    )
+    .sort((a, b) => (a.key < b.key ? -1 : a.key > b.key ? 1 : 0));
+}
+
+function groupCount(data, map) {
+  return groupReduce(data, map, (p, v) => p + 1, () => 0);
+}
+
+function groupSum(data, map) {
+  return groupReduce(data, map, (p, v) => (p += map(v)), () => 0);
+}
+
 var payments = null;
 beforeEach(() => {
   payments = crossfilter(someData);
@@ -273,4 +300,163 @@ describe("typedCrossfilter", () => {
     dimMap[33].filterAll();
     expect(payments.allFiltered()).toEqual(someData);
   });
+
+  test("group, default mapping, default reducer, no filter", () => {
+    expect(payments).toBeDefined();
+
+    var quantity = payments.dimension(r => r.quantity, Int32Array);
+    var tip = payments.dimension(r => r.tip, Int32Array);
+    var type = payments.dimension(r => r.type, "enum");
+    var total = payments.dimension(r => r.total, Int32Array);
+
+    _.each(
+      {
+        tip: tip.group(r => r),
+        type: type.group(),
+        total: total.group(),
+        quantity: quantity.group()
+      },
+      (grp, k) => {
+        const whatWeExpect = groupCount(someData, v => v[k]);
+        expect(grp.all()).toEqual(whatWeExpect);
+        expect(grp.size()).toEqual(whatWeExpect.length);
+        expect(grp.dispose()).toEqual(grp);
+      }
+    );
+  });
+
+  test("group, custom map, default reducer, no filters", () => {
+    expect(payments).toBeDefined();
+
+    // custom mapping in groups only works for scalar types.  Enums do not
+    // currently implement it.
+
+    const tip = payments.dimension(r => r.tip, Int32Array);
+    const totalX10 = payments.dimension(r => r.total * 10, Int32Array);
+    const type = payments.dimension(r => r.type, "enum");
+
+    const paymentsByTip_A = tip.group();
+    const paymentsByTip_B = tip.group(r => 10 * r);
+    const paymentsByType = type.group(); // identity only
+    const paymentsByTotalX10_A = totalX10.group();
+    const paymentsByTotalX10_B = totalX10.group(r => r / 10);
+
+    expect(paymentsByTip_A.all()).toEqual(groupCount(someData, v => v.tip));
+    expect(paymentsByTip_B.all()).toEqual(
+      groupCount(someData, v => 10 * v.tip)
+    );
+    expect(paymentsByType.all()).toEqual(groupCount(someData, v => v.type));
+    expect(paymentsByTotalX10_A.all()).toEqual(
+      groupCount(someData, v => 10 * v.total)
+    );
+    expect(paymentsByTotalX10_B.all()).toEqual(
+      groupCount(someData, v => (10 * v.total) / 10)
+    );
+
+    for (let i of [
+      paymentsByTip_A,
+      paymentsByTip_B,
+      paymentsByType,
+      paymentsByTotalX10_A,
+      paymentsByTotalX10_B,
+      tip,
+      totalX10,
+      type
+    ]) {
+      expect(i.dispose()).toEqual(i);
+    }
+  });
+
+  test("group, default map, custom reducer, no filters", () => {
+    expect(payments).toBeDefined();
+
+    const total = payments.dimension(r => r.total, Float32Array);
+    const type = payments.dimension(r => r.type, "enum");
+
+    const paymentsByTotal = total.group();
+    const paymentsByType = type.group();
+
+    // reduceCount
+    expect(paymentsByTotal.reduceCount()).toEqual(paymentsByTotal);
+    expect(paymentsByTotal.all()).toEqual(groupCount(someData, v => v.total));
+
+    // reduceSum
+    expect(paymentsByTotal.reduceSum(v => v.total)).toEqual(paymentsByTotal);
+    expect(paymentsByTotal.all()).toEqual(groupSum(someData, v => v.total));
+
+    // use custom reducers (my reducers) - count by three, init 1
+    expect(
+      paymentsByTotal.reduce((p, v) => (p += 3), (p, v) => (p -= 3), () => 1)
+    ).toEqual(paymentsByTotal);
+    expect(paymentsByTotal.all()).toEqual(
+      groupReduce(someData, v => v.total, (p, v) => p + 3, () => 1)
+    );
+
+    for (let i of [paymentsByTotal, paymentsByType, type]) {
+      expect(i.dispose()).toEqual(i);
+    }
+  });
+
+  test("group, default map, default reducer, filters", () => {
+    // From the docs:
+    // Note: a grouping intersects the crossfilter's current filters, except for the
+    // associated dimension's filter. Thus, group methods consider only records that
+    // satisfy every filter except this dimension's filter. So, if the crossfilter of
+    // payments is filtered by type and total, then group by total only observes the
+    // filter by type.
+
+    expect(payments).toBeDefined();
+
+    const tip = payments.dimension(r => r.tip, Int32Array);
+    const total = payments.dimension(r => r.total, Int32Array);
+    const type = payments.dimension(r => r.type, "enum");
+
+    const paymentsByTip = tip.group();
+    const paymentsByTotal = total.group();
+    const paymentsByType = type.group();
+
+    // 1. confirm that changing the filter on a dimension does NOT change that
+    // dimensions groups.
+    {
+      tip.filterAll(), total.filterAll(), type.filterAll();
+      let before = _.cloneDeep(paymentsByTip.all());
+      tip.filterExact(0);
+      expect(paymentsByTip.all()).toEqual(before);
+    }
+
+    // 2. confirm that changing a filter on a different dimension DOES change
+    // all other groups.
+    {
+      tip.filterAll(), total.filterAll(), type.filterAll();
+      const before = _.cloneDeep([paymentsByTotal.all(), paymentsByType.all()]);
+      tip.filterExact(0);
+      const after = [paymentsByTotal.all(), paymentsByType.all()];
+      expect(after).not.toEqual(before);
+      expect(after).toEqual([
+        groupReduce(
+          someData,
+          v => v.total,
+          (p, v) => (v.tip !== 0 ? p : p + 1),
+          () => 0
+        ),
+        groupReduce(
+          someData,
+          v => v.type,
+          (p, v) => (v.tip !== 0 ? p : p + 1),
+          () => 0
+        )
+      ]);
+    }
+
+    for (let i of [
+      paymentsByTip,
+      paymentsByTotal,
+      paymentsByType,
+      tip,
+      total,
+      type
+    ]) {
+      expect(i.dispose()).toEqual(i);
+    }
+  });
 });

From 8f56f0ce6e35a8913b925e3d88a8afda946e14e8 Mon Sep 17 00:00:00 2001
From: bkmartinjr <bruce@chanzuckerberg.com>
Date: Sat, 4 Aug 2018 13:05:50 -0700
Subject: [PATCH 5/6] crossfilters:  reverse ES6 module change; add group
 support for enumerated dimensions

---
 client/src/util/typedCrossfilter/bitArray.js  |   2 +-
 client/src/util/typedCrossfilter/index.js     | 118 +++++++++---------
 .../typedCrossfilter/positiveIntervals.js     |   2 +-
 client/src/util/typedCrossfilter/util.js      |  18 ++-
 4 files changed, 77 insertions(+), 63 deletions(-)

diff --git a/client/src/util/typedCrossfilter/bitArray.js b/client/src/util/typedCrossfilter/bitArray.js
index 2fa46acb..667f359c 100644
--- a/client/src/util/typedCrossfilter/bitArray.js
+++ b/client/src/util/typedCrossfilter/bitArray.js
@@ -251,4 +251,4 @@ class BitArray {
   }
 }
 
-export default BitArray;
+module.exports = BitArray;
diff --git a/client/src/util/typedCrossfilter/index.js b/client/src/util/typedCrossfilter/index.js
index 3d3b413e..57a4de56 100644
--- a/client/src/util/typedCrossfilter/index.js
+++ b/client/src/util/typedCrossfilter/index.js
@@ -29,9 +29,20 @@ more complex API.  In a few cases, elements of that API were incorporated.
 
 */
 
-import PositiveIntervals from "./positiveIntervals";
-import BitArray from "./bitArray";
-import {fillRange, lowerBound, lowerBoundIndirect, upperBound, upperBoundIndirect} from "./util";
+var PositiveIntervals = require("./positiveIntervals");
+var BitArray = require("./bitArray");
+var Util = require("./util");
+
+class NotImplementedError extends Error {
+  constructor(...params) {
+    super(...params);
+
+    // Maintains proper stack trace for where our error was thrown (only available on V8)
+    if (Error.captureStackTrace) {
+      Error.captureStackTrace(this, NotImplementedError);
+    }
+  }
+}
 
 class TypedCrossfilter {
   constructor(data) {
@@ -118,7 +129,7 @@ class ScalarDimension {
     this.value = array;
 
     // create sort index
-    this.index = fillRange(new Uint32Array(this.crossfilter.data.length));
+    this.index = Util.fillRange(new Uint32Array(this.crossfilter.data.length));
     this.index.sort((a, b) => array[a] - array[b]);
 
     // groups, if any
@@ -149,20 +160,6 @@ class ScalarDimension {
   _updateFilters(newFilter) {
     newFilter = PositiveIntervals.canonicalize(newFilter);
 
-    // XXX removed optimization for now
-    // special case optimization - select all/none can bypass
-    // more complex work and just clobber everything.
-    //
-    // if (newFilter.length === 0) {
-    //   this.crossfilter.selection.deselectAll(this._id);
-    // } else if (
-    //   newFilter.length === 1 &&
-    //   newFilter[0][0] === 0 &&
-    //   newFilter[0][1] == this.index.length
-    // ) {
-    //   this.crossfilter.selection.selectAll(this._id);
-    // } else {
-
     const adds = PositiveIntervals.difference(newFilter, this.currentFilter);
     const dels = PositiveIntervals.difference(this.currentFilter, newFilter);
 
@@ -190,23 +187,20 @@ class ScalarDimension {
       f.dim.groups.forEach(grp => grp._updateReduceAdd(this, adds))
     );
 
-    // XXX removed optimization
-    // }
-
     this.currentFilter = newFilter;
   }
 
   // filter by value - exact match
   filterExact(value) {
     const newFilter = [
-      lowerBoundIndirect(
+      Util.lowerBoundIndirect(
         this.value,
         this.index,
         value,
         0,
         this.value.length
       ),
-      upperBoundIndirect(
+      Util.upperBoundIndirect(
         this.value,
         this.index,
         value,
@@ -227,14 +221,14 @@ class ScalarDimension {
     const newFilter = [];
     for (let v = 0, len = values.length; v < len; v++) {
       const intv = [
-        lowerBoundIndirect(
+        Util.lowerBoundIndirect(
           this.value,
           this.index,
           values[v],
           0,
           this.value.length
         ),
-        upperBoundIndirect(
+        Util.upperBoundIndirect(
           this.value,
           this.index,
           values[v],
@@ -253,14 +247,14 @@ class ScalarDimension {
   filterRange(range) {
     const newFilter = [];
     const intv = [
-      lowerBoundIndirect(
+      Util.lowerBoundIndirect(
         this.value,
         this.index,
         range[0],
         0,
         this.value.length
       ),
-      upperBoundIndirect(
+      Util.upperBoundIndirect(
         this.value,
         this.index,
         range[1],
@@ -380,7 +374,7 @@ class EnumDimension extends ScalarDimension {
     const enumLen = this.enumIndex.length;
     for (let i = 0; i < len; i++) {
       const v = value(data[i]);
-      const e = lowerBound(this.enumIndex, v, 0, enumLen);
+      const e = Util.lowerBound(this.enumIndex, v, 0, enumLen);
       array[i] = e;
     }
     return array;
@@ -388,14 +382,14 @@ class EnumDimension extends ScalarDimension {
 
   filterExact(value) {
     return super.filterExact(
-      lowerBound(this.enumIndex, value, 0, this.enumIndex.length)
+      Util.lowerBound(this.enumIndex, value, 0, this.enumIndex.length)
     );
   }
 
   filterEnum(values) {
     return super.filterEnum(
       values.map(v =>
-        lowerBound(this.enumIndex, v, 0, this.enumIndex.length)
+        Util.lowerBound(this.enumIndex, v, 0, this.enumIndex.length)
       )
     );
   }
@@ -403,7 +397,7 @@ class EnumDimension extends ScalarDimension {
   filterRange(range) {
     return super.filterEnum(
       range.map(v =>
-        lowerBound(this.enumIndex, v, 0, this.enumIndex.length)
+        Util.lowerBound(this.enumIndex, v, 0, this.enumIndex.length)
       )
     );
   }
@@ -417,31 +411,13 @@ class EnumDimension extends ScalarDimension {
 
 // Groups!  Map/reduce
 //
-// XXX: potential optimizations not implemented
-//  - groupValue may be identity - could skip creating separate group map
-//  - only works for scalars so far (no enum)
-//
 class ScalarGroup {
   constructor(groupValue, groupValueType, dimension) {
     // parent dimension
     this.dimension = dimension;
 
-    // groupValue is optional.  Defaults to identity.  Used to perform
-    // initial map operation.
-    //
-    if (groupValue === undefined) {
-      this.mapValue = dimension.value;
-    } else {
-      const data = dimension.value;
-      const len = data.length;
-      const array = new groupValueType(dimension.value.length);
-      for (let i = 0; i < len; i++) {
-        array[i] = groupValue(data[i]);
-      }
-      this.mapValue = array;
-    }
-
-    this.groups = [];
+    // generate group names from dimension values
+    this.mapValue = this._map(groupValue, groupValueType, dimension);
 
     // group index is mapping from data record index to group index
     this.groupIndex = new Uint32Array(dimension.crossfilter.data.length);
@@ -453,6 +429,24 @@ class ScalarGroup {
     this._reduce();
   }
 
+  // internal support function - map all dimension values to group values.
+  //
+  _map(groupValue, groupValueType, dimension) {
+    // groupValue is optional.  Defaults to identity.  Used to perform
+    // initial map operation.
+    //
+    // identity: save some memory...
+    if (groupValue === undefined) return dimension.value;
+
+    const data = dimension.value;
+    const len = data.length;
+    const mapValue = new groupValueType(dimension.value.length);
+    for (let i = 0; i < len; i++) {
+      mapValue[i] = groupValue(data[i]);
+    }
+    return mapValue;
+  }
+
   // Update the group reduction incrementally.  Called when *any* dimension filter
   // changes.  Guaranteed to be called AFTER the crossfilter is updated.
   //
@@ -461,8 +455,6 @@ class ScalarGroup {
   //  * intv: interval list of newly selected values on `dim` (adds)
   //
   _updateReduceAdd(dim, intv) {
-    // console.log("_updateReduceAdd", dim, intv);
-
     // ignore updates to self, as we don't reduce inclusive of our filter
     if (dim === this.dimension || intv.length === 0) return;
 
@@ -489,8 +481,6 @@ class ScalarGroup {
   //  * intv: interval list of previously selected values on `dim` (dels)
   //
   _updateReduceDel(dim, intv) {
-    // console.log("_updateReduceDel", dim, intv);
-
     // ignore updates to self, as we don't reduce inclusive of our filter
     if (dim === this.dimension || intv.length === 0) return;
 
@@ -584,10 +574,26 @@ class EnumGroup extends ScalarGroup {
     super(groupValue, groupValueType, dimension);
   }
 
+  _map(groupValue, groupValueType, dimension) {
+    // groupValue is optional.  Defaults to identity.  Used to perform
+    // initial map operation.
+    //
+    // identity: save some memory
+    if (groupValue === undefined) return dimension.value;
+
+    // non-identity mapping unsupported for EnumDimension/EnumGroup.
+    // XXX: this could be implemented, but would require another index
+    // array to map from the group names/keys back to the dimension values.
+    // With this, we just rely on the dimensions `enumIndex` to map from
+    // enumeration value to the record.
+    throw new NotImplementedError("enumerated group mapping not implemented");
+  }
+
   all() {
     const res = [];
     this.groups.forEach(e =>
       res.push({
+        // XXX: assumes identity group map - see comment in _map()
         key: this.dimension.enumIndex[e.key],
         value: e.value
       })
@@ -609,4 +615,4 @@ crossfilter.TypedCrossfilter = TypedCrossfilter;
 crossfilter.ScalarDimension = ScalarDimension;
 crossfilter.EnumDimension = EnumDimension;
 
-export default crossfilter;
+module.exports = crossfilter;
diff --git a/client/src/util/typedCrossfilter/positiveIntervals.js b/client/src/util/typedCrossfilter/positiveIntervals.js
index 5e54bf7f..6e5d074a 100644
--- a/client/src/util/typedCrossfilter/positiveIntervals.js
+++ b/client/src/util/typedCrossfilter/positiveIntervals.js
@@ -129,4 +129,4 @@ class PositiveIntervals {
   }
 }
 
-export default PositiveIntervals;
+module.exports = PositiveIntervals;
diff --git a/client/src/util/typedCrossfilter/util.js b/client/src/util/typedCrossfilter/util.js
index 18b2c21e..deffcdec 100644
--- a/client/src/util/typedCrossfilter/util.js
+++ b/client/src/util/typedCrossfilter/util.js
@@ -8,7 +8,7 @@
 // fill an array or typedarray with a sequential range of numbers,
 // starting with `start`
 //
-export function fillRange(arr, start = 0) {
+function fillRange(arr, start = 0) {
   for (let i = 0, len = arr.length; i < len; i++) {
     arr[i] = i + start;
   }
@@ -30,7 +30,7 @@ export function fillRange(arr, start = 0) {
 // a factory version of lowerBound that takes an accessor (rather than having
 // a special-cased version for lining the indirection).
 //
-export function lowerBound(valueArray, value, first, last) {
+function lowerBound(valueArray, value, first, last) {
   // this is just a binary search
   while (first < last) {
     const middle = (first + last) >>> 1;
@@ -45,7 +45,7 @@ export function lowerBound(valueArray, value, first, last) {
 
 // Inlined performance optimization - used to indirect through a sort map.
 //
-export function lowerBoundIndirect(valueArray, indexArray, value, first, last) {
+function lowerBoundIndirect(valueArray, indexArray, value, first, last) {
   // this is just a binary search
   while (first < last) {
     const middle = (first + last) >>> 1;
@@ -69,7 +69,7 @@ export function lowerBoundIndirect(valueArray, indexArray, value, first, last) {
 //    C++: upper_bound()
 //    Python: bisect.bisect_right()
 //
-export function upperBound(valueArray, value, first, last) {
+function upperBound(valueArray, value, first, last) {
   // this is just a binary search
   while (first < last) {
     const middle = (first + last) >>> 1;
@@ -84,7 +84,7 @@ export function upperBound(valueArray, value, first, last) {
 
 // Inline performance optimization
 //
-export function upperBoundIndirect(valueArray, indexArray, value, first, last) {
+function upperBoundIndirect(valueArray, indexArray, value, first, last) {
   // this is just a binary search
   while (first < last) {
     const middle = (first + last) >>> 1;
@@ -96,3 +96,11 @@ export function upperBoundIndirect(valueArray, indexArray, value, first, last) {
   }
   return first;
 }
+
+module.exports = {
+  fillRange,
+  lowerBound,
+  lowerBoundIndirect,
+  upperBound,
+  upperBoundIndirect
+};

From 8119962c283802523b3ba65ea0f90145d7edf5ea Mon Sep 17 00:00:00 2001
From: bkmartinjr <bruce@chanzuckerberg.com>
Date: Wed, 8 Aug 2018 10:45:48 -0700
Subject: [PATCH 6/6] revert the module change reversion

---
 client/src/util/typedCrossfilter/bitArray.js  |  2 +-
 client/src/util/typedCrossfilter/index.js     | 58 +++++++------------
 .../typedCrossfilter/positiveIntervals.js     |  2 +-
 client/src/util/typedCrossfilter/util.js      | 18 ++----
 4 files changed, 28 insertions(+), 52 deletions(-)

diff --git a/client/src/util/typedCrossfilter/bitArray.js b/client/src/util/typedCrossfilter/bitArray.js
index 667f359c..2fa46acb 100644
--- a/client/src/util/typedCrossfilter/bitArray.js
+++ b/client/src/util/typedCrossfilter/bitArray.js
@@ -251,4 +251,4 @@ class BitArray {
   }
 }
 
-module.exports = BitArray;
+export default BitArray;
diff --git a/client/src/util/typedCrossfilter/index.js b/client/src/util/typedCrossfilter/index.js
index 57a4de56..7c19d10b 100644
--- a/client/src/util/typedCrossfilter/index.js
+++ b/client/src/util/typedCrossfilter/index.js
@@ -29,9 +29,15 @@ more complex API.  In a few cases, elements of that API were incorporated.
 
 */
 
-var PositiveIntervals = require("./positiveIntervals");
-var BitArray = require("./bitArray");
-var Util = require("./util");
+import PositiveIntervals from "./positiveIntervals";
+import BitArray from "./bitArray";
+import {
+  fillRange,
+  lowerBound,
+  lowerBoundIndirect,
+  upperBound,
+  upperBoundIndirect
+} from "./util";
 
 class NotImplementedError extends Error {
   constructor(...params) {
@@ -129,7 +135,7 @@ class ScalarDimension {
     this.value = array;
 
     // create sort index
-    this.index = Util.fillRange(new Uint32Array(this.crossfilter.data.length));
+    this.index = fillRange(new Uint32Array(this.crossfilter.data.length));
     this.index.sort((a, b) => array[a] - array[b]);
 
     // groups, if any
@@ -193,20 +199,8 @@ class ScalarDimension {
   // filter by value - exact match
   filterExact(value) {
     const newFilter = [
-      Util.lowerBoundIndirect(
-        this.value,
-        this.index,
-        value,
-        0,
-        this.value.length
-      ),
-      Util.upperBoundIndirect(
-        this.value,
-        this.index,
-        value,
-        0,
-        this.value.length
-      )
+      lowerBoundIndirect(this.value, this.index, value, 0, this.value.length),
+      upperBoundIndirect(this.value, this.index, value, 0, this.value.length)
     ];
     if (newFilter[0] <= newFilter[1]) {
       this._updateFilters([newFilter]);
@@ -221,14 +215,14 @@ class ScalarDimension {
     const newFilter = [];
     for (let v = 0, len = values.length; v < len; v++) {
       const intv = [
-        Util.lowerBoundIndirect(
+        lowerBoundIndirect(
           this.value,
           this.index,
           values[v],
           0,
           this.value.length
         ),
-        Util.upperBoundIndirect(
+        upperBoundIndirect(
           this.value,
           this.index,
           values[v],
@@ -247,20 +241,14 @@ class ScalarDimension {
   filterRange(range) {
     const newFilter = [];
     const intv = [
-      Util.lowerBoundIndirect(
+      lowerBoundIndirect(
         this.value,
         this.index,
         range[0],
         0,
         this.value.length
       ),
-      Util.upperBoundIndirect(
-        this.value,
-        this.index,
-        range[1],
-        0,
-        this.value.length
-      )
+      upperBoundIndirect(this.value, this.index, range[1], 0, this.value.length)
     ];
     if (intv[0] < intv[1]) newFilter.push(intv);
     this._updateFilters(newFilter);
@@ -374,7 +362,7 @@ class EnumDimension extends ScalarDimension {
     const enumLen = this.enumIndex.length;
     for (let i = 0; i < len; i++) {
       const v = value(data[i]);
-      const e = Util.lowerBound(this.enumIndex, v, 0, enumLen);
+      const e = lowerBound(this.enumIndex, v, 0, enumLen);
       array[i] = e;
     }
     return array;
@@ -382,23 +370,19 @@ class EnumDimension extends ScalarDimension {
 
   filterExact(value) {
     return super.filterExact(
-      Util.lowerBound(this.enumIndex, value, 0, this.enumIndex.length)
+      lowerBound(this.enumIndex, value, 0, this.enumIndex.length)
     );
   }
 
   filterEnum(values) {
     return super.filterEnum(
-      values.map(v =>
-        Util.lowerBound(this.enumIndex, v, 0, this.enumIndex.length)
-      )
+      values.map(v => lowerBound(this.enumIndex, v, 0, this.enumIndex.length))
     );
   }
 
   filterRange(range) {
     return super.filterEnum(
-      range.map(v =>
-        Util.lowerBound(this.enumIndex, v, 0, this.enumIndex.length)
-      )
+      range.map(v => lowerBound(this.enumIndex, v, 0, this.enumIndex.length))
     );
   }
 
@@ -615,4 +599,4 @@ crossfilter.TypedCrossfilter = TypedCrossfilter;
 crossfilter.ScalarDimension = ScalarDimension;
 crossfilter.EnumDimension = EnumDimension;
 
-module.exports = crossfilter;
+export default crossfilter;
diff --git a/client/src/util/typedCrossfilter/positiveIntervals.js b/client/src/util/typedCrossfilter/positiveIntervals.js
index 6e5d074a..5e54bf7f 100644
--- a/client/src/util/typedCrossfilter/positiveIntervals.js
+++ b/client/src/util/typedCrossfilter/positiveIntervals.js
@@ -129,4 +129,4 @@ class PositiveIntervals {
   }
 }
 
-module.exports = PositiveIntervals;
+export default PositiveIntervals;
diff --git a/client/src/util/typedCrossfilter/util.js b/client/src/util/typedCrossfilter/util.js
index deffcdec..18b2c21e 100644
--- a/client/src/util/typedCrossfilter/util.js
+++ b/client/src/util/typedCrossfilter/util.js
@@ -8,7 +8,7 @@
 // fill an array or typedarray with a sequential range of numbers,
 // starting with `start`
 //
-function fillRange(arr, start = 0) {
+export function fillRange(arr, start = 0) {
   for (let i = 0, len = arr.length; i < len; i++) {
     arr[i] = i + start;
   }
@@ -30,7 +30,7 @@ function fillRange(arr, start = 0) {
 // a factory version of lowerBound that takes an accessor (rather than having
 // a special-cased version for lining the indirection).
 //
-function lowerBound(valueArray, value, first, last) {
+export function lowerBound(valueArray, value, first, last) {
   // this is just a binary search
   while (first < last) {
     const middle = (first + last) >>> 1;
@@ -45,7 +45,7 @@ function lowerBound(valueArray, value, first, last) {
 
 // Inlined performance optimization - used to indirect through a sort map.
 //
-function lowerBoundIndirect(valueArray, indexArray, value, first, last) {
+export function lowerBoundIndirect(valueArray, indexArray, value, first, last) {
   // this is just a binary search
   while (first < last) {
     const middle = (first + last) >>> 1;
@@ -69,7 +69,7 @@ function lowerBoundIndirect(valueArray, indexArray, value, first, last) {
 //    C++: upper_bound()
 //    Python: bisect.bisect_right()
 //
-function upperBound(valueArray, value, first, last) {
+export function upperBound(valueArray, value, first, last) {
   // this is just a binary search
   while (first < last) {
     const middle = (first + last) >>> 1;
@@ -84,7 +84,7 @@ function upperBound(valueArray, value, first, last) {
 
 // Inline performance optimization
 //
-function upperBoundIndirect(valueArray, indexArray, value, first, last) {
+export function upperBoundIndirect(valueArray, indexArray, value, first, last) {
   // this is just a binary search
   while (first < last) {
     const middle = (first + last) >>> 1;
@@ -96,11 +96,3 @@ function upperBoundIndirect(valueArray, indexArray, value, first, last) {
   }
   return first;
 }
-
-module.exports = {
-  fillRange,
-  lowerBound,
-  lowerBoundIndirect,
-  upperBound,
-  upperBoundIndirect
-};