Merge pull request #164 from chanzuckerberg/bkmartinjr-crossfiltergroups

add groups functionality to crossfilter

client/__tests__/util/typedCrossfilter.test.js

@@ -101,6 +101,33 @@ const someData = [
   }
 ];
 
+function groupReduce(data, valueMap, valueReduce, valueInit) {
+  return _
+    .reduce(
+      data,
+      (acc, value) => {
+        const k = valueMap(value);
+        let r = _.find(acc, o => o.key === k);
+        if (!r) {
+          r = { key: k, value: valueInit() };
+          acc.push(r);
+        }
+        r.value = valueReduce(r.value, value);
+        return acc;
+      },
+      []
+    )
+    .sort((a, b) => (a.key < b.key ? -1 : a.key > b.key ? 1 : 0));
+}
+
+function groupCount(data, map) {
+  return groupReduce(data, map, (p, v) => p + 1, () => 0);
+}
+
+function groupSum(data, map) {
+  return groupReduce(data, map, (p, v) => (p += map(v)), () => 0);
+}
+
 var payments = null;
 beforeEach(() => {
   payments = crossfilter(someData);
@@ -273,4 +300,163 @@ describe("typedCrossfilter", () => {
     dimMap[33].filterAll();
     expect(payments.allFiltered()).toEqual(someData);
   });
+
+  test("group, default mapping, default reducer, no filter", () => {
+    expect(payments).toBeDefined();
+
+    var quantity = payments.dimension(r => r.quantity, Int32Array);
+    var tip = payments.dimension(r => r.tip, Int32Array);
+    var type = payments.dimension(r => r.type, "enum");
+    var total = payments.dimension(r => r.total, Int32Array);
+
+    _.each(
+      {
+        tip: tip.group(r => r),
+        type: type.group(),
+        total: total.group(),
+        quantity: quantity.group()
+      },
+      (grp, k) => {
+        const whatWeExpect = groupCount(someData, v => v[k]);
+        expect(grp.all()).toEqual(whatWeExpect);
+        expect(grp.size()).toEqual(whatWeExpect.length);
+        expect(grp.dispose()).toEqual(grp);
+      }
+    );
+  });
+
+  test("group, custom map, default reducer, no filters", () => {
+    expect(payments).toBeDefined();
+
+    // custom mapping in groups only works for scalar types.  Enums do not
+    // currently implement it.
+
+    const tip = payments.dimension(r => r.tip, Int32Array);
+    const totalX10 = payments.dimension(r => r.total * 10, Int32Array);
+    const type = payments.dimension(r => r.type, "enum");
+
+    const paymentsByTip_A = tip.group();
+    const paymentsByTip_B = tip.group(r => 10 * r);
+    const paymentsByType = type.group(); // identity only
+    const paymentsByTotalX10_A = totalX10.group();
+    const paymentsByTotalX10_B = totalX10.group(r => r / 10);
+
+    expect(paymentsByTip_A.all()).toEqual(groupCount(someData, v => v.tip));
+    expect(paymentsByTip_B.all()).toEqual(
+      groupCount(someData, v => 10 * v.tip)
+    );
+    expect(paymentsByType.all()).toEqual(groupCount(someData, v => v.type));
+    expect(paymentsByTotalX10_A.all()).toEqual(
+      groupCount(someData, v => 10 * v.total)
+    );
+    expect(paymentsByTotalX10_B.all()).toEqual(
+      groupCount(someData, v => (10 * v.total) / 10)
+    );
+
+    for (let i of [
+      paymentsByTip_A,
+      paymentsByTip_B,
+      paymentsByType,
+      paymentsByTotalX10_A,
+      paymentsByTotalX10_B,
+      tip,
+      totalX10,
+      type
+    ]) {
+      expect(i.dispose()).toEqual(i);
+    }
+  });
+
+  test("group, default map, custom reducer, no filters", () => {
+    expect(payments).toBeDefined();
+
+    const total = payments.dimension(r => r.total, Float32Array);
+    const type = payments.dimension(r => r.type, "enum");
+
+    const paymentsByTotal = total.group();
+    const paymentsByType = type.group();
+
+    // reduceCount
+    expect(paymentsByTotal.reduceCount()).toEqual(paymentsByTotal);
+    expect(paymentsByTotal.all()).toEqual(groupCount(someData, v => v.total));
+
+    // reduceSum
+    expect(paymentsByTotal.reduceSum(v => v.total)).toEqual(paymentsByTotal);
+    expect(paymentsByTotal.all()).toEqual(groupSum(someData, v => v.total));
+
+    // use custom reducers (my reducers) - count by three, init 1
+    expect(
+      paymentsByTotal.reduce((p, v) => (p += 3), (p, v) => (p -= 3), () => 1)
+    ).toEqual(paymentsByTotal);
+    expect(paymentsByTotal.all()).toEqual(
+      groupReduce(someData, v => v.total, (p, v) => p + 3, () => 1)
+    );
+
+    for (let i of [paymentsByTotal, paymentsByType, type]) {
+      expect(i.dispose()).toEqual(i);
+    }
+  });
+
+  test("group, default map, default reducer, filters", () => {
+    // From the docs:
+    // Note: a grouping intersects the crossfilter's current filters, except for the
+    // associated dimension's filter. Thus, group methods consider only records that
+    // satisfy every filter except this dimension's filter. So, if the crossfilter of
+    // payments is filtered by type and total, then group by total only observes the
+    // filter by type.
+
+    expect(payments).toBeDefined();
+
+    const tip = payments.dimension(r => r.tip, Int32Array);
+    const total = payments.dimension(r => r.total, Int32Array);
+    const type = payments.dimension(r => r.type, "enum");
+
+    const paymentsByTip = tip.group();
+    const paymentsByTotal = total.group();
+    const paymentsByType = type.group();
+
+    // 1. confirm that changing the filter on a dimension does NOT change that
+    // dimensions groups.
+    {
+      tip.filterAll(), total.filterAll(), type.filterAll();
+      let before = _.cloneDeep(paymentsByTip.all());
+      tip.filterExact(0);
+      expect(paymentsByTip.all()).toEqual(before);
+    }
+
+    // 2. confirm that changing a filter on a different dimension DOES change
+    // all other groups.
+    {
+      tip.filterAll(), total.filterAll(), type.filterAll();
+      const before = _.cloneDeep([paymentsByTotal.all(), paymentsByType.all()]);
+      tip.filterExact(0);
+      const after = [paymentsByTotal.all(), paymentsByType.all()];
+      expect(after).not.toEqual(before);
+      expect(after).toEqual([
+        groupReduce(
+          someData,
+          v => v.total,
+          (p, v) => (v.tip !== 0 ? p : p + 1),
+          () => 0
+        ),
+        groupReduce(
+          someData,
+          v => v.type,
+          (p, v) => (v.tip !== 0 ? p : p + 1),
+          () => 0
+        )
+      ]);
+    }
+
+    for (let i of [
+      paymentsByTip,
+      paymentsByTotal,
+      paymentsByType,
+      tip,
+      total,
+      type
+    ]) {
+      expect(i.dispose()).toEqual(i);
+    }
+  });
 });

client/package.json

@@ -112,6 +112,7 @@
   "jest": {
     "testMatch": [
       "**/__tests__/**/?(*.)(spec|test).js?(x)"
-    ]
+    ],
+    "testURL": "http://localhost/"
   }
 }

client/src/util/typedCrossfilter/bitArray.js

@@ -116,7 +116,7 @@ class BitArray {
     // all selection tests assume unallocated dimensions are zero valued.
     this.deselectAll(dim);
     const col = dim >>> 5;
-    this.bitmask[col] &= ~(1 << (dim % 32));
+    this.bitmask[col] &= ~(1 << dim % 32);
     this.dimensionCount--;
   }
 
@@ -134,12 +134,37 @@ class BitArray {
     return true;
   }
 
+  // return true if this index is selected in ALL dimensions IGNORING dim
+  //
+  isSelectedIgnoringDim(index, dim) {
+    const ignoreOffset = dim >>> 5;
+    const ignoreMask = ~(1 << dim % 32);
+
+    const width = this.width;
+    const length = this.length;
+    const bitarray = this.bitarray;
+
+    for (let w = 0; w < width; w++) {
+      const bitmask = this.bitmask[w];
+      if (w === ignoreOffset) {
+        if (
+          bitmask &&
+          (bitarray[w * length + index] & ignoreMask) !== (bitmask & ignoreMask)
+        )
+          return false;
+      } else {
+        if (bitmask && bitarray[w * length + index] !== bitmask) return false;
+      }
+    }
+    return true;
+  }
+
   // select index on dimension
   //
   selectOne(dim, index) {
     const col = dim >>> 5;
     const before = this.bitarray[col * this.length + index];
-    const after = before | (1 << (dim % 32));
+    const after = before | (1 << dim % 32);
     this.bitarray[col * this.length + index] = after;
   }
 
@@ -148,7 +173,7 @@ class BitArray {
   deselectOne(dim, index) {
     const col = dim >>> 5;
     const before = this.bitarray[col * this.length + index];
-    const after = before & ~(1 << (dim % 32));
+    const after = before & ~(1 << dim % 32);
     this.bitarray[col * this.length + index] = after;
   }
 
@@ -158,7 +183,7 @@ class BitArray {
     let col = dim >> 5;
     const bitmask = this.bitmask[col];
     const bitarray = this.bitarray;
-    const one = 1 << (dim % 32);
+    const one = 1 << dim % 32;
     for (let i = col * this.length, len = i + this.length; i < len; i++) {
       bitarray[i] |= one;
     }
@@ -170,7 +195,7 @@ class BitArray {
     let col = dim >> 5;
     const bitmask = this.bitmask[col];
     const bitarray = this.bitarray;
-    const zero = ~(1 << (dim % 32));
+    const zero = ~(1 << dim % 32);
     for (let i = col * this.length, len = i + this.length; i < len; i++) {
       bitarray[i] &= zero;
     }
@@ -184,7 +209,7 @@ class BitArray {
     const first = range[0];
     const last = range[1];
     const bitarray = this.bitarray;
-    const one = 1 << (dim % 32);
+    const one = 1 << dim % 32;
     const offset = col * this.length;
     for (let i = first; i < last; i++) {
       bitarray[offset + indirect[i]] |= one;
@@ -198,7 +223,7 @@ class BitArray {
     const first = range[0];
     const last = range[1];
     const bitarray = this.bitarray;
-    const zero = ~(1 << (dim % 32));
+    const zero = ~(1 << dim % 32);
     const offset = col * this.length;
     for (let i = first; i < last; i++) {
       bitarray[offset + indirect[i]] &= zero;

client/src/util/typedCrossfilter/index.js

@@ -31,7 +31,24 @@ more complex API.  In a few cases, elements of that API were incorporated.
 
 import PositiveIntervals from "./positiveIntervals";
 import BitArray from "./bitArray";
-import {fillRange, lowerBound, lowerBoundIndirect, upperBound, upperBoundIndirect} from "./util";
+import {
+  fillRange,
+  lowerBound,
+  lowerBoundIndirect,
+  upperBound,
+  upperBoundIndirect
+} from "./util";
+
+class NotImplementedError extends Error {
+  constructor(...params) {
+    super(...params);
+
+    // Maintains proper stack trace for where our error was thrown (only available on V8)
+    if (Error.captureStackTrace) {
+      Error.captureStackTrace(this, NotImplementedError);
+    }
+  }
+}
 
 class TypedCrossfilter {
   constructor(data) {
@@ -120,6 +137,9 @@ class ScalarDimension {
     // create sort index
     this.index = fillRange(new Uint32Array(this.crossfilter.data.length));
     this.index.sort((a, b) => array[a] - array[b]);
+
+    // groups, if any
+    this.groups = [];
   }
 
   _createValueArray(value, array) {
@@ -134,44 +154,44 @@ class ScalarDimension {
 
   dispose() {
     this.crossfilter._freeDimension(this._id);
+    return this;
   }
 
   id() {
     return this._id;
   }
 
+  // Argument is an array of intervals indicating records newly selected/filtered
+  //
   _updateFilters(newFilter) {
     newFilter = PositiveIntervals.canonicalize(newFilter);
 
-    // special case optimization - select all/none can bypass
+    const adds = PositiveIntervals.difference(newFilter, this.currentFilter);
-    // more complex work and just clobber everything.
+    const dels = PositiveIntervals.difference(this.currentFilter, newFilter);
-    //
+
-    if (newFilter.length === 0) {
+    this.crossfilter.filters.forEach(f =>
-      this.crossfilter.selection.deselectAll(this._id);
+      f.dim.groups.forEach(grp => grp._updateReduceDel(this, dels))
-    } else if (
+    );
-      newFilter.length === 1 &&
+
-      newFilter[0][0] === 0 &&
+    dels.forEach(interval =>
-      newFilter[0][1] == this.index.length
+      this.crossfilter.selection.deselectIndirectFromRange(
-    ) {
+        this._id,
-      this.crossfilter.selection.selectAll(this._id);
+        this.index,
-    } else {
+        interval
-      const adds = PositiveIntervals.difference(newFilter, this.currentFilter);
+      )
-      const dels = PositiveIntervals.difference(this.currentFilter, newFilter);
+    );
-      dels.forEach(interval =>
+
-        this.crossfilter.selection.deselectIndirectFromRange(
+    adds.forEach(interval =>
-          this._id,
+      this.crossfilter.selection.selectIndirectFromRange(
-          this.index,
+        this._id,
-          interval
+        this.index,
-        )
+        interval
-      );
+      )
-      adds.forEach(interval =>
+    );
-        this.crossfilter.selection.selectIndirectFromRange(
+
-          this._id,
+    this.crossfilter.filters.forEach(f =>
-          this.index,
+      f.dim.groups.forEach(grp => grp._updateReduceAdd(this, adds))
-          interval
+    );
-        )
-      );
-    }
 
     this.currentFilter = newFilter;
   }
@@ -179,20 +199,8 @@ class ScalarDimension {
   // filter by value - exact match
   filterExact(value) {
     const newFilter = [
-      lowerBoundIndirect(
+      lowerBoundIndirect(this.value, this.index, value, 0, this.value.length),
-        this.value,
+      upperBoundIndirect(this.value, this.index, value, 0, this.value.length)
-        this.index,
-        value,
-        0,
-        this.value.length
-      ),
-      upperBoundIndirect(
-        this.value,
-        this.index,
-        value,
-        0,
-        this.value.length
-      )
     ];
     if (newFilter[0] <= newFilter[1]) {
       this._updateFilters([newFilter]);
@@ -240,13 +248,7 @@ class ScalarDimension {
         0,
         this.value.length
       ),
-      upperBoundIndirect(
+      upperBoundIndirect(this.value, this.index, range[1], 0, this.value.length)
-        this.value,
-        this.index,
-        range[1],
-        0,
-        this.value.length
-      )
     ];
     if (intv[0] < intv[1]) newFilter.push(intv);
     this._updateFilters(newFilter);
@@ -323,6 +325,16 @@ class ScalarDimension {
 
     return ret;
   }
+
+  group(groupValue) {
+    const grp = new ScalarGroup(groupValue, this.value.constructor, this);
+    this.groups.push(grp);
+    return grp;
+  }
+
+  _freeGroup(group) {
+    this.groups = this.groups.filter(e => e !== group);
+  }
 }
 
 // Ordered enumeration - supports any sortable enumerable type, eg,
@@ -364,19 +376,215 @@ class EnumDimension extends ScalarDimension {
 
   filterEnum(values) {
     return super.filterEnum(
-      values.map(v =>
+      values.map(v => lowerBound(this.enumIndex, v, 0, this.enumIndex.length))
-        lowerBound(this.enumIndex, v, 0, this.enumIndex.length)
-      )
     );
   }
 
   filterRange(range) {
     return super.filterEnum(
-      range.map(v =>
+      range.map(v => lowerBound(this.enumIndex, v, 0, this.enumIndex.length))
-        lowerBound(this.enumIndex, v, 0, this.enumIndex.length)
-      )
     );
   }
+
+  group(groupValue) {
+    const grp = new EnumGroup(groupValue, this.value.constructor, this);
+    this.groups.push(grp);
+    return grp;
+  }
+}
+
+// Groups!  Map/reduce
+//
+class ScalarGroup {
+  constructor(groupValue, groupValueType, dimension) {
+    // parent dimension
+    this.dimension = dimension;
+
+    // generate group names from dimension values
+    this.mapValue = this._map(groupValue, groupValueType, dimension);
+
+    // group index is mapping from data record index to group index
+    this.groupIndex = new Uint32Array(dimension.crossfilter.data.length);
+
+    // default to counting
+    this.reduceCount();
+
+    // Creates this.groups
+    this._reduce();
+  }
+
+  // internal support function - map all dimension values to group values.
+  //
+  _map(groupValue, groupValueType, dimension) {
+    // groupValue is optional.  Defaults to identity.  Used to perform
+    // initial map operation.
+    //
+    // identity: save some memory...
+    if (groupValue === undefined) return dimension.value;
+
+    const data = dimension.value;
+    const len = data.length;
+    const mapValue = new groupValueType(dimension.value.length);
+    for (let i = 0; i < len; i++) {
+      mapValue[i] = groupValue(data[i]);
+    }
+    return mapValue;
+  }
+
+  // Update the group reduction incrementally.  Called when *any* dimension filter
+  // changes.  Guaranteed to be called AFTER the crossfilter is updated.
+  //
+  // Arguments:
+  //  * dim: the dimension that is changing
+  //  * intv: interval list of newly selected values on `dim` (adds)
+  //
+  _updateReduceAdd(dim, intv) {
+    // ignore updates to self, as we don't reduce inclusive of our filter
+    if (dim === this.dimension || intv.length === 0) return;
+
+    // Each item in the range was just added to `dim`.  It was NOT previously
+    // selected - reduceAdd if it is now selected.
+    const selection = this.dimension.crossfilter.selection;
+    const data = this.dimension.crossfilter.data;
+    intv.forEach(rng => {
+      for (let r = rng[0]; r < rng[1]; r++) {
+        const i = dim.index[r];
+        if (selection.isSelectedIgnoringDim(i, this.dimension.id())) {
+          const group = this.groups[this.groupIndex[i]];
+          group.value = this.reduceAdd(group.value, data[i]);
+        }
+      }
+    });
+  }
+
+  // Update the group reduction incrementally.  Called when *any* dimension filter
+  // changes.   Guaranteed to be called BEFORE the crossfilter is updated.
+  //
+  // Arguments:
+  //  * dim: the dimension that is changing
+  //  * intv: interval list of previously selected values on `dim` (dels)
+  //
+  _updateReduceDel(dim, intv) {
+    // ignore updates to self, as we don't reduce inclusive of our filter
+    if (dim === this.dimension || intv.length === 0) return;
+
+    // Each item in the range will be remved from `dim`.  reduceRemove if it
+    // is currently selected.
+    const selection = this.dimension.crossfilter.selection;
+    const data = this.dimension.crossfilter.data;
+    intv.forEach(rng => {
+      for (let r = rng[0]; r < rng[1]; r++) {
+        const i = dim.index[r];
+        if (selection.isSelectedIgnoringDim(i, this.dimension.id())) {
+          const group = this.groups[this.groupIndex[i]];
+          group.value = this.reduceRemove(group.value, data[i]);
+        }
+      }
+    });
+  }
+
+  // Reduce the entire data set, creating both the group index and the
+  // groups data.
+  //
+  _reduce() {
+    const dimension = this.dimension;
+    const data = dimension.crossfilter.data;
+
+    // Create groups
+    const groupNames = new Set(this.mapValue);
+    this.groups = [];
+    const groupIndexByName = {};
+    groupNames.forEach(name => {
+      this.groups.push({ key: name, value: this.reduceInitial() });
+      groupIndexByName[name] = this.groups.length - 1;
+    });
+
+    // Create groupIndex - index map between data record index and group index
+    for (let i = 0, len = this.mapValue.length; i < len; i++) {
+      this.groupIndex[i] = groupIndexByName[this.mapValue[i]];
+    }
+
+    // reduce all filtered records, IGNORING the current dimension's filter
+    const selection = dimension.crossfilter.selection;
+    for (let i = 0, len = data.length; i < len; i++) {
+      if (selection.isSelectedIgnoringDim(i, dimension.id())) {
+        const group = this.groups[this.groupIndex[i]];
+        group.value = this.reduceAdd(group.value, data[i]);
+      }
+    }
+  }
+
+  dispose() {
+    this.dimension._freeGroup(this);
+    return this;
+  }
+
+  // return number of distinct values in the group, independent of any filters.
+  //
+  size() {
+    return this.groups.length;
+  }
+
+  // Set the reduce functions and return the grouping.
+  //
+  reduce(add, remove, initial) {
+    this.reduceAdd = add;
+    this.reduceRemove = remove;
+    this.reduceInitial = initial;
+    this._reduce();
+    return this;
+  }
+
+  // set the reduce functions to count records.
+  reduceCount() {
+    return this.reduce((p, v) => p + 1, (p, v) => p - 1, () => 0);
+  }
+
+  // set the reduce functions to sum records using specified value accessor.
+  //
+  reduceSum(value) {
+    return this.reduce((p, v) => p + value(v), (p, v) => p - value(v), () => 0);
+  }
+
+  all() {
+    const res = [...this.groups];
+    res.sort((a, b) => (a.key < b.key ? -1 : a.key > b.key ? 1 : 0));
+    return res;
+  }
+}
+
+class EnumGroup extends ScalarGroup {
+  constructor(groupValue, groupValueType, dimension) {
+    super(groupValue, groupValueType, dimension);
+  }
+
+  _map(groupValue, groupValueType, dimension) {
+    // groupValue is optional.  Defaults to identity.  Used to perform
+    // initial map operation.
+    //
+    // identity: save some memory
+    if (groupValue === undefined) return dimension.value;
+
+    // non-identity mapping unsupported for EnumDimension/EnumGroup.
+    // XXX: this could be implemented, but would require another index
+    // array to map from the group names/keys back to the dimension values.
+    // With this, we just rely on the dimensions `enumIndex` to map from
+    // enumeration value to the record.
+    throw new NotImplementedError("enumerated group mapping not implemented");
+  }
+
+  all() {
+    const res = [];
+    this.groups.forEach(e =>
+      res.push({
+        // XXX: assumes identity group map - see comment in _map()
+        key: this.dimension.enumIndex[e.key],
+        value: e.value
+      })
+    );
+    res.sort((a, b) => (a.key < b.key ? -1 : a.key > b.key ? 1 : 0));
+    return res;
+  }
 }
 
 // Wrapper for backwards compat with crossfilter.