diff --git a/client/__tests__/util/typedCrossfilter.test.js b/client/__tests__/util/typedCrossfilter.test.js index 00e68495..1ebbcaf1 100644 --- a/client/__tests__/util/typedCrossfilter.test.js +++ b/client/__tests__/util/typedCrossfilter.test.js @@ -101,6 +101,33 @@ const someData = [ } ]; +function groupReduce(data, valueMap, valueReduce, valueInit) { + return _ + .reduce( + data, + (acc, value) => { + const k = valueMap(value); + let r = _.find(acc, o => o.key === k); + if (!r) { + r = { key: k, value: valueInit() }; + acc.push(r); + } + r.value = valueReduce(r.value, value); + return acc; + }, + [] + ) + .sort((a, b) => (a.key < b.key ? -1 : a.key > b.key ? 1 : 0)); +} + +function groupCount(data, map) { + return groupReduce(data, map, (p, v) => p + 1, () => 0); +} + +function groupSum(data, map) { + return groupReduce(data, map, (p, v) => (p += map(v)), () => 0); +} + var payments = null; beforeEach(() => { payments = crossfilter(someData); @@ -273,4 +300,163 @@ describe("typedCrossfilter", () => { dimMap[33].filterAll(); expect(payments.allFiltered()).toEqual(someData); }); + + test("group, default mapping, default reducer, no filter", () => { + expect(payments).toBeDefined(); + + var quantity = payments.dimension(r => r.quantity, Int32Array); + var tip = payments.dimension(r => r.tip, Int32Array); + var type = payments.dimension(r => r.type, "enum"); + var total = payments.dimension(r => r.total, Int32Array); + + _.each( + { + tip: tip.group(r => r), + type: type.group(), + total: total.group(), + quantity: quantity.group() + }, + (grp, k) => { + const whatWeExpect = groupCount(someData, v => v[k]); + expect(grp.all()).toEqual(whatWeExpect); + expect(grp.size()).toEqual(whatWeExpect.length); + expect(grp.dispose()).toEqual(grp); + } + ); + }); + + test("group, custom map, default reducer, no filters", () => { + expect(payments).toBeDefined(); + + // custom mapping in groups only works for scalar types. Enums do not + // currently implement it. + + const tip = payments.dimension(r => r.tip, Int32Array); + const totalX10 = payments.dimension(r => r.total * 10, Int32Array); + const type = payments.dimension(r => r.type, "enum"); + + const paymentsByTip_A = tip.group(); + const paymentsByTip_B = tip.group(r => 10 * r); + const paymentsByType = type.group(); // identity only + const paymentsByTotalX10_A = totalX10.group(); + const paymentsByTotalX10_B = totalX10.group(r => r / 10); + + expect(paymentsByTip_A.all()).toEqual(groupCount(someData, v => v.tip)); + expect(paymentsByTip_B.all()).toEqual( + groupCount(someData, v => 10 * v.tip) + ); + expect(paymentsByType.all()).toEqual(groupCount(someData, v => v.type)); + expect(paymentsByTotalX10_A.all()).toEqual( + groupCount(someData, v => 10 * v.total) + ); + expect(paymentsByTotalX10_B.all()).toEqual( + groupCount(someData, v => (10 * v.total) / 10) + ); + + for (let i of [ + paymentsByTip_A, + paymentsByTip_B, + paymentsByType, + paymentsByTotalX10_A, + paymentsByTotalX10_B, + tip, + totalX10, + type + ]) { + expect(i.dispose()).toEqual(i); + } + }); + + test("group, default map, custom reducer, no filters", () => { + expect(payments).toBeDefined(); + + const total = payments.dimension(r => r.total, Float32Array); + const type = payments.dimension(r => r.type, "enum"); + + const paymentsByTotal = total.group(); + const paymentsByType = type.group(); + + // reduceCount + expect(paymentsByTotal.reduceCount()).toEqual(paymentsByTotal); + expect(paymentsByTotal.all()).toEqual(groupCount(someData, v => v.total)); + + // reduceSum + expect(paymentsByTotal.reduceSum(v => v.total)).toEqual(paymentsByTotal); + expect(paymentsByTotal.all()).toEqual(groupSum(someData, v => v.total)); + + // use custom reducers (my reducers) - count by three, init 1 + expect( + paymentsByTotal.reduce((p, v) => (p += 3), (p, v) => (p -= 3), () => 1) + ).toEqual(paymentsByTotal); + expect(paymentsByTotal.all()).toEqual( + groupReduce(someData, v => v.total, (p, v) => p + 3, () => 1) + ); + + for (let i of [paymentsByTotal, paymentsByType, type]) { + expect(i.dispose()).toEqual(i); + } + }); + + test("group, default map, default reducer, filters", () => { + // From the docs: + // Note: a grouping intersects the crossfilter's current filters, except for the + // associated dimension's filter. Thus, group methods consider only records that + // satisfy every filter except this dimension's filter. So, if the crossfilter of + // payments is filtered by type and total, then group by total only observes the + // filter by type. + + expect(payments).toBeDefined(); + + const tip = payments.dimension(r => r.tip, Int32Array); + const total = payments.dimension(r => r.total, Int32Array); + const type = payments.dimension(r => r.type, "enum"); + + const paymentsByTip = tip.group(); + const paymentsByTotal = total.group(); + const paymentsByType = type.group(); + + // 1. confirm that changing the filter on a dimension does NOT change that + // dimensions groups. + { + tip.filterAll(), total.filterAll(), type.filterAll(); + let before = _.cloneDeep(paymentsByTip.all()); + tip.filterExact(0); + expect(paymentsByTip.all()).toEqual(before); + } + + // 2. confirm that changing a filter on a different dimension DOES change + // all other groups. + { + tip.filterAll(), total.filterAll(), type.filterAll(); + const before = _.cloneDeep([paymentsByTotal.all(), paymentsByType.all()]); + tip.filterExact(0); + const after = [paymentsByTotal.all(), paymentsByType.all()]; + expect(after).not.toEqual(before); + expect(after).toEqual([ + groupReduce( + someData, + v => v.total, + (p, v) => (v.tip !== 0 ? p : p + 1), + () => 0 + ), + groupReduce( + someData, + v => v.type, + (p, v) => (v.tip !== 0 ? p : p + 1), + () => 0 + ) + ]); + } + + for (let i of [ + paymentsByTip, + paymentsByTotal, + paymentsByType, + tip, + total, + type + ]) { + expect(i.dispose()).toEqual(i); + } + }); }); diff --git a/client/package.json b/client/package.json index 77833ea9..e27bbc87 100644 --- a/client/package.json +++ b/client/package.json @@ -112,6 +112,7 @@ "jest": { "testMatch": [ "**/__tests__/**/?(*.)(spec|test).js?(x)" - ] + ], + "testURL": "http://localhost/" } } diff --git a/client/src/util/typedCrossfilter/bitArray.js b/client/src/util/typedCrossfilter/bitArray.js index 9e4f3a5a..2fa46acb 100644 --- a/client/src/util/typedCrossfilter/bitArray.js +++ b/client/src/util/typedCrossfilter/bitArray.js @@ -116,7 +116,7 @@ class BitArray { // all selection tests assume unallocated dimensions are zero valued. this.deselectAll(dim); const col = dim >>> 5; - this.bitmask[col] &= ~(1 << (dim % 32)); + this.bitmask[col] &= ~(1 << dim % 32); this.dimensionCount--; } @@ -134,12 +134,37 @@ class BitArray { return true; } + // return true if this index is selected in ALL dimensions IGNORING dim + // + isSelectedIgnoringDim(index, dim) { + const ignoreOffset = dim >>> 5; + const ignoreMask = ~(1 << dim % 32); + + const width = this.width; + const length = this.length; + const bitarray = this.bitarray; + + for (let w = 0; w < width; w++) { + const bitmask = this.bitmask[w]; + if (w === ignoreOffset) { + if ( + bitmask && + (bitarray[w * length + index] & ignoreMask) !== (bitmask & ignoreMask) + ) + return false; + } else { + if (bitmask && bitarray[w * length + index] !== bitmask) return false; + } + } + return true; + } + // select index on dimension // selectOne(dim, index) { const col = dim >>> 5; const before = this.bitarray[col * this.length + index]; - const after = before | (1 << (dim % 32)); + const after = before | (1 << dim % 32); this.bitarray[col * this.length + index] = after; } @@ -148,7 +173,7 @@ class BitArray { deselectOne(dim, index) { const col = dim >>> 5; const before = this.bitarray[col * this.length + index]; - const after = before & ~(1 << (dim % 32)); + const after = before & ~(1 << dim % 32); this.bitarray[col * this.length + index] = after; } @@ -158,7 +183,7 @@ class BitArray { let col = dim >> 5; const bitmask = this.bitmask[col]; const bitarray = this.bitarray; - const one = 1 << (dim % 32); + const one = 1 << dim % 32; for (let i = col * this.length, len = i + this.length; i < len; i++) { bitarray[i] |= one; } @@ -170,7 +195,7 @@ class BitArray { let col = dim >> 5; const bitmask = this.bitmask[col]; const bitarray = this.bitarray; - const zero = ~(1 << (dim % 32)); + const zero = ~(1 << dim % 32); for (let i = col * this.length, len = i + this.length; i < len; i++) { bitarray[i] &= zero; } @@ -184,7 +209,7 @@ class BitArray { const first = range[0]; const last = range[1]; const bitarray = this.bitarray; - const one = 1 << (dim % 32); + const one = 1 << dim % 32; const offset = col * this.length; for (let i = first; i < last; i++) { bitarray[offset + indirect[i]] |= one; @@ -198,7 +223,7 @@ class BitArray { const first = range[0]; const last = range[1]; const bitarray = this.bitarray; - const zero = ~(1 << (dim % 32)); + const zero = ~(1 << dim % 32); const offset = col * this.length; for (let i = first; i < last; i++) { bitarray[offset + indirect[i]] &= zero; diff --git a/client/src/util/typedCrossfilter/index.js b/client/src/util/typedCrossfilter/index.js index b2d38db4..7c19d10b 100644 --- a/client/src/util/typedCrossfilter/index.js +++ b/client/src/util/typedCrossfilter/index.js @@ -31,7 +31,24 @@ more complex API. In a few cases, elements of that API were incorporated. import PositiveIntervals from "./positiveIntervals"; import BitArray from "./bitArray"; -import {fillRange, lowerBound, lowerBoundIndirect, upperBound, upperBoundIndirect} from "./util"; +import { + fillRange, + lowerBound, + lowerBoundIndirect, + upperBound, + upperBoundIndirect +} from "./util"; + +class NotImplementedError extends Error { + constructor(...params) { + super(...params); + + // Maintains proper stack trace for where our error was thrown (only available on V8) + if (Error.captureStackTrace) { + Error.captureStackTrace(this, NotImplementedError); + } + } +} class TypedCrossfilter { constructor(data) { @@ -120,6 +137,9 @@ class ScalarDimension { // create sort index this.index = fillRange(new Uint32Array(this.crossfilter.data.length)); this.index.sort((a, b) => array[a] - array[b]); + + // groups, if any + this.groups = []; } _createValueArray(value, array) { @@ -134,44 +154,44 @@ class ScalarDimension { dispose() { this.crossfilter._freeDimension(this._id); + return this; } id() { return this._id; } + // Argument is an array of intervals indicating records newly selected/filtered + // _updateFilters(newFilter) { newFilter = PositiveIntervals.canonicalize(newFilter); - // special case optimization - select all/none can bypass - // more complex work and just clobber everything. - // - if (newFilter.length === 0) { - this.crossfilter.selection.deselectAll(this._id); - } else if ( - newFilter.length === 1 && - newFilter[0][0] === 0 && - newFilter[0][1] == this.index.length - ) { - this.crossfilter.selection.selectAll(this._id); - } else { - const adds = PositiveIntervals.difference(newFilter, this.currentFilter); - const dels = PositiveIntervals.difference(this.currentFilter, newFilter); - dels.forEach(interval => - this.crossfilter.selection.deselectIndirectFromRange( - this._id, - this.index, - interval - ) - ); - adds.forEach(interval => - this.crossfilter.selection.selectIndirectFromRange( - this._id, - this.index, - interval - ) - ); - } + const adds = PositiveIntervals.difference(newFilter, this.currentFilter); + const dels = PositiveIntervals.difference(this.currentFilter, newFilter); + + this.crossfilter.filters.forEach(f => + f.dim.groups.forEach(grp => grp._updateReduceDel(this, dels)) + ); + + dels.forEach(interval => + this.crossfilter.selection.deselectIndirectFromRange( + this._id, + this.index, + interval + ) + ); + + adds.forEach(interval => + this.crossfilter.selection.selectIndirectFromRange( + this._id, + this.index, + interval + ) + ); + + this.crossfilter.filters.forEach(f => + f.dim.groups.forEach(grp => grp._updateReduceAdd(this, adds)) + ); this.currentFilter = newFilter; } @@ -179,20 +199,8 @@ class ScalarDimension { // filter by value - exact match filterExact(value) { const newFilter = [ - lowerBoundIndirect( - this.value, - this.index, - value, - 0, - this.value.length - ), - upperBoundIndirect( - this.value, - this.index, - value, - 0, - this.value.length - ) + lowerBoundIndirect(this.value, this.index, value, 0, this.value.length), + upperBoundIndirect(this.value, this.index, value, 0, this.value.length) ]; if (newFilter[0] <= newFilter[1]) { this._updateFilters([newFilter]); @@ -240,13 +248,7 @@ class ScalarDimension { 0, this.value.length ), - upperBoundIndirect( - this.value, - this.index, - range[1], - 0, - this.value.length - ) + upperBoundIndirect(this.value, this.index, range[1], 0, this.value.length) ]; if (intv[0] < intv[1]) newFilter.push(intv); this._updateFilters(newFilter); @@ -323,6 +325,16 @@ class ScalarDimension { return ret; } + + group(groupValue) { + const grp = new ScalarGroup(groupValue, this.value.constructor, this); + this.groups.push(grp); + return grp; + } + + _freeGroup(group) { + this.groups = this.groups.filter(e => e !== group); + } } // Ordered enumeration - supports any sortable enumerable type, eg, @@ -364,19 +376,215 @@ class EnumDimension extends ScalarDimension { filterEnum(values) { return super.filterEnum( - values.map(v => - lowerBound(this.enumIndex, v, 0, this.enumIndex.length) - ) + values.map(v => lowerBound(this.enumIndex, v, 0, this.enumIndex.length)) ); } filterRange(range) { return super.filterEnum( - range.map(v => - lowerBound(this.enumIndex, v, 0, this.enumIndex.length) - ) + range.map(v => lowerBound(this.enumIndex, v, 0, this.enumIndex.length)) ); } + + group(groupValue) { + const grp = new EnumGroup(groupValue, this.value.constructor, this); + this.groups.push(grp); + return grp; + } +} + +// Groups! Map/reduce +// +class ScalarGroup { + constructor(groupValue, groupValueType, dimension) { + // parent dimension + this.dimension = dimension; + + // generate group names from dimension values + this.mapValue = this._map(groupValue, groupValueType, dimension); + + // group index is mapping from data record index to group index + this.groupIndex = new Uint32Array(dimension.crossfilter.data.length); + + // default to counting + this.reduceCount(); + + // Creates this.groups + this._reduce(); + } + + // internal support function - map all dimension values to group values. + // + _map(groupValue, groupValueType, dimension) { + // groupValue is optional. Defaults to identity. Used to perform + // initial map operation. + // + // identity: save some memory... + if (groupValue === undefined) return dimension.value; + + const data = dimension.value; + const len = data.length; + const mapValue = new groupValueType(dimension.value.length); + for (let i = 0; i < len; i++) { + mapValue[i] = groupValue(data[i]); + } + return mapValue; + } + + // Update the group reduction incrementally. Called when *any* dimension filter + // changes. Guaranteed to be called AFTER the crossfilter is updated. + // + // Arguments: + // * dim: the dimension that is changing + // * intv: interval list of newly selected values on `dim` (adds) + // + _updateReduceAdd(dim, intv) { + // ignore updates to self, as we don't reduce inclusive of our filter + if (dim === this.dimension || intv.length === 0) return; + + // Each item in the range was just added to `dim`. It was NOT previously + // selected - reduceAdd if it is now selected. + const selection = this.dimension.crossfilter.selection; + const data = this.dimension.crossfilter.data; + intv.forEach(rng => { + for (let r = rng[0]; r < rng[1]; r++) { + const i = dim.index[r]; + if (selection.isSelectedIgnoringDim(i, this.dimension.id())) { + const group = this.groups[this.groupIndex[i]]; + group.value = this.reduceAdd(group.value, data[i]); + } + } + }); + } + + // Update the group reduction incrementally. Called when *any* dimension filter + // changes. Guaranteed to be called BEFORE the crossfilter is updated. + // + // Arguments: + // * dim: the dimension that is changing + // * intv: interval list of previously selected values on `dim` (dels) + // + _updateReduceDel(dim, intv) { + // ignore updates to self, as we don't reduce inclusive of our filter + if (dim === this.dimension || intv.length === 0) return; + + // Each item in the range will be remved from `dim`. reduceRemove if it + // is currently selected. + const selection = this.dimension.crossfilter.selection; + const data = this.dimension.crossfilter.data; + intv.forEach(rng => { + for (let r = rng[0]; r < rng[1]; r++) { + const i = dim.index[r]; + if (selection.isSelectedIgnoringDim(i, this.dimension.id())) { + const group = this.groups[this.groupIndex[i]]; + group.value = this.reduceRemove(group.value, data[i]); + } + } + }); + } + + // Reduce the entire data set, creating both the group index and the + // groups data. + // + _reduce() { + const dimension = this.dimension; + const data = dimension.crossfilter.data; + + // Create groups + const groupNames = new Set(this.mapValue); + this.groups = []; + const groupIndexByName = {}; + groupNames.forEach(name => { + this.groups.push({ key: name, value: this.reduceInitial() }); + groupIndexByName[name] = this.groups.length - 1; + }); + + // Create groupIndex - index map between data record index and group index + for (let i = 0, len = this.mapValue.length; i < len; i++) { + this.groupIndex[i] = groupIndexByName[this.mapValue[i]]; + } + + // reduce all filtered records, IGNORING the current dimension's filter + const selection = dimension.crossfilter.selection; + for (let i = 0, len = data.length; i < len; i++) { + if (selection.isSelectedIgnoringDim(i, dimension.id())) { + const group = this.groups[this.groupIndex[i]]; + group.value = this.reduceAdd(group.value, data[i]); + } + } + } + + dispose() { + this.dimension._freeGroup(this); + return this; + } + + // return number of distinct values in the group, independent of any filters. + // + size() { + return this.groups.length; + } + + // Set the reduce functions and return the grouping. + // + reduce(add, remove, initial) { + this.reduceAdd = add; + this.reduceRemove = remove; + this.reduceInitial = initial; + this._reduce(); + return this; + } + + // set the reduce functions to count records. + reduceCount() { + return this.reduce((p, v) => p + 1, (p, v) => p - 1, () => 0); + } + + // set the reduce functions to sum records using specified value accessor. + // + reduceSum(value) { + return this.reduce((p, v) => p + value(v), (p, v) => p - value(v), () => 0); + } + + all() { + const res = [...this.groups]; + res.sort((a, b) => (a.key < b.key ? -1 : a.key > b.key ? 1 : 0)); + return res; + } +} + +class EnumGroup extends ScalarGroup { + constructor(groupValue, groupValueType, dimension) { + super(groupValue, groupValueType, dimension); + } + + _map(groupValue, groupValueType, dimension) { + // groupValue is optional. Defaults to identity. Used to perform + // initial map operation. + // + // identity: save some memory + if (groupValue === undefined) return dimension.value; + + // non-identity mapping unsupported for EnumDimension/EnumGroup. + // XXX: this could be implemented, but would require another index + // array to map from the group names/keys back to the dimension values. + // With this, we just rely on the dimensions `enumIndex` to map from + // enumeration value to the record. + throw new NotImplementedError("enumerated group mapping not implemented"); + } + + all() { + const res = []; + this.groups.forEach(e => + res.push({ + // XXX: assumes identity group map - see comment in _map() + key: this.dimension.enumIndex[e.key], + value: e.value + }) + ); + res.sort((a, b) => (a.key < b.key ? -1 : a.key > b.key ? 1 : 0)); + return res; + } } // Wrapper for backwards compat with crossfilter.