From 996b06ceccf238784000af3eb2199b3b30745fc0 Mon Sep 17 00:00:00 2001 From: Bruce Martin Date: Wed, 20 Mar 2019 14:32:35 -0700 Subject: [PATCH] refactoring - immutable crossfilter (#647) * immutable crossfilter * PR review changes --- .../__tests__/util/stateManager/world.test.js | 51 +- .../util/typedCrossfilter/crossfilter.test.js | 330 ++++++++ .../typedCrossfilter/typedCrossfilter.test.js | 590 ------------- .../src/components/continuous/continuous.js | 3 +- .../geneExpression/expressionButtons.js | 3 +- client/src/components/graph/graph.js | 19 +- .../src/components/scatterplot/scatterplot.js | 11 +- client/src/reducers/controls.js | 216 ++--- .../src/util/stateManager/controlsHelpers.js | 40 +- client/src/util/stateManager/world.js | 75 +- client/src/util/typedCrossfilter/bitArray.js | 26 + .../src/util/typedCrossfilter/crossfilter.js | 594 +++++++++++++ client/src/util/typedCrossfilter/index.js | 777 +----------------- 13 files changed, 1153 insertions(+), 1582 deletions(-) create mode 100644 client/__tests__/util/typedCrossfilter/crossfilter.test.js delete mode 100644 client/__tests__/util/typedCrossfilter/typedCrossfilter.test.js create mode 100644 client/src/util/typedCrossfilter/crossfilter.js diff --git a/client/__tests__/util/stateManager/world.test.js b/client/__tests__/util/stateManager/world.test.js index 94c1b9c8..a4eeed0a 100644 --- a/client/__tests__/util/stateManager/world.test.js +++ b/client/__tests__/util/stateManager/world.test.js @@ -3,6 +3,7 @@ import * as Universe from "../../../src/util/stateManager/universe"; import * as World from "../../../src/util/stateManager/world"; import * as Dataframe from "../../../src/util/dataframe"; import Crossfilter from "../../../src/util/typedCrossfilter"; +import { DimTypes } from "../../../src/util/typedCrossfilter/crossfilter"; import * as REST from "./sampleResponses"; import { obsAnnoDimensionName, @@ -26,15 +27,15 @@ const defaultBigBang = () => { /* create world */ const world = World.createWorldFromEntireUniverse(universe); /* create crossfilter */ - const crossfilter = Crossfilter(world.obsAnnotations); - /* create dimension map */ - const dimensionMap = World.createObsDimensionMap(crossfilter, world); + const crossfilter = World.createObsDimensions( + new Crossfilter(world.obsAnnotations), + world + ); return { universe, world, - crossfilter, - dimensionMap + crossfilter }; }; @@ -71,13 +72,16 @@ describe("createWorldFromCurrentSelection", () => { const { universe, world: originalWorld, - crossfilter, - dimensionMap + crossfilter: originalCrossfilter } = defaultBigBang(); /* mock a selection */ - dimensionMap[obsAnnoDimensionName("field1")].filterRange([0, 5]); - dimensionMap[obsAnnoDimensionName("field3")].filterExact(false); + let crossfilter = originalCrossfilter + .select(obsAnnoDimensionName("field1"), { mode: "range", lo: 0, hi: 5 }) + .select(obsAnnoDimensionName("field3"), { + mode: "exact", + values: [false] + }); /* create the world from the selection */ const world = World.createWorldFromCurrentSelection( @@ -86,7 +90,7 @@ describe("createWorldFromCurrentSelection", () => { crossfilter ); expect(world).toBeDefined(); - expect(world.nObs).toEqual(crossfilter.countFiltered()); + expect(world.nObs).toEqual(crossfilter.countSelected()); /* calculate expected values and match against result @@ -136,43 +140,32 @@ describe("createObsDimensionMap", () => { - check that dimension typing is sane */ - const { dimensionMap } = defaultBigBang(); + const { crossfilter } = defaultBigBang(); const annotationNames = _.map( REST.schema.schema.annotations.obs, c => c.name ); const schemaByObsName = _.keyBy(REST.schema.schema.annotations.obs, "name"); - expect(dimensionMap).toBeDefined(); + expect(crossfilter).toBeDefined(); annotationNames.forEach(name => { - const dim = dimensionMap[obsAnnoDimensionName(name)]; + const dim = crossfilter.dimensions[obsAnnoDimensionName(name)]; if (name === "name") { expect(dim).toBeUndefined(); } else { const { type } = schemaByObsName[name]; if (type === "string" || type === "boolean" || type === "categorical") { - expect(dim).toBeInstanceOf(Crossfilter.EnumDimension); + expect(dim.dim).toBeInstanceOf(DimTypes.enum); } else { - expect(dim).toBeInstanceOf(Crossfilter.ScalarDimension); + expect(dim.dim).toBeInstanceOf(DimTypes.scalar); } } }); - expect(dimensionMap[layoutDimensionName("XY")]).toBeInstanceOf( - Crossfilter.SpatialDimension - ); + expect( + crossfilter.dimensions[layoutDimensionName("XY")].dim + ).toBeInstanceOf(DimTypes.spatial); }); }); -describe("createVarDataDimension", () => { - /* create default universe */ - const { world, crossfilter } = defaultBigBang(); - world.varData = world.varData.withCol( - "GENE", - Float32Array.from(_.range(world.nObs)) - ); - const result = World.createVarDataDimension(world, crossfilter, "GENE"); - expect(result).toBeInstanceOf(Crossfilter.ScalarDimension); -}); - describe("worldEqUniverse", () => { const { universe, world } = defaultBigBang(); const result = World.worldEqUniverse(world, universe); diff --git a/client/__tests__/util/typedCrossfilter/crossfilter.test.js b/client/__tests__/util/typedCrossfilter/crossfilter.test.js new file mode 100644 index 00000000..fefa1410 --- /dev/null +++ b/client/__tests__/util/typedCrossfilter/crossfilter.test.js @@ -0,0 +1,330 @@ +import _ from "lodash"; +import { polygonContains } from "d3"; + +import Crossfilter from "../../../src/util/typedCrossfilter"; + +const someData = [ + { + date: "2011-11-14T16:17:54Z", + quantity: 2, + total: 190, + tip: 100, + type: "tab", + productIDs: ["001"], + coords: [0, 0] + }, + { + date: "2011-11-14T16:20:19Z", + quantity: 2, + total: 190, + tip: 100, + type: "tab", + productIDs: ["001", "005"], + coords: [0.4, 0.4] + }, + { + date: "2011-11-14T16:28:54Z", + quantity: 1, + total: 300, + tip: 200, + type: "visa", + productIDs: ["004", "005"], + coords: [0.3, 0.1] + }, + { + date: "2011-11-14T16:30:43Z", + quantity: 2, + total: 90, + tip: 0, + type: "tab", + productIDs: ["001", "002"], + coords: [0.392, 0.1] + }, + { + date: "2011-11-14T16:48:46Z", + quantity: 2, + total: 90, + tip: 0, + type: "tab", + productIDs: ["005"], + coords: [0.7, 0.0482] + }, + { + date: "2011-11-14T16:53:41Z", + quantity: 2, + total: 90, + tip: 0, + type: "tab", + productIDs: ["001", "004", "005"], + coords: [0.9999, 1.0] + }, + { + date: "2011-11-14T16:54:06Z", + quantity: 1, + total: 100, + tip: 0, + type: "cash", + productIDs: ["001", "002", "003", "004", "005"], + coords: [0.384, 0.6938] + }, + { + date: "2011-11-14T16:58:03Z", + quantity: 2, + total: 90, + tip: 0, + type: "tab", + productIDs: ["001"], + coords: [0.4822, 0.482] + }, + { + date: "2011-11-14T17:07:21Z", + quantity: 2, + total: 90, + tip: 0, + type: "tab", + productIDs: ["004", "005"], + coords: [0.2234, 0] + }, + { + date: "2011-11-14T17:22:59Z", + quantity: 2, + total: 90, + tip: 0, + type: "tab", + productIDs: ["001", "002", "004", "005"], + coords: [0.382, 0.38485] + }, + { + date: "2011-11-14T17:25:45Z", + quantity: 2, + total: 200, + tip: 0, + type: "cash", + productIDs: ["002"], + coords: [0.998, 0.8472] + }, + { + date: "2011-11-14T17:29:52Z", + quantity: 1, + total: 200, + tip: 100, + type: "visa", + productIDs: ["004"], + coords: [0.8273, 0.3384] + } +]; + +let payments = null; +beforeEach(() => { + payments = new Crossfilter(someData); +}); + +describe("ImmutableTypedCrossfilter", () => { + test("create crossfilter", () => { + expect(payments).toBeDefined(); + expect(payments.size()).toEqual(someData.length); + expect(payments.all()).toEqual(someData); + + const p = payments + .addDimension("quantity", "scalar", (i, d) => d[i].quantity, Int32Array) + .select("quantity", { mode: "all" }); + expect(p).toBeDefined(); + expect(p.all()).toEqual(someData); + expect(p.size()).toEqual(someData.length); + expect(p.isElementSelected(0)).toBeTruthy(); + expect(p.countSelected()).toEqual(someData.length); + expect(p.allSelected()).toEqual(someData); + }); + + test("immutability", () => { + /* + the following should return a new crossfilter: + - addDimension() + - delDimension() + - select + */ + const p2 = payments.addDimension( + "quantity", + "scalar", + (i, data) => data[i].quantity, + Int32Array + ); + + expect(payments).not.toBe(p2); + const p3 = p2.select("quantity", { mode: "all" }); + expect(p3).not.toBe(p2); + + const p4 = p3.delDimension("quantity"); + expect(p4).not.toBe(p3); + }); + + test("select all and none", () => { + let p = payments + .addDimension("quantity", "scalar", (i, d) => d[i].quantity, Int32Array) + .addDimension("tip", "scalar", (i, d) => d[i].tip, Float32Array) + .addDimension("total", "scalar", (i, d) => d[i].total, Float32Array) + .addDimension("type", "enum", (i, d) => d[i].type); + expect(p).toBeDefined(); + + /* expect all records to be selected - default init state */ + expect(p.allSelected()).toEqual(someData); + expect(p.countSelected()).toEqual(someData.length); + expect(p.allSelectedMask()).toEqual( + new Uint8Array(someData.length).fill(1) + ); + expect(p.fillByIsSelected(new Uint8Array(someData.length), 99, 0)).toEqual( + new Uint8Array(someData.length).fill(99) + ); + for (let i = 0; i < someData.length; i += 1) { + expect(p.isElementSelected(i)).toBeTruthy(); + } + + /* expect a selectAll on one dimension to change nothing */ + p = p.select("tip", { mode: "all" }); + expect(p.allSelected()).toEqual(someData); + + /* ditto */ + p = p.select("quantity", { mode: "all" }); + expect(p.allSelected()).toEqual(someData); + + /* select none on one dimension */ + p = p.select("type", { mode: "none" }); + expect(p.allSelected()).toEqual([]); + expect(p.countSelected()).toEqual(0); + expect(p.allSelectedMask()).toEqual( + new Uint8Array(someData.length).fill(0) + ); + expect(p.fillByIsSelected(new Uint8Array(someData.length), 99, 0)).toEqual( + new Uint8Array(someData.length).fill(0) + ); + for (let i = 0; i < someData.length; i += 1) { + expect(p.isElementSelected(i)).toBeFalsy(); + } + + p = p.select("quantity", { mode: "none" }); + expect(p.allSelected()).toEqual([]); + + // invert the first none; should have no effect because type is + // still not filtered. + p = p.select("quantity", { mode: "all" }); + expect(p.allSelected()).toEqual([]); + + /* select all of type; should select all records */ + p = p.select("type", { mode: "all" }); + expect(p.allSelected()).toEqual(someData); + }); + + describe("scalar dimension", () => { + let p; + beforeEach(() => { + p = payments + .addDimension("quantity", "scalar", (i, d) => d[i].quantity, Int32Array) + .addDimension("tip", "scalar", (i, d) => d[i].tip, Float32Array) + .select("tip", { mode: "all" }); + }); + + /* + select modes: all, none, exact, range + */ + test("all", () => { + expect(p.select("quantity", { mode: "all" }).countSelected()).toEqual( + someData.length + ); + }); + test("none", () => { + expect(p.select("quantity", { mode: "none" }).countSelected()).toEqual(0); + }); + test.each([[[]], [[2]], [[2, 1]], [[9, 82]], [[0, 1]]])("exact: %p", v => + expect( + p.select("quantity", { mode: "exact", values: v }).countSelected() + ).toEqual(_.filter(someData, d => v.includes(d.quantity)).length) + ); + test.each([[0, 1], [1, 2], [0, 99], [99, 100000]])("range %p", (lo, hi) => + expect( + p.select("quantity", { mode: "range", lo, hi }).countSelected() + ).toEqual( + _.filter(someData, d => d.quantity >= lo && d.quantity < hi).length + ) + ); + test("bad mode", () => { + expect(() => p.select("type", { mode: "bad mode" })).toThrow(Error); + }); + }); + + describe("enum dimension", () => { + let p; + beforeEach(() => { + p = payments.addDimension("type", "enum", (i, d) => d[i].type); + }); + + test("all", () => { + expect(p.select("type", { mode: "all" }).countSelected()).toEqual( + someData.length + ); + }); + test("none", () => { + expect(p.select("type", { mode: "none" }).countSelected()).toEqual(0); + }); + test.each([ + [[]], + [["tab"]], + [["visa"]], + [["visa", "tab"]], + [["cash", "tab", "visa"]] + ])("exact: %p", v => + expect( + p.select("type", { mode: "exact", values: v }).countSelected() + ).toEqual(_.filter(someData, d => v.includes(d.type)).length) + ); + test("range", () => { + expect(() => p.select("type", { mode: "range", lo: 0, hi: 9 })).toThrow( + Error + ); + }); + test("bad mode", () => { + expect(() => p.select("type", { mode: "bad mode" })).toThrow(Error); + }); + }); + + describe("spatial dimension", () => { + let p; + beforeEach(() => { + const X = someData.map(r => r.coords[0]); + const Y = someData.map(r => r.coords[1]); + p = payments.addDimension("coords", "spatial", X, Y); + }); + + test("all", () => { + expect(p.select("coords", { mode: "all" }).countSelected()).toEqual( + someData.length + ); + }); + test("none", () => { + expect(p.select("coords", { mode: "none" }).countSelected()).toEqual(0); + }); + test.each([[0, 0, 1, 1], [0, 0, 0.5, 0.5], [0.5, 0.5, 1, 1]])( + "within-rect %d %d %d %d", + (x0, y0, x1, y1) => { + expect( + p + .select("coords", { mode: "within-rect", x0, y0, x1, y1 }) + .allSelected() + ).toEqual( + _.filter(someData, d => { + const [x, y] = d.coords; + return x0 <= x && x < x1 && y0 <= y && y < y1; + }) + ); + } + ); + + test.each([ + [[[0, 0], [0, 1], [1, 1], [1, 0]]], + [[[0, 0], [0, 0.5], [0.5, 0.5], [0.5, 0]]] + ])("within-polygon %p", polygon => { + expect( + p.select("coords", { mode: "within-polygon", polygon }).allSelected() + ).toEqual(_.filter(someData, d => polygonContains(polygon, d.coords))); + }); + }); +}); diff --git a/client/__tests__/util/typedCrossfilter/typedCrossfilter.test.js b/client/__tests__/util/typedCrossfilter/typedCrossfilter.test.js deleted file mode 100644 index f7fa2f12..00000000 --- a/client/__tests__/util/typedCrossfilter/typedCrossfilter.test.js +++ /dev/null @@ -1,590 +0,0 @@ -// jshint esversion: 6 -import _ from "lodash"; -import crossfilter from "../../../src/util/typedCrossfilter"; - -const someData = [ - { - date: "2011-11-14T16:17:54Z", - quantity: 2, - total: 190, - tip: 100, - type: "tab", - productIDs: ["001"] - }, - { - date: "2011-11-14T16:20:19Z", - quantity: 2, - total: 190, - tip: 100, - type: "tab", - productIDs: ["001", "005"] - }, - { - date: "2011-11-14T16:28:54Z", - quantity: 1, - total: 300, - tip: 200, - type: "visa", - productIDs: ["004", "005"] - }, - { - date: "2011-11-14T16:30:43Z", - quantity: 2, - total: 90, - tip: 0, - type: "tab", - productIDs: ["001", "002"] - }, - { - date: "2011-11-14T16:48:46Z", - quantity: 2, - total: 90, - tip: 0, - type: "tab", - productIDs: ["005"] - }, - { - date: "2011-11-14T16:53:41Z", - quantity: 2, - total: 90, - tip: 0, - type: "tab", - productIDs: ["001", "004", "005"] - }, - { - date: "2011-11-14T16:54:06Z", - quantity: 1, - total: 100, - tip: 0, - type: "cash", - productIDs: ["001", "002", "003", "004", "005"] - }, - { - date: "2011-11-14T16:58:03Z", - quantity: 2, - total: 90, - tip: 0, - type: "tab", - productIDs: ["001"] - }, - { - date: "2011-11-14T17:07:21Z", - quantity: 2, - total: 90, - tip: 0, - type: "tab", - productIDs: ["004", "005"] - }, - { - date: "2011-11-14T17:22:59Z", - quantity: 2, - total: 90, - tip: 0, - type: "tab", - productIDs: ["001", "002", "004", "005"] - }, - { - date: "2011-11-14T17:25:45Z", - quantity: 2, - total: 200, - tip: 0, - type: "cash", - productIDs: ["002"] - }, - { - date: "2011-11-14T17:29:52Z", - quantity: 1, - total: 200, - tip: 100, - type: "visa", - productIDs: ["004"] - } -]; - -function groupReduce(data, valueMap, valueReduce, valueInit) { - return _.reduce( - data, - (acc, value) => { - const k = valueMap(value); - let r = _.find(acc, o => o.key === k); - if (!r) { - r = { key: k, value: valueInit() }; - acc.push(r); - } - r.value = valueReduce(r.value, value); - return acc; - }, - [] - ).sort((a, b) => (a.key < b.key ? -1 : a.key > b.key ? 1 : 0)); -} - -function groupCount(data, map) { - return groupReduce(data, map, p => p + 1, () => 0); -} - -function groupSum(data, map) { - return groupReduce( - data, - map, - (p, v) => { - p += map(v); - return p; - }, - () => 0 - ); -} - -let payments = null; -beforeEach(() => { - payments = crossfilter(someData); -}); - -describe("typedCrossfilter", () => { - test("alloc and free", () => { - expect(payments).toBeDefined(); - expect(payments.size()).toEqual(someData.length); - expect(payments.all()).toEqual(someData); - - const quantity = payments.dimension( - crossfilter.ScalarDimension, - (i, data) => data[i].quantity, - Int32Array - ); - expect(quantity).toBeDefined(); - expect(quantity.id()).toBeDefined(); - - quantity.dispose(); - expect(payments.size()).toEqual(someData.length); - expect(payments.all()).toEqual(someData); - }); - - test("filterAll and filterNone", () => { - expect(payments).toBeDefined(); - const quantity = payments.dimension( - crossfilter.ScalarDimension, - (i, data) => data[i].quantity, - Int32Array - ); - const tip = payments.dimension( - crossfilter.ScalarDimension, - (i, data) => data[i].tip, - Float32Array - ); - const total = payments.dimension( - crossfilter.ScalarDimension, - (i, data) => data[i].total, - Float32Array - ); - const type = payments.dimension( - crossfilter.EnumDimension, - (i, data) => data[i].type - ); - - expect(quantity).toBeDefined(); - expect(tip).toBeDefined(); - expect(total).toBeDefined(); - expect(type).toBeDefined(); - - // initially, all should be filtered - expect(payments.allFiltered()).toHaveLength(payments.size()); - expect(payments.allFiltered()).toEqual(payments.all()); - expect(payments.countFiltered()).toEqual(someData.length); - - // filterAll - tip.filterAll(); // should change nothing - expect(payments.allFiltered()).toEqual(payments.all()); - expect(payments.countFiltered()).toEqual(someData.length); - - // ditto - total.filterAll(); - expect(payments.allFiltered()).toEqual(payments.all()); - expect(payments.countFiltered()).toEqual(someData.length); - - // filterNone - type.filterNone(); - expect(payments.allFiltered()).toEqual([]); - expect(payments.countFiltered()).toEqual(0); - - quantity.filterNone(); - expect(payments.allFiltered()).toEqual([]); - expect(payments.countFiltered()).toEqual(0); - - // invert the first none; should have no effect because type is - // still not filtered - quantity.filterAll(); - expect(payments.allFiltered()).toEqual([]); - expect(payments.countFiltered()).toEqual(0); - - // filter all of type; should select all - type.filterAll(); - expect(payments.allFiltered()).toEqual(payments.all()); - expect(payments.countFiltered()).toEqual(payments.size()); - }); - - test("filterExact", () => { - expect(payments).toBeDefined(); - const quantity = payments.dimension( - crossfilter.ScalarDimension, - (i, data) => data[i].quantity, - Int32Array - ); - const tip = payments.dimension( - crossfilter.ScalarDimension, - (i, data) => data[i].tip, - Float32Array - ); - const type = payments.dimension( - crossfilter.EnumDimension, - (i, data) => data[i].type - ); - - quantity.filterExact(1); - expect(payments.countFiltered()).toEqual( - _.countBy(someData, "quantity")[1] - ); - expect(payments.allFiltered()).toEqual(_.filter(someData, { quantity: 1 })); - - tip.filterExact(0); - expect(payments.allFiltered()).toEqual( - _.filter(someData, { tip: 0, quantity: 1 }) - ); - - type.filterExact("cash"); - expect(payments.allFiltered()).toEqual( - _.filter(someData, { tip: 0, quantity: 1, type: "cash" }) - ); - }); - - test("filterRange", () => { - expect(payments).toBeDefined(); - const quantity = payments.dimension( - crossfilter.ScalarDimension, - (i, data) => data[i].quantity, - Int32Array - ); - const tip = payments.dimension( - crossfilter.ScalarDimension, - (i, data) => data[i].tip, - Float32Array - ); - const total = payments.dimension( - crossfilter.ScalarDimension, - (i, data) => data[i].total, - Float32Array - ); - const type = payments.dimension( - crossfilter.EnumDimension, - (i, data) => data[i].type - ); - - tip.filterRange([0, 91]); - expect(payments.allFiltered()).toEqual( - _(someData) - .filter(r => r.tip >= 0 && r.tip < 91) - .value() - ); - - tip.filterRange([0, 90]); - expect(payments.allFiltered()).toEqual( - _(someData) - .filter(r => r.tip >= 0 && r.tip < 90) - .value() - ); - - tip.filterRange([1, 90]); - expect(payments.allFiltered()).toEqual( - _(someData) - .filter(r => r.tip >= 1 && r.tip < 91) - .value() - ); - }); - - test("filterEnum", () => { - expect(payments).toBeDefined(); - const quantity = payments.dimension( - crossfilter.ScalarDimension, - (i, data) => data[i].quantity, - Int32Array - ); - const tip = payments.dimension( - crossfilter.ScalarDimension, - (i, data) => data[i].tip, - Float32Array - ); - const total = payments.dimension( - crossfilter.ScalarDimension, - (i, data) => data[i].total, - Float32Array - ); - const type = payments.dimension( - crossfilter.EnumDimension, - (i, data) => data[i].type - ); - - type.filterEnum(["tab", "cash"]); - expect(payments.allFiltered()).toEqual( - _(someData) - .filter(r => r.type === "cash" || r.type === "tab") - .value() - ); - - tip.filterEnum([0, 100]); - expect(payments.allFiltered()).toEqual( - _(someData) - .filter(r => r.type === "cash" || r.type === "tab") - .filter(r => r.tip === 0 || r.tip === 100) - .value() - ); - }); - - test("more than 32 dimensions", () => { - expect(payments).toBeDefined(); - const quantity = payments.dimension( - crossfilter.ScalarDimension, - (i, data) => data[i].quantity, - Int32Array - ); - const tip = payments.dimension( - crossfilter.ScalarDimension, - (i, data) => data[i].tip, - Float32Array - ); - const total = payments.dimension( - crossfilter.ScalarDimension, - (i, data) => data[i].total, - Float32Array - ); - const type = payments.dimension( - crossfilter.EnumDimension, - (i, data) => data[i].type - ); - - // Create a bunch of fake dimensions to ensure we can handle > 32 - let dimMap = {}; - for (let i = 0; i < 65; i++) { - dimMap[i] = payments.dimension( - crossfilter.ScalarDimension, - () => Math.random(), - Float32Array - ); - expect(dimMap[i]).toBeDefined(); - expect(dimMap[i].id()).toBeDefined(); - } - - // everything should start as selected/filtered - expect(payments.countFiltered()).toEqual(someData.length); - - dimMap[0].filterAll(); - dimMap[64].filterAll(); - expect(payments.countFiltered()).toEqual(someData.length); - - dimMap[33].filterNone(); - expect(payments.allFiltered()).toEqual([]); - - dimMap[33].filterAll(); - expect(payments.allFiltered()).toEqual(someData); - }); - - test("group, default mapping, default reducer, no filter", () => { - expect(payments).toBeDefined(); - - const quantity = payments.dimension( - crossfilter.ScalarDimension, - (i, data) => data[i].quantity, - Int32Array - ); - const tip = payments.dimension( - crossfilter.ScalarDimension, - (i, data) => data[i].tip, - Int32Array - ); - const type = payments.dimension( - crossfilter.EnumDimension, - (i, data) => data[i].type - ); - const total = payments.dimension( - crossfilter.ScalarDimension, - (i, data) => data[i].total, - Int32Array - ); - - _.each( - { - tip: tip.group(r => r), - type: type.group(), - total: total.group(), - quantity: quantity.group() - }, - (grp, k) => { - const whatWeExpect = groupCount(someData, v => v[k]); - expect(grp.all()).toEqual(whatWeExpect); - expect(grp.size()).toEqual(whatWeExpect.length); - expect(grp.dispose()).toEqual(grp); - } - ); - }); - - test("group, custom map, default reducer, no filters", () => { - expect(payments).toBeDefined(); - - // custom mapping in groups only works for scalar types. Enums do not - // currently implement it. - - const tip = payments.dimension( - crossfilter.ScalarDimension, - (i, data) => data[i].tip, - Int32Array - ); - const totalX10 = payments.dimension( - crossfilter.ScalarDimension, - (i, data) => data[i].total * 10, - Int32Array - ); - const type = payments.dimension( - crossfilter.EnumDimension, - (i, data) => data[i].type - ); - - const paymentsByTip_A = tip.group(); - const paymentsByTip_B = tip.group(r => 10 * r); - const paymentsByType = type.group(); // identity only - const paymentsByTotalX10_A = totalX10.group(); - const paymentsByTotalX10_B = totalX10.group(r => r / 10); - - expect(paymentsByTip_A.all()).toEqual(groupCount(someData, v => v.tip)); - expect(paymentsByTip_B.all()).toEqual( - groupCount(someData, v => 10 * v.tip) - ); - expect(paymentsByType.all()).toEqual(groupCount(someData, v => v.type)); - expect(paymentsByTotalX10_A.all()).toEqual( - groupCount(someData, v => 10 * v.total) - ); - expect(paymentsByTotalX10_B.all()).toEqual( - groupCount(someData, v => (10 * v.total) / 10) - ); - - for (let i of [ - paymentsByTip_A, - paymentsByTip_B, - paymentsByType, - paymentsByTotalX10_A, - paymentsByTotalX10_B, - tip, - totalX10, - type - ]) { - expect(i.dispose()).toEqual(i); - } - }); - - test("group, default map, custom reducer, no filters", () => { - expect(payments).toBeDefined(); - - const total = payments.dimension( - crossfilter.ScalarDimension, - (i, data) => data[i].total, - Float32Array - ); - const type = payments.dimension( - crossfilter.EnumDimension, - (i, data) => data[i].type - ); - - const paymentsByTotal = total.group(); - const paymentsByType = type.group(); - - // reduceCount - expect(paymentsByTotal.reduceCount()).toEqual(paymentsByTotal); - expect(paymentsByTotal.all()).toEqual(groupCount(someData, v => v.total)); - - // reduceSum - expect(paymentsByTotal.reduceSum(v => v.total)).toEqual(paymentsByTotal); - expect(paymentsByTotal.all()).toEqual(groupSum(someData, v => v.total)); - - // use custom reducers (my reducers) - count by three, init 1 - expect( - paymentsByTotal.reduce((p, v) => (p += 3), (p, v) => (p -= 3), () => 1) - ).toEqual(paymentsByTotal); - expect(paymentsByTotal.all()).toEqual( - groupReduce(someData, v => v.total, (p, v) => p + 3, () => 1) - ); - - for (let i of [paymentsByTotal, paymentsByType, type]) { - expect(i.dispose()).toEqual(i); - } - }); - - test("group, default map, default reducer, filters", () => { - // From the docs: - // Note: a grouping intersects the crossfilter's current filters, except for the - // associated dimension's filter. Thus, group methods consider only records that - // satisfy every filter except this dimension's filter. So, if the crossfilter of - // payments is filtered by type and total, then group by total only observes the - // filter by type. - - expect(payments).toBeDefined(); - - const tip = payments.dimension( - crossfilter.ScalarDimension, - (i, data) => data[i].tip, - Int32Array - ); - const total = payments.dimension( - crossfilter.ScalarDimension, - (i, data) => data[i].total, - Int32Array - ); - const type = payments.dimension( - crossfilter.EnumDimension, - (i, data) => data[i].type - ); - - const paymentsByTip = tip.group(); - const paymentsByTotal = total.group(); - const paymentsByType = type.group(); - - // 1. confirm that changing the filter on a dimension does NOT change that - // dimensions groups. - { - tip.filterAll(), total.filterAll(), type.filterAll(); - let before = _.cloneDeep(paymentsByTip.all()); - tip.filterExact(0); - expect(paymentsByTip.all()).toEqual(before); - } - - // 2. confirm that changing a filter on a different dimension DOES change - // all other groups. - { - tip.filterAll(), total.filterAll(), type.filterAll(); - const before = _.cloneDeep([paymentsByTotal.all(), paymentsByType.all()]); - tip.filterExact(0); - const after = [paymentsByTotal.all(), paymentsByType.all()]; - expect(after).not.toEqual(before); - expect(after).toEqual([ - groupReduce( - someData, - v => v.total, - (p, v) => (v.tip !== 0 ? p : p + 1), - () => 0 - ), - groupReduce( - someData, - v => v.type, - (p, v) => (v.tip !== 0 ? p : p + 1), - () => 0 - ) - ]); - } - - for (let i of [ - paymentsByTip, - paymentsByTotal, - paymentsByType, - tip, - total, - type - ]) { - expect(i.dispose()).toEqual(i); - } - }); -}); diff --git a/client/src/components/continuous/continuous.js b/client/src/components/continuous/continuous.js index 7d994889..81659b14 100644 --- a/client/src/components/continuous/continuous.js +++ b/client/src/components/continuous/continuous.js @@ -11,8 +11,7 @@ import HistogramBrush from "../brushableHistogram"; @connect(state => ({ obsAnnotations: _.get(state.controls.world, "obsAnnotations", null), colorAccessor: state.controls.colorAccessor, - colorScale: state.controls.colors.scale, - selectionUpdate: _.get(state.controls, "crossfilter.updateTime", null), + colorScale: state.controls.colorScale, schema: _.get(state.controls.world, "schema", null) })) class Continuous extends React.Component { diff --git a/client/src/components/geneExpression/expressionButtons.js b/client/src/components/geneExpression/expressionButtons.js index cd545fb6..fe47c02b 100644 --- a/client/src/components/geneExpression/expressionButtons.js +++ b/client/src/components/geneExpression/expressionButtons.js @@ -10,8 +10,7 @@ import CellSetButton from "./cellSetButtons"; @connect(state => ({ differential: state.differential, world: state.controls.world, - crossfilter: state.controls.crossfilter, - selectionUpdate: _.get(state.controls, "crossfilter.updateTime", null) + crossfilter: state.controls.crossfilter })) class Expression extends React.Component { constructor(props) { diff --git a/client/src/components/graph/graph.js b/client/src/components/graph/graph.js index b8c16e31..2884dff8 100644 --- a/client/src/components/graph/graph.js +++ b/client/src/components/graph/graph.js @@ -32,7 +32,6 @@ import { World } from "../../util/stateManager"; responsive: state.responsive, colorRGB: _.get(state.controls, "colors.rgb", null), opacityForDeselectedCells: state.controls.opacityForDeselectedCells, - selectionUpdate: _.get(state.controls, "crossfilter.updateTime", null), resettingInterface: state.controls.resettingInterface, userDefinedGenes: state.controls.userDefinedGenes, diffexpGenes: state.controls.diffexpGenes, @@ -103,13 +102,7 @@ class Graph extends React.Component { componentDidUpdate(prevProps) { const { renderCache } = this; - const { - world, - crossfilter, - colorRGB, - responsive, - selectionUpdate - } = this.props; + const { world, crossfilter, colorRGB, responsive } = this.props; const { reglRender, mode, @@ -173,11 +166,11 @@ class Graph extends React.Component { // Sizes for each point - updates are triggered only when selected // obs change - if (!renderCache.sizes || selectionUpdate !== prevProps.selectionUpdate) { + if (!renderCache.sizes || crossfilter !== prevProps.crossfilter) { if (!renderCache.sizes) { renderCache.sizes = new Float32Array(nObs); } - crossfilter.fillByIsFiltered(renderCache.sizes, 4, 0.2); + crossfilter.fillByIsSelected(renderCache.sizes, 4, 0.2); sizeBuffer({ data: renderCache.sizes, dimension: 1 }); } @@ -242,7 +235,7 @@ class Graph extends React.Component { if (!crossfilter || !world || !universe) { return false; } - const nothingSelected = crossfilter.countFiltered() === crossfilter.size(); + const nothingSelected = crossfilter.countSelected() === crossfilter.size(); const nothingColoredBy = !colorAccessor; const noGenes = userDefinedGenes.length === 0 && diffexpGenes.length === 0; const scatterNotDpl = !scatterplotXXaccessor || !scatterplotYYaccessor; @@ -450,8 +443,8 @@ class Graph extends React.Component { data-testid="subset-button" disabled={ crossfilter && - (crossfilter.countFiltered() === 0 || - crossfilter.countFiltered() === crossfilter.size()) + (crossfilter.countSelected() === 0 || + crossfilter.countSelected() === crossfilter.size()) } style={{ marginRight: 10 }} onClick={() => { diff --git a/client/src/components/scatterplot/scatterplot.js b/client/src/components/scatterplot/scatterplot.js index 71438f7c..b4691161 100644 --- a/client/src/components/scatterplot/scatterplot.js +++ b/client/src/components/scatterplot/scatterplot.js @@ -58,9 +58,7 @@ import finiteExtent from "../../util/finiteExtent"; expressionX, expressionY, - crossfilter, - // updated whenever the crossfilter selection is updated - selectionUpdate: _.get(state.controls, "crossfilter.updateTime", null) + crossfilter }; }) class Scatterplot extends React.Component { @@ -136,8 +134,7 @@ class Scatterplot extends React.Component { scatterplotYYaccessor, expressionX, expressionY, - colorRGB, - selectionUpdate + colorRGB } = this.props; const { reglRender, @@ -210,11 +207,11 @@ class Scatterplot extends React.Component { // Sizes for each point - updates are triggered only when selected // obs change - if (!renderCache.sizes || selectionUpdate !== prevProps.selctionUpdate) { + if (!renderCache.sizes || crossfilter !== prevProps.crossfilter) { if (!renderCache.sizes) { renderCache.sizes = new Float32Array(cellCount); } - crossfilter.fillByIsFiltered(renderCache.sizes, 4, 0.2); + crossfilter.fillByIsSelected(renderCache.sizes, 4, 0.2); sizeBuffer({ data: renderCache.sizes, dimension: 1 }); } diff --git a/client/src/reducers/controls.js b/client/src/reducers/controls.js index 570505f1..6a43235e 100644 --- a/client/src/reducers/controls.js +++ b/client/src/reducers/controls.js @@ -35,7 +35,6 @@ const Controls = ( world: null, categoricalSelectionState: null, crossfilter: null, - dimensionMap: null, userDefinedGenes: [], userDefinedGenesLoading: false, diffexpGenes: [], @@ -95,8 +94,10 @@ const Controls = ( state, world ); - const crossfilter = Crossfilter(world.obsAnnotations); - const dimensionMap = World.createObsDimensionMap(crossfilter, world); + const crossfilter = World.createObsDimensions( + new Crossfilter(world.obsAnnotations), + world + ); WorldUtil.clearCaches(); return { @@ -104,11 +105,10 @@ const Controls = ( loading: false, error: null, universe, - fullUniverseCache: { world, crossfilter, dimensionMap }, + fullUniverseCache: { world, crossfilter }, world, categoricalSelectionState, crossfilter, - dimensionMap, colorMode, colorAccessor: null, colors, @@ -116,40 +116,35 @@ const Controls = ( }; } case "reset World to eq Universe": { + /* + 1. Reset world & crossfilter, using previously created objects which were + stashed in `fullUniverseCache` + 2. Add crossfilter dimension for all userDefined and diffexp genes/varData, + as they are not part of the cached crossfilter. + 3. Compute categorical selection summary + 4. Reset all WorldUtil caches + 5. Reset color-by + */ const { userDefinedGenes, diffexpGenes, fullUniverseCache } = state; - const { world, crossfilter } = fullUniverseCache; - // reset all crossfilter dimensions - _.forEach(fullUniverseCache.dimensionMap, dim => dim.filterAll()); + const { world } = fullUniverseCache; + const crossfilter = ControlsHelpers.createGeneDimensions( + userDefinedGenes, + diffexpGenes, + world, + fullUniverseCache.crossfilter + ); const colorMode = null; const colors = createColors(world, colorMode); const categoricalSelectionState = ControlsHelpers.createCategoricalSelectionState( state, world ); - - /* free dimensions not in cache (otherwise they leak) */ - _.forEach(state.dimensionMap, (dim, dimName) => { - if (!fullUniverseCache.dimensionMap[dimName]) { - dim.dispose(); - } - }); - const dimensionMap = { - ...fullUniverseCache.dimensionMap, - ...ControlsHelpers.createGenesDimMap( - userDefinedGenes, - diffexpGenes, - world, - crossfilter - ) - }; WorldUtil.clearCaches(); - return { ...state, world, categoricalSelectionState, crossfilter, - dimensionMap, colorMode, colorAccessor: null, colors, @@ -175,16 +170,15 @@ const Controls = ( state, world ); - const crossfilter = Crossfilter(world.obsAnnotations); - const dimensionMap = { - ...World.createObsDimensionMap(crossfilter, world), - ...ControlsHelpers.createGenesDimMap( - userDefinedGenes, - diffexpGenes, - world, - crossfilter - ) - }; + let crossfilter = new Crossfilter(world.obsAnnotations); + crossfilter = World.createObsDimensions(crossfilter, world); + crossfilter = ControlsHelpers.createGeneDimensions( + userDefinedGenes, + diffexpGenes, + world, + crossfilter + ); + WorldUtil.clearCaches(); return { @@ -194,8 +188,7 @@ const Controls = ( world, colors, categoricalSelectionState, - crossfilter, - dimensionMap + crossfilter }; } case "expression load success": { @@ -276,56 +269,57 @@ const Controls = ( }; } case "request user defined gene success": { - const { world, crossfilter, dimensionMap, userDefinedGenes } = state; + const { world, crossfilter: oldCrossfilter, userDefinedGenes } = state; const _userDefinedGenes = userDefinedGenes.slice(); const gene = action.data.genes[0]; - dimensionMap[ - userDefinedDimensionName(gene) - ] = World.createVarDataDimension(world, crossfilter, gene); - + const crossfilter = oldCrossfilter.addDimension( + userDefinedDimensionName(gene), + "scalar", + world.varData.col(gene).asArray(), + Float32Array + ); return { ...state, - dimensionMap, + crossfilter, userDefinedGenes: _userDefinedGenes, userDefinedGenesLoading: false }; } case "request differential expression success": { - const { world, crossfilter, dimensionMap } = state; + const { world, crossfilter: oldCrossfilter } = state; const _diffexpGenes = []; action.data.forEach(d => { _diffexpGenes.push(world.varAnnotations.at(d[0], "name")); }); + let crossfilter = oldCrossfilter; _.forEach(_diffexpGenes, gene => { - dimensionMap[diffexpDimensionName(gene)] = World.createVarDataDimension( - world, - crossfilter, - gene + crossfilter = crossfilter.addDimension( + diffexpDimensionName(gene), + "scalar", + world.varData.col(gene).asArray(), + Float32Array ); }); return { ...state, - dimensionMap, + crossfilter, diffexpGenes: _diffexpGenes }; } case "clear differential expression": { - const { world, dimensionMap } = state; - const _dimensionMap = dimensionMap; + const { world } = state; + let { crossfilter } = state; _.forEach(action.diffExp, values => { const name = world.varAnnotations.at(values[0], "name"); - // clean up crossfilter dimensions - const dimension = dimensionMap[diffexpDimensionName(name)]; - dimension.dispose(); - delete dimensionMap[diffexpDimensionName(name)]; + crossfilter = crossfilter.delDimension(diffexpDimensionName(name)); }); return { ...state, - dimensionMap: _dimensionMap, + crossfilter, diffexpGenes: [] }; } @@ -342,34 +336,29 @@ const Controls = ( }; } case "clear user defined gene": { - const { userDefinedGenes, dimensionMap } = state; + const { userDefinedGenes, crossfilter: oldCrossfilter } = state; const newUserDefinedGenes = _.filter( userDefinedGenes, d => d !== action.data ); - - const dimension = dimensionMap[userDefinedDimensionName(action.data)]; - dimension.dispose(); - delete dimensionMap[userDefinedDimensionName(action.data)]; - + const crossfilter = oldCrossfilter.delDimension( + userDefinedDimensionName(action.data) + ); return { ...state, - dimensionMap, + crossfilter, userDefinedGenes: newUserDefinedGenes }; } case "clear all user defined genes": { - const { userDefinedGenes, dimensionMap } = state; - + const { userDefinedGenes } = state; + let { crossfilter } = state; _.forEach(userDefinedGenes, gene => { - const dimension = dimensionMap[userDefinedDimensionName(gene)]; - dimension.dispose(); - delete dimensionMap[userDefinedDimensionName(gene)]; + crossfilter = crossfilter.delDimension(userDefinedDimensionName(gene)); }); - return { ...state, - dimensionMap, + crossfilter, userDefinedGenes: [] }; } @@ -395,34 +384,49 @@ const Controls = ( User Events *******************************/ case "graph brush selection change": { - state.dimensionMap[layoutDimensionName("XY")].filterWithinRect( - action.brushCoords.northwest, - action.brushCoords.southeast - ); + const name = layoutDimensionName("XY"); + const [x0, y0] = action.brushCoords.northwest; + const [x1, y1] = action.brushCoords.southeast; + const crossfilter = state.crossfilter.select(name, { + mode: "within-rect", + x0, + y0, + x1, + y1 + }); return { ...state, + crossfilter, graphBrushSelection: action.brushCoords }; } case "lasso deselect": case "graph brush deselect": { - state.dimensionMap[layoutDimensionName("XY")].filterAll(); + const name = layoutDimensionName("XY"); + const crossfilter = state.crossfilter.select(name, { mode: "all" }); return { ...state, + crossfilter, graphBrushSelection: null }; } case "lasso selection": { const { polygon } = action; - const dXY = state.dimensionMap[layoutDimensionName("XY")]; + const name = layoutDimensionName("XY"); + const { crossfilter: oldCrossfilter } = state; + let crossfilter; if (polygon.length < 3) { // single point or a line is not a polygon, and is therefore a deselect - dXY.filterAll(); + crossfilter = oldCrossfilter.select(name, { mode: "all" }); } else { - dXY.filterWithinPolygon(polygon); + crossfilter = oldCrossfilter.select(name, { + mode: "within-polygon", + polygon + }); } return { - ...state + ...state, + crossfilter }; } case "continuous metadata histogram brush": { @@ -430,15 +434,17 @@ const Controls = ( action.continuousNamespace, action.selection ); + let { crossfilter } = state; // action.selection: metadata name being selected // action.range: filter range, or null if deselected if (!action.range) { - state.dimensionMap[name].filterAll(); + crossfilter = crossfilter.select(name, { mode: "all" }); } else { - state.dimensionMap[name].filterRange(action.range); + const [lo, hi] = action.range; + crossfilter = crossfilter.select(name, { mode: "range", lo, hi }); } - return { ...state }; + return { ...state, crossfilter }; } case "change opacity deselected cells in 2d graph background": return { @@ -476,13 +482,15 @@ const Controls = ( // update the filter to match all selected options const cat = newCategoricalSelectionState[action.metadataField]; - state.dimensionMap[obsAnnoDimensionName(action.metadataField)].filterEnum( - ControlsHelpers.selectedValuesForCategory(cat) - ); - + const dName = obsAnnoDimensionName(action.metadataField); + const crossfilter = state.crossfilter.select(dName, { + mode: "exact", + values: ControlsHelpers.selectedValuesForCategory(cat) + }); return { ...state, - categoricalSelectionState: newCategoricalSelectionState + categoricalSelectionState: newCategoricalSelectionState, + crossfilter }; } case "categorical metadata filter deselect": { @@ -500,13 +508,15 @@ const Controls = ( // update the filter to match all selected options const cat = newCategoricalSelectionState[action.metadataField]; - state.dimensionMap[obsAnnoDimensionName(action.metadataField)].filterEnum( - ControlsHelpers.selectedValuesForCategory(cat) - ); - + const dName = obsAnnoDimensionName(action.metadataField); + const crossfilter = state.crossfilter.select(dName, { + mode: "exact", + values: ControlsHelpers.selectedValuesForCategory(cat) + }); return { ...state, - categoricalSelectionState: newCategoricalSelectionState + categoricalSelectionState: newCategoricalSelectionState, + crossfilter }; } case "categorical metadata filter none of these": { @@ -520,12 +530,14 @@ const Controls = ( ).fill(false) } }; - state.dimensionMap[ - obsAnnoDimensionName(action.metadataField) - ].filterNone(); + const dName = obsAnnoDimensionName(action.metadataField); + const crossfilter = state.crossfilter.select(dName, { + mode: "none" + }); return { ...state, - categoricalSelectionState: newCategoricalSelectionState + categoricalSelectionState: newCategoricalSelectionState, + crossfilter }; } case "categorical metadata filter all of these": { @@ -539,12 +551,14 @@ const Controls = ( ).fill(true) } }; - state.dimensionMap[ - obsAnnoDimensionName(action.metadataField) - ].filterAll(); + const dName = obsAnnoDimensionName(action.metadataField); + const crossfilter = state.crossfilter.select(dName, { + mode: "all" + }); return { ...state, - categoricalSelectionState: newCategoricalSelectionState + categoricalSelectionState: newCategoricalSelectionState, + crossfilter }; } diff --git a/client/src/util/stateManager/controlsHelpers.js b/client/src/util/stateManager/controlsHelpers.js index dc03ba4b..48b04544 100644 --- a/client/src/util/stateManager/controlsHelpers.js +++ b/client/src/util/stateManager/controlsHelpers.js @@ -98,29 +98,35 @@ export function selectedValuesForCategory(categorySelectionState) { } /* -build a crossfilter dimension map for all gene expression related dimensions. +build a crossfilter dimensions for all gene expression related dimensions. */ -export function createGenesDimMap( +export function createGeneDimensions( userDefinedGenes, diffexpGenes, world, crossfilter ) { - function _createGenesDimMap(genes, nameCreator) { - return genes.reduce((acc, gene) => { - acc[nameCreator(gene)] = World.createVarDataDimension( - world, - crossfilter, - gene - ); - return acc; - }, {}); - } - - return { - ..._createGenesDimMap(userDefinedGenes, userDefinedDimensionName), - ..._createGenesDimMap(diffexpGenes, diffexpDimensionName) - }; + crossfilter = userDefinedGenes.reduce( + (xflt, gene) => + xflt.addDimension( + userDefinedDimensionName(gene), + "scalar", + world.varData.col(gene).asArray(), + Float32Array + ), + crossfilter + ); + crossfilter = diffexpGenes.reduce( + (xflt, gene) => + xflt.addDimension( + diffexpDimensionName(gene), + "scalar", + world.varData.col(gene).asArray(), + Float32Array + ), + crossfilter + ); + return crossfilter; } export function pruneVarDataCache(varData, needed) { diff --git a/client/src/util/stateManager/world.js b/client/src/util/stateManager/world.js index 8104a48b..9ac3124c 100644 --- a/client/src/util/stateManager/world.js +++ b/client/src/util/stateManager/world.js @@ -1,8 +1,6 @@ // jshint esversion: 6 -import _ from "lodash"; import { layoutDimensionName, obsAnnoDimensionName } from "../nameCreators"; -import Crossfilter from "../typedCrossfilter"; import * as Dataframe from "../dataframe"; /* @@ -98,7 +96,7 @@ export function createWorldFromCurrentSelection(universe, world, crossfilter) { newWorld.varAnnotations = universe.varAnnotations; /* now subset/cut obs */ - const mask = crossfilter.allFilteredMask(); + const mask = crossfilter.allSelectedMask(); newWorld.obsAnnotations = world.obsAnnotations.isubsetMask(mask); newWorld.obsLayout = world.obsLayout.isubsetMask(mask); newWorld.nObs = newWorld.obsAnnotations.dims[0]; @@ -139,63 +137,32 @@ function deduceDimensionType(attributes, fieldName) { return dimensionType; } -/* - Return a crossfilter dimension for the specified world & named gene. - - NOTE: this assumes that the expression data was already loaded, - by calling an appropriate action creator. - - Caller needs to *save* this dimension somewhere for it to be later used. - Dimension must be destroyed by calling dimension.dispose() - when it is no longer needed - (it will not be garbage collected without this call) -*/ -export function createVarDataDimension(world, crossfilter, name) { - return crossfilter.dimension( - Crossfilter.ScalarDimension, - world.varData.col(name).asArray(), - Float32Array - ); -} - -export function createObsDimensionMap(crossfilter, world) { +export function createObsDimensions(crossfilter, world) { /* - create and return a crossfilter dimension for every obs annotation - for which we have a supported type. + create and return a crossfilter with a dimension for every obs annotation + for which we have a supported type, *except* 'name' */ const { schema, obsLayout, obsAnnotations } = world; + const annoList = schema.annotations.obs.filter(anno => anno.name !== "name"); + crossfilter = annoList.reduce((xfltr, anno) => { + const dimType = deduceDimensionType(anno, anno.name); + const colData = obsAnnotations.col(anno.name).asArray(); + const name = obsAnnoDimensionName(anno.name); + if (dimType === "enum") { + return xfltr.addDimension(name, "enum", colData); + } + if (dimType) { + return xfltr.addDimension(name, "scalar", colData, dimType); + } + return xfltr; + }, crossfilter); - // Create a crossfilter dimension for all obs annotations *except* 'name' - const dimensionMap = _(schema.annotations.obs) - .filter(anno => anno.name !== "name") - .transform((result, anno) => { - const dimType = deduceDimensionType(anno, anno.name); - const colData = obsAnnotations.col(anno.name).asArray(); - if (dimType === "enum") { - result[obsAnnoDimensionName(anno.name)] = crossfilter.dimension( - Crossfilter.EnumDimension, - colData - ); - } else if (dimType) { - result[obsAnnoDimensionName(anno.name)] = crossfilter.dimension( - Crossfilter.ScalarDimension, - colData, - dimType - ); - } // else ignore the annotation - }, {}) - .value(); - - /* - Add crossfilter dimensions allowing filtering on layout - */ - dimensionMap[layoutDimensionName("XY")] = crossfilter.dimension( - Crossfilter.SpatialDimension, + return crossfilter.addDimension( + layoutDimensionName("XY"), + "spatial", obsLayout.col("X").asArray(), obsLayout.col("Y").asArray() ); - - return dimensionMap; } export function worldEqUniverse(world, universe) { @@ -206,7 +173,7 @@ export function getSelectedByIndex(crossfilter) { /* return array of obsIndex, containing all selected obs/cells. */ - const selected = crossfilter.allFilteredMask(); // array of bool-ish + const selected = crossfilter.allSelectedMask(); // array of bool-ish const keys = crossfilter.data.rowIndex.keys(); // row keys, aka universe rowIndex const set = new Int32Array(selected.length); diff --git a/client/src/util/typedCrossfilter/bitArray.js b/client/src/util/typedCrossfilter/bitArray.js index 1a72a7d0..51ff809a 100644 --- a/client/src/util/typedCrossfilter/bitArray.js +++ b/client/src/util/typedCrossfilter/bitArray.js @@ -211,6 +211,19 @@ class BitArray { } } + // select range of indices on a dimension + // + selectFromRange(dim, range) { + const col = dim >>> 5; + const first = range[0]; + const last = range[1]; + const one = 1 << dim % 32; + const offset = col * this.length; + for (let i = first; i < last; i += 1) { + this.bitarray[offset + i] |= one; + } + } + // select range of indices on a dimension, indirect through a sort map. // Indirect functions are used to map between sort and natural order. // @@ -225,6 +238,19 @@ class BitArray { } } + // deselect range of indices on a dimension + // + deselectFromRange(dim, range) { + const col = dim >>> 5; + const first = range[0]; + const last = range[1]; + const zero = ~(1 << dim % 32); + const offset = col * this.length; + for (let i = first; i < last; i += 1) { + this.bitarray[offset + i] &= zero; + } + } + // deselect range of indices on a dimension, indirect through a sort map. // deselectIndirectFromRange(dim, indirect, range) { diff --git a/client/src/util/typedCrossfilter/crossfilter.js b/client/src/util/typedCrossfilter/crossfilter.js new file mode 100644 index 00000000..616f5a4a --- /dev/null +++ b/client/src/util/typedCrossfilter/crossfilter.js @@ -0,0 +1,594 @@ +import { polygonContains } from "d3"; + +import PositiveIntervals from "./positiveIntervals"; +import BitArray from "./bitArray"; +import { sort } from "./sort"; +import { + makeSortIndex, + lowerBound, + lowerBoundIndirect, + upperBoundIndirect +} from "./util"; + +class NotImplementedError extends Error { + constructor(...params) { + super(...params); + + // Maintains proper stack trace for where our error was thrown (only available on V8) + if (Error.captureStackTrace) { + Error.captureStackTrace(this, NotImplementedError); + } + } +} + +export default class ImmutableTypedCrossfilter { + constructor(data, dimensions = {}, selectionCache = null) { + /* + Typically, parameter 'data' is one of: + - Array of objects/records + - Dataframe (util/dataframe) + Other parameters are only used internally. + + Object field description: + - data: reference to the array of records in the crossfilter + - selectionBitArray: bit array containing the flatted selection state + of all dimensions. This is lazily created and is effectively + a perfomance cache. Methods which return a new crossfilter, + such as select(), addDimention() and delDimension(), will pass + the cache forward to the new object, as the typical "immutable API" + usage pattern is to retain the new crossfilter and discard the old. + - dimensions: contains each dimension and its current state: + - id: bit offset in the cached bit array + - dim: the dimension object + - name: the dimension name + - selection: the dimension's current selection + */ + this.data = data; + this.selectionCache = selectionCache; /* BitArray */ + this.dimensions = dimensions; /* name: { id, dim, name, selection } */ + } + + size() { + return this.data.length; + } + + all() { + return this.data; + } + + dimensionNames() { + /* return array of all dimensions (by name) */ + return Object.keys(this.dimensions); + } + + addDimension(name, type, ...rest) { + /* + Add a new dimension to this crossfilter, of type DimensionType. + Remainder of parameters are dimension-type-specific. + */ + const { data, selectionCache } = this; + + if (this.dimensions[name] !== undefined) { + throw new Error(`Adding duplicate dimension name ${name}`); + } + + this.selectionCache = null; // pass ownership to new crossfilter + + let id; + if (selectionCache) { + id = selectionCache.allocDimension(); + selectionCache.selectAll(id); + } + const DimensionType = DimTypes[type]; + const dim = new DimensionType(name, data, ...rest); + const dimensions = { + ...this.dimensions, + [name]: { + id, + dim, + name, + selection: dim.select({ mode: "all" }) + } + }; + return new ImmutableTypedCrossfilter(data, dimensions, selectionCache); + } + + delDimension(name) { + const { data, selectionCache } = this; + const dimensions = { ...this.dimensions }; + if (dimensions[name] === undefined) { + throw new ReferenceError(`Unable to delete unknown dimension ${name}`); + } + + const { id } = dimensions[name]; + delete dimensions[name]; + this.selectionCache = null; // pass ownership to new crossfilter + if (selectionCache) { + selectionCache.freeDimension(id); + } + return new ImmutableTypedCrossfilter(data, dimensions, selectionCache); + } + + select(name, spec) { + /* + select on named dimension, as indicated by `spec`. Spec is an object + specifying the selection, and must contain at least a `mode` field. + + Examples: + select("foo", {mode: "all"}); + select("bar", {mode: "none"}); + select("mumble", {mode: "exact", values: "blue"}); + select("mumble", {mode: "exact", values: ["red", "green", "blue"]}); + select("blort", {mode: "range", lo: 0, hi: 999.99}); + */ + const { data, selectionCache } = this; + this.selectionCache = null; + const dimensions = { ...this.dimensions }; + const { dim, id, selection: oldSelection } = dimensions[name]; + const newSelection = dim.select(spec); + newSelection.ranges = PositiveIntervals.canonicalize(newSelection.ranges); + dimensions[name] = { id, dim, name, selection: newSelection }; + ImmutableTypedCrossfilter._dimSelnHasUpdated( + selectionCache, + id, + newSelection, + oldSelection + ); + return new ImmutableTypedCrossfilter(data, dimensions, selectionCache); + } + + static _dimSelnHasUpdated(selectionCache, id, newSeln, oldSeln) { + /* + Selection has updated from oldSeln to newSeln. Update the + bit array if it exists. If not, we will lazy create it when + needed. + */ + if (selectionCache) { + /* + if both new and old selection use the same index, we can + perform an incremental update. If the index changed, we have + to do a suboptimal full deselect/select. + */ + let adds; + let dels; + if (newSeln.index === oldSeln.index) { + adds = PositiveIntervals.difference(newSeln.ranges, oldSeln.ranges); + dels = PositiveIntervals.difference(oldSeln.ranges, newSeln.ranges); + } else { + // console.log("suboptimal selection update - index changed"); + adds = newSeln.ranges; + dels = oldSeln.ranges; + } + + /* + allow dimensions to return selected ranges in either dimension sort + order (indirect via index), or in original record order. + + If sort index exists in the dimension, assume sort ordered ranges. + */ + if (oldSeln.index) { + dels.forEach(interval => + selectionCache.deselectIndirectFromRange(id, oldSeln.index, interval) + ); + } else { + dels.forEach(interval => + selectionCache.deselectFromRange(id, interval) + ); + } + + if (newSeln.index) { + adds.forEach(interval => + selectionCache.selectIndirectFromRange(id, newSeln.index, interval) + ); + } else { + adds.forEach(interval => selectionCache.selectFromRange(id, interval)); + } + } + } + + _getSelectionCache() { + if (!this.selectionCache) { + // console.log("...rebuilding crossfilter cache..."); + const selectionCache = new BitArray(this.data.length); + Object.keys(this.dimensions).forEach(name => { + const { selection } = this.dimensions[name]; + const id = selectionCache.allocDimension(); + this.dimensions[name].id = id; + const { ranges, index } = selection; + ranges.forEach(range => { + if (index) { + selectionCache.selectIndirectFromRange(id, index, range); + } else { + selectionCache.selectFromRange(id, range); + } + }); + }); + this.selectionCache = selectionCache; + } + return this.selectionCache; + } + + allSelected() { + /* + return array of all records currently selected by all dimensions + */ + const selectionCache = this._getSelectionCache(); + const { data } = this; + if (Array.isArray(data)) { + const res = []; + for (let i = 0, len = data.length; i < len; i += 1) { + if (selectionCache.isSelected(i)) { + res.push(data[i]); + } + } + return res; + } + /* else, Dataframe-like */ + return data.isubsetMask(this.allSelectedMask()); + } + + allSelectedMask() { + /* + return Uint8Array containing selection state (truthy/falsey) for each record. + */ + const selectionCache = this._getSelectionCache(); + return selectionCache.fillBySelection( + new Uint8Array(this.data.length), + 1, + 0 + ); + } + + countSelected() { + /* + return number of records selected on all dimensions + */ + const selectionCache = this._getSelectionCache(); + return selectionCache.selectionCount(); + } + + isElementSelected(i) { + /* + return truthy/falsey if this record is selected on all dimensions + */ + const selectionCache = this._getSelectionCache(); + return selectionCache.isSelected(i); + } + + fillByIsSelected(array, selectedValue, deselectedValue) { + /* + fill array with one of two values, based upon selection state. + */ + const selectionCache = this._getSelectionCache(); + return selectionCache.fillBySelection( + array, + selectedValue, + deselectedValue + ); + } +} + +/* +Base dimension object. + +A Dimension is an index, accessed via a select() method. The protocol +for a dimension: + - constructor - first param is name, remainder is whatever params are + required to initialize the dimension. + - select - one and only param is the selection specifier. Returns an + array of record IDs. + - name - the dimension name/label. +*/ +class _ImmutableBaseDimension { + constructor(name) { + this.name = name; + } + + /* eslint-disable class-methods-use-this */ + select(spec) { + const { mode } = spec; + if (mode === undefined) { + throw new Error("select spec does not contain 'mode'"); + } + throw new Error(`select mode ${mode} not implemented`); + } + /* eslint-enable class-methods-use-this */ +} + +class ImmutableScalarDimension extends _ImmutableBaseDimension { + constructor(name, data, value, ValueArrayType) { + super(name); + + // Three modes - caller can provide a pre-created value array, + // a map function which will create it, or another array which + // will used with an identity map function. + let array; + if (value instanceof ValueArrayType) { + // user has provided the final typed array - just use it + if (value.length !== data.length) { + throw new RangeError( + "ScalarDimension values length must equal crossfilter data record count" + ); + } + array = value; + } else if (value instanceof Function) { + // Create value array from user-provided map function. + array = this._createValueArray( + data, + value, + new ValueArrayType(data.length) + ); + } else if (isArrayOrTypedArray(value)) { + // Create value array from user-provided array. Typically used + // only by enumerated dimensions + array = this._createValueArray( + data, + i => value[i], + new ValueArrayType(data.length) + ); + } else { + throw new NotImplementedError( + "dimension value must be function or value array type" + ); + } + this.value = array; + + // create sort index + this.index = makeSortIndex(array); + } + + /* eslint-disable class-methods-use-this */ + _createValueArray(data, mapf, array) { + // create dimension value array + const len = data.length; + const larray = array; + for (let i = 0; i < len; i += 1) { + larray[i] = mapf(i, data); + } + return larray; + } + /* eslint-enable class-methods-use-this */ + + select(spec) { + const { mode } = spec; + const { index } = this; + switch (mode) { + case "all": + return { ranges: [[0, this.value.length]], index }; + case "none": + return { ranges: [], index }; + case "exact": + return this.selectExact(spec); + case "range": + return this.selectRange(spec); + default: + return super.select(spec); + } + } + + selectExact(spec) { + const { value, index } = this; + let { values } = spec; + if (!Array.isArray(values)) { + values = [values]; + } + const ranges = []; + for (let v = 0, len = values.length; v < len; v += 1) { + const r = [ + lowerBoundIndirect(value, index, values[v], 0, value.length), + upperBoundIndirect(value, index, values[v], 0, value.length) + ]; + if (r[0] <= r[1]) { + ranges.push(r); + } + } + return { ranges, index }; + } + + selectRange(spec) { + const { value, index } = this; + /* [lo, hi) */ + const { lo, hi } = spec; + const ranges = []; + const r = [ + lowerBoundIndirect(value, index, lo, 0, value.length), + lowerBoundIndirect(value, index, hi, 0, value.length) + ]; + if (r[0] < r[1]) ranges.push(r); + return { ranges, index }; + } +} + +class ImmutableEnumDimension extends ImmutableScalarDimension { + constructor(name, data, value) { + super(name, data, value, Uint32Array); + } + + _createValueArray(data, mapf, array) { + const len = data.length; + const larray = array; + + // create enumeration table - mapping between the value + // and the enum. + const s = new Set(); + for (let i = 0; i < len; i += 1) { + s.add(mapf(i, data)); + } + const enumIndex = sort(Array.from(s)); + this.enumIndex = enumIndex; + + // create dimension value array + const enumLen = enumIndex.length; + for (let i = 0; i < len; i += 1) { + const v = mapf(i, data); + const e = lowerBound(enumIndex, v, 0, enumLen); + larray[i] = e; + } + return larray; + } + + selectExact(spec) { + const { enumIndex } = this; + const { values } = spec; + return super.selectExact({ + mode: spec.mode, + values: values.map(v => lowerBound(enumIndex, v, 0, enumIndex.length)) + }); + } + + /* eslint-disable class-methods-use-this */ + selectRange() { + throw new Error("range selection unsupported on Enumerated dimension"); + } + /* eslint-enable class-methods-use-this */ +} + +class ImmutableSpatialDimension extends _ImmutableBaseDimension { + constructor(name, data, X, Y) { + super(name); + + if (X.length !== Y.length && X.length !== data.length) { + throw new RangeError( + "SpatialDimension values must have same dimensionality as crossfilter" + ); + } + this.X = X; + this.Y = Y; + + this.Xindex = makeSortIndex(X); + this.Yindex = makeSortIndex(Y); + } + + select(spec) { + const { mode } = spec; + switch (mode) { + case "all": + return { ranges: [[0, this.X.length]], index: null }; + case "none": + return { ranges: [], index: null }; + case "within-rect": + return this.selectWithinRect(spec); + case "within-polygon": + return this.selectWithinPolygon(spec); + default: + return super.select(spec); + } + } + + selectWithinRect(spec) { + /* + { mode: "within-rect", x0: 1, y0: 0, x1: 3, y1: 9 } + */ + const { x0, y0, x1, y1 } = spec; + const { X, Y } = this; + const ranges = []; + let start = -1; + for (let i = 0, l = X.length; i < l; i += 1) { + const x = X[i]; + const y = Y[i]; + const inside = x0 <= x && x < x1 && y0 <= y && y < y1; + if (inside && start === -1) start = i; + if (!inside && start !== -1) { + ranges.push([start, i]); + start = -1; + } + } + if (start !== -1) ranges.push([start, X.length]); + return { ranges, index: null }; + } + + /* + Relatively brute force filter by polygon. + + Currently uses d3.polygonContains() to test for polygon inclusion, which itself + uses a ray casting (crossing number) algorithm. There are a series of optimizations + to make this faster: + * first sliced by X or Y, using an index on the axis + * then the polygon bounding box is used for trivial rejection + * then the polygon test is applied + */ + + selectWithinPolygon(spec) { + /* + { mode: "within-polygon", polygon: [ [x0, y0], ... ] } + */ + const { polygon } = spec; + const [minX, minY, maxX, maxY] = polygonBoundingBox(polygon); + const { X, Y, Xindex, Yindex } = this; + const { length } = X; + let slice; + let index; + if (maxY - minY > maxX - minX) { + slice = [ + lowerBoundIndirect(X, Xindex, minX, 0, length), + lowerBoundIndirect(X, Xindex, maxX, 0, length) + ]; + index = Xindex; + } else { + slice = [ + lowerBoundIndirect(Y, Yindex, minY, 0, length), + lowerBoundIndirect(Y, Yindex, maxY, 0, length) + ]; + index = Yindex; + } + + const ranges = []; + let start = -1; + for (let i = slice[0], e = slice[1]; i < e; i += 1) { + const rid = index[i]; + const x = X[rid]; + const y = Y[rid]; + const inside = + minX <= x && + x < maxX && + minY <= y && + y < maxY && + withinPolygon(polygon, x, y); + + if (inside && start === -1) start = i; + if (!inside && start !== -1) { + ranges.push([start, i]); + start = -1; + } + } + if (start !== -1) ranges.push([start, slice[1]]); + return { ranges, index }; + } +} + +/* Helpers */ +export const DimTypes = { + scalar: ImmutableScalarDimension, + enum: ImmutableEnumDimension, + spatial: ImmutableSpatialDimension +}; + +function isArrayOrTypedArray(x) { + return ( + Array.isArray(x) || + (ArrayBuffer.isView(x) && + Object.prototype.toString.call(x) !== "[object DataView]") + ); +} + +/* return bounding box of the polygon */ +function polygonBoundingBox(polygon) { + let minX = Number.MAX_VALUE; + let minY = Number.MAX_VALUE; + let maxX = Number.MIN_VALUE; + let maxY = Number.MIN_VALUE; + for (let i = 0, l = polygon.length; i < l; i += 1) { + const point = polygon[i]; + const [x, y] = point; + if (x < minX) minX = x; + if (y < minY) minY = y; + if (x > maxX) maxX = x; + if (y > maxY) maxY = y; + } + return [minX, minY, maxX, maxY]; +} + +function withinPolygon(polygon, x, y) { + // TODO XXX replace + return polygonContains(polygon, [x, y]); +} diff --git a/client/src/util/typedCrossfilter/index.js b/client/src/util/typedCrossfilter/index.js index 99888314..9c66e56e 100644 --- a/client/src/util/typedCrossfilter/index.js +++ b/client/src/util/typedCrossfilter/index.js @@ -1,7 +1,5 @@ -// jshint esversion: 6 - /* -Typedarray Crossfilter - a re-implementation of a subset of crossfilter, with +Crossfilter - a re-implementation of a subset of crossfilter, with time/space optimizations predicated upon the following assumptions: - dimensions are uniformly typed, and all values must be of that type - dimension values must be a primitive type (int, float, string). Arrays @@ -11,14 +9,18 @@ time/space optimizations predicated upon the following assumptions: want to do that, you have to create the new crossfilter, using the new data, from scratch. -The actual backing store for a dimension is a TypedArray, enabling significant +In addition, this implementation is easier to use with a "redux" style +app, as all operations on the crossfilter are immutable (ie, return a +new crossfilter). + +The actual backing store for a dimension is a TypedArray, enabling performance improvements over the original crossfilter. There are also a handful of new methods, primarily to take advantage of the performance (eg, crossfilter.fillBySelection) -Helpful documents (this module tries to follow the original API as much -as is feasable): +Helpful documents (this module follows similar concepts as the original, +but deviates from the API): https://github.com/square/crossfilter/ http://square.github.io/crossfilter/ @@ -26,766 +28,7 @@ There is also a newer, community supported fork of crossfilter, with a more complex API. In a few cases, elements of that API were incorporated. https://github.com/square/crossfilter/ +See test cases for some concrete examples. */ -// XXX replace -import { polygonContains } from "d3"; -import PositiveIntervals from "./positiveIntervals"; -import BitArray from "./bitArray"; -import { - makeSortIndex, - lowerBound, - lowerBoundIndirect, - upperBoundIndirect -} from "./util"; - -function isArrayOrTypedArray(x) { - return ( - Array.isArray(x) || - (ArrayBuffer.isView(x) && - Object.prototype.toString.call(x) !== "[object DataView]") - ); -} - -class NotImplementedError extends Error { - constructor(...params) { - super(...params); - - // Maintains proper stack trace for where our error was thrown (only available on V8) - if (Error.captureStackTrace) { - Error.captureStackTrace(this, NotImplementedError); - } - } -} - -class TypedCrossfilter { - constructor(data) { - /* - Typically, data is one of: - - Array of objects/records - - Dataframe (util/dataframe) - */ - this.data = data; - - // filters: array of { id, dimension } - this.filters = []; - this.selection = new BitArray(data.length); - this.updateTime = 0; - } - - size() { - return this.data.length; - } - - all() { - return this.data; - } - - /* - Create a crossfilter dimension, upon which filtering (subselection) can - be done. Each dimension is typed, and has a particular set of filtering - semantics. - * ScalarDimension - backed by TypedArray values, supporting filtering - by value (within a value range, or one or more exact values) - * EnumDimension - backed by an enumeration (eg, strings, bools), filtering - by one or more enum categories. - * SpatialDimension - backed by 2D points, filter by containment within - various shapes (currently supports within Rectangle and within Polygon). - Call this method to create a dimension, passing arguments appropriate for - the dimension constructor. - */ - dimension(DimensionType, ...rest) { - const id = this.selection.allocDimension(); - const dim = new DimensionType(this, id, ...rest); - this.filters.push({ id, dim }); - dim.filterAll(); - return dim; - } - - _freeDimension(id) { - this.selection.freeDimension(id); - this.filters = this.filters.filter(f => f._id !== id); - } - - // return array of all records that are selected/filtered - // by all dimensions. - allFiltered() { - const { data, selection } = this; - if (Array.isArray(data)) { - const res = []; - for (let i = 0, len = data.length; i < len; i += 1) { - if (selection.isSelected(i)) { - res.push(data[i]); - } - } - return res; - } - /* else, Dataframe-like */ - return data.isubsetMask(this.allFilteredMask()); - } - - // return Uint8array containing selection state (truthy/falsey) for each record. - // - allFilteredMask() { - return this.selection.fillBySelection( - new Uint8Array(this.data.length), - 1, - 0 - ); - } - - countFiltered() { - return this.selection.selectionCount(); - } - - isElementFiltered(i) { - return this.selection.isSelected(i); - } - - // fill array with one of two values, based upon selection state - fillByIsFiltered(array, selectedValue, deselectedValue) { - return this.selection.fillBySelection( - array, - selectedValue, - deselectedValue - ); - } -} - -// Base dimension type - not exported. -class _Dimension { - constructor(xfltr, id) { - this.crossfilter = xfltr; - this._id = id; - this.groups = []; - } - - dispose() { - this.crossfilter._freeDimension(this._id); - return this; - } - - id() { - return this._id; - } - - _filterUpdate() { - this.crossfilter.updateTime += 1; - } -} - -// Scalar dimension type - value must be a scalar type (eg, int, float), -// and value array must be a TypedArray. -// -class ScalarDimension extends _Dimension { - constructor(xfltr, id, value, ValueArrayType) { - super(xfltr, id); - - // current selection filter, expressed as PostiveIntervals. - this.currentFilter = []; - - // Two modes - caller can provide a pre-created value array, - // or a map function which will create it. - let array; - if (value instanceof ValueArrayType) { - // user has provided the final typed array - just use it - if (value.length !== this.crossfilter.data.length) { - throw new RangeError( - "ScalarDimension values length must equal crossfilter data record count" - ); - } - array = value; - } else if (value instanceof Function) { - // Create value array from user-provided map function. - array = this._createValueArray( - value, - new ValueArrayType(this.crossfilter.data.length) - ); - } else if (isArrayOrTypedArray(value)) { - // Create value array from user-provided array. Typically used - // only by enumerated dimensions - array = this._createValueArray( - i => value[i], - new ValueArrayType(this.crossfilter.data.length) - ); - } else { - throw new NotImplementedError( - "dimension value must be function or value array type" - ); - } - this.value = array; - - // create sort index - this.index = makeSortIndex(array); - } - - _createValueArray(value, array) { - // create dimension value array - const { data } = this.crossfilter; - const len = data.length; - const larray = array; - for (let i = 0; i < len; i += 1) { - larray[i] = value(i, data); - } - return larray; - } - - // Argument is an array of intervals indicating records newly selected/filtered - // - _updateFilters(newFilter) { - const cNewFilter = PositiveIntervals.canonicalize(newFilter); - - const adds = PositiveIntervals.difference(cNewFilter, this.currentFilter); - const dels = PositiveIntervals.difference(this.currentFilter, cNewFilter); - - this.crossfilter.filters.forEach(f => - f.dim.groups.forEach(grp => grp._updateReduceDel(this, dels)) - ); - - dels.forEach(interval => - this.crossfilter.selection.deselectIndirectFromRange( - this._id, - this.index, - interval - ) - ); - - adds.forEach(interval => - this.crossfilter.selection.selectIndirectFromRange( - this._id, - this.index, - interval - ) - ); - - this.crossfilter.filters.forEach(f => - f.dim.groups.forEach(grp => grp._updateReduceAdd(this, adds)) - ); - - this.currentFilter = cNewFilter; - this._filterUpdate(); - } - - // filter by value - exact match - filterExact(value) { - const newFilter = [ - lowerBoundIndirect(this.value, this.index, value, 0, this.value.length), - upperBoundIndirect(this.value, this.index, value, 0, this.value.length) - ]; - if (newFilter[0] <= newFilter[1]) { - this._updateFilters([newFilter]); - } else { - this._updateFilters([]); - } - return this; - } - - // filter by a set of values, eg. enum. - filterEnum(values) { - const newFilter = []; - for (let v = 0, len = values.length; v < len; v += 1) { - const intv = [ - lowerBoundIndirect( - this.value, - this.index, - values[v], - 0, - this.value.length - ), - upperBoundIndirect( - this.value, - this.index, - values[v], - 0, - this.value.length - ) - ]; - if (intv[0] <= intv[1]) newFilter.push(intv); - } - this._updateFilters(newFilter); - return this; - } - - // filter by value range [lo, hi) - // lo: inclusive, hi: exclusive - filterRange(range) { - const newFilter = []; - const intv = [ - lowerBoundIndirect( - this.value, - this.index, - range[0], - 0, - this.value.length - ), - upperBoundIndirect(this.value, this.index, range[1], 0, this.value.length) - ]; - if (intv[0] < intv[1]) newFilter.push(intv); - this._updateFilters(newFilter); - return this; - } - - // select all - equivalent of selecting all in this dimension - filterAll() { - this._updateFilters([[0, this.value.length]]); - return this; - } - - // select none - filterNone() { - this._updateFilters([]); - } - - // return top k records, starting with offset, in descending order. - // Order is this dimension's sort order - top(k, offset = 0) { - const { data, selection } = this.crossfilter; - const { index } = this; - const len = index.length; - const ret = []; - let i = 0; - let skip = 0; - let found = 0; - - // skip up to offset records - for (i = len - 1; i >= 0 && skip < offset; i -= 1) { - if (selection.isSelected(index[i])) { - skip += 1; - } - } - - // grab up to k records - for (; i >= 0 && found < k; i -= 1) { - if (selection.isSelected(index[i])) { - ret.push(data[index[i]]); - found += 1; - } - } - - return ret; - } - - // return bottom k records, starting with offset, in ascending order. - // Order is this dimension's sort order - bottom(k, offset = 0) { - const { data, selection } = this.crossfilter; - const { index } = this; - const len = index.length; - const ret = []; - let skip = 0; - let found = 0; - let i = 0; - - // skip up to offset records - for (i = 0; i < len && skip < offset; i += 1) { - if (selection.isSelected(index[i])) { - skip += 1; - } - } - - // grab up to k records - for (; i < len && found < k; i += 1) { - if (selection.isSelected(index[i])) { - ret.push(data[index[i]]); - found += 1; - } - } - - return ret; - } - - group(groupValue) { - const grp = new ScalarGroup(groupValue, this.value.constructor, this); - this.groups.push(grp); - return grp; - } - - _freeGroup(group) { - this.groups = this.groups.filter(e => e !== group); - } -} - -// Ordered enumeration - supports any sortable enumerable type, eg, -// strings, which can be mapped into an fixed numeric range [0..n). -// -class EnumDimension extends ScalarDimension { - constructor(xfltr, id, value) { - super(xfltr, id, value, Uint32Array); - } - - _createValueArray(value, array) { - const { data } = this.crossfilter; - const len = data.length; - const larray = array; - - // create enumeration table - mapping between the value - // and the enum. - const s = new Set(); - for (let i = 0; i < len; i += 1) { - s.add(value(i, data)); - } - this.enumIndex = Array.from(s); - this.enumIndex.sort(); - - // create dimension value array - const enumLen = this.enumIndex.length; - for (let i = 0; i < len; i += 1) { - const v = value(i, data); - const e = lowerBound(this.enumIndex, v, 0, enumLen); - larray[i] = e; - } - return larray; - } - - filterExact(value) { - return super.filterExact( - lowerBound(this.enumIndex, value, 0, this.enumIndex.length) - ); - } - - filterEnum(values) { - return super.filterEnum( - values.map(v => lowerBound(this.enumIndex, v, 0, this.enumIndex.length)) - ); - } - - filterRange(range) { - return super.filterEnum( - range.map(v => lowerBound(this.enumIndex, v, 0, this.enumIndex.length)) - ); - } - - group(groupValue) { - const grp = new EnumGroup(groupValue, this.value.constructor, this); - this.groups.push(grp); - return grp; - } -} - -/* -Super simple 2D spatial dimension, supporting basic "filter within" -operations. -*/ -class SpatialDimension extends _Dimension { - constructor(xfltr, id, X, Y) { - super(xfltr, id); - - if (X.length !== Y.length && X.length !== this.crossfilter.data.length) { - throw new RangeError( - "SpatialDimension values must have same dimensionality as crossfilter" - ); - } - this.X = X; - this.Y = Y; - - this.Xindex = makeSortIndex(X); - this.Yindex = makeSortIndex(Y); - } - - filterAll() { - this.crossfilter.selection.selectAll(this._id); - this._filterUpdate(); - } - - filterNone() { - this.crossfilter.selection.deselectAll(this._id); - this._filterUpdate(); - } - - /* - this could be smarter, but we don't currently use it... - */ - filterWithinRect(northwest, southeast) { - const [x0, y0] = northwest; - const [x1, y1] = southeast; - const { X, Y } = this; - const seln = this.crossfilter.selection; - const { _id } = this; - seln.deselectAll(_id); - for (let i = 0, l = this.X.length; i < l; i += 1) { - const x = X[i]; - const y = Y[i]; - if (x0 <= x && x < x1 && y0 <= y && y < y1) { - seln.selectOne(_id, i); - } - } - this._filterUpdate(); - } - - /* - Relatively brute force filter by polygon. Polygon is array of points, where - each point is [x,y]. Eg, [[x0,y0], [x1,y1], ...]. - - Currently uses d3.polygonContains() to test for polygon inclusion, which itself - uses a ray casting (crossing number) algorithm. There are a series of optimizations - to make this faster: - * first sliced by X or Y, using an index on the axis - * then the polygon bounding box is used for trivial rejection - * then the polygon test is applied - */ - filterWithinPolygon(polygon) { - /* return bounding box of the polygon */ - function polygonBoundingBox(pg) { - let minX = Number.MAX_VALUE; - let minY = Number.MAX_VALUE; - let maxX = Number.MIN_VALUE; - let maxY = Number.MIN_VALUE; - for (let i = 0, l = pg.length; i < l; i += 1) { - const p = pg[i]; - const x = p[0]; - const y = p[1]; - if (x < minX) minX = x; - if (y < minY) minY = y; - if (x > maxX) maxX = x; - if (y > maxY) maxY = y; - } - return [minX, minY, maxX, maxY]; - } - - const [minX, minY, maxX, maxY] = polygonBoundingBox(polygon); - const { X, Y } = this; - let slice; - let index; - if (maxY - minY > maxX - minX) { - slice = [ - lowerBoundIndirect(X, this.Xindex, minX, 0, X.length), - upperBoundIndirect(X, this.Xindex, maxX, 0, X.length) - ]; - index = this.Xindex; - } else { - slice = [ - lowerBoundIndirect(Y, this.Yindex, minY, 0, Y.length), - upperBoundIndirect(Y, this.Yindex, maxY, 0, Y.length) - ]; - index = this.Yindex; - } - - const seln = this.crossfilter.selection; - const { _id } = this; - const testWithin = polygonContains; // d3.polygonContains() - seln.deselectAll(_id); - - for (let i = slice[0], e = slice[1]; i < e; i += 1) { - const rid = index[i]; - const x = X[rid]; - const y = Y[rid]; - if ( - minX <= x && - x < maxX && - minY <= y && - y < maxY && - testWithin(polygon, [x, y]) - ) { - seln.selectOne(_id, rid); - } - } - this._filterUpdate(); - } -} - -// Groups! Map/reduce -// -class ScalarGroup { - constructor(groupValue, groupValueType, dimension) { - // parent dimension - this.dimension = dimension; - - // generate group names from dimension values - this.mapValue = this.constructor._map( - groupValue, - groupValueType, - dimension - ); - - // group index is mapping from data record index to group index - this.groupIndex = new Uint32Array(dimension.crossfilter.data.length); - - // default to counting - this.reduceCount(); - - // Creates this.groups - this._reduce(); - } - - // internal support function - map all dimension values to group values. - // - static _map(groupValue, GroupValueType, dimension) { - // groupValue is optional. Defaults to identity. Used to perform - // initial map operation. - // - // identity: save some memory... - if (groupValue === undefined) return dimension.value; - - const data = dimension.value; - const len = data.length; - const mapValue = new GroupValueType(dimension.value.length); - for (let i = 0; i < len; i += 1) { - mapValue[i] = groupValue(data[i]); - } - return mapValue; - } - - // Update the group reduction incrementally. Called when *any* dimension filter - // changes. Guaranteed to be called AFTER the crossfilter is updated. - // - // Arguments: - // * dim: the dimension that is changing - // * intv: interval list of newly selected values on `dim` (adds) - // - _updateReduceAdd(dim, intv) { - // ignore updates to self, as we don't reduce inclusive of our filter - if (dim === this.dimension || intv.length === 0) return; - - // Each item in the range was just added to `dim`. It was NOT previously - // selected - reduceAdd if it is now selected. - const { data, selection } = this.dimension.crossfilter; - intv.forEach(rng => { - for (let r = rng[0]; r < rng[1]; r += 1) { - const i = dim.index[r]; - if (selection.isSelectedIgnoringDim(i, this.dimension.id())) { - const group = this.groups[this.groupIndex[i]]; - group.value = this.reduceAdd(group.value, data[i]); - } - } - }); - } - - // Update the group reduction incrementally. Called when *any* dimension filter - // changes. Guaranteed to be called BEFORE the crossfilter is updated. - // - // Arguments: - // * dim: the dimension that is changing - // * intv: interval list of previously selected values on `dim` (dels) - // - _updateReduceDel(dim, intv) { - // ignore updates to self, as we don't reduce inclusive of our filter - if (dim === this.dimension || intv.length === 0) return; - - // Each item in the range will be remved from `dim`. reduceRemove if it - // is currently selected. - const { data, selection } = this.dimension.crossfilter; - intv.forEach(rng => { - for (let r = rng[0]; r < rng[1]; r += 1) { - const i = dim.index[r]; - if (selection.isSelectedIgnoringDim(i, this.dimension.id())) { - const group = this.groups[this.groupIndex[i]]; - group.value = this.reduceRemove(group.value, data[i]); - } - } - }); - } - - // Reduce the entire data set, creating both the group index and the - // groups data. - // - _reduce() { - const { dimension } = this; - const { data } = dimension.crossfilter; - - // Create groups - const groupNames = new Set(this.mapValue); - this.groups = []; - const groupIndexByName = {}; - groupNames.forEach(name => { - this.groups.push({ key: name, value: this.reduceInitial() }); - groupIndexByName[name] = this.groups.length - 1; - }); - - // Create groupIndex - index map between data record index and group index - for (let i = 0, len = this.mapValue.length; i < len; i += 1) { - this.groupIndex[i] = groupIndexByName[this.mapValue[i]]; - } - - // reduce all filtered records, IGNORING the current dimension's filter - const { selection } = dimension.crossfilter; - for (let i = 0, len = data.length; i < len; i += 1) { - if (selection.isSelectedIgnoringDim(i, dimension.id())) { - const group = this.groups[this.groupIndex[i]]; - group.value = this.reduceAdd(group.value, data[i]); - } - } - } - - dispose() { - this.dimension._freeGroup(this); - return this; - } - - // return number of distinct values in the group, independent of any filters. - // - size() { - return this.groups.length; - } - - // Set the reduce functions and return the grouping. - // - reduce(add, remove, initial) { - this.reduceAdd = add; - this.reduceRemove = remove; - this.reduceInitial = initial; - this._reduce(); - return this; - } - - // set the reduce functions to count records. - reduceCount() { - return this.reduce(p => p + 1, p => p - 1, () => 0); - } - - // set the reduce functions to sum records using specified value accessor. - // - reduceSum(value) { - return this.reduce((p, v) => p + value(v), (p, v) => p - value(v), () => 0); - } - - all() { - const res = [...this.groups]; - res.sort((a, b) => (a.key < b.key ? -1 : a.key > b.key ? 1 : 0)); - return res; - } -} - -class EnumGroup extends ScalarGroup { - static _map(groupValue, groupValueType, dimension) { - // groupValue is optional. Defaults to identity. Used to perform - // initial map operation. - // - // identity: save some memory - if (groupValue === undefined) return dimension.value; - - // non-identity mapping unsupported for EnumDimension/EnumGroup. - // XXX: this could be implemented, but would require another index - // array to map from the group names/keys back to the dimension values. - // With this, we just rely on the dimensions `enumIndex` to map from - // enumeration value to the record. - throw new NotImplementedError("enumerated group mapping not implemented"); - } - - all() { - const res = []; - this.groups.forEach(e => - res.push({ - // XXX: assumes identity group map - see comment in _map() - key: this.dimension.enumIndex[e.key], - value: e.value - }) - ); - res.sort((a, b) => (a.key < b.key ? -1 : a.key > b.key ? 1 : 0)); - return res; - } -} - -// Wrapper for backwards compat with crossfilter. -// -function crossfilter(data) { - return new TypedCrossfilter(data); -} - -crossfilter.PositiveIntervals = PositiveIntervals; -crossfilter.BitArray = BitArray; -crossfilter.TypedCrossfilter = TypedCrossfilter; -crossfilter.ScalarDimension = ScalarDimension; -crossfilter.EnumDimension = EnumDimension; -crossfilter.SpatialDimension = SpatialDimension; - -export default crossfilter; +export { default } from "./crossfilter";