fork/cellxgene
1
0
Fork 0
mirror of https://github.com/chanzuckerberg/cellxgene.git synced 2026-09-21 08:18:13 +08:00
Code Issues Packages Projects Releases Wiki Activity

refactoring - immutable crossfilter (#647)

Browse Source 
* immutable crossfilter

* PR review changes
This commit is contained in:
Bruce Martin
2019-03-20 14:32:35 -07:00
committed by GitHub
parent 571b7387e7
commit 996b06cecc
13 changed files with 1153 additions and 1582 deletions
Download Patch File Download Diff File Expand all files Collapse all files
client/__tests__/util/stateManager/world.test.js
+22 -29
View File
@@ -3,6 +3,7 @@ import * as Universe from "../../../src/util/stateManager/universe";
import * as World from "../../../src/util/stateManager/world";
import * as Dataframe from "../../../src/util/dataframe";
import Crossfilter from "../../../src/util/typedCrossfilter";
import { DimTypes } from "../../../src/util/typedCrossfilter/crossfilter";
import * as REST from "./sampleResponses";
import {
obsAnnoDimensionName,
@@ -26,15 +27,15 @@ const defaultBigBang = () => {
/* create world */
const world = World.createWorldFromEntireUniverse(universe);
/* create crossfilter */
const crossfilter = Crossfilter(world.obsAnnotations);
/* create dimension map */
const dimensionMap = World.createObsDimensionMap(crossfilter, world);
const crossfilter = World.createObsDimensions(
new Crossfilter(world.obsAnnotations),
world
);
return {
universe,
world,
crossfilter,
dimensionMap
crossfilter
};
};
@@ -71,13 +72,16 @@ describe("createWorldFromCurrentSelection", () => {
const {
universe,
world: originalWorld,
crossfilter,
dimensionMap
crossfilter: originalCrossfilter
} = defaultBigBang();
/* mock a selection */
dimensionMap[obsAnnoDimensionName("field1")].filterRange([0, 5]);
dimensionMap[obsAnnoDimensionName("field3")].filterExact(false);
let crossfilter = originalCrossfilter
.select(obsAnnoDimensionName("field1"), { mode: "range", lo: 0, hi: 5 })
.select(obsAnnoDimensionName("field3"), {
mode: "exact",
values: [false]
});
/* create the world from the selection */
const world = World.createWorldFromCurrentSelection(
@@ -86,7 +90,7 @@ describe("createWorldFromCurrentSelection", () => {
crossfilter
);
expect(world).toBeDefined();
expect(world.nObs).toEqual(crossfilter.countFiltered());
expect(world.nObs).toEqual(crossfilter.countSelected());
/*
calculate expected values and match against result
@@ -136,43 +140,32 @@ describe("createObsDimensionMap", () => {
- check that dimension typing is sane
*/
const { dimensionMap } = defaultBigBang();
const { crossfilter } = defaultBigBang();
const annotationNames = _.map(
REST.schema.schema.annotations.obs,
c => c.name
);
const schemaByObsName = _.keyBy(REST.schema.schema.annotations.obs, "name");
expect(dimensionMap).toBeDefined();
expect(crossfilter).toBeDefined();
annotationNames.forEach(name => {
const dim = dimensionMap[obsAnnoDimensionName(name)];
const dim = crossfilter.dimensions[obsAnnoDimensionName(name)];
if (name === "name") {
expect(dim).toBeUndefined();
} else {
const { type } = schemaByObsName[name];
if (type === "string" || type === "boolean" || type === "categorical") {
expect(dim).toBeInstanceOf(Crossfilter.EnumDimension);
expect(dim.dim).toBeInstanceOf(DimTypes.enum);
} else {
expect(dim).toBeInstanceOf(Crossfilter.ScalarDimension);
expect(dim.dim).toBeInstanceOf(DimTypes.scalar);
}
}
});
expect(dimensionMap[layoutDimensionName("XY")]).toBeInstanceOf(
Crossfilter.SpatialDimension
);
expect(
crossfilter.dimensions[layoutDimensionName("XY")].dim
).toBeInstanceOf(DimTypes.spatial);
});
});
describe("createVarDataDimension", () => {
/* create default universe */
const { world, crossfilter } = defaultBigBang();
world.varData = world.varData.withCol(
"GENE",
Float32Array.from(_.range(world.nObs))
);
const result = World.createVarDataDimension(world, crossfilter, "GENE");
expect(result).toBeInstanceOf(Crossfilter.ScalarDimension);
});
describe("worldEqUniverse", () => {
const { universe, world } = defaultBigBang();
const result = World.worldEqUniverse(world, universe);
client/__tests__/util/typedCrossfilter/crossfilter.test.js
+330
View File
@@ -0,0 +1,330 @@
import _ from "lodash";
import { polygonContains } from "d3";
import Crossfilter from "../../../src/util/typedCrossfilter";
const someData = [
{
date: "2011-11-14T16:17:54Z",
quantity: 2,
total: 190,
tip: 100,
type: "tab",
productIDs: ["001"],
coords: [0, 0]
},
{
date: "2011-11-14T16:20:19Z",
quantity: 2,
total: 190,
tip: 100,
type: "tab",
productIDs: ["001", "005"],
coords: [0.4, 0.4]
},
{
date: "2011-11-14T16:28:54Z",
quantity: 1,
total: 300,
tip: 200,
type: "visa",
productIDs: ["004", "005"],
coords: [0.3, 0.1]
},
{
date: "2011-11-14T16:30:43Z",
quantity: 2,
total: 90,
tip: 0,
type: "tab",
productIDs: ["001", "002"],
coords: [0.392, 0.1]
},
{
date: "2011-11-14T16:48:46Z",
quantity: 2,
total: 90,
tip: 0,
type: "tab",
productIDs: ["005"],
coords: [0.7, 0.0482]
},
{
date: "2011-11-14T16:53:41Z",
quantity: 2,
total: 90,
tip: 0,
type: "tab",
productIDs: ["001", "004", "005"],
coords: [0.9999, 1.0]
},
{
date: "2011-11-14T16:54:06Z",
quantity: 1,
total: 100,
tip: 0,
type: "cash",
productIDs: ["001", "002", "003", "004", "005"],
coords: [0.384, 0.6938]
},
{
date: "2011-11-14T16:58:03Z",
quantity: 2,
total: 90,
tip: 0,
type: "tab",
productIDs: ["001"],
coords: [0.4822, 0.482]
},
{
date: "2011-11-14T17:07:21Z",
quantity: 2,
total: 90,
tip: 0,
type: "tab",
productIDs: ["004", "005"],
coords: [0.2234, 0]
},
{
date: "2011-11-14T17:22:59Z",
quantity: 2,
total: 90,
tip: 0,
type: "tab",
productIDs: ["001", "002", "004", "005"],
coords: [0.382, 0.38485]
},
{
date: "2011-11-14T17:25:45Z",
quantity: 2,
total: 200,
tip: 0,
type: "cash",
productIDs: ["002"],
coords: [0.998, 0.8472]
},
{
date: "2011-11-14T17:29:52Z",
quantity: 1,
total: 200,
tip: 100,
type: "visa",
productIDs: ["004"],
coords: [0.8273, 0.3384]
}
];
let payments = null;
beforeEach(() => {
payments = new Crossfilter(someData);
});
describe("ImmutableTypedCrossfilter", () => {
test("create crossfilter", () => {
expect(payments).toBeDefined();
expect(payments.size()).toEqual(someData.length);
expect(payments.all()).toEqual(someData);
const p = payments
.addDimension("quantity", "scalar", (i, d) => d[i].quantity, Int32Array)
.select("quantity", { mode: "all" });
expect(p).toBeDefined();
expect(p.all()).toEqual(someData);
expect(p.size()).toEqual(someData.length);
expect(p.isElementSelected(0)).toBeTruthy();
expect(p.countSelected()).toEqual(someData.length);
expect(p.allSelected()).toEqual(someData);
});
test("immutability", () => {
/*
the following should return a new crossfilter:
- addDimension()
- delDimension()
- select
*/
const p2 = payments.addDimension(
"quantity",
"scalar",
(i, data) => data[i].quantity,
Int32Array
);
expect(payments).not.toBe(p2);
const p3 = p2.select("quantity", { mode: "all" });
expect(p3).not.toBe(p2);
const p4 = p3.delDimension("quantity");
expect(p4).not.toBe(p3);
});
test("select all and none", () => {
let p = payments
.addDimension("quantity", "scalar", (i, d) => d[i].quantity, Int32Array)
.addDimension("tip", "scalar", (i, d) => d[i].tip, Float32Array)
.addDimension("total", "scalar", (i, d) => d[i].total, Float32Array)
.addDimension("type", "enum", (i, d) => d[i].type);
expect(p).toBeDefined();
/* expect all records to be selected - default init state */
expect(p.allSelected()).toEqual(someData);
expect(p.countSelected()).toEqual(someData.length);
expect(p.allSelectedMask()).toEqual(
new Uint8Array(someData.length).fill(1)
);
expect(p.fillByIsSelected(new Uint8Array(someData.length), 99, 0)).toEqual(
new Uint8Array(someData.length).fill(99)
);
for (let i = 0; i < someData.length; i += 1) {
expect(p.isElementSelected(i)).toBeTruthy();
}
/* expect a selectAll on one dimension to change nothing */
p = p.select("tip", { mode: "all" });
expect(p.allSelected()).toEqual(someData);
/* ditto */
p = p.select("quantity", { mode: "all" });
expect(p.allSelected()).toEqual(someData);
/* select none on one dimension */
p = p.select("type", { mode: "none" });
expect(p.allSelected()).toEqual([]);
expect(p.countSelected()).toEqual(0);
expect(p.allSelectedMask()).toEqual(
new Uint8Array(someData.length).fill(0)
);
expect(p.fillByIsSelected(new Uint8Array(someData.length), 99, 0)).toEqual(
new Uint8Array(someData.length).fill(0)
);
for (let i = 0; i < someData.length; i += 1) {
expect(p.isElementSelected(i)).toBeFalsy();
}
p = p.select("quantity", { mode: "none" });
expect(p.allSelected()).toEqual([]);
// invert the first none; should have no effect because type is
// still not filtered.
p = p.select("quantity", { mode: "all" });
expect(p.allSelected()).toEqual([]);
/* select all of type; should select all records */
p = p.select("type", { mode: "all" });
expect(p.allSelected()).toEqual(someData);
});
describe("scalar dimension", () => {
let p;
beforeEach(() => {
p = payments
.addDimension("quantity", "scalar", (i, d) => d[i].quantity, Int32Array)
.addDimension("tip", "scalar", (i, d) => d[i].tip, Float32Array)
.select("tip", { mode: "all" });
});
/*
select modes: all, none, exact, range
*/
test("all", () => {
expect(p.select("quantity", { mode: "all" }).countSelected()).toEqual(
someData.length
);
});
test("none", () => {
expect(p.select("quantity", { mode: "none" }).countSelected()).toEqual(0);
});
test.each([[[]], [[2]], [[2, 1]], [[9, 82]], [[0, 1]]])("exact: %p", v =>
expect(
p.select("quantity", { mode: "exact", values: v }).countSelected()
).toEqual(_.filter(someData, d => v.includes(d.quantity)).length)
);
test.each([[0, 1], [1, 2], [0, 99], [99, 100000]])("range %p", (lo, hi) =>
expect(
p.select("quantity", { mode: "range", lo, hi }).countSelected()
).toEqual(
_.filter(someData, d => d.quantity >= lo && d.quantity < hi).length
)
);
test("bad mode", () => {
expect(() => p.select("type", { mode: "bad mode" })).toThrow(Error);
});
});
describe("enum dimension", () => {
let p;
beforeEach(() => {
p = payments.addDimension("type", "enum", (i, d) => d[i].type);
});
test("all", () => {
expect(p.select("type", { mode: "all" }).countSelected()).toEqual(
someData.length
);
});
test("none", () => {
expect(p.select("type", { mode: "none" }).countSelected()).toEqual(0);
});
test.each([
[[]],
[["tab"]],
[["visa"]],
[["visa", "tab"]],
[["cash", "tab", "visa"]]
])("exact: %p", v =>
expect(
p.select("type", { mode: "exact", values: v }).countSelected()
).toEqual(_.filter(someData, d => v.includes(d.type)).length)
);
test("range", () => {
expect(() => p.select("type", { mode: "range", lo: 0, hi: 9 })).toThrow(
Error
);
});
test("bad mode", () => {
expect(() => p.select("type", { mode: "bad mode" })).toThrow(Error);
});
});
describe("spatial dimension", () => {
let p;
beforeEach(() => {
const X = someData.map(r => r.coords[0]);
const Y = someData.map(r => r.coords[1]);
p = payments.addDimension("coords", "spatial", X, Y);
});
test("all", () => {
expect(p.select("coords", { mode: "all" }).countSelected()).toEqual(
someData.length
);
});
test("none", () => {
expect(p.select("coords", { mode: "none" }).countSelected()).toEqual(0);
});
test.each([[0, 0, 1, 1], [0, 0, 0.5, 0.5], [0.5, 0.5, 1, 1]])(
"within-rect %d %d %d %d",
(x0, y0, x1, y1) => {
expect(
p
.select("coords", { mode: "within-rect", x0, y0, x1, y1 })
.allSelected()
).toEqual(
_.filter(someData, d => {
const [x, y] = d.coords;
return x0 <= x && x < x1 && y0 <= y && y < y1;
})
);
}
);
test.each([
[[[0, 0], [0, 1], [1, 1], [1, 0]]],
[[[0, 0], [0, 0.5], [0.5, 0.5], [0.5, 0]]]
])("within-polygon %p", polygon => {
expect(
p.select("coords", { mode: "within-polygon", polygon }).allSelected()
).toEqual(_.filter(someData, d => polygonContains(polygon, d.coords)));
});
});
});
client/__tests__/util/typedCrossfilter/typedCrossfilter.test.js
-590
View File
@@ -1,590 +0,0 @@
// jshint esversion: 6
import _ from "lodash";
import crossfilter from "../../../src/util/typedCrossfilter";
const someData = [
{
date: "2011-11-14T16:17:54Z",
quantity: 2,
total: 190,
tip: 100,
type: "tab",
productIDs: ["001"]
},
{
date: "2011-11-14T16:20:19Z",
quantity: 2,
total: 190,
tip: 100,
type: "tab",
productIDs: ["001", "005"]
},
{
date: "2011-11-14T16:28:54Z",
quantity: 1,
total: 300,
tip: 200,
type: "visa",
productIDs: ["004", "005"]
},
{
date: "2011-11-14T16:30:43Z",
quantity: 2,
total: 90,
tip: 0,
type: "tab",
productIDs: ["001", "002"]
},
{
date: "2011-11-14T16:48:46Z",
quantity: 2,
total: 90,
tip: 0,
type: "tab",
productIDs: ["005"]
},
{
date: "2011-11-14T16:53:41Z",
quantity: 2,
total: 90,
tip: 0,
type: "tab",
productIDs: ["001", "004", "005"]
},
{
date: "2011-11-14T16:54:06Z",
quantity: 1,
total: 100,
tip: 0,
type: "cash",
productIDs: ["001", "002", "003", "004", "005"]
},
{
date: "2011-11-14T16:58:03Z",
quantity: 2,
total: 90,
tip: 0,
type: "tab",
productIDs: ["001"]
},
{
date: "2011-11-14T17:07:21Z",
quantity: 2,
total: 90,
tip: 0,
type: "tab",
productIDs: ["004", "005"]
},
{
date: "2011-11-14T17:22:59Z",
quantity: 2,
total: 90,
tip: 0,
type: "tab",
productIDs: ["001", "002", "004", "005"]
},
{
date: "2011-11-14T17:25:45Z",
quantity: 2,
total: 200,
tip: 0,
type: "cash",
productIDs: ["002"]
},
{
date: "2011-11-14T17:29:52Z",
quantity: 1,
total: 200,
tip: 100,
type: "visa",
productIDs: ["004"]
}
];
function groupReduce(data, valueMap, valueReduce, valueInit) {
return _.reduce(
data,
(acc, value) => {
const k = valueMap(value);
let r = _.find(acc, o => o.key === k);
if (!r) {
r = { key: k, value: valueInit() };
acc.push(r);
}
r.value = valueReduce(r.value, value);
return acc;
},
[]
).sort((a, b) => (a.key < b.key ? -1 : a.key > b.key ? 1 : 0));
}
function groupCount(data, map) {
return groupReduce(data, map, p => p + 1, () => 0);
}
function groupSum(data, map) {
return groupReduce(
data,
map,
(p, v) => {
p += map(v);
return p;
},
() => 0
);
}
let payments = null;
beforeEach(() => {
payments = crossfilter(someData);
});
describe("typedCrossfilter", () => {
test("alloc and free", () => {
expect(payments).toBeDefined();
expect(payments.size()).toEqual(someData.length);
expect(payments.all()).toEqual(someData);
const quantity = payments.dimension(
crossfilter.ScalarDimension,
(i, data) => data[i].quantity,
Int32Array
);
expect(quantity).toBeDefined();
expect(quantity.id()).toBeDefined();
quantity.dispose();
expect(payments.size()).toEqual(someData.length);
expect(payments.all()).toEqual(someData);
});
test("filterAll and filterNone", () => {
expect(payments).toBeDefined();
const quantity = payments.dimension(
crossfilter.ScalarDimension,
(i, data) => data[i].quantity,
Int32Array
);
const tip = payments.dimension(
crossfilter.ScalarDimension,
(i, data) => data[i].tip,
Float32Array
);
const total = payments.dimension(
crossfilter.ScalarDimension,
(i, data) => data[i].total,
Float32Array
);
const type = payments.dimension(
crossfilter.EnumDimension,
(i, data) => data[i].type
);
expect(quantity).toBeDefined();
expect(tip).toBeDefined();
expect(total).toBeDefined();
expect(type).toBeDefined();
// initially, all should be filtered
expect(payments.allFiltered()).toHaveLength(payments.size());
expect(payments.allFiltered()).toEqual(payments.all());
expect(payments.countFiltered()).toEqual(someData.length);
// filterAll
tip.filterAll(); // should change nothing
expect(payments.allFiltered()).toEqual(payments.all());
expect(payments.countFiltered()).toEqual(someData.length);
// ditto
total.filterAll();
expect(payments.allFiltered()).toEqual(payments.all());
expect(payments.countFiltered()).toEqual(someData.length);
// filterNone
type.filterNone();
expect(payments.allFiltered()).toEqual([]);
expect(payments.countFiltered()).toEqual(0);
quantity.filterNone();
expect(payments.allFiltered()).toEqual([]);
expect(payments.countFiltered()).toEqual(0);
// invert the first none; should have no effect because type is
// still not filtered
quantity.filterAll();
expect(payments.allFiltered()).toEqual([]);
expect(payments.countFiltered()).toEqual(0);
// filter all of type; should select all
type.filterAll();
expect(payments.allFiltered()).toEqual(payments.all());
expect(payments.countFiltered()).toEqual(payments.size());
});
test("filterExact", () => {
expect(payments).toBeDefined();
const quantity = payments.dimension(
crossfilter.ScalarDimension,
(i, data) => data[i].quantity,
Int32Array
);
const tip = payments.dimension(
crossfilter.ScalarDimension,
(i, data) => data[i].tip,
Float32Array
);
const type = payments.dimension(
crossfilter.EnumDimension,
(i, data) => data[i].type
);
quantity.filterExact(1);
expect(payments.countFiltered()).toEqual(
_.countBy(someData, "quantity")[1]
);
expect(payments.allFiltered()).toEqual(_.filter(someData, { quantity: 1 }));
tip.filterExact(0);
expect(payments.allFiltered()).toEqual(
_.filter(someData, { tip: 0, quantity: 1 })
);
type.filterExact("cash");
expect(payments.allFiltered()).toEqual(
_.filter(someData, { tip: 0, quantity: 1, type: "cash" })
);
});
test("filterRange", () => {
expect(payments).toBeDefined();
const quantity = payments.dimension(
crossfilter.ScalarDimension,
(i, data) => data[i].quantity,
Int32Array
);
const tip = payments.dimension(
crossfilter.ScalarDimension,
(i, data) => data[i].tip,
Float32Array
);
const total = payments.dimension(
crossfilter.ScalarDimension,
(i, data) => data[i].total,
Float32Array
);
const type = payments.dimension(
crossfilter.EnumDimension,
(i, data) => data[i].type
);
tip.filterRange([0, 91]);
expect(payments.allFiltered()).toEqual(
_(someData)
.filter(r => r.tip >= 0 && r.tip < 91)
.value()
);
tip.filterRange([0, 90]);
expect(payments.allFiltered()).toEqual(
_(someData)
.filter(r => r.tip >= 0 && r.tip < 90)
.value()
);
tip.filterRange([1, 90]);
expect(payments.allFiltered()).toEqual(
_(someData)
.filter(r => r.tip >= 1 && r.tip < 91)
.value()
);
});
test("filterEnum", () => {
expect(payments).toBeDefined();
const quantity = payments.dimension(
crossfilter.ScalarDimension,
(i, data) => data[i].quantity,
Int32Array
);
const tip = payments.dimension(
crossfilter.ScalarDimension,
(i, data) => data[i].tip,
Float32Array
);
const total = payments.dimension(
crossfilter.ScalarDimension,
(i, data) => data[i].total,
Float32Array
);
const type = payments.dimension(
crossfilter.EnumDimension,
(i, data) => data[i].type
);
type.filterEnum(["tab", "cash"]);
expect(payments.allFiltered()).toEqual(
_(someData)
.filter(r => r.type === "cash" || r.type === "tab")
.value()
);
tip.filterEnum([0, 100]);
expect(payments.allFiltered()).toEqual(
_(someData)
.filter(r => r.type === "cash" || r.type === "tab")
.filter(r => r.tip === 0 || r.tip === 100)
.value()
);
});
test("more than 32 dimensions", () => {
expect(payments).toBeDefined();
const quantity = payments.dimension(
crossfilter.ScalarDimension,
(i, data) => data[i].quantity,
Int32Array
);
const tip = payments.dimension(
crossfilter.ScalarDimension,
(i, data) => data[i].tip,
Float32Array
);
const total = payments.dimension(
crossfilter.ScalarDimension,
(i, data) => data[i].total,
Float32Array
);
const type = payments.dimension(
crossfilter.EnumDimension,
(i, data) => data[i].type
);
// Create a bunch of fake dimensions to ensure we can handle > 32
let dimMap = {};
for (let i = 0; i < 65; i++) {
dimMap[i] = payments.dimension(
crossfilter.ScalarDimension,
() => Math.random(),
Float32Array
);
expect(dimMap[i]).toBeDefined();
expect(dimMap[i].id()).toBeDefined();
}
// everything should start as selected/filtered
expect(payments.countFiltered()).toEqual(someData.length);
dimMap[0].filterAll();
dimMap[64].filterAll();
expect(payments.countFiltered()).toEqual(someData.length);
dimMap[33].filterNone();
expect(payments.allFiltered()).toEqual([]);
dimMap[33].filterAll();
expect(payments.allFiltered()).toEqual(someData);
});
test("group, default mapping, default reducer, no filter", () => {
expect(payments).toBeDefined();
const quantity = payments.dimension(
crossfilter.ScalarDimension,
(i, data) => data[i].quantity,
Int32Array
);
const tip = payments.dimension(
crossfilter.ScalarDimension,
(i, data) => data[i].tip,
Int32Array
);
const type = payments.dimension(
crossfilter.EnumDimension,
(i, data) => data[i].type
);
const total = payments.dimension(
crossfilter.ScalarDimension,
(i, data) => data[i].total,
Int32Array
);
_.each(
{
tip: tip.group(r => r),
type: type.group(),
total: total.group(),
quantity: quantity.group()
},
(grp, k) => {
const whatWeExpect = groupCount(someData, v => v[k]);
expect(grp.all()).toEqual(whatWeExpect);
expect(grp.size()).toEqual(whatWeExpect.length);
expect(grp.dispose()).toEqual(grp);
}
);
});
test("group, custom map, default reducer, no filters", () => {
expect(payments).toBeDefined();
// custom mapping in groups only works for scalar types. Enums do not
// currently implement it.
const tip = payments.dimension(
crossfilter.ScalarDimension,
(i, data) => data[i].tip,
Int32Array
);
const totalX10 = payments.dimension(
crossfilter.ScalarDimension,
(i, data) => data[i].total * 10,
Int32Array
);
const type = payments.dimension(
crossfilter.EnumDimension,
(i, data) => data[i].type
);
const paymentsByTip_A = tip.group();
const paymentsByTip_B = tip.group(r => 10 * r);
const paymentsByType = type.group(); // identity only
const paymentsByTotalX10_A = totalX10.group();
const paymentsByTotalX10_B = totalX10.group(r => r / 10);
expect(paymentsByTip_A.all()).toEqual(groupCount(someData, v => v.tip));
expect(paymentsByTip_B.all()).toEqual(
groupCount(someData, v => 10 * v.tip)
);
expect(paymentsByType.all()).toEqual(groupCount(someData, v => v.type));
expect(paymentsByTotalX10_A.all()).toEqual(
groupCount(someData, v => 10 * v.total)
);
expect(paymentsByTotalX10_B.all()).toEqual(
groupCount(someData, v => (10 * v.total) / 10)
);
for (let i of [
paymentsByTip_A,
paymentsByTip_B,
paymentsByType,
paymentsByTotalX10_A,
paymentsByTotalX10_B,
tip,
totalX10,
type
]) {
expect(i.dispose()).toEqual(i);
}
});
test("group, default map, custom reducer, no filters", () => {
expect(payments).toBeDefined();
const total = payments.dimension(
crossfilter.ScalarDimension,
(i, data) => data[i].total,
Float32Array
);
const type = payments.dimension(
crossfilter.EnumDimension,
(i, data) => data[i].type
);
const paymentsByTotal = total.group();
const paymentsByType = type.group();
// reduceCount
expect(paymentsByTotal.reduceCount()).toEqual(paymentsByTotal);
expect(paymentsByTotal.all()).toEqual(groupCount(someData, v => v.total));
// reduceSum
expect(paymentsByTotal.reduceSum(v => v.total)).toEqual(paymentsByTotal);
expect(paymentsByTotal.all()).toEqual(groupSum(someData, v => v.total));
// use custom reducers (my reducers) - count by three, init 1
expect(
paymentsByTotal.reduce((p, v) => (p += 3), (p, v) => (p -= 3), () => 1)
).toEqual(paymentsByTotal);
expect(paymentsByTotal.all()).toEqual(
groupReduce(someData, v => v.total, (p, v) => p + 3, () => 1)
);
for (let i of [paymentsByTotal, paymentsByType, type]) {
expect(i.dispose()).toEqual(i);
}
});
test("group, default map, default reducer, filters", () => {
// From the docs:
// Note: a grouping intersects the crossfilter's current filters, except for the
// associated dimension's filter. Thus, group methods consider only records that
// satisfy every filter except this dimension's filter. So, if the crossfilter of
// payments is filtered by type and total, then group by total only observes the
// filter by type.
expect(payments).toBeDefined();
const tip = payments.dimension(
crossfilter.ScalarDimension,
(i, data) => data[i].tip,
Int32Array
);
const total = payments.dimension(
crossfilter.ScalarDimension,
(i, data) => data[i].total,
Int32Array
);
const type = payments.dimension(
crossfilter.EnumDimension,
(i, data) => data[i].type
);
const paymentsByTip = tip.group();
const paymentsByTotal = total.group();
const paymentsByType = type.group();
// 1. confirm that changing the filter on a dimension does NOT change that
// dimensions groups.
{
tip.filterAll(), total.filterAll(), type.filterAll();
let before = _.cloneDeep(paymentsByTip.all());
tip.filterExact(0);
expect(paymentsByTip.all()).toEqual(before);
}
// 2. confirm that changing a filter on a different dimension DOES change
// all other groups.
{
tip.filterAll(), total.filterAll(), type.filterAll();
const before = _.cloneDeep([paymentsByTotal.all(), paymentsByType.all()]);
tip.filterExact(0);
const after = [paymentsByTotal.all(), paymentsByType.all()];
expect(after).not.toEqual(before);
expect(after).toEqual([
groupReduce(
someData,
v => v.total,
(p, v) => (v.tip !== 0 ? p : p + 1),
() => 0
),
groupReduce(
someData,
v => v.type,
(p, v) => (v.tip !== 0 ? p : p + 1),
() => 0
)
]);
}
for (let i of [
paymentsByTip,
paymentsByTotal,
paymentsByType,
tip,
total,
type
]) {
expect(i.dispose()).toEqual(i);
}
});
});
client/src/components/continuous/continuous.js
+1 -2
View File
@@ -11,8 +11,7 @@ import HistogramBrush from "../brushableHistogram";
@connect(state => ({
obsAnnotations: _.get(state.controls.world, "obsAnnotations", null),
colorAccessor: state.controls.colorAccessor,
colorScale: state.controls.colors.scale,
selectionUpdate: _.get(state.controls, "crossfilter.updateTime", null),
colorScale: state.controls.colorScale,
schema: _.get(state.controls.world, "schema", null)
}))
class Continuous extends React.Component {
client/src/components/geneExpression/expressionButtons.js
+1 -2
View File
@@ -10,8 +10,7 @@ import CellSetButton from "./cellSetButtons";
@connect(state => ({
differential: state.differential,
world: state.controls.world,
crossfilter: state.controls.crossfilter,
selectionUpdate: _.get(state.controls, "crossfilter.updateTime", null)
crossfilter: state.controls.crossfilter
}))
class Expression extends React.Component {
constructor(props) {
client/src/components/graph/graph.js
+6 -13
View File
@@ -32,7 +32,6 @@ import { World } from "../../util/stateManager";
responsive: state.responsive,
colorRGB: _.get(state.controls, "colors.rgb", null),
opacityForDeselectedCells: state.controls.opacityForDeselectedCells,
selectionUpdate: _.get(state.controls, "crossfilter.updateTime", null),
resettingInterface: state.controls.resettingInterface,
userDefinedGenes: state.controls.userDefinedGenes,
diffexpGenes: state.controls.diffexpGenes,
@@ -103,13 +102,7 @@ class Graph extends React.Component {
componentDidUpdate(prevProps) {
const { renderCache } = this;
const {
world,
crossfilter,
colorRGB,
responsive,
selectionUpdate
} = this.props;
const { world, crossfilter, colorRGB, responsive } = this.props;
const {
reglRender,
mode,
@@ -173,11 +166,11 @@ class Graph extends React.Component {
// Sizes for each point - updates are triggered only when selected
// obs change
if (!renderCache.sizes || selectionUpdate !== prevProps.selectionUpdate) {
if (!renderCache.sizes || crossfilter !== prevProps.crossfilter) {
if (!renderCache.sizes) {
renderCache.sizes = new Float32Array(nObs);
}
crossfilter.fillByIsFiltered(renderCache.sizes, 4, 0.2);
crossfilter.fillByIsSelected(renderCache.sizes, 4, 0.2);
sizeBuffer({ data: renderCache.sizes, dimension: 1 });
}
@@ -242,7 +235,7 @@ class Graph extends React.Component {
if (!crossfilter || !world || !universe) {
return false;
}
const nothingSelected = crossfilter.countFiltered() === crossfilter.size();
const nothingSelected = crossfilter.countSelected() === crossfilter.size();
const nothingColoredBy = !colorAccessor;
const noGenes = userDefinedGenes.length === 0 && diffexpGenes.length === 0;
const scatterNotDpl = !scatterplotXXaccessor || !scatterplotYYaccessor;
@@ -450,8 +443,8 @@ class Graph extends React.Component {
data-testid="subset-button"
disabled={
crossfilter &&
(crossfilter.countFiltered() === 0 ||
crossfilter.countFiltered() === crossfilter.size())
(crossfilter.countSelected() === 0 ||
crossfilter.countSelected() === crossfilter.size())
}
style={{ marginRight: 10 }}
onClick={() => {
client/src/components/scatterplot/scatterplot.js
+4 -7
View File
@@ -58,9 +58,7 @@ import finiteExtent from "../../util/finiteExtent";
expressionX,
expressionY,
crossfilter,
// updated whenever the crossfilter selection is updated
selectionUpdate: _.get(state.controls, "crossfilter.updateTime", null)
crossfilter
};
})
class Scatterplot extends React.Component {
@@ -136,8 +134,7 @@ class Scatterplot extends React.Component {
scatterplotYYaccessor,
expressionX,
expressionY,
colorRGB,
selectionUpdate
colorRGB
} = this.props;
const {
reglRender,
@@ -210,11 +207,11 @@ class Scatterplot extends React.Component {
// Sizes for each point - updates are triggered only when selected
// obs change
if (!renderCache.sizes || selectionUpdate !== prevProps.selctionUpdate) {
if (!renderCache.sizes || crossfilter !== prevProps.crossfilter) {
if (!renderCache.sizes) {
renderCache.sizes = new Float32Array(cellCount);
}
crossfilter.fillByIsFiltered(renderCache.sizes, 4, 0.2);
crossfilter.fillByIsSelected(renderCache.sizes, 4, 0.2);
sizeBuffer({ data: renderCache.sizes, dimension: 1 });
}
client/src/reducers/controls.js
Vendored
+115 -101
View File
@@ -35,7 +35,6 @@ const Controls = (
world: null,
categoricalSelectionState: null,
crossfilter: null,
dimensionMap: null,
userDefinedGenes: [],
userDefinedGenesLoading: false,
diffexpGenes: [],
@@ -95,8 +94,10 @@ const Controls = (
state,
world
);
const crossfilter = Crossfilter(world.obsAnnotations);
const dimensionMap = World.createObsDimensionMap(crossfilter, world);
const crossfilter = World.createObsDimensions(
new Crossfilter(world.obsAnnotations),
world
);
WorldUtil.clearCaches();
return {
@@ -104,11 +105,10 @@ const Controls = (
loading: false,
error: null,
universe,
fullUniverseCache: { world, crossfilter, dimensionMap },
fullUniverseCache: { world, crossfilter },
world,
categoricalSelectionState,
crossfilter,
dimensionMap,
colorMode,
colorAccessor: null,
colors,
@@ -116,40 +116,35 @@ const Controls = (
};
}
case "reset World to eq Universe": {
/*
1. Reset world & crossfilter, using previously created objects which were
stashed in `fullUniverseCache`
2. Add crossfilter dimension for all userDefined and diffexp genes/varData,
as they are not part of the cached crossfilter.
3. Compute categorical selection summary
4. Reset all WorldUtil caches
5. Reset color-by
*/
const { userDefinedGenes, diffexpGenes, fullUniverseCache } = state;
const { world, crossfilter } = fullUniverseCache;
// reset all crossfilter dimensions
_.forEach(fullUniverseCache.dimensionMap, dim => dim.filterAll());
const { world } = fullUniverseCache;
const crossfilter = ControlsHelpers.createGeneDimensions(
userDefinedGenes,
diffexpGenes,
world,
fullUniverseCache.crossfilter
);
const colorMode = null;
const colors = createColors(world, colorMode);
const categoricalSelectionState = ControlsHelpers.createCategoricalSelectionState(
state,
world
);
/* free dimensions not in cache (otherwise they leak) */
_.forEach(state.dimensionMap, (dim, dimName) => {
if (!fullUniverseCache.dimensionMap[dimName]) {
dim.dispose();
}
});
const dimensionMap = {
...fullUniverseCache.dimensionMap,
...ControlsHelpers.createGenesDimMap(
userDefinedGenes,
diffexpGenes,
world,
crossfilter
)
};
WorldUtil.clearCaches();
return {
...state,
world,
categoricalSelectionState,
crossfilter,
dimensionMap,
colorMode,
colorAccessor: null,
colors,
@@ -175,16 +170,15 @@ const Controls = (
state,
world
);
const crossfilter = Crossfilter(world.obsAnnotations);
const dimensionMap = {
...World.createObsDimensionMap(crossfilter, world),
...ControlsHelpers.createGenesDimMap(
userDefinedGenes,
diffexpGenes,
world,
crossfilter
)
};
let crossfilter = new Crossfilter(world.obsAnnotations);
crossfilter = World.createObsDimensions(crossfilter, world);
crossfilter = ControlsHelpers.createGeneDimensions(
userDefinedGenes,
diffexpGenes,
world,
crossfilter
);
WorldUtil.clearCaches();
return {
@@ -194,8 +188,7 @@ const Controls = (
world,
colors,
categoricalSelectionState,
crossfilter,
dimensionMap
crossfilter
};
}
case "expression load success": {
@@ -276,56 +269,57 @@ const Controls = (
};
}
case "request user defined gene success": {
const { world, crossfilter, dimensionMap, userDefinedGenes } = state;
const { world, crossfilter: oldCrossfilter, userDefinedGenes } = state;
const _userDefinedGenes = userDefinedGenes.slice();
const gene = action.data.genes[0];
dimensionMap[
userDefinedDimensionName(gene)
] = World.createVarDataDimension(world, crossfilter, gene);
const crossfilter = oldCrossfilter.addDimension(
userDefinedDimensionName(gene),
"scalar",
world.varData.col(gene).asArray(),
Float32Array
);
return {
...state,
dimensionMap,
crossfilter,
userDefinedGenes: _userDefinedGenes,
userDefinedGenesLoading: false
};
}
case "request differential expression success": {
const { world, crossfilter, dimensionMap } = state;
const { world, crossfilter: oldCrossfilter } = state;
const _diffexpGenes = [];
action.data.forEach(d => {
_diffexpGenes.push(world.varAnnotations.at(d[0], "name"));
});
let crossfilter = oldCrossfilter;
_.forEach(_diffexpGenes, gene => {
dimensionMap[diffexpDimensionName(gene)] = World.createVarDataDimension(
world,
crossfilter,
gene
crossfilter = crossfilter.addDimension(
diffexpDimensionName(gene),
"scalar",
world.varData.col(gene).asArray(),
Float32Array
);
});
return {
...state,
dimensionMap,
crossfilter,
diffexpGenes: _diffexpGenes
};
}
case "clear differential expression": {
const { world, dimensionMap } = state;
const _dimensionMap = dimensionMap;
const { world } = state;
let { crossfilter } = state;
_.forEach(action.diffExp, values => {
const name = world.varAnnotations.at(values[0], "name");
// clean up crossfilter dimensions
const dimension = dimensionMap[diffexpDimensionName(name)];
dimension.dispose();
delete dimensionMap[diffexpDimensionName(name)];
crossfilter = crossfilter.delDimension(diffexpDimensionName(name));
});
return {
...state,
dimensionMap: _dimensionMap,
crossfilter,
diffexpGenes: []
};
}
@@ -342,34 +336,29 @@ const Controls = (
};
}
case "clear user defined gene": {
const { userDefinedGenes, dimensionMap } = state;
const { userDefinedGenes, crossfilter: oldCrossfilter } = state;
const newUserDefinedGenes = _.filter(
userDefinedGenes,
d => d !== action.data
);
const dimension = dimensionMap[userDefinedDimensionName(action.data)];
dimension.dispose();
delete dimensionMap[userDefinedDimensionName(action.data)];
const crossfilter = oldCrossfilter.delDimension(
userDefinedDimensionName(action.data)
);
return {
...state,
dimensionMap,
crossfilter,
userDefinedGenes: newUserDefinedGenes
};
}
case "clear all user defined genes": {
const { userDefinedGenes, dimensionMap } = state;
const { userDefinedGenes } = state;
let { crossfilter } = state;
_.forEach(userDefinedGenes, gene => {
const dimension = dimensionMap[userDefinedDimensionName(gene)];
dimension.dispose();
delete dimensionMap[userDefinedDimensionName(gene)];
crossfilter = crossfilter.delDimension(userDefinedDimensionName(gene));
});
return {
...state,
dimensionMap,
crossfilter,
userDefinedGenes: []
};
}
@@ -395,34 +384,49 @@ const Controls = (
User Events
*******************************/
case "graph brush selection change": {
state.dimensionMap[layoutDimensionName("XY")].filterWithinRect(
action.brushCoords.northwest,
action.brushCoords.southeast
);
const name = layoutDimensionName("XY");
const [x0, y0] = action.brushCoords.northwest;
const [x1, y1] = action.brushCoords.southeast;
const crossfilter = state.crossfilter.select(name, {
mode: "within-rect",
x0,
y0,
x1,
y1
});
return {
...state,
crossfilter,
graphBrushSelection: action.brushCoords
};
}
case "lasso deselect":
case "graph brush deselect": {
state.dimensionMap[layoutDimensionName("XY")].filterAll();
const name = layoutDimensionName("XY");
const crossfilter = state.crossfilter.select(name, { mode: "all" });
return {
...state,
crossfilter,
graphBrushSelection: null
};
}
case "lasso selection": {
const { polygon } = action;
const dXY = state.dimensionMap[layoutDimensionName("XY")];
const name = layoutDimensionName("XY");
const { crossfilter: oldCrossfilter } = state;
let crossfilter;
if (polygon.length < 3) {
// single point or a line is not a polygon, and is therefore a deselect
dXY.filterAll();
crossfilter = oldCrossfilter.select(name, { mode: "all" });
} else {
dXY.filterWithinPolygon(polygon);
crossfilter = oldCrossfilter.select(name, {
mode: "within-polygon",
polygon
});
}
return {
...state
...state,
crossfilter
};
}
case "continuous metadata histogram brush": {
@@ -430,15 +434,17 @@ const Controls = (
action.continuousNamespace,
action.selection
);
let { crossfilter } = state;
// action.selection: metadata name being selected
// action.range: filter range, or null if deselected
if (!action.range) {
state.dimensionMap[name].filterAll();
crossfilter = crossfilter.select(name, { mode: "all" });
} else {
state.dimensionMap[name].filterRange(action.range);
const [lo, hi] = action.range;
crossfilter = crossfilter.select(name, { mode: "range", lo, hi });
}
return { ...state };
return { ...state, crossfilter };
}
case "change opacity deselected cells in 2d graph background":
return {
@@ -476,13 +482,15 @@ const Controls = (
// update the filter to match all selected options
const cat = newCategoricalSelectionState[action.metadataField];
state.dimensionMap[obsAnnoDimensionName(action.metadataField)].filterEnum(
ControlsHelpers.selectedValuesForCategory(cat)
);
const dName = obsAnnoDimensionName(action.metadataField);
const crossfilter = state.crossfilter.select(dName, {
mode: "exact",
values: ControlsHelpers.selectedValuesForCategory(cat)
});
return {
...state,
categoricalSelectionState: newCategoricalSelectionState
categoricalSelectionState: newCategoricalSelectionState,
crossfilter
};
}
case "categorical metadata filter deselect": {
@@ -500,13 +508,15 @@ const Controls = (
// update the filter to match all selected options
const cat = newCategoricalSelectionState[action.metadataField];
state.dimensionMap[obsAnnoDimensionName(action.metadataField)].filterEnum(
ControlsHelpers.selectedValuesForCategory(cat)
);
const dName = obsAnnoDimensionName(action.metadataField);
const crossfilter = state.crossfilter.select(dName, {
mode: "exact",
values: ControlsHelpers.selectedValuesForCategory(cat)
});
return {
...state,
categoricalSelectionState: newCategoricalSelectionState
categoricalSelectionState: newCategoricalSelectionState,
crossfilter
};
}
case "categorical metadata filter none of these": {
@@ -520,12 +530,14 @@ const Controls = (
).fill(false)
}
};
state.dimensionMap[
obsAnnoDimensionName(action.metadataField)
].filterNone();
const dName = obsAnnoDimensionName(action.metadataField);
const crossfilter = state.crossfilter.select(dName, {
mode: "none"
});
return {
...state,
categoricalSelectionState: newCategoricalSelectionState
categoricalSelectionState: newCategoricalSelectionState,
crossfilter
};
}
case "categorical metadata filter all of these": {
@@ -539,12 +551,14 @@ const Controls = (
).fill(true)
}
};
state.dimensionMap[
obsAnnoDimensionName(action.metadataField)
].filterAll();
const dName = obsAnnoDimensionName(action.metadataField);
const crossfilter = state.crossfilter.select(dName, {
mode: "all"
});
return {
...state,
categoricalSelectionState: newCategoricalSelectionState
categoricalSelectionState: newCategoricalSelectionState,
crossfilter
};
}
client/src/util/stateManager/controlsHelpers.js
+23 -17
View File
@@ -98,29 +98,35 @@ export function selectedValuesForCategory(categorySelectionState) {
}
/*
build a crossfilter dimension map for all gene expression related dimensions.
build a crossfilter dimensions for all gene expression related dimensions.
*/
export function createGenesDimMap(
export function createGeneDimensions(
userDefinedGenes,
diffexpGenes,
world,
crossfilter
) {
function _createGenesDimMap(genes, nameCreator) {
return genes.reduce((acc, gene) => {
acc[nameCreator(gene)] = World.createVarDataDimension(
world,
crossfilter,
gene
);
return acc;
}, {});
}
return {
..._createGenesDimMap(userDefinedGenes, userDefinedDimensionName),
..._createGenesDimMap(diffexpGenes, diffexpDimensionName)
};
crossfilter = userDefinedGenes.reduce(
(xflt, gene) =>
xflt.addDimension(
userDefinedDimensionName(gene),
"scalar",
world.varData.col(gene).asArray(),
Float32Array
),
crossfilter
);
crossfilter = diffexpGenes.reduce(
(xflt, gene) =>
xflt.addDimension(
diffexpDimensionName(gene),
"scalar",
world.varData.col(gene).asArray(),
Float32Array
),
crossfilter
);
return crossfilter;
}
export function pruneVarDataCache(varData, needed) {
client/src/util/stateManager/world.js
+21 -54
View File
@@ -1,8 +1,6 @@
// jshint esversion: 6
import _ from "lodash";
import { layoutDimensionName, obsAnnoDimensionName } from "../nameCreators";
import Crossfilter from "../typedCrossfilter";
import * as Dataframe from "../dataframe";
/*
@@ -98,7 +96,7 @@ export function createWorldFromCurrentSelection(universe, world, crossfilter) {
newWorld.varAnnotations = universe.varAnnotations;
/* now subset/cut obs */
const mask = crossfilter.allFilteredMask();
const mask = crossfilter.allSelectedMask();
newWorld.obsAnnotations = world.obsAnnotations.isubsetMask(mask);
newWorld.obsLayout = world.obsLayout.isubsetMask(mask);
newWorld.nObs = newWorld.obsAnnotations.dims[0];
@@ -139,63 +137,32 @@ function deduceDimensionType(attributes, fieldName) {
return dimensionType;
}
/*
Return a crossfilter dimension for the specified world & named gene.
NOTE: this assumes that the expression data was already loaded,
by calling an appropriate action creator.
Caller needs to *save* this dimension somewhere for it to be later used.
Dimension must be destroyed by calling dimension.dispose()
when it is no longer needed
(it will not be garbage collected without this call)
*/
export function createVarDataDimension(world, crossfilter, name) {
return crossfilter.dimension(
Crossfilter.ScalarDimension,
world.varData.col(name).asArray(),
Float32Array
);
}
export function createObsDimensionMap(crossfilter, world) {
export function createObsDimensions(crossfilter, world) {
/*
create and return a crossfilter dimension for every obs annotation
for which we have a supported type.
create and return a crossfilter with a dimension for every obs annotation
for which we have a supported type, *except* 'name'
*/
const { schema, obsLayout, obsAnnotations } = world;
const annoList = schema.annotations.obs.filter(anno => anno.name !== "name");
crossfilter = annoList.reduce((xfltr, anno) => {
const dimType = deduceDimensionType(anno, anno.name);
const colData = obsAnnotations.col(anno.name).asArray();
const name = obsAnnoDimensionName(anno.name);
if (dimType === "enum") {
return xfltr.addDimension(name, "enum", colData);
}
if (dimType) {
return xfltr.addDimension(name, "scalar", colData, dimType);
}
return xfltr;
}, crossfilter);
// Create a crossfilter dimension for all obs annotations *except* 'name'
const dimensionMap = _(schema.annotations.obs)
.filter(anno => anno.name !== "name")
.transform((result, anno) => {
const dimType = deduceDimensionType(anno, anno.name);
const colData = obsAnnotations.col(anno.name).asArray();
if (dimType === "enum") {
result[obsAnnoDimensionName(anno.name)] = crossfilter.dimension(
Crossfilter.EnumDimension,
colData
);
} else if (dimType) {
result[obsAnnoDimensionName(anno.name)] = crossfilter.dimension(
Crossfilter.ScalarDimension,
colData,
dimType
);
} // else ignore the annotation
}, {})
.value();
/*
Add crossfilter dimensions allowing filtering on layout
*/
dimensionMap[layoutDimensionName("XY")] = crossfilter.dimension(
Crossfilter.SpatialDimension,
return crossfilter.addDimension(
layoutDimensionName("XY"),
"spatial",
obsLayout.col("X").asArray(),
obsLayout.col("Y").asArray()
);
return dimensionMap;
}
export function worldEqUniverse(world, universe) {
@@ -206,7 +173,7 @@ export function getSelectedByIndex(crossfilter) {
/*
return array of obsIndex, containing all selected obs/cells.
*/
const selected = crossfilter.allFilteredMask(); // array of bool-ish
const selected = crossfilter.allSelectedMask(); // array of bool-ish
const keys = crossfilter.data.rowIndex.keys(); // row keys, aka universe rowIndex
const set = new Int32Array(selected.length);
client/src/util/typedCrossfilter/bitArray.js
+26
View File
@@ -211,6 +211,19 @@ class BitArray {
}
}
// select range of indices on a dimension
//
selectFromRange(dim, range) {
const col = dim >>> 5;
const first = range[0];
const last = range[1];
const one = 1 << dim % 32;
const offset = col * this.length;
for (let i = first; i < last; i += 1) {
this.bitarray[offset + i] |= one;
}
}
// select range of indices on a dimension, indirect through a sort map.
// Indirect functions are used to map between sort and natural order.
//
@@ -225,6 +238,19 @@ class BitArray {
}
}
// deselect range of indices on a dimension
//
deselectFromRange(dim, range) {
const col = dim >>> 5;
const first = range[0];
const last = range[1];
const zero = ~(1 << dim % 32);
const offset = col * this.length;
for (let i = first; i < last; i += 1) {
this.bitarray[offset + i] &= zero;
}
}
// deselect range of indices on a dimension, indirect through a sort map.
//
deselectIndirectFromRange(dim, indirect, range) {
client/src/util/typedCrossfilter/crossfilter.js
+594
View File
@@ -0,0 +1,594 @@
import { polygonContains } from "d3";
import PositiveIntervals from "./positiveIntervals";
import BitArray from "./bitArray";
import { sort } from "./sort";
import {
makeSortIndex,
lowerBound,
lowerBoundIndirect,
upperBoundIndirect
} from "./util";
class NotImplementedError extends Error {
constructor(...params) {
super(...params);
// Maintains proper stack trace for where our error was thrown (only available on V8)
if (Error.captureStackTrace) {
Error.captureStackTrace(this, NotImplementedError);
}
}
}
export default class ImmutableTypedCrossfilter {
constructor(data, dimensions = {}, selectionCache = null) {
/*
Typically, parameter 'data' is one of:
- Array of objects/records
- Dataframe (util/dataframe)
Other parameters are only used internally.
Object field description:
- data: reference to the array of records in the crossfilter
- selectionBitArray: bit array containing the flatted selection state
of all dimensions. This is lazily created and is effectively
a perfomance cache. Methods which return a new crossfilter,
such as select(), addDimention() and delDimension(), will pass
the cache forward to the new object, as the typical "immutable API"
usage pattern is to retain the new crossfilter and discard the old.
- dimensions: contains each dimension and its current state:
- id: bit offset in the cached bit array
- dim: the dimension object
- name: the dimension name
- selection: the dimension's current selection
*/
this.data = data;
this.selectionCache = selectionCache; /* BitArray */
this.dimensions = dimensions; /* name: { id, dim, name, selection } */
}
size() {
return this.data.length;
}
all() {
return this.data;
}
dimensionNames() {
/* return array of all dimensions (by name) */
return Object.keys(this.dimensions);
}
addDimension(name, type, ...rest) {
/*
Add a new dimension to this crossfilter, of type DimensionType.
Remainder of parameters are dimension-type-specific.
*/
const { data, selectionCache } = this;
if (this.dimensions[name] !== undefined) {
throw new Error(`Adding duplicate dimension name ${name}`);
}
this.selectionCache = null; // pass ownership to new crossfilter
let id;
if (selectionCache) {
id = selectionCache.allocDimension();
selectionCache.selectAll(id);
}
const DimensionType = DimTypes[type];
const dim = new DimensionType(name, data, ...rest);
const dimensions = {
...this.dimensions,
[name]: {
id,
dim,
name,
selection: dim.select({ mode: "all" })
}
};
return new ImmutableTypedCrossfilter(data, dimensions, selectionCache);
}
delDimension(name) {
const { data, selectionCache } = this;
const dimensions = { ...this.dimensions };
if (dimensions[name] === undefined) {
throw new ReferenceError(`Unable to delete unknown dimension ${name}`);
}
const { id } = dimensions[name];
delete dimensions[name];
this.selectionCache = null; // pass ownership to new crossfilter
if (selectionCache) {
selectionCache.freeDimension(id);
}
return new ImmutableTypedCrossfilter(data, dimensions, selectionCache);
}
select(name, spec) {
/*
select on named dimension, as indicated by `spec`. Spec is an object
specifying the selection, and must contain at least a `mode` field.
Examples:
select("foo", {mode: "all"});
select("bar", {mode: "none"});
select("mumble", {mode: "exact", values: "blue"});
select("mumble", {mode: "exact", values: ["red", "green", "blue"]});
select("blort", {mode: "range", lo: 0, hi: 999.99});
*/
const { data, selectionCache } = this;
this.selectionCache = null;
const dimensions = { ...this.dimensions };
const { dim, id, selection: oldSelection } = dimensions[name];
const newSelection = dim.select(spec);
newSelection.ranges = PositiveIntervals.canonicalize(newSelection.ranges);
dimensions[name] = { id, dim, name, selection: newSelection };
ImmutableTypedCrossfilter._dimSelnHasUpdated(
selectionCache,
id,
newSelection,
oldSelection
);
return new ImmutableTypedCrossfilter(data, dimensions, selectionCache);
}
static _dimSelnHasUpdated(selectionCache, id, newSeln, oldSeln) {
/*
Selection has updated from oldSeln to newSeln. Update the
bit array if it exists. If not, we will lazy create it when
needed.
*/
if (selectionCache) {
/*
if both new and old selection use the same index, we can
perform an incremental update. If the index changed, we have
to do a suboptimal full deselect/select.
*/
let adds;
let dels;
if (newSeln.index === oldSeln.index) {
adds = PositiveIntervals.difference(newSeln.ranges, oldSeln.ranges);
dels = PositiveIntervals.difference(oldSeln.ranges, newSeln.ranges);
} else {
// console.log("suboptimal selection update - index changed");
adds = newSeln.ranges;
dels = oldSeln.ranges;
}
/*
allow dimensions to return selected ranges in either dimension sort
order (indirect via index), or in original record order.
If sort index exists in the dimension, assume sort ordered ranges.
*/
if (oldSeln.index) {
dels.forEach(interval =>
selectionCache.deselectIndirectFromRange(id, oldSeln.index, interval)
);
} else {
dels.forEach(interval =>
selectionCache.deselectFromRange(id, interval)
);
}
if (newSeln.index) {
adds.forEach(interval =>
selectionCache.selectIndirectFromRange(id, newSeln.index, interval)
);
} else {
adds.forEach(interval => selectionCache.selectFromRange(id, interval));
}
}
}
_getSelectionCache() {
if (!this.selectionCache) {
// console.log("...rebuilding crossfilter cache...");
const selectionCache = new BitArray(this.data.length);
Object.keys(this.dimensions).forEach(name => {
const { selection } = this.dimensions[name];
const id = selectionCache.allocDimension();
this.dimensions[name].id = id;
const { ranges, index } = selection;
ranges.forEach(range => {
if (index) {
selectionCache.selectIndirectFromRange(id, index, range);
} else {
selectionCache.selectFromRange(id, range);
}
});
});
this.selectionCache = selectionCache;
}
return this.selectionCache;
}
allSelected() {
/*
return array of all records currently selected by all dimensions
*/
const selectionCache = this._getSelectionCache();
const { data } = this;
if (Array.isArray(data)) {
const res = [];
for (let i = 0, len = data.length; i < len; i += 1) {
if (selectionCache.isSelected(i)) {
res.push(data[i]);
}
}
return res;
}
/* else, Dataframe-like */
return data.isubsetMask(this.allSelectedMask());
}
allSelectedMask() {
/*
return Uint8Array containing selection state (truthy/falsey) for each record.
*/
const selectionCache = this._getSelectionCache();
return selectionCache.fillBySelection(
new Uint8Array(this.data.length),
1,
0
);
}
countSelected() {
/*
return number of records selected on all dimensions
*/
const selectionCache = this._getSelectionCache();
return selectionCache.selectionCount();
}
isElementSelected(i) {
/*
return truthy/falsey if this record is selected on all dimensions
*/
const selectionCache = this._getSelectionCache();
return selectionCache.isSelected(i);
}
fillByIsSelected(array, selectedValue, deselectedValue) {
/*
fill array with one of two values, based upon selection state.
*/
const selectionCache = this._getSelectionCache();
return selectionCache.fillBySelection(
array,
selectedValue,
deselectedValue
);
}
}
/*
Base dimension object.
A Dimension is an index, accessed via a select() method. The protocol
for a dimension:
- constructor - first param is name, remainder is whatever params are
required to initialize the dimension.
- select - one and only param is the selection specifier. Returns an
array of record IDs.
- name - the dimension name/label.
*/
class _ImmutableBaseDimension {
constructor(name) {
this.name = name;
}
/* eslint-disable class-methods-use-this */
select(spec) {
const { mode } = spec;
if (mode === undefined) {
throw new Error("select spec does not contain 'mode'");
}
throw new Error(`select mode ${mode} not implemented`);
}
/* eslint-enable class-methods-use-this */
}
class ImmutableScalarDimension extends _ImmutableBaseDimension {
constructor(name, data, value, ValueArrayType) {
super(name);
// Three modes - caller can provide a pre-created value array,
// a map function which will create it, or another array which
// will used with an identity map function.
let array;
if (value instanceof ValueArrayType) {
// user has provided the final typed array - just use it
if (value.length !== data.length) {
throw new RangeError(
"ScalarDimension values length must equal crossfilter data record count"
);
}
array = value;
} else if (value instanceof Function) {
// Create value array from user-provided map function.
array = this._createValueArray(
data,
value,
new ValueArrayType(data.length)
);
} else if (isArrayOrTypedArray(value)) {
// Create value array from user-provided array. Typically used
// only by enumerated dimensions
array = this._createValueArray(
data,
i => value[i],
new ValueArrayType(data.length)
);
} else {
throw new NotImplementedError(
"dimension value must be function or value array type"
);
}
this.value = array;
// create sort index
this.index = makeSortIndex(array);
}
/* eslint-disable class-methods-use-this */
_createValueArray(data, mapf, array) {
// create dimension value array
const len = data.length;
const larray = array;
for (let i = 0; i < len; i += 1) {
larray[i] = mapf(i, data);
}
return larray;
}
/* eslint-enable class-methods-use-this */
select(spec) {
const { mode } = spec;
const { index } = this;
switch (mode) {
case "all":
return { ranges: [[0, this.value.length]], index };
case "none":
return { ranges: [], index };
case "exact":
return this.selectExact(spec);
case "range":
return this.selectRange(spec);
default:
return super.select(spec);
}
}
selectExact(spec) {
const { value, index } = this;
let { values } = spec;
if (!Array.isArray(values)) {
values = [values];
}
const ranges = [];
for (let v = 0, len = values.length; v < len; v += 1) {
const r = [
lowerBoundIndirect(value, index, values[v], 0, value.length),
upperBoundIndirect(value, index, values[v], 0, value.length)
];
if (r[0] <= r[1]) {
ranges.push(r);
}
}
return { ranges, index };
}
selectRange(spec) {
const { value, index } = this;
/* [lo, hi) */
const { lo, hi } = spec;
const ranges = [];
const r = [
lowerBoundIndirect(value, index, lo, 0, value.length),
lowerBoundIndirect(value, index, hi, 0, value.length)
];
if (r[0] < r[1]) ranges.push(r);
return { ranges, index };
}
}
class ImmutableEnumDimension extends ImmutableScalarDimension {
constructor(name, data, value) {
super(name, data, value, Uint32Array);
}
_createValueArray(data, mapf, array) {
const len = data.length;
const larray = array;
// create enumeration table - mapping between the value
// and the enum.
const s = new Set();
for (let i = 0; i < len; i += 1) {
s.add(mapf(i, data));
}
const enumIndex = sort(Array.from(s));
this.enumIndex = enumIndex;
// create dimension value array
const enumLen = enumIndex.length;
for (let i = 0; i < len; i += 1) {
const v = mapf(i, data);
const e = lowerBound(enumIndex, v, 0, enumLen);
larray[i] = e;
}
return larray;
}
selectExact(spec) {
const { enumIndex } = this;
const { values } = spec;
return super.selectExact({
mode: spec.mode,
values: values.map(v => lowerBound(enumIndex, v, 0, enumIndex.length))
});
}
/* eslint-disable class-methods-use-this */
selectRange() {
throw new Error("range selection unsupported on Enumerated dimension");
}
/* eslint-enable class-methods-use-this */
}
class ImmutableSpatialDimension extends _ImmutableBaseDimension {
constructor(name, data, X, Y) {
super(name);
if (X.length !== Y.length && X.length !== data.length) {
throw new RangeError(
"SpatialDimension values must have same dimensionality as crossfilter"
);
}
this.X = X;
this.Y = Y;
this.Xindex = makeSortIndex(X);
this.Yindex = makeSortIndex(Y);
}
select(spec) {
const { mode } = spec;
switch (mode) {
case "all":
return { ranges: [[0, this.X.length]], index: null };
case "none":
return { ranges: [], index: null };
case "within-rect":
return this.selectWithinRect(spec);
case "within-polygon":
return this.selectWithinPolygon(spec);
default:
return super.select(spec);
}
}
selectWithinRect(spec) {
/*
{ mode: "within-rect", x0: 1, y0: 0, x1: 3, y1: 9 }
*/
const { x0, y0, x1, y1 } = spec;
const { X, Y } = this;
const ranges = [];
let start = -1;
for (let i = 0, l = X.length; i < l; i += 1) {
const x = X[i];
const y = Y[i];
const inside = x0 <= x && x < x1 && y0 <= y && y < y1;
if (inside && start === -1) start = i;
if (!inside && start !== -1) {
ranges.push([start, i]);
start = -1;
}
}
if (start !== -1) ranges.push([start, X.length]);
return { ranges, index: null };
}
/*
Relatively brute force filter by polygon.
Currently uses d3.polygonContains() to test for polygon inclusion, which itself
uses a ray casting (crossing number) algorithm. There are a series of optimizations
to make this faster:
* first sliced by X or Y, using an index on the axis
* then the polygon bounding box is used for trivial rejection
* then the polygon test is applied
*/
selectWithinPolygon(spec) {
/*
{ mode: "within-polygon", polygon: [ [x0, y0], ... ] }
*/
const { polygon } = spec;
const [minX, minY, maxX, maxY] = polygonBoundingBox(polygon);
const { X, Y, Xindex, Yindex } = this;
const { length } = X;
let slice;
let index;
if (maxY - minY > maxX - minX) {
slice = [
lowerBoundIndirect(X, Xindex, minX, 0, length),
lowerBoundIndirect(X, Xindex, maxX, 0, length)
];
index = Xindex;
} else {
slice = [
lowerBoundIndirect(Y, Yindex, minY, 0, length),
lowerBoundIndirect(Y, Yindex, maxY, 0, length)
];
index = Yindex;
}
const ranges = [];
let start = -1;
for (let i = slice[0], e = slice[1]; i < e; i += 1) {
const rid = index[i];
const x = X[rid];
const y = Y[rid];
const inside =
minX <= x &&
x < maxX &&
minY <= y &&
y < maxY &&
withinPolygon(polygon, x, y);
if (inside && start === -1) start = i;
if (!inside && start !== -1) {
ranges.push([start, i]);
start = -1;
}
}
if (start !== -1) ranges.push([start, slice[1]]);
return { ranges, index };
}
}
/* Helpers */
export const DimTypes = {
scalar: ImmutableScalarDimension,
enum: ImmutableEnumDimension,
spatial: ImmutableSpatialDimension
};
function isArrayOrTypedArray(x) {
return (
Array.isArray(x) ||
(ArrayBuffer.isView(x) &&
Object.prototype.toString.call(x) !== "[object DataView]")
);
}
/* return bounding box of the polygon */
function polygonBoundingBox(polygon) {
let minX = Number.MAX_VALUE;
let minY = Number.MAX_VALUE;
let maxX = Number.MIN_VALUE;
let maxY = Number.MIN_VALUE;
for (let i = 0, l = polygon.length; i < l; i += 1) {
const point = polygon[i];
const [x, y] = point;
if (x < minX) minX = x;
if (y < minY) minY = y;
if (x > maxX) maxX = x;
if (y > maxY) maxY = y;
}
return [minX, minY, maxX, maxY];
}
function withinPolygon(polygon, x, y) {
// TODO XXX replace
return polygonContains(polygon, [x, y]);
}
client/src/util/typedCrossfilter/index.js
+10 -767
View File
@@ -1,7 +1,5 @@
// jshint esversion: 6
/*
Typedarray Crossfilter - a re-implementation of a subset of crossfilter, with
Crossfilter - a re-implementation of a subset of crossfilter, with
time/space optimizations predicated upon the following assumptions:
- dimensions are uniformly typed, and all values must be of that type
- dimension values must be a primitive type (int, float, string). Arrays
@@ -11,14 +9,18 @@ time/space optimizations predicated upon the following assumptions:
want to do that, you have to create the new crossfilter, using the new
data, from scratch.
The actual backing store for a dimension is a TypedArray, enabling significant
In addition, this implementation is easier to use with a "redux" style
app, as all operations on the crossfilter are immutable (ie, return a
new crossfilter).
The actual backing store for a dimension is a TypedArray, enabling
performance improvements over the original crossfilter.
There are also a handful of new methods, primarily to take advantage of the
performance (eg, crossfilter.fillBySelection)
Helpful documents (this module tries to follow the original API as much
as is feasable):
Helpful documents (this module follows similar concepts as the original,
but deviates from the API):
https://github.com/square/crossfilter/
http://square.github.io/crossfilter/
@@ -26,766 +28,7 @@ There is also a newer, community supported fork of crossfilter, with a
more complex API. In a few cases, elements of that API were incorporated.
https://github.com/square/crossfilter/
See test cases for some concrete examples.
*/
// XXX replace
import { polygonContains } from "d3";
import PositiveIntervals from "./positiveIntervals";
import BitArray from "./bitArray";
import {
makeSortIndex,
lowerBound,
lowerBoundIndirect,
upperBoundIndirect
} from "./util";
function isArrayOrTypedArray(x) {
return (
Array.isArray(x) ||
(ArrayBuffer.isView(x) &&
Object.prototype.toString.call(x) !== "[object DataView]")
);
}
class NotImplementedError extends Error {
constructor(...params) {
super(...params);
// Maintains proper stack trace for where our error was thrown (only available on V8)
if (Error.captureStackTrace) {
Error.captureStackTrace(this, NotImplementedError);
}
}
}
class TypedCrossfilter {
constructor(data) {
/*
Typically, data is one of:
- Array of objects/records
- Dataframe (util/dataframe)
*/
this.data = data;
// filters: array of { id, dimension }
this.filters = [];
this.selection = new BitArray(data.length);
this.updateTime = 0;
}
size() {
return this.data.length;
}
all() {
return this.data;
}
/*
Create a crossfilter dimension, upon which filtering (subselection) can
be done. Each dimension is typed, and has a particular set of filtering
semantics.
* ScalarDimension - backed by TypedArray values, supporting filtering
by value (within a value range, or one or more exact values)
* EnumDimension - backed by an enumeration (eg, strings, bools), filtering
by one or more enum categories.
* SpatialDimension - backed by 2D points, filter by containment within
various shapes (currently supports within Rectangle and within Polygon).
Call this method to create a dimension, passing arguments appropriate for
the dimension constructor.
*/
dimension(DimensionType, ...rest) {
const id = this.selection.allocDimension();
const dim = new DimensionType(this, id, ...rest);
this.filters.push({ id, dim });
dim.filterAll();
return dim;
}
_freeDimension(id) {
this.selection.freeDimension(id);
this.filters = this.filters.filter(f => f._id !== id);
}
// return array of all records that are selected/filtered
// by all dimensions.
allFiltered() {
const { data, selection } = this;
if (Array.isArray(data)) {
const res = [];
for (let i = 0, len = data.length; i < len; i += 1) {
if (selection.isSelected(i)) {
res.push(data[i]);
}
}
return res;
}
/* else, Dataframe-like */
return data.isubsetMask(this.allFilteredMask());
}
// return Uint8array containing selection state (truthy/falsey) for each record.
//
allFilteredMask() {
return this.selection.fillBySelection(
new Uint8Array(this.data.length),
1,
0
);
}
countFiltered() {
return this.selection.selectionCount();
}
isElementFiltered(i) {
return this.selection.isSelected(i);
}
// fill array with one of two values, based upon selection state
fillByIsFiltered(array, selectedValue, deselectedValue) {
return this.selection.fillBySelection(
array,
selectedValue,
deselectedValue
);
}
}
// Base dimension type - not exported.
class _Dimension {
constructor(xfltr, id) {
this.crossfilter = xfltr;
this._id = id;
this.groups = [];
}
dispose() {
this.crossfilter._freeDimension(this._id);
return this;
}
id() {
return this._id;
}
_filterUpdate() {
this.crossfilter.updateTime += 1;
}
}
// Scalar dimension type - value must be a scalar type (eg, int, float),
// and value array must be a TypedArray.
//
class ScalarDimension extends _Dimension {
constructor(xfltr, id, value, ValueArrayType) {
super(xfltr, id);
// current selection filter, expressed as PostiveIntervals.
this.currentFilter = [];
// Two modes - caller can provide a pre-created value array,
// or a map function which will create it.
let array;
if (value instanceof ValueArrayType) {
// user has provided the final typed array - just use it
if (value.length !== this.crossfilter.data.length) {
throw new RangeError(
"ScalarDimension values length must equal crossfilter data record count"
);
}
array = value;
} else if (value instanceof Function) {
// Create value array from user-provided map function.
array = this._createValueArray(
value,
new ValueArrayType(this.crossfilter.data.length)
);
} else if (isArrayOrTypedArray(value)) {
// Create value array from user-provided array. Typically used
// only by enumerated dimensions
array = this._createValueArray(
i => value[i],
new ValueArrayType(this.crossfilter.data.length)
);
} else {
throw new NotImplementedError(
"dimension value must be function or value array type"
);
}
this.value = array;
// create sort index
this.index = makeSortIndex(array);
}
_createValueArray(value, array) {
// create dimension value array
const { data } = this.crossfilter;
const len = data.length;
const larray = array;
for (let i = 0; i < len; i += 1) {
larray[i] = value(i, data);
}
return larray;
}
// Argument is an array of intervals indicating records newly selected/filtered
//
_updateFilters(newFilter) {
const cNewFilter = PositiveIntervals.canonicalize(newFilter);
const adds = PositiveIntervals.difference(cNewFilter, this.currentFilter);
const dels = PositiveIntervals.difference(this.currentFilter, cNewFilter);
this.crossfilter.filters.forEach(f =>
f.dim.groups.forEach(grp => grp._updateReduceDel(this, dels))
);
dels.forEach(interval =>
this.crossfilter.selection.deselectIndirectFromRange(
this._id,
this.index,
interval
)
);
adds.forEach(interval =>
this.crossfilter.selection.selectIndirectFromRange(
this._id,
this.index,
interval
)
);
this.crossfilter.filters.forEach(f =>
f.dim.groups.forEach(grp => grp._updateReduceAdd(this, adds))
);
this.currentFilter = cNewFilter;
this._filterUpdate();
}
// filter by value - exact match
filterExact(value) {
const newFilter = [
lowerBoundIndirect(this.value, this.index, value, 0, this.value.length),
upperBoundIndirect(this.value, this.index, value, 0, this.value.length)
];
if (newFilter[0] <= newFilter[1]) {
this._updateFilters([newFilter]);
} else {
this._updateFilters([]);
}
return this;
}
// filter by a set of values, eg. enum.
filterEnum(values) {
const newFilter = [];
for (let v = 0, len = values.length; v < len; v += 1) {
const intv = [
lowerBoundIndirect(
this.value,
this.index,
values[v],
0,
this.value.length
),
upperBoundIndirect(
this.value,
this.index,
values[v],
0,
this.value.length
)
];
if (intv[0] <= intv[1]) newFilter.push(intv);
}
this._updateFilters(newFilter);
return this;
}
// filter by value range [lo, hi)
// lo: inclusive, hi: exclusive
filterRange(range) {
const newFilter = [];
const intv = [
lowerBoundIndirect(
this.value,
this.index,
range[0],
0,
this.value.length
),
upperBoundIndirect(this.value, this.index, range[1], 0, this.value.length)
];
if (intv[0] < intv[1]) newFilter.push(intv);
this._updateFilters(newFilter);
return this;
}
// select all - equivalent of selecting all in this dimension
filterAll() {
this._updateFilters([[0, this.value.length]]);
return this;
}
// select none
filterNone() {
this._updateFilters([]);
}
// return top k records, starting with offset, in descending order.
// Order is this dimension's sort order
top(k, offset = 0) {
const { data, selection } = this.crossfilter;
const { index } = this;
const len = index.length;
const ret = [];
let i = 0;
let skip = 0;
let found = 0;
// skip up to offset records
for (i = len - 1; i >= 0 && skip < offset; i -= 1) {
if (selection.isSelected(index[i])) {
skip += 1;
}
}
// grab up to k records
for (; i >= 0 && found < k; i -= 1) {
if (selection.isSelected(index[i])) {
ret.push(data[index[i]]);
found += 1;
}
}
return ret;
}
// return bottom k records, starting with offset, in ascending order.
// Order is this dimension's sort order
bottom(k, offset = 0) {
const { data, selection } = this.crossfilter;
const { index } = this;
const len = index.length;
const ret = [];
let skip = 0;
let found = 0;
let i = 0;
// skip up to offset records
for (i = 0; i < len && skip < offset; i += 1) {
if (selection.isSelected(index[i])) {
skip += 1;
}
}
// grab up to k records
for (; i < len && found < k; i += 1) {
if (selection.isSelected(index[i])) {
ret.push(data[index[i]]);
found += 1;
}
}
return ret;
}
group(groupValue) {
const grp = new ScalarGroup(groupValue, this.value.constructor, this);
this.groups.push(grp);
return grp;
}
_freeGroup(group) {
this.groups = this.groups.filter(e => e !== group);
}
}
// Ordered enumeration - supports any sortable enumerable type, eg,
// strings, which can be mapped into an fixed numeric range [0..n).
//
class EnumDimension extends ScalarDimension {
constructor(xfltr, id, value) {
super(xfltr, id, value, Uint32Array);
}
_createValueArray(value, array) {
const { data } = this.crossfilter;
const len = data.length;
const larray = array;
// create enumeration table - mapping between the value
// and the enum.
const s = new Set();
for (let i = 0; i < len; i += 1) {
s.add(value(i, data));
}
this.enumIndex = Array.from(s);
this.enumIndex.sort();
// create dimension value array
const enumLen = this.enumIndex.length;
for (let i = 0; i < len; i += 1) {
const v = value(i, data);
const e = lowerBound(this.enumIndex, v, 0, enumLen);
larray[i] = e;
}
return larray;
}
filterExact(value) {
return super.filterExact(
lowerBound(this.enumIndex, value, 0, this.enumIndex.length)
);
}
filterEnum(values) {
return super.filterEnum(
values.map(v => lowerBound(this.enumIndex, v, 0, this.enumIndex.length))
);
}
filterRange(range) {
return super.filterEnum(
range.map(v => lowerBound(this.enumIndex, v, 0, this.enumIndex.length))
);
}
group(groupValue) {
const grp = new EnumGroup(groupValue, this.value.constructor, this);
this.groups.push(grp);
return grp;
}
}
/*
Super simple 2D spatial dimension, supporting basic "filter within"
operations.
*/
class SpatialDimension extends _Dimension {
constructor(xfltr, id, X, Y) {
super(xfltr, id);
if (X.length !== Y.length && X.length !== this.crossfilter.data.length) {
throw new RangeError(
"SpatialDimension values must have same dimensionality as crossfilter"
);
}
this.X = X;
this.Y = Y;
this.Xindex = makeSortIndex(X);
this.Yindex = makeSortIndex(Y);
}
filterAll() {
this.crossfilter.selection.selectAll(this._id);
this._filterUpdate();
}
filterNone() {
this.crossfilter.selection.deselectAll(this._id);
this._filterUpdate();
}
/*
this could be smarter, but we don't currently use it...
*/
filterWithinRect(northwest, southeast) {
const [x0, y0] = northwest;
const [x1, y1] = southeast;
const { X, Y } = this;
const seln = this.crossfilter.selection;
const { _id } = this;
seln.deselectAll(_id);
for (let i = 0, l = this.X.length; i < l; i += 1) {
const x = X[i];
const y = Y[i];
if (x0 <= x && x < x1 && y0 <= y && y < y1) {
seln.selectOne(_id, i);
}
}
this._filterUpdate();
}
/*
Relatively brute force filter by polygon. Polygon is array of points, where
each point is [x,y]. Eg, [[x0,y0], [x1,y1], ...].
Currently uses d3.polygonContains() to test for polygon inclusion, which itself
uses a ray casting (crossing number) algorithm. There are a series of optimizations
to make this faster:
* first sliced by X or Y, using an index on the axis
* then the polygon bounding box is used for trivial rejection
* then the polygon test is applied
*/
filterWithinPolygon(polygon) {
/* return bounding box of the polygon */
function polygonBoundingBox(pg) {
let minX = Number.MAX_VALUE;
let minY = Number.MAX_VALUE;
let maxX = Number.MIN_VALUE;
let maxY = Number.MIN_VALUE;
for (let i = 0, l = pg.length; i < l; i += 1) {
const p = pg[i];
const x = p[0];
const y = p[1];
if (x < minX) minX = x;
if (y < minY) minY = y;
if (x > maxX) maxX = x;
if (y > maxY) maxY = y;
}
return [minX, minY, maxX, maxY];
}
const [minX, minY, maxX, maxY] = polygonBoundingBox(polygon);
const { X, Y } = this;
let slice;
let index;
if (maxY - minY > maxX - minX) {
slice = [
lowerBoundIndirect(X, this.Xindex, minX, 0, X.length),
upperBoundIndirect(X, this.Xindex, maxX, 0, X.length)
];
index = this.Xindex;
} else {
slice = [
lowerBoundIndirect(Y, this.Yindex, minY, 0, Y.length),
upperBoundIndirect(Y, this.Yindex, maxY, 0, Y.length)
];
index = this.Yindex;
}
const seln = this.crossfilter.selection;
const { _id } = this;
const testWithin = polygonContains; // d3.polygonContains()
seln.deselectAll(_id);
for (let i = slice[0], e = slice[1]; i < e; i += 1) {
const rid = index[i];
const x = X[rid];
const y = Y[rid];
if (
minX <= x &&
x < maxX &&
minY <= y &&
y < maxY &&
testWithin(polygon, [x, y])
) {
seln.selectOne(_id, rid);
}
}
this._filterUpdate();
}
}
// Groups! Map/reduce
//
class ScalarGroup {
constructor(groupValue, groupValueType, dimension) {
// parent dimension
this.dimension = dimension;
// generate group names from dimension values
this.mapValue = this.constructor._map(
groupValue,
groupValueType,
dimension
);
// group index is mapping from data record index to group index
this.groupIndex = new Uint32Array(dimension.crossfilter.data.length);
// default to counting
this.reduceCount();
// Creates this.groups
this._reduce();
}
// internal support function - map all dimension values to group values.
//
static _map(groupValue, GroupValueType, dimension) {
// groupValue is optional. Defaults to identity. Used to perform
// initial map operation.
//
// identity: save some memory...
if (groupValue === undefined) return dimension.value;
const data = dimension.value;
const len = data.length;
const mapValue = new GroupValueType(dimension.value.length);
for (let i = 0; i < len; i += 1) {
mapValue[i] = groupValue(data[i]);
}
return mapValue;
}
// Update the group reduction incrementally. Called when *any* dimension filter
// changes. Guaranteed to be called AFTER the crossfilter is updated.
//
// Arguments:
// * dim: the dimension that is changing
// * intv: interval list of newly selected values on `dim` (adds)
//
_updateReduceAdd(dim, intv) {
// ignore updates to self, as we don't reduce inclusive of our filter
if (dim === this.dimension || intv.length === 0) return;
// Each item in the range was just added to `dim`. It was NOT previously
// selected - reduceAdd if it is now selected.
const { data, selection } = this.dimension.crossfilter;
intv.forEach(rng => {
for (let r = rng[0]; r < rng[1]; r += 1) {
const i = dim.index[r];
if (selection.isSelectedIgnoringDim(i, this.dimension.id())) {
const group = this.groups[this.groupIndex[i]];
group.value = this.reduceAdd(group.value, data[i]);
}
}
});
}
// Update the group reduction incrementally. Called when *any* dimension filter
// changes. Guaranteed to be called BEFORE the crossfilter is updated.
//
// Arguments:
// * dim: the dimension that is changing
// * intv: interval list of previously selected values on `dim` (dels)
//
_updateReduceDel(dim, intv) {
// ignore updates to self, as we don't reduce inclusive of our filter
if (dim === this.dimension || intv.length === 0) return;
// Each item in the range will be remved from `dim`. reduceRemove if it
// is currently selected.
const { data, selection } = this.dimension.crossfilter;
intv.forEach(rng => {
for (let r = rng[0]; r < rng[1]; r += 1) {
const i = dim.index[r];
if (selection.isSelectedIgnoringDim(i, this.dimension.id())) {
const group = this.groups[this.groupIndex[i]];
group.value = this.reduceRemove(group.value, data[i]);
}
}
});
}
// Reduce the entire data set, creating both the group index and the
// groups data.
//
_reduce() {
const { dimension } = this;
const { data } = dimension.crossfilter;
// Create groups
const groupNames = new Set(this.mapValue);
this.groups = [];
const groupIndexByName = {};
groupNames.forEach(name => {
this.groups.push({ key: name, value: this.reduceInitial() });
groupIndexByName[name] = this.groups.length - 1;
});
// Create groupIndex - index map between data record index and group index
for (let i = 0, len = this.mapValue.length; i < len; i += 1) {
this.groupIndex[i] = groupIndexByName[this.mapValue[i]];
}
// reduce all filtered records, IGNORING the current dimension's filter
const { selection } = dimension.crossfilter;
for (let i = 0, len = data.length; i < len; i += 1) {
if (selection.isSelectedIgnoringDim(i, dimension.id())) {
const group = this.groups[this.groupIndex[i]];
group.value = this.reduceAdd(group.value, data[i]);
}
}
}
dispose() {
this.dimension._freeGroup(this);
return this;
}
// return number of distinct values in the group, independent of any filters.
//
size() {
return this.groups.length;
}
// Set the reduce functions and return the grouping.
//
reduce(add, remove, initial) {
this.reduceAdd = add;
this.reduceRemove = remove;
this.reduceInitial = initial;
this._reduce();
return this;
}
// set the reduce functions to count records.
reduceCount() {
return this.reduce(p => p + 1, p => p - 1, () => 0);
}
// set the reduce functions to sum records using specified value accessor.
//
reduceSum(value) {
return this.reduce((p, v) => p + value(v), (p, v) => p - value(v), () => 0);
}
all() {
const res = [...this.groups];
res.sort((a, b) => (a.key < b.key ? -1 : a.key > b.key ? 1 : 0));
return res;
}
}
class EnumGroup extends ScalarGroup {
static _map(groupValue, groupValueType, dimension) {
// groupValue is optional. Defaults to identity. Used to perform
// initial map operation.
//
// identity: save some memory
if (groupValue === undefined) return dimension.value;
// non-identity mapping unsupported for EnumDimension/EnumGroup.
// XXX: this could be implemented, but would require another index
// array to map from the group names/keys back to the dimension values.
// With this, we just rely on the dimensions `enumIndex` to map from
// enumeration value to the record.
throw new NotImplementedError("enumerated group mapping not implemented");
}
all() {
const res = [];
this.groups.forEach(e =>
res.push({
// XXX: assumes identity group map - see comment in _map()
key: this.dimension.enumIndex[e.key],
value: e.value
})
);
res.sort((a, b) => (a.key < b.key ? -1 : a.key > b.key ? 1 : 0));
return res;
}
}
// Wrapper for backwards compat with crossfilter.
//
function crossfilter(data) {
return new TypedCrossfilter(data);
}
crossfilter.PositiveIntervals = PositiveIntervals;
crossfilter.BitArray = BitArray;
crossfilter.TypedCrossfilter = TypedCrossfilter;
crossfilter.ScalarDimension = ScalarDimension;
crossfilter.EnumDimension = EnumDimension;
crossfilter.SpatialDimension = SpatialDimension;
export default crossfilter;
export { default } from "./crossfilter";