From cc661f31e1de879b92b81bc2840dda2324c31019 Mon Sep 17 00:00:00 2001 From: bkmartinjr Date: Wed, 23 May 2018 20:59:13 -0700 Subject: [PATCH 1/8] first cut at high performance crossfilter --- src/actions/index.js | 2 +- src/components/continuous/continuous.js | 1 - src/components/continuous/histogramBrush.js | 4 +- src/components/continuous/parallel.js | 6 +- src/components/expression/cellSetButtons.js | 8 +- .../expression/expressionButtons.js | 8 +- src/components/graph/graph.js | 108 +-- src/components/scatterplot/scatterplot.js | 23 +- src/middleware/updateCellColors.js | 26 +- .../updateCellSelectionMiddleware.js | 165 +--- src/reducers/controls.js | 339 ++++++-- src/reducers/index.js | 4 +- src/util/typedCrossfilter.js | 785 ++++++++++++++++++ 13 files changed, 1128 insertions(+), 351 deletions(-) create mode 100644 src/util/typedCrossfilter.js diff --git a/src/actions/index.js b/src/actions/index.js index 9cd0e823..fc6cc398 100644 --- a/src/actions/index.js +++ b/src/actions/index.js @@ -89,7 +89,7 @@ const initialize = () => { // function cleanupExpressionResponse(data) { const s = store.getState(); - const metadata = s.controls.currentCellSelectionMap; + const metadata = s.controls.allCellsMetadataMap; let errorFound = false; data.data.cells = _.filter(data.data.cells, cell => { if (!errorFound && !metadata[cell.cellname]) { diff --git a/src/components/continuous/continuous.js b/src/components/continuous/continuous.js index 86f38e5a..226b6d0f 100644 --- a/src/components/continuous/continuous.js +++ b/src/components/continuous/continuous.js @@ -38,7 +38,6 @@ import { margin, width, height, createDimensions } from "./util"; colorAccessor: state.controls.colorAccessor, colorScale: state.controls.colorScale, graphBrushSelection: state.controls.graphBrushSelection, - currentCellSelection: state.controls.currentCellSelection, axesHaveBeenDrawn: state.controls.axesHaveBeenDrawn }; }) diff --git a/src/components/continuous/histogramBrush.js b/src/components/continuous/histogramBrush.js index 4ac59c34..fb4bd750 100644 --- a/src/components/continuous/histogramBrush.js +++ b/src/components/continuous/histogramBrush.js @@ -26,7 +26,7 @@ import { connect } from "react-redux"; return { colorAccessor: state.controls.colorAccessor, colorScale: state.controls.colorScale, - currentCellSelection: state.controls.currentCellSelection + allCellsMetadata: state.controls.allCellsMetadata }; }) class HistogramBrush extends React.Component { @@ -52,7 +52,7 @@ class HistogramBrush extends React.Component { calcHistogramCache(nextProps) { // recalculate expensive stuff const allValuesForContinuousFieldAsArray = _.map( - nextProps.currentCellSelection, + nextProps.allCellsMetadata, nextProps.metadataField ); diff --git a/src/components/continuous/parallel.js b/src/components/continuous/parallel.js index ce3bd03e..5ffa3463 100644 --- a/src/components/continuous/parallel.js +++ b/src/components/continuous/parallel.js @@ -36,7 +36,7 @@ import { margin, width, height, createDimensions } from "./util"; colorAccessor: state.controls.colorAccessor, colorScale: state.controls.colorScale, graphBrushSelection: state.controls.graphBrushSelection, - currentCellSelection: state.controls.currentCellSelection, + allCellsMetadata: state.controls.allCellsMetadata, axesHaveBeenDrawn: state.controls.axesHaveBeenDrawn }; }) @@ -96,7 +96,7 @@ class Parallel extends React.Component { /* https://stackoverflow.com/questions/23123138/perform-debounce-in-react-js */ if ( nextProps.ranges && - nextProps.currentCellSelection && + nextProps.allCellsMetadata && nextProps.axesHaveBeenDrawn ) { if (this.state._drawLinesCanvas) { @@ -106,7 +106,7 @@ class Parallel extends React.Component { this.state.ctx.clearRect(0, 0, width, height); const _drawLinesCanvas = drawLinesCanvas( - nextProps.currentCellSelection, + nextProps.allCellsMetadata, this.state.dimensions, this.state.xscale, this.state.ctx, diff --git a/src/components/expression/cellSetButtons.js b/src/components/expression/cellSetButtons.js index f3e8c32e..5291ba7e 100644 --- a/src/components/expression/cellSetButtons.js +++ b/src/components/expression/cellSetButtons.js @@ -9,13 +9,7 @@ import actions from "../../actions"; @connect() class CellSetButton extends React.Component { set() { - const set = []; - - _.each(this.props.currentCellSelection, cell => { - if (cell["__selected__"]) { - set.push(cell.CellName); - } - }); + const set = _.map(this.props.crossfilter.cells.allFiltered(), "CellName"); this.props.dispatch({ type: diff --git a/src/components/expression/expressionButtons.js b/src/components/expression/expressionButtons.js index 50c1c91c..1f0aa481 100644 --- a/src/components/expression/expressionButtons.js +++ b/src/components/expression/expressionButtons.js @@ -8,8 +8,8 @@ import CellSetButton from "./cellSetButtons"; @connect(state => { return { - currentCellSelection: state.controls.currentCellSelection, - differential: state.differential + differential: state.differential, + crossfilter: state.controls.crossfilter }; }) class Expression extends React.Component { @@ -43,7 +43,9 @@ class Expression extends React.Component {
There are currently {" " + - _.filter(this.props.currentCellSelection, "__selected__").length + + (this.props.crossfilter + ? this.props.crossfilter.cells.countFiltered() + : 0) + " "} cells selected, click a cell set button to store them.
diff --git a/src/components/graph/graph.js b/src/components/graph/graph.js index 8a8efd92..2fc602ab 100644 --- a/src/components/graph/graph.js +++ b/src/components/graph/graph.js @@ -22,31 +22,11 @@ import FaSave from "react-icons/lib/fa/download"; /* https://bl.ocks.org/mbostock/9078690 - quadtree for onClick / hover selections */ @connect(state => { - const vertices = - state.cells.cells && state.cells.cells.data.graph - ? state.cells.cells.data.graph - : null; - const ranges = - state.cells.cells && state.cells.cells.data.ranges - ? state.cells.cells.data.ranges - : null; - const metadata = - state.cells.cells && state.cells.cells.data.metadata - ? state.cells.cells.data.metadata - : null; - return { - ranges, - vertices, - metadata, - colorAccessor: state.controls.colorAccessor, - colorScale: state.controls.colorScale, - continuousSelection: state.controls.continuousSelection, - graphVec: state.controls.graphVec, - currentCellSelection: state.controls.currentCellSelection, - graphBrushSelection: state.controls.graphBrushSelection, + allCellsMetadata: state.controls.allCellsMetadata, opacityForDeselectedCells: state.controls.opacityForDeselectedCells, - responsive: state.responsive + responsive: state.responsive, + crossfilter: state.controls.crossfilter }; }) class Graph extends React.Component { @@ -55,6 +35,10 @@ class Graph extends React.Component { this.count = 0; this.inverse = mat4.identity([]); this.graphPaddingTop = 100; + this.renderCache = { + positions: null, + colors: null + }; this.state = { drawn: false, svg: null, @@ -104,42 +88,60 @@ class Graph extends React.Component { } componentWillReceiveProps(nextProps) { - if (this.state.regl && nextProps.vertices) { - /* update regl */ - const vertices = nextProps.currentCellSelection; - const vertexCount = vertices.length; - const positions = new Float32Array(2 * vertexCount); - const colors = new Float32Array(3 * vertexCount); - const sizes = new Float32Array(vertexCount); + if (this.state.regl && nextProps.crossfilter) { + /* update the regl state */ + const crossfilter = nextProps.crossfilter.cells; + const cells = crossfilter.all(); + const cellCount = cells.length; - // d3.scaleLinear().domain([0,1]).range([-1,1]) - const glScaleX = scaleLinear([0, 1], [-1, 1]); - // d3.scaleLinear().domain([0,1]).range([1,-1]) - const glScaleY = scaleLinear([0, 1], [1, -1]); + // X/Y positions for each point - a cached value that only + // changes if we have loaded entirely new cell data + // + if ( + !this.renderCache.positions || + this.props.crossfilter.cells != nextProps.crossfilter.cells + ) { + const positions = new Float32Array(2 * cellCount); - /* - Construct Vectors - */ - const graphVec = nextProps.graphVec; - for (var i = 0; i < vertexCount; i++) { - const cell = vertices[i]; - const cellIdx = cell.__cellIndex__; - const x = glScaleX(graphVec[2 * cellIdx]); - const y = glScaleY(graphVec[2 * cellIdx + 1]); - positions[2 * i] = x; - positions[2 * i + 1] = y; + // d3.scaleLinear().domain([0,1]).range([-1,1]) + const glScaleX = scaleLinear([0, 1], [-1, 1]); + // d3.scaleLinear().domain([0,1]).range([1,-1]) + const glScaleY = scaleLinear([0, 1], [1, -1]); - colors.set(cell.__colorRGB__, 3 * i); - - sizes[i] = cell.__selected__ - ? 4 - : 0.2; /* make this a function of the number of total cells, including regraph */ + for (let i = 0; i < cellCount; i++) { + positions[2 * i] = glScaleX(cells[i].__x__); + positions[2 * i + 1] = glScaleY(cells[i].__y__); + } + this.renderCache.positions = positions; } - this.state.pointBuffer({ data: positions, dimension: 2 }); - this.state.colorBuffer({ data: colors, dimension: 3 }); + // Colors for each point - a cached value that only changes when + // the cell metadata changes (done by updateCellColors middleware). + // NOTE: this is a slightly pessimistic assumption, as the metadata + // could have changed for some other reason, but for now color is + // the only metadata that changes client-side. If this is problematic, + // we could add some sort of color-specific indicator to the app state. + if ( + !this.renderCache.colors || + this.props.allCellsMetadata != nextProps.allCellsMetadata + ) { + const colors = new Float32Array(3 * cellCount); + for (let i = 0; i < cellCount; i++) { + colors.set(cells[i].__colorRGB__, 3 * i); + } + this.renderCache.colors = colors; + } + + const sizes = new Float32Array(cellCount); + crossfilter.fillByIsFiltered(sizes, 4, 0.2); + + this.state.pointBuffer({ + data: this.renderCache.positions, + dimension: 2 + }); + this.state.colorBuffer({ data: this.renderCache.colors, dimension: 3 }); this.state.sizeBuffer({ data: sizes, dimension: 1 }); - this.count = vertexCount; + this.count = cellCount; } if ( diff --git a/src/components/scatterplot/scatterplot.js b/src/components/scatterplot/scatterplot.js index d853d869..c69a4acc 100644 --- a/src/components/scatterplot/scatterplot.js +++ b/src/components/scatterplot/scatterplot.js @@ -39,11 +39,10 @@ import { margin, width, height, createDimensions } from "./util"; initializeRanges, colorAccessor: state.controls.colorAccessor, colorScale: state.controls.colorScale, - currentCellSelection: state.controls.currentCellSelection, - currentCellSelectionMap: state.controls.currentCellSelectionMap, scatterplotXXaccessor: state.controls.scatterplotXXaccessor, scatterplotYYaccessor: state.controls.scatterplotYYaccessor, opacityForDeselectedCells: state.controls.opacityForDeselectedCells, + crossfilter: state.controls.crossfilter, differential: state.differential, expression: state.expression }; @@ -124,7 +123,6 @@ class Scatterplot extends React.Component { this.state.pointBuffer && this.state.colorBuffer && this.state.sizeBuffer && - this.props.currentCellSelection && this.props.expression.data && this.props.expression.data.genes && this.props.scatterplotXXaccessor && @@ -132,8 +130,7 @@ class Scatterplot extends React.Component { this.state.xScale && this.state.yScale ) { - const currentCellSelectionMap = this.props.currentCellSelectionMap; - + const crossfilter = this.props.crossfilter.cells; const data = this.props.expression.data; const cells = data.cells; const genes = data.genes; @@ -158,9 +155,8 @@ class Scatterplot extends React.Component { /* Construct Vectors */ - for (var i = 0; i < cellCount; i++) { + for (let i = 0; i < cellCount; i++) { const cell = cells[i]; - const cellMetadata = currentCellSelectionMap[cell.cellname]; positions[2 * i] = glScaleX( this.state.xScale(cell.e[geneXXaccessorIndex]) @@ -168,14 +164,15 @@ class Scatterplot extends React.Component { positions[2 * i + 1] = glScaleY( this.state.yScale(cell.e[geneYYaccessorIndex]) ); - - colors.set(cellMetadata.__colorRGB__, 3 * i); - - sizes[i] = cellMetadata.__selected__ - ? 4 - : 0.2; /* make this a function of the number of total cells, including regraph */ } + for (let i = 0; i < cellCount; i++) { + const metadata = this.props.metadata[i]; + colors.set(metadata.__colorRGB__, 3 * i); + } + + crossfilter.fillByIsFiltered(sizes, 4, 0.2); + this.state.pointBuffer({ data: positions, dimension: 2 }); this.state.colorBuffer({ data: colors, dimension: 3 }); this.state.sizeBuffer({ data: sizes, dimension: 1 }); diff --git a/src/middleware/updateCellColors.js b/src/middleware/updateCellColors.js index f6a8ba9f..25831b97 100644 --- a/src/middleware/updateCellColors.js +++ b/src/middleware/updateCellColors.js @@ -20,7 +20,7 @@ import { parseRGB } from "../util/parseRGB"; This is nice because we keep a lot of filtering business logic centralized (what it means in practice to be selected) */ -const updateCellSelectionMiddleware = store => { +const updateCellColorsMiddleware = store => { return next => { return action => { const s = store.getState(); @@ -37,7 +37,7 @@ const updateCellSelectionMiddleware = store => { ); /* if the cells haven't loaded or the action wasn't a color change, bail */ } - let currentSelectionWithUpdatedColors = s.controls.currentCellSelection.slice( + let allCellsMetadataWithUpdatedColors = s.controls.allCellsMetadata.slice( 0 ); let colorScale; @@ -46,7 +46,7 @@ const updateCellSelectionMiddleware = store => { in plain language... (a) once the cells have loaded. - (b) each time a user changes a color control we need to update currentCellSelection colors + (b) each time a user changes a color control we need to update allCellsMetadata colors This is available to all the draw functions as cell["__color__"] and cell["__colorRGB__"] */ @@ -54,8 +54,8 @@ const updateCellSelectionMiddleware = store => { if (action.type === "color by categorical metadata") { colorScale = d3.scaleOrdinal().range(globals.ordinalColors); - for (let i = 0; i < currentSelectionWithUpdatedColors.length; i++) { - const cell = currentSelectionWithUpdatedColors[i]; + for (let i = 0; i < allCellsMetadataWithUpdatedColors.length; i++) { + const cell = allCellsMetadataWithUpdatedColors[i]; let c = colorScale(cell[action.colorAccessor]); cell.__color__ = c; cell.__colorRGB__ = parseRGB(c); @@ -68,10 +68,10 @@ const updateCellSelectionMiddleware = store => { .domain([0, action.rangeMaxForColorAccessor]) .range([1, 0]); - _.each(currentSelectionWithUpdatedColors, (cell, i) => { + _.each(allCellsMetadataWithUpdatedColors, (cell, i) => { let c = d3.interpolateViridis(colorScale(cell[action.colorAccessor])); - currentSelectionWithUpdatedColors[i]["__color__"] = c; - currentSelectionWithUpdatedColors[i]["__colorRGB__"] = parseRGB(c); + allCellsMetadataWithUpdatedColors[i]["__color__"] = c; + allCellsMetadataWithUpdatedColors[i]["__colorRGB__"] = parseRGB(c); }); } @@ -113,12 +113,12 @@ const updateCellSelectionMiddleware = store => { 0 ]); /* invert viridis... probably pass this scale through to others */ - _.each(currentSelectionWithUpdatedColors, (cell, i) => { + _.each(allCellsMetadataWithUpdatedColors, (cell, i) => { let c = d3.interpolateViridis( colorScale(expressionMap[cell.CellName][indexOfGene]) ); - currentSelectionWithUpdatedColors[i]["__color__"] = c; - currentSelectionWithUpdatedColors[i]["__colorRGB__"] = parseRGB(c); + allCellsMetadataWithUpdatedColors[i]["__color__"] = c; + allCellsMetadataWithUpdatedColors[i]["__colorRGB__"] = parseRGB(c); }); } @@ -126,7 +126,7 @@ const updateCellSelectionMiddleware = store => { append the result of all the filters to the action the user just triggered */ let modifiedAction = Object.assign({}, action, { - currentSelectionWithUpdatedColors, + allCellsMetadataWithUpdatedColors, colorScale }); @@ -135,4 +135,4 @@ const updateCellSelectionMiddleware = store => { }; }; -export default updateCellSelectionMiddleware; +export default updateCellColorsMiddleware; diff --git a/src/middleware/updateCellSelectionMiddleware.js b/src/middleware/updateCellSelectionMiddleware.js index 9e8dea24..b2c1f6de 100644 --- a/src/middleware/updateCellSelectionMiddleware.js +++ b/src/middleware/updateCellSelectionMiddleware.js @@ -48,7 +48,7 @@ const updateCellSelectionMiddleware = store => { - make a FRESH copy of all of the cells - metadata has cellname and index, and that's all we ever need to reference cell info */ - let newSelection = s.controls.currentCellSelection.slice(0); + let newSelection = s.controls.allCellsMetadata.slice(0); // _.forEach(newSelection, cell => (cell.__selected__ = true)); for (let i = 0; i < newSelection.length; i++) { newSelection[i].__selected__ = true; @@ -58,7 +58,7 @@ const updateCellSelectionMiddleware = store => { in plain language... (a) once the cells have loaded. - (b) each time a user changes ANY control we need to update currentCellSelection + (b) each time a user changes ANY control we need to update allCellsMetadata there are two states: 1. control state we already know about (state.foo) @@ -66,34 +66,6 @@ const updateCellSelectionMiddleware = store => { */ - if ( - /* is there a 2d graph brush selection ? */ - action.type === "graph brush selection change" || - s.controls.graphBrushSelection - ) { - const graphBrushSelection /* it exists, so is it new or old */ = - action.type === "graph brush selection change" - ? action.brushCoords - : s.controls.graphBrushSelection; - - const graphVec = s.controls.graphVec; - for (let i = 0; i < newSelection.length; i++) { - const cell = newSelection[i]; - const cellId = cell.__cellIndex__; - const x = graphVec[2 * cellId]; - const y = graphVec[2 * cellId + 1]; - - const pointIsInsideBrushBounds = - x >= graphBrushSelection.northwest[0] && - x <= graphBrushSelection.southeast[0] && - y <= graphBrushSelection.northwest[1] && - y >= graphBrushSelection.southeast[1]; - - if (!pointIsInsideBrushBounds) { - cell.__selected__ = false; - } - } - } if ( (action.type === "continuous selection using parallel coords brushing" && @@ -118,139 +90,8 @@ const updateCellSelectionMiddleware = store => { }); } - /* - Continuous histograms ___---^^^^--[------__]__---___ - - Create newContinuousUserDefinedRanges - Filter based on them - */ - - let newContinuousUserDefinedRanges = - s.controls.continuousUserDefinedRanges; - - /* check if this is the action and take care of that metadata field */ - if (action.type === "continuous metadata histogram brush") { - /* - was this a deselect? if so it will be null - was it a select? set the new range [20, 50] - we overload this because it is less if statements thru the whole system - but it's invisible here, thus comment. - */ - newContinuousUserDefinedRanges[action.selection] = action.range; - } - - let activeContinuousHistogramFilters = []; - - _.each(newContinuousUserDefinedRanges, (value, key, i) => { - if (value !== null) { - activeContinuousHistogramFilters.push(key); - } - }); - - /* see if there are others from previous... */ - if (activeContinuousHistogramFilters.length > 0) { - _.each(activeContinuousHistogramFilters, key => { - _.each(newSelection, (cell, i) => { - if ( - +cell[key] < newContinuousUserDefinedRanges[key][0] || - +cell[key] > newContinuousUserDefinedRanges[key][1] - ) { - newSelection[i]["__selected__"] = false; - } - }); - }); - } - - /* - 1. figure out if the users have unchecked boxes - 2. put them in an array - 3. filter on them - */ - - let newCategoricalAsBooleansMap = s.controls.categoricalAsBooleansMap; - - /* - ...spread for merge: https://github.com/reactjs/redux/issues/432 - - we do the update here instead of the reducer because we need it for the reactive computation - */ - if (action.type === "categorical metadata filter select") { - newCategoricalAsBooleansMap = { - ...s.controls.categoricalAsBooleansMap, - [action.metadataField]: { - ...s.controls.categoricalAsBooleansMap[action.metadataField], - [action.value]: true - } - }; - } else if (action.type === "categorical metadata filter deselect") { - newCategoricalAsBooleansMap = { - ...s.controls.categoricalAsBooleansMap, - [action.metadataField]: { - ...s.controls.categoricalAsBooleansMap[action.metadataField], - [action.value]: false - } - }; - } else if (action.type === "categorical metadata filter none of these") { - const metadataFieldWithAllOfTheseValueSelected = {}; - - /* set EVERYTHING to false in this intermediate object */ - _.each( - s.controls.categoricalAsBooleansMap[action.metadataField], - (isActive, option) => { - metadataFieldWithAllOfTheseValueSelected[option] = false; - } - ); - - newCategoricalAsBooleansMap = { - ...s.controls.categoricalAsBooleansMap, - [action.metadataField]: metadataFieldWithAllOfTheseValueSelected - }; - } else if (action.type === "categorical metadata filter all of these") { - const metadataFieldWithAllOfTheseValueSelected = {}; - - /* set EVERYTHING to true in this intermediate object */ - _.each( - s.controls.categoricalAsBooleansMap[action.metadataField], - (isActive, option) => { - metadataFieldWithAllOfTheseValueSelected[option] = true; - } - ); - - newCategoricalAsBooleansMap = { - ...s.controls.categoricalAsBooleansMap, - [action.metadataField]: metadataFieldWithAllOfTheseValueSelected - }; - } - - const inactiveCategories = []; - _.each(newCategoricalAsBooleansMap, (options, category) => { - _.each(options, (isActive, option) => { - if (!isActive) { - inactiveCategories.push({ category, option }); - } - }); - }); - - if (inactiveCategories.length > 0) { - _.each(inactiveCategories, d => { - if ( - s.controls.categoricalAsCellsMap[d.category] && - s.controls.categoricalAsCellsMap[d.category][d.option] - ) { - _.forEach( - s.controls.categoricalAsCellsMap[d.category][d.option], - c => { - c.__selected__ = false; - } - ); - } - }); - } - let modifiedAction = Object.assign({}, action, { - newSelection, - newCategoricalAsBooleansMap, - newContinuousUserDefinedRanges + newSelection }); /* append the result of all the filters to the action the user just triggered */ return next(modifiedAction); diff --git a/src/reducers/controls.js b/src/reducers/controls.js index 47d43ee1..256a1097 100644 --- a/src/reducers/controls.js +++ b/src/reducers/controls.js @@ -1,16 +1,78 @@ // jshint esversion: 6 import _ from "lodash"; import { parseRGB } from "../util/parseRGB"; +var crossfilter = require("../util/typedCrossfilter"); + +// In the case where the REST server does not implement data schema +// declaration, we attempt to deduce it by sniffing the data. +// +function createSchemaByDataSniffing(ranges) { + let schema = {}; + _.forEach(ranges, (value, key) => { + schema[key] = { + displayname: key, + variabletype: value.options ? "categorical" : "continuous" + }; + + // Metadata field type is inferred by sniffing the data. This has some risks. + // Caveats: + // * Values have been converted to native JS objects by the JSON parser. + // * Lots of assumptions about he REST API behaving properly (eg, min/max + // are the same type, etc). + let type; + if (schema[key].variabletype === "continuous" && value.range) { + // Use min/max as a proxy for all data. + const min = value.range.min; + const max = value.range.max; + type = + typeof min !== "number" || typeof max !== "number" + ? "string" + : Number.isSafeInteger(min) && Number.isSafeInteger(max) + ? "int" + : "float"; + } else { + // use an option value as a proxy for all data + const aVal = value.options[0]; + type = + typeof aVal !== "number" + ? "string" + : Number.isSafeInteger(aVal) ? "int" : "float"; + } + schema[key].type = type; + }); + return schema; +} + +// Deduce the correct crossfilter dimension type from a metadata +// schema description. +// +function deduceDimensionType(attributes, fieldName) { + let dimensionType; + if (attributes.type === "string") { + dimensionType = "enum"; + } else if (attributes.type === "int") { + dimensionType = Int32Array; + } else if (attributes.type === "float") { + dimensionType = Float32Array; + } else { + console.error( + `Warning - REST API returned unknown metadata schema (${ + attributes.type + }) for field ${fieldName}.` + ); + // skip it - we don't know what to do with this type + } + return dimensionType; +} const Controls = ( state = { _ranges: null /* this comes from initialize, this is universe */, allGeneNames: null, allCellsOnClient: null /* this comes from cells endpoint, this is world */, - currentCellSelection: null /* this comes from user actions, all draw components use this, it is created by middleware */, - graphVec: null, + allCellsMetadata: null /* this comes from user actions, all draw components use this, it is created by middleware */, + crossfilter: null /* the current user selection state */, categoricalAsBooleansMap: null, - categoricalAsCellsMap: null, colorAccessor: null, colorScale: null, opacityForDeselectedCells: 0.2, @@ -28,118 +90,157 @@ const Controls = ( /********************************** Keep a copy of 'universe' ***********************************/ - case "initialize success": + case "initialize success": { + if (!action.data.data.schema) { + console.error("Warning - REST API omitted schema description."); + } return Object.assign({}, state, { _ranges: action.data.data.ranges, - allGeneNames: action.data.data.genes + allGeneNames: action.data.data.genes, + schema: action.data.data.schema }); - case "request cells success": - // Store the graph in a linear array for fast access. Index into - // the array by "cell index", which is stored as metadata field - // __cellIndex__. - // - // Code below relies on the REST API guarantee that the graph and - // metadata are returned as arrays with the same order and length. - // - const graphVec = new Float32Array(2 * action.data.data.graph.length); - _.each(action.data.data.graph, (g, i) => { - graphVec[2 * i] = g[1]; - graphVec[2 * i + 1] = g[2]; - }); - - const currentCellSelection = action.data.data.metadata.slice(0); - const currentCellSelectionMap = _.keyBy(currentCellSelection, "CellName"); + } + case "request cells success": { + const allCellsMetadata = action.data.data.metadata.slice(0); + const allCellsMetadataMap = _.keyBy(allCellsMetadata, "CellName"); /* construct a copy of the ranges object that only has categorical replace all counts with bool flags ie., everything starts out checked we mutate this map in the actions below - */ - const categoricalAsBooleansMap = {}, - categoricalAsCellsMap = {}; - const continuousUserDefinedRanges = {}; + */ + const categoricalAsBooleansMap = {}; _.each(action.data.data.ranges, (value, key) => { if ( key !== "CellName" && value.options /* it's categorical, it has options instead of ranges */ ) { - const optionsAsBooleans = {}, - optionsAsCells = {}; + const optionsAsBooleans = {}; _.each(value.options, (_value, _key) => { optionsAsBooleans[_key] = true; - optionsAsCells[_key] = []; }); categoricalAsBooleansMap[key] = optionsAsBooleans; - categoricalAsCellsMap[key] = optionsAsCells; - } else if (key !== "CellName" && value.range) { - continuousUserDefinedRanges[key] = null; } }); - _.each(currentCellSelection, (cell, idx) => { + const graph = action.data.data.graph; + _.each(allCellsMetadata, (cell, idx) => { cell.__cellIndex__ = idx; - cell.__selected__ = true; cell.__color__ = "rgba(0,0,0,1)"; /* initial color for all cells in all charts */ cell.__colorRGB__ = parseRGB(cell.__color__); - - // Add each cell to its categorical metadata set. - _.forEach(cell, (_value, key) => { - if ( - categoricalAsCellsMap[key] && - categoricalAsCellsMap[key][_value] - ) { - const s = categoricalAsCellsMap[key][_value]; - if (s) s.push(cell); - } - }); + cell.__x__ = graph[idx][1]; + cell.__y__ = graph[idx][2]; }); + // Build the selection crossfilter. + // + let cellsCrossfilter = crossfilter(allCellsMetadata); + let cellsDimensionsMap = {}; + cellsDimensionsMap.x = cellsCrossfilter.dimension( + r => r.__x__, + Float32Array + ); + cellsDimensionsMap.y = cellsCrossfilter.dimension( + r => r.__y__, + Float32Array + ); + + // Now walk the schema and make an appropriate dimension for each + // metadata field. This is a simplistic mapping, and could be + // optmized to use smaller scalars (to save memory) or larger + // floating point where precision is needed. + // + // If we don't have a schema (bad server!), fake it by inferring + // important fields from the ranges element. + // + if (!state.schema) { + state.schema = createSchemaByDataSniffing(action.data.data.ranges); + } + _.forEach(state.schema, (attributes, key) => { + if (key !== "CellName") { + const dimensionType = deduceDimensionType(attributes, key); + if (dimensionType) { + cellsDimensionsMap[key] = cellsCrossfilter.dimension( + r => r[key], + dimensionType + ); + } + } + }); return Object.assign({}, state, { + // this is only used as a flag that data has loaded. Could be + // removed (other variables would suffice for the same test). allCellsOnClient: action.data.data, - currentCellSelection, - currentCellSelectionMap, - graphVec, + allCellsMetadata, + allCellsMetadataMap, categoricalAsBooleansMap, - categoricalAsCellsMap, - continuousUserDefinedRanges, + crossfilter: { + cells: cellsCrossfilter, + dimensionMap: cellsDimensionsMap + }, graphBrushSelection: null /* if we are getting new cells from the server, the layout (probably? definitely?) just changed, so this is now irrelevant, and we WILL need to call a function to reset state of this kind when cells success happens */ }); + } /* * * * * * * * * * * * * * * * * * User events * * * * * * * * * * * * * * * * * */ - case "parallel coordinates axes have been drawn": + case "parallel coordinates axes have been drawn": { return Object.assign({}, state, { axesHaveBeenDrawn: true }); + } case "continuous selection using parallel coords brushing": { return Object.assign({}, state, { continuousSelection: action.data, - currentCellSelection: - action.newSelection /* this comes from middleware */ + crossfilter: { + ...state.crossfilter + } }); } - case "graph brush selection change": + case "graph brush selection change": { + state.crossfilter.dimensionMap.x.filterRange([ + action.brushCoords.northwest[0], + action.brushCoords.southeast[0] + ]); + state.crossfilter.dimensionMap.y.filterRange([ + action.brushCoords.southeast[1], + action.brushCoords.northwest[1] + ]); return Object.assign({}, state, { - graphBrushSelection: - action.brushCoords /* this has already been applied in middleware but store it for next time */, - currentCellSelection: - action.newSelection /* this comes from middleware */ + graphBrushSelection: action.brushCoords, + crossfilter: { + ...state.crossfilter + } }); - case "graph brush deselect": + } + case "graph brush deselect": { + state.crossfilter.dimensionMap.x.filterAll(); + state.crossfilter.dimensionMap.y.filterAll(); return Object.assign({}, state, { graphBrushSelection: null, - currentCellSelection: - action.newSelection /* this comes from middleware */ + crossfilter: { + ...state.crossfilter + } }); - case "continuous metadata histogram brush": + } + case "continuous metadata histogram brush": { + // action.selection: metadata name being selected + // action.range: filter range, or null if deselected + if (!action.range) { + state.crossfilter.dimensionMap[action.selection].filterAll(); + } else { + state.crossfilter.dimensionMap[action.selection].filterRange( + action.range + ); + } return Object.assign({}, state, { - newContinuousUserDefinedRanges: - action.newContinuousUserDefinedRanges /* this has already been applied in middleware but store it for next time */, - currentCellSelection: - action.newSelection /* this comes from middleware */ + crossfilter: { + ...state.crossfilter + } }); + } case "change opacity deselected cells in 2d graph background": return Object.assign({}, state, { opacityForDeselectedCells: action.data @@ -147,57 +248,113 @@ const Controls = ( /******************************* Categorical metadata *******************************/ - case "categorical metadata filter select": + case "categorical metadata filter select": { + const newCategoricalAsBooleansMap = { + ...state.categoricalAsBooleansMap, + [action.metadataField]: { + ...state.categoricalAsBooleansMap[action.metadataField], + [action.value]: true + } + }; + // update the filter for the one category that changed state + state.crossfilter.dimensionMap[action.metadataField].filterEnum( + _.filter( + _.map( + newCategoricalAsBooleansMap[action.metadataField], + (val, key) => (val ? key : false) + ) + ) + ); return Object.assign({}, state, { - categoricalAsBooleansMap: - action.newCategoricalAsBooleansMap /* this comes from middleware */, - currentCellSelection: - action.newSelection /* this comes from middleware */ + categoricalAsBooleansMap: newCategoricalAsBooleansMap, + crossfilter: { + ...state.crossfilter + } }); - case "categorical metadata filter deselect": + } + case "categorical metadata filter deselect": { + const newCategoricalAsBooleansMap = { + ...state.categoricalAsBooleansMap, + [action.metadataField]: { + ...state.categoricalAsBooleansMap[action.metadataField], + [action.value]: false + } + }; + // update the filter for the one category that changed state + state.crossfilter.dimensionMap[action.metadataField].filterEnum( + _.filter( + _.map( + newCategoricalAsBooleansMap[action.metadataField], + (val, key) => (val ? key : false) + ) + ) + ); return Object.assign({}, state, { - categoricalAsBooleansMap: - action.newCategoricalAsBooleansMap /* this comes from middleware */, - currentCellSelection: - action.newSelection /* this comes from middleware */ + categoricalAsBooleansMap: newCategoricalAsBooleansMap, + crossfilter: { + ...state.crossfilter + } }); - case "categorical metadata filter none of these": + } + case "categorical metadata filter none of these": { + const newCategoricalAsBooleansMap = { + ...state.categoricalAsBooleansMap + }; + _.forEach( + newCategoricalAsBooleansMap[action.metadataField], + (v, k, c) => { + c[k] = false; + } + ); + state.crossfilter.dimensionMap[action.metadataField].filterNone(); return Object.assign({}, state, { - categoricalAsBooleansMap: - action.newCategoricalAsBooleansMap /* this comes from middleware */, - currentCellSelection: - action.newSelection /* this comes from middleware */ + categoricalAsBooleansMap: newCategoricalAsBooleansMap, + crossfilter: { + ...state.crossfilter + } }); - case "categorical metadata filter all of these": + } + case "categorical metadata filter all of these": { + const newCategoricalAsBooleansMap = { + ...state.categoricalAsBooleansMap + }; + _.forEach( + newCategoricalAsBooleansMap[action.metadataField], + (v, k, c) => { + c[k] = true; + } + ); + state.crossfilter.dimensionMap[action.metadataField].filterAll(); return Object.assign({}, state, { - categoricalAsBooleansMap: - action.newCategoricalAsBooleansMap /* this comes from middleware */, - currentCellSelection: - action.newSelection /* this comes from middleware */ + categoricalAsBooleansMap: newCategoricalAsBooleansMap, + crossfilter: { + ...state.crossfilter + } }); + } /******************************* Color Scale *******************************/ case "color by continuous metadata": return Object.assign({}, state, { colorAccessor: action.colorAccessor, - currentCellSelection: - action.currentSelectionWithUpdatedColors /* this comes from middleware */, + allCellsMetadata: + action.allCellsMetadataWithUpdatedColors /* this comes from middleware */, colorScale: action.colorScale }); case "color by expression": return Object.assign({}, state, { colorAccessor: action.gene, - currentCellSelection: - action.currentSelectionWithUpdatedColors /* this comes from middleware */, + allCellsMetadata: + action.allCellsMetadataWithUpdatedColors /* this comes from middleware */, colorScale: action.colorScale }); case "color by categorical metadata": return Object.assign({}, state, { colorAccessor: action.colorAccessor /* pass the scale through additionally, and it's a legend! */, - currentCellSelection: - action.currentSelectionWithUpdatedColors /* this comes from middleware */, + allCellsMetadata: + action.allCellsMetadataWithUpdatedColors /* this comes from middleware */, colorScale: action.colorScale }); case "store current cell selection as differential set 1": diff --git a/src/reducers/index.js b/src/reducers/index.js index 807490c1..ea44deec 100644 --- a/src/reducers/index.js +++ b/src/reducers/index.js @@ -1,7 +1,7 @@ // jshint esversion: 6 import { combineReducers, createStore, applyMiddleware } from "redux"; import updateURLMiddleware from "../middleware/updateURLMiddleware"; -import updateCellSelectionMiddleware from "../middleware/updateCellSelectionMiddleware"; +// import updateCellSelectionMiddleware from "../middleware/updateCellSelectionMiddleware"; import updateCellColors from "../middleware/updateCellColors"; import thunk from "redux-thunk"; @@ -28,7 +28,7 @@ let store = createStore( applyMiddleware( thunk, updateURLMiddleware, - updateCellSelectionMiddleware, + // updateCellSelectionMiddleware, updateCellColors ) ); diff --git a/src/util/typedCrossfilter.js b/src/util/typedCrossfilter.js new file mode 100644 index 00000000..4181eefe --- /dev/null +++ b/src/util/typedCrossfilter.js @@ -0,0 +1,785 @@ +"use strict"; +// jshint esversion: 6 + +/* +Typedarray Crossfilter - a re-implementation of a subset of crossfilter, with +major time/space optimizations predicated upon the following assumptions: + - dimensions are uniformly typed, and all values must be of that type + - dimension values must be a primitive type (int, float, string). Arrays + or other complex types not supported. + - dimension creation requires call-provided type declaration + - no support for adding/removing data to an existing crossfilter. If you + want to do that, you have to create the new crossfilter, using the new + data, from scratch. + +The actual backing store for a dimension is a TypedArray, enabling significant +performance improvements over the original crossfilter. + +There are also a handful of new methods, primarily to take advantage of the +performance (eg, crossfilter.fillBySelection) + +Helpful documents (this code tries to follow the original API as much +as is feasable): + https://github.com/square/crossfilter/ + http://square.github.io/crossfilter/ + +There is also a newer, community supported fork of crossfilter, with a +more complex API. In a few cases, elements of that API were incorporated. + https://github.com/square/crossfilter/ + +*/ + +/* + Utility functions, private to this module +*/ + +// fill an array or typedarray with a sequential range of numbers, +// starting with `start` +// +function fillRange(arr, start = 0) { + for (let i = 0, len = arr.length; i < len; i++) { + arr[i] = i + start; + } + return arr; +} + +// Search for `value` in the sorted array `tarr`, in the range [first, last). +// Return the first (left most) index where tarr[index] >= value. +// +// In other words, return array index I where: +// tarr[i] < value for all tarr[lo:I] +// tarr[i] >= value for all tarr[I:last] +// +// Essentially the same thing as: +// C++: lower_bound() +// Python: bisect.bisect_left() +// +function lowerBound(valueArray, value, first, last) { + // this is just a binary search + while (first < last) { + const middle = (first + last) >>> 1; + if (valueArray[middle] < value) { + first = middle + 1; + } else { + last = middle; + } + } + return first; +} + +// XXX: it is likely that there would be minimal performance hit from creating +// a factory version of lowerBound that takes an accessor (rather than having +// a special-cased version for lining the indirection). +// +// Benchmarking shows this manual inlining is up to 4X faster than an accessor. +// The real issue is how often we call it. +// +function lowerBoundIndirect(valueArray, indexArray, value, first, last) { + // this is just a binary search + while (first < last) { + const middle = (first + last) >>> 1; + if (valueArray[indexArray[middle]] < value) { + first = middle + 1; + } else { + last = middle; + } + } + return first; +} + +// Search for `value in the sorted array `tarr`, in the range [first, last). +// Return the first value where tarr[index] > value. +// +// In other words, return array index I, where: +// tarr[i] <= value for all tarr[lo:I] +// tarr[i] > value for all tarr[I:last] +// +// Essentially the same thing as: +// C++: upper_bound() +// Python: bisect.bisect_right() +// +function upperBound(valueArray, value, first, last) { + // this is just a binary search + while (first < last) { + const middle = (first + last) >>> 1; + if (valueArray[middle] > value) { + last = middle; + } else { + first = middle + 1; + } + } + return first; +} + +function upperBoundIndirect(valueArray, indexArray, value, first, last) { + // this is just a binary search + while (first < last) { + const middle = (first + last) >>> 1; + if (valueArray[indexArray[middle]] > value) { + last = middle; + } else { + first = middle + 1; + } + } + return first; +} + +// Interval operations - very simple version of interval set relationship +// operators. An interval is a multi-interval list of [min, max], +// where min and max are mandatory. Constraints: +// * min <= max +// * Legal intervals: [], [ [0, 1], ... ] +// * all min and max values must be >= 0 +// * Not legal: [ [] ] +// +// Code assumes intervals have a low cardinality; many operations are done +// with a brute force scan. +// +class PositiveIntervals { + // Canonicalize - ensure that: + // 1. no overlapping intervals + // 2. sorted in order of interval min. + // + static canonicalize(A) { + if (A.length <= 1) return A; + let copy = A.slice(); + copy.sort((a, b) => a[0] - b[0]); + const res = []; + res.push(copy[0]); + for (let i = 1, len = copy.length; i < len; i++) { + if (copy[i][0] > res[res.length - 1][1]) { + // non-overlapping, add to result + res.push(copy[i]); + } else if (copy[i][1] > res[res.length - 1][1]) { + // merge this into previous + res[res.length - 1][1] = copy[i][1]; + } + } + return res; + } + + // Return interval with values belonging to both A and B. + // + static union(A, B) { + return PositiveIntervals.canonicalize([...A, ...B]); + } + + static _flatten(A, B) { + let points = []; /* point, A, start */ + for (let a = 0; a < A.length; a++) { + points.push([A[a][0], true, true]); + points.push([A[a][1], true, false]); + } + for (let b = 0; b < B.length; b++) { + points.push([B[b][0], false, true]); + points.push([B[b][1], false, false]); + } + // Sort order: point, then start + points.sort((a, b) => (a[0] !== b[0] ? a[0] - b[0] : a[2] ? 1 : -1)); + return points; + } + + // A - B, ie, the interval with all values in A that are not in B. + // + static difference(A, B) { + // Corner cases + if (A.length === 0 || B.length === 0) { + return PositiveIntervals.canonicalize(A); + } + + A = PositiveIntervals.canonicalize(A); + B = PositiveIntervals.canonicalize(B); + + const points = PositiveIntervals._flatten(A, B); + const res = []; + let aDepth = 0; + let depth = 0; + let intervalStart; + let prevPoint; + for (let i = 0; i < points.length; i++) { + const p = points[i]; + const before = depth; + const delta = p[2] ? 1 : -1; + depth += delta; + if (p[1]) aDepth += delta; + + if (i === points.length - 1 || p[0] !== points[i + 1][0]) { + if (aDepth === 1 && depth === 1) { + intervalStart = p[0]; + } else if (intervalStart !== undefined) { + res.push([intervalStart, p[0]]); + intervalStart = undefined; + } + } + prevPoint = p[0]; + } + // guaranteed to be in canonical form + return res; + } + + // Return interval with values belonging to A or B. + // + static intersection(A, B) { + if (A.length === 0 || B.length === 0) { + return []; + } + + A = PositiveIntervals.canonicalize(A); + B = PositiveIntervals.canonicalize(B); + + const points = PositiveIntervals._flatten(A, B); + const res = []; + let depth = 0; + let intervalStart; + for (let i = 0; i < points.length; i++) { + const p = points[i]; + const before = depth; + depth += p[2] ? 1 : -1; + if (depth === 2) { + intervalStart = p[0]; + } else if (intervalStart !== undefined) { + res.push([intervalStart, p[0]]); + intervalStart = undefined; + } + } + // guaranteed to be in canonical form + return res; + } +} + +// BitArray is a 2D bitarray with size [length, nBitWidth]. +// Each bit is referred to as a `dimension`. Dimensions may be +// dynamically allocated and deallocated. The overall length +// of the BitArray is fixed at creation time (for simplicity). +// +// Organization of the bitarray is dimension-major. +// +// Primary operations on the BitArray are: +// - set & clear dimension +// - test dimension +// - various performance or convenience test operations +// +// The underlying data structure uses TypedArrays for performance. +// +class BitArray { + constructor(length) { + // Initially allocate a 32 bit wide array. allocDimension() will expand + // as necessary. + // + // Int32Array is (counterintuitively) used to accomadate JS numeric casting + // (to/from primitive number type). + // + + // Fixed for the life of this object. + this.length = length; + + // Bitarray width. width is always greater than 32*dimensionCount. + this.width = 1; // underlying number of 32 bit arrays + this.dimensionCount = 0; // num allocated dimensions + + this.bitmask = new Int32Array(this.width); // dimension allocation mask + this.bitarray = new Int32Array(this.width * this.length); + + console.log("a", wasmHelpersModule); + } + + get selectionCount() { + return this.countAllOnes(); + } + + countAllOnes() { + let count = 0; + for (let i = 0; i < this.width; i++) { + const bitmask = this.bitmask[i]; + for (let j = i * this.length, len = j + this.length; j < len; j++) { + if (this.bitarray[i * this.length + j] === bitmask) count++; + } + } + return count; + } + + // count trailing zeros + static ctz(v) { + let c = 32; + v &= -v; // isolate lowest non-zero bit + if (v) c--; + if (v & 0x0000ffff) c -= 16; + if (v & 0x00ff00ff) c -= 8; + if (v & 0x0f0f0f0f) c -= 4; + if (v & 0x33333333) c -= 2; + if (v & 0x55555555) c -= 1; + return c; + } + + // find a free dimension. Return undefined if none + _findFreeDimension() { + let dim; + for (let col = 0; col < this.width; col++) { + const bitmask = this.bitmask[col]; + const lowestZeroBit = ~this.bitmask[col] & -~this.bitmask[col]; + if (lowestZeroBit) { + this.bitmask[col] |= lowestZeroBit; + dim = 32 * col + BitArray.ctz(lowestZeroBit); + } + } + return dim; + } + + // allocate and return the dimension ID (bit position) + allocDimension() { + let dim = this._findFreeDimension(); + + // if we did not find free dimension, expand the bitarray. + if (dim === undefined) { + this.width++; + + const biggerBitArray = new Int32Array(this.width * this.length); + biggerBitArray.set(this.bitarray); + this.bitarray = biggerBitArray; + + const biggerBitmask = new Int32Array(this.width); + biggerBitmask.set(this.bitmask); + this.bitmask = biggerBitmask; + + dim = this._findFreeDimension(); + } + + this.dimensionCount++; + return dim; + } + + freeDimension(dim) { + // all selection tests assume unallocated dimensions are zero valued. + this.deselectAll(dim); + const col = dim >>> 5; + this.bitmask[col] &= ~(1 << (dim % 32)); + this.dimensionCount--; + } + + isSelected(index) { + const width = this.width; + const length = this.length; + const bitarray = this.bitarray; + + for (let w = 0; w < width; w++) { + const bitmask = this.bitmask[w]; + if (!bitmask || bitarray[w * length + index] !== bitmask) return false; + } + return true; + } + + selectOne(dim, index) { + const col = dim >>> 5; + const before = this.bitarray[col * this.length + index]; + const after = before | (1 << (dim % 32)); + this.bitarray[col] = after; + } + + deselectOne(dim, index) { + const col = dim >>> 5; + const before = this.bitarray[col * this.length + index]; + const after = before & ~(1 << (dim % 32)); + this.bitarray[col] = after; + } + + selectAll(dim) { + let col = dim >> 5; + const bitmask = this.bitmask[col]; + const bitarray = this.bitarray; + const one = 1 << (dim % 32); + for (let i = col * this.length, len = i + this.length; i < len; i++) { + bitarray[i] |= one; + } + } + + deselectAll(dim) { + let col = dim >> 5; + const bitmask = this.bitmask[col]; + const bitarray = this.bitarray; + const zero = ~(1 << (dim % 32)); + for (let i = col * this.length, len = i + this.length; i < len; i++) { + bitarray[i] &= zero; + } + } + + // indirect functions are used to map between sort and natural order + selectIndirectFromRange(dim, indirect, range) { + const col = dim >>> 5; + const first = range[0]; + const last = range[1]; + const bitarray = this.bitarray; + const one = 1 << (dim % 32); + const offset = col * this.length; + for (let i = first; i < last; i++) { + bitarray[offset + indirect[i]] |= one; + } + } + + deselectIndirectFromRange(dim, indirect, range) { + const col = dim >>> 5; + const first = range[0]; + const last = range[1]; + const bitarray = this.bitarray; + const zero = ~(1 << (dim % 32)); + const offset = col * this.length; + for (let i = first; i < last; i++) { + bitarray[offset + indirect[i]] &= zero; + } + } + + // Fill the array with selected|deselected value based upon the + // current selection state. + fillBySelection(result, selectedValue, deselectedValue) { + // special case (width === 1) for performance + if (this.width === 1) { + const bitmask = this.bitmask[0]; + const bitarray = this.bitarray; + for (let i = 0, len = this.length; i < len; i++) { + result[i] = bitarray[i] === bitmask ? selectedValue : deselectedValue; + } + } else { + for (let i = 0, len = this.length; i < len; i++) { + result[i] = this.isSelected(i) ? selectedValue : deselectedValue; + } + } + return result; + } +} + +class TypedCrossfilter { + constructor(data) { + this.data = data; + + // filters: array of { id, dimension } + this.filters = []; + this.selection = new BitArray(data.length); + } + + size() { + return this.data.length; + } + + all() { + return this.data; + } + + dimension(value, valueArrayType) { + const id = this.selection.allocDimension(); + let dim; + if (valueArrayType === "enum") { + dim = new EnumDimension(value, this, id); + } else { + dim = new ScalarDimension(value, valueArrayType, this, id); + } + this.filters.push({ id, dim }); + dim.filterAll(); + return dim; + } + + _freeDimension(id) { + this.selection.freeDimension(id); + this.filters = this.filters.filter(f => f.id != id); + } + + // return array of all records that are selected/filtered + // by all dimensions. + allFiltered() { + const selection = this.selection; + const res = []; + for (let i = 0, len = this.data.length; i < len; i++) { + if (selection.isSelected(i)) { + res.push(this.data[i]); + } + } + return res; + } + + countFiltered() { + return this.selection.selectionCount; + } + + isElementFiltered(i) { + return this.selection.isSelected(i); + } + + // fill array with one of two values, based upon selection state + fillByIsFiltered(array, selectedValue, deselectedValue) { + return this.selection.fillBySelection( + array, + selectedValue, + deselectedValue + ); + } +} + +// Base dimension type - value must be a scalar type (eg, int, float), +// and value array must be a TypedArray. +// +class ScalarDimension { + constructor(value, valueArrayType, crossfilter, id) { + this.crossfilter = crossfilter; + this.id = id; + + // current selection filter, expressed as PostiveIntervals. + this.currentFilter = []; + + // Create value array + const array = this._createValueArray( + value, + new valueArrayType(this.crossfilter.data.length) + ); + this.value = array; + + // create sort index + this.index = fillRange(new Uint32Array(this.crossfilter.data.length)); + this.index.sort((a, b) => array[a] - array[b]); + } + + _createValueArray(value, array) { + // create dimension value array + const data = this.crossfilter.data; + const len = data.length; + for (let i = 0; i < len; i++) { + array[i] = value(data[i]); + } + return array; + } + + dispose() { + this.crossfilter._freeDimension(this.id); + } + + id() { + return this.id; + } + + _updateFilters(newFilter) { + newFilter = PositiveIntervals.canonicalize(newFilter); + + // special case optimization - select all/none can bypass + // more complex work and just clobber everything. + // + if (newFilter.length === 0) { + this.crossfilter.selection.deselectAll(this.id); + } else if ( + newFilter.length === 1 && + newFilter[0][0] === 0 && + newFilter[0][1] == this.index.length + ) { + this.crossfilter.selection.selectAll(this.id); + } else { + const adds = PositiveIntervals.difference(newFilter, this.currentFilter); + const dels = PositiveIntervals.difference(this.currentFilter, newFilter); + dels.forEach(interval => + this.crossfilter.selection.deselectIndirectFromRange( + this.id, + this.index, + interval + ) + ); + adds.forEach(interval => + this.crossfilter.selection.selectIndirectFromRange( + this.id, + this.index, + interval + ) + ); + } + + this.currentFilter = newFilter; + } + + // filter by value - exact match + filterExact(value) { + const newFilter = [ + lowerBoundIndirect(this.value, this.index, value, 0, this.value.length), + upperBoundIndirect(this.value, this.index, value, 0, this.value.length) + ]; + if (newFilter[0] <= newFilter[1]) { + this._updateFilters([newFilter]); + } else { + this._updateFilters([]); + } + return this; + } + + // filter by a set of values, eg. enum. + filterEnum(values) { + const newFilter = []; + for (let v = 0, len = values.length; v < len; v++) { + const intv = [ + lowerBoundIndirect( + this.value, + this.index, + values[v], + 0, + this.value.length + ), + upperBoundIndirect( + this.value, + this.index, + values[v], + 0, + this.value.length + ) + ]; + if (intv[0] <= intv[1]) newFilter.push(intv); + } + this._updateFilters(newFilter); + return this; + } + + // filter by value range [lo, hi) + // lo: inclusive, hi: exclusive + filterRange(range) { + const newFilter = []; + const intv = [ + lowerBoundIndirect( + this.value, + this.index, + range[0], + 0, + this.value.length + ), + upperBoundIndirect(this.value, this.index, range[1], 0, this.value.length) + ]; + if (intv[0] < intv[1]) newFilter.push(intv); + this._updateFilters(newFilter); + return this; + } + + // select all - equivalent of selecting all in this dimension + filterAll() { + this._updateFilters([[0, this.value.length]]); + return this; + } + + // select none + filterNone() { + this._updateFilters([]); + } + + // return top k records, starting with offset, in descending order. + // Order is this dimension's sort order + top(k, offset = 0) { + const data = this.crossfilter.data; + const selection = this.crossfilter.selection; + const index = this.index; + const len = index.length; + const ret = []; + let i = 0; + let skip = 0; + let found = 0; + + // skip up to offset records + for (i = len - 1; 0 <= i && skip < offset; i--) { + if (selection.isSelected(index[i])) { + skip++; + } + } + + // grab up to k records + for (; 0 <= i && found < k; i--) { + if (selection.isSelected(index[i])) { + ret.push(data[index[i]]); + found++; + } + } + + return ret; + } + + // return bottom k records, starting with offset, in ascending order. + // Order is this dimension's sort order + bottom(k, offset = 0) { + const data = this.crossfilter.data; + const selection = this.crossfilter.selection; + const index = this.index; + const len = index.length; + const ret = []; + let skip = 0; + let found = 0; + let i = 0; + + // skip up to offset records + for (i = 0; i < len && skip < offset; i++) { + if (selection.isSelected(index[i])) { + skip++; + } + } + + // grab up to k records + for (; i < len && found < k; i++) { + if (selection.isSelected(index[i])) { + ret.push(data[index[i]]); + found++; + } + } + + return ret; + } +} + +// Ordered enumeration - supports any sortable enumerable type, eg, +// strings, which can be mapped into an fixed numeric range [0..n). +// +class EnumDimension extends ScalarDimension { + constructor(value, crossfilter, id) { + super(value, Uint32Array, crossfilter, id); + } + + _createValueArray(value, array) { + const data = this.crossfilter.data; + const len = data.length; + + // create enumeration table - mapping between the value + // and the enum. + const s = new Set(); + for (let i = 0; i < len; i++) { + s.add(value(data[i])); + } + this.enumIndex = Array.from(s); + this.enumIndex.sort(); + + // create dimension value array + const enumLen = this.enumIndex.length; + for (let i = 0; i < len; i++) { + const v = value(data[i]); + const e = lowerBound(this.enumIndex, v, 0, enumLen); + array[i] = e; + } + return array; + } + + filterExact(value) { + return super.filterExact( + lowerBound(this.enumIndex, value, 0, this.enumIndex.length) + ); + } + + filterEnum(values) { + return super.filterEnum( + values.map(v => lowerBound(this.enumIndex, v, 0, this.enumIndex.length)) + ); + } + + filterRange(range) { + return super.filterEnum( + range.map(v => lowerBound(this.enumIndex, v, 0, this.enumIndex.length)) + ); + } +} + +// Wrapper for backwards compat with crossfilter. +// +function crossfilter(data) { + return new TypedCrossfilter(data); +} + +crossfilter.PositiveIntervals = PositiveIntervals; +crossfilter.BitArray = BitArray; +crossfilter.TypedCrossfilter = TypedCrossfilter; +crossfilter.ScalarDimension = ScalarDimension; +crossfilter.EnumDimension = EnumDimension; + +module.exports = crossfilter; From 007e1d3a667fefb68b20c10da654fad3afcd760b Mon Sep 17 00:00:00 2001 From: bkmartinjr Date: Wed, 23 May 2018 21:01:28 -0700 Subject: [PATCH 2/8] remove detritus --- src/util/typedCrossfilter.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/util/typedCrossfilter.js b/src/util/typedCrossfilter.js index 4181eefe..72fdf40c 100644 --- a/src/util/typedCrossfilter.js +++ b/src/util/typedCrossfilter.js @@ -279,8 +279,6 @@ class BitArray { this.bitmask = new Int32Array(this.width); // dimension allocation mask this.bitarray = new Int32Array(this.width * this.length); - - console.log("a", wasmHelpersModule); } get selectionCount() { From 7874e6a68b2290a9eccce6813d27223d75794104 Mon Sep 17 00:00:00 2001 From: bkmartinjr Date: Fri, 25 May 2018 17:27:50 -0700 Subject: [PATCH 3/8] better caching of graph regl data --- src/components/graph/graph.js | 42 +++++++++++++++++++++++------------ 1 file changed, 28 insertions(+), 14 deletions(-) diff --git a/src/components/graph/graph.js b/src/components/graph/graph.js index 2fc602ab..8ca3596c 100644 --- a/src/components/graph/graph.js +++ b/src/components/graph/graph.js @@ -101,18 +101,26 @@ class Graph extends React.Component { !this.renderCache.positions || this.props.crossfilter.cells != nextProps.crossfilter.cells ) { - const positions = new Float32Array(2 * cellCount); + if (!this.renderCache.positions) + this.renderCache.positions = new Float32Array(2 * cellCount); // d3.scaleLinear().domain([0,1]).range([-1,1]) const glScaleX = scaleLinear([0, 1], [-1, 1]); // d3.scaleLinear().domain([0,1]).range([1,-1]) const glScaleY = scaleLinear([0, 1], [1, -1]); - for (let i = 0; i < cellCount; i++) { + for ( + let i = 0, positions = this.renderCache.positions; + i < cellCount; + i++ + ) { positions[2 * i] = glScaleX(cells[i].__x__); positions[2 * i + 1] = glScaleY(cells[i].__y__); } - this.renderCache.positions = positions; + this.state.pointBuffer({ + data: this.renderCache.positions, + dimension: 2 + }); } // Colors for each point - a cached value that only changes when @@ -125,22 +133,28 @@ class Graph extends React.Component { !this.renderCache.colors || this.props.allCellsMetadata != nextProps.allCellsMetadata ) { - const colors = new Float32Array(3 * cellCount); - for (let i = 0; i < cellCount; i++) { + if (!this.renderCache.colors) + this.renderCache.colors = new Float32Array(3 * cellCount); + for (let i = 0, colors = this.renderCache.colors; i < cellCount; i++) { colors.set(cells[i].__colorRGB__, 3 * i); } - this.renderCache.colors = colors; + this.state.colorBuffer({ data: this.renderCache.colors, dimension: 3 }); } - const sizes = new Float32Array(cellCount); - crossfilter.fillByIsFiltered(sizes, 4, 0.2); + // Sizes for each point - this is presumed to change each time the + // component receives new props. Almost always a true assumption, as + // most property upates are due to changes driving a crossfilter + // selection set change. + // + if ( + !this.renderCache.sizes || + this.props.crossfilter.cells != nextProps.crossfilter.cells + ) { + this.renderCache.sizes = new Float32Array(cellCount); + } + crossfilter.fillByIsFiltered(this.renderCache.sizes, 4, 0.2); + this.state.sizeBuffer({ data: this.renderCache.sizes, dimension: 1 }); - this.state.pointBuffer({ - data: this.renderCache.positions, - dimension: 2 - }); - this.state.colorBuffer({ data: this.renderCache.colors, dimension: 3 }); - this.state.sizeBuffer({ data: sizes, dimension: 1 }); this.count = cellCount; } From 2c79e98809aab87e60316e6a5fa324d46f037011 Mon Sep 17 00:00:00 2001 From: bkmartinjr Date: Sun, 27 May 2018 08:02:47 -0700 Subject: [PATCH 4/8] reorganize TypedCrossfilter code --- src/util/typedCrossfilter.js | 783 ------------------ src/util/typedCrossfilter/bitArray.js | 223 +++++ src/util/typedCrossfilter/index.js | 394 +++++++++ .../typedCrossfilter/positiveIntervals.js | 132 +++ src/util/typedCrossfilter/util.js | 106 +++ 5 files changed, 855 insertions(+), 783 deletions(-) delete mode 100644 src/util/typedCrossfilter.js create mode 100644 src/util/typedCrossfilter/bitArray.js create mode 100644 src/util/typedCrossfilter/index.js create mode 100644 src/util/typedCrossfilter/positiveIntervals.js create mode 100644 src/util/typedCrossfilter/util.js diff --git a/src/util/typedCrossfilter.js b/src/util/typedCrossfilter.js deleted file mode 100644 index 72fdf40c..00000000 --- a/src/util/typedCrossfilter.js +++ /dev/null @@ -1,783 +0,0 @@ -"use strict"; -// jshint esversion: 6 - -/* -Typedarray Crossfilter - a re-implementation of a subset of crossfilter, with -major time/space optimizations predicated upon the following assumptions: - - dimensions are uniformly typed, and all values must be of that type - - dimension values must be a primitive type (int, float, string). Arrays - or other complex types not supported. - - dimension creation requires call-provided type declaration - - no support for adding/removing data to an existing crossfilter. If you - want to do that, you have to create the new crossfilter, using the new - data, from scratch. - -The actual backing store for a dimension is a TypedArray, enabling significant -performance improvements over the original crossfilter. - -There are also a handful of new methods, primarily to take advantage of the -performance (eg, crossfilter.fillBySelection) - -Helpful documents (this code tries to follow the original API as much -as is feasable): - https://github.com/square/crossfilter/ - http://square.github.io/crossfilter/ - -There is also a newer, community supported fork of crossfilter, with a -more complex API. In a few cases, elements of that API were incorporated. - https://github.com/square/crossfilter/ - -*/ - -/* - Utility functions, private to this module -*/ - -// fill an array or typedarray with a sequential range of numbers, -// starting with `start` -// -function fillRange(arr, start = 0) { - for (let i = 0, len = arr.length; i < len; i++) { - arr[i] = i + start; - } - return arr; -} - -// Search for `value` in the sorted array `tarr`, in the range [first, last). -// Return the first (left most) index where tarr[index] >= value. -// -// In other words, return array index I where: -// tarr[i] < value for all tarr[lo:I] -// tarr[i] >= value for all tarr[I:last] -// -// Essentially the same thing as: -// C++: lower_bound() -// Python: bisect.bisect_left() -// -function lowerBound(valueArray, value, first, last) { - // this is just a binary search - while (first < last) { - const middle = (first + last) >>> 1; - if (valueArray[middle] < value) { - first = middle + 1; - } else { - last = middle; - } - } - return first; -} - -// XXX: it is likely that there would be minimal performance hit from creating -// a factory version of lowerBound that takes an accessor (rather than having -// a special-cased version for lining the indirection). -// -// Benchmarking shows this manual inlining is up to 4X faster than an accessor. -// The real issue is how often we call it. -// -function lowerBoundIndirect(valueArray, indexArray, value, first, last) { - // this is just a binary search - while (first < last) { - const middle = (first + last) >>> 1; - if (valueArray[indexArray[middle]] < value) { - first = middle + 1; - } else { - last = middle; - } - } - return first; -} - -// Search for `value in the sorted array `tarr`, in the range [first, last). -// Return the first value where tarr[index] > value. -// -// In other words, return array index I, where: -// tarr[i] <= value for all tarr[lo:I] -// tarr[i] > value for all tarr[I:last] -// -// Essentially the same thing as: -// C++: upper_bound() -// Python: bisect.bisect_right() -// -function upperBound(valueArray, value, first, last) { - // this is just a binary search - while (first < last) { - const middle = (first + last) >>> 1; - if (valueArray[middle] > value) { - last = middle; - } else { - first = middle + 1; - } - } - return first; -} - -function upperBoundIndirect(valueArray, indexArray, value, first, last) { - // this is just a binary search - while (first < last) { - const middle = (first + last) >>> 1; - if (valueArray[indexArray[middle]] > value) { - last = middle; - } else { - first = middle + 1; - } - } - return first; -} - -// Interval operations - very simple version of interval set relationship -// operators. An interval is a multi-interval list of [min, max], -// where min and max are mandatory. Constraints: -// * min <= max -// * Legal intervals: [], [ [0, 1], ... ] -// * all min and max values must be >= 0 -// * Not legal: [ [] ] -// -// Code assumes intervals have a low cardinality; many operations are done -// with a brute force scan. -// -class PositiveIntervals { - // Canonicalize - ensure that: - // 1. no overlapping intervals - // 2. sorted in order of interval min. - // - static canonicalize(A) { - if (A.length <= 1) return A; - let copy = A.slice(); - copy.sort((a, b) => a[0] - b[0]); - const res = []; - res.push(copy[0]); - for (let i = 1, len = copy.length; i < len; i++) { - if (copy[i][0] > res[res.length - 1][1]) { - // non-overlapping, add to result - res.push(copy[i]); - } else if (copy[i][1] > res[res.length - 1][1]) { - // merge this into previous - res[res.length - 1][1] = copy[i][1]; - } - } - return res; - } - - // Return interval with values belonging to both A and B. - // - static union(A, B) { - return PositiveIntervals.canonicalize([...A, ...B]); - } - - static _flatten(A, B) { - let points = []; /* point, A, start */ - for (let a = 0; a < A.length; a++) { - points.push([A[a][0], true, true]); - points.push([A[a][1], true, false]); - } - for (let b = 0; b < B.length; b++) { - points.push([B[b][0], false, true]); - points.push([B[b][1], false, false]); - } - // Sort order: point, then start - points.sort((a, b) => (a[0] !== b[0] ? a[0] - b[0] : a[2] ? 1 : -1)); - return points; - } - - // A - B, ie, the interval with all values in A that are not in B. - // - static difference(A, B) { - // Corner cases - if (A.length === 0 || B.length === 0) { - return PositiveIntervals.canonicalize(A); - } - - A = PositiveIntervals.canonicalize(A); - B = PositiveIntervals.canonicalize(B); - - const points = PositiveIntervals._flatten(A, B); - const res = []; - let aDepth = 0; - let depth = 0; - let intervalStart; - let prevPoint; - for (let i = 0; i < points.length; i++) { - const p = points[i]; - const before = depth; - const delta = p[2] ? 1 : -1; - depth += delta; - if (p[1]) aDepth += delta; - - if (i === points.length - 1 || p[0] !== points[i + 1][0]) { - if (aDepth === 1 && depth === 1) { - intervalStart = p[0]; - } else if (intervalStart !== undefined) { - res.push([intervalStart, p[0]]); - intervalStart = undefined; - } - } - prevPoint = p[0]; - } - // guaranteed to be in canonical form - return res; - } - - // Return interval with values belonging to A or B. - // - static intersection(A, B) { - if (A.length === 0 || B.length === 0) { - return []; - } - - A = PositiveIntervals.canonicalize(A); - B = PositiveIntervals.canonicalize(B); - - const points = PositiveIntervals._flatten(A, B); - const res = []; - let depth = 0; - let intervalStart; - for (let i = 0; i < points.length; i++) { - const p = points[i]; - const before = depth; - depth += p[2] ? 1 : -1; - if (depth === 2) { - intervalStart = p[0]; - } else if (intervalStart !== undefined) { - res.push([intervalStart, p[0]]); - intervalStart = undefined; - } - } - // guaranteed to be in canonical form - return res; - } -} - -// BitArray is a 2D bitarray with size [length, nBitWidth]. -// Each bit is referred to as a `dimension`. Dimensions may be -// dynamically allocated and deallocated. The overall length -// of the BitArray is fixed at creation time (for simplicity). -// -// Organization of the bitarray is dimension-major. -// -// Primary operations on the BitArray are: -// - set & clear dimension -// - test dimension -// - various performance or convenience test operations -// -// The underlying data structure uses TypedArrays for performance. -// -class BitArray { - constructor(length) { - // Initially allocate a 32 bit wide array. allocDimension() will expand - // as necessary. - // - // Int32Array is (counterintuitively) used to accomadate JS numeric casting - // (to/from primitive number type). - // - - // Fixed for the life of this object. - this.length = length; - - // Bitarray width. width is always greater than 32*dimensionCount. - this.width = 1; // underlying number of 32 bit arrays - this.dimensionCount = 0; // num allocated dimensions - - this.bitmask = new Int32Array(this.width); // dimension allocation mask - this.bitarray = new Int32Array(this.width * this.length); - } - - get selectionCount() { - return this.countAllOnes(); - } - - countAllOnes() { - let count = 0; - for (let i = 0; i < this.width; i++) { - const bitmask = this.bitmask[i]; - for (let j = i * this.length, len = j + this.length; j < len; j++) { - if (this.bitarray[i * this.length + j] === bitmask) count++; - } - } - return count; - } - - // count trailing zeros - static ctz(v) { - let c = 32; - v &= -v; // isolate lowest non-zero bit - if (v) c--; - if (v & 0x0000ffff) c -= 16; - if (v & 0x00ff00ff) c -= 8; - if (v & 0x0f0f0f0f) c -= 4; - if (v & 0x33333333) c -= 2; - if (v & 0x55555555) c -= 1; - return c; - } - - // find a free dimension. Return undefined if none - _findFreeDimension() { - let dim; - for (let col = 0; col < this.width; col++) { - const bitmask = this.bitmask[col]; - const lowestZeroBit = ~this.bitmask[col] & -~this.bitmask[col]; - if (lowestZeroBit) { - this.bitmask[col] |= lowestZeroBit; - dim = 32 * col + BitArray.ctz(lowestZeroBit); - } - } - return dim; - } - - // allocate and return the dimension ID (bit position) - allocDimension() { - let dim = this._findFreeDimension(); - - // if we did not find free dimension, expand the bitarray. - if (dim === undefined) { - this.width++; - - const biggerBitArray = new Int32Array(this.width * this.length); - biggerBitArray.set(this.bitarray); - this.bitarray = biggerBitArray; - - const biggerBitmask = new Int32Array(this.width); - biggerBitmask.set(this.bitmask); - this.bitmask = biggerBitmask; - - dim = this._findFreeDimension(); - } - - this.dimensionCount++; - return dim; - } - - freeDimension(dim) { - // all selection tests assume unallocated dimensions are zero valued. - this.deselectAll(dim); - const col = dim >>> 5; - this.bitmask[col] &= ~(1 << (dim % 32)); - this.dimensionCount--; - } - - isSelected(index) { - const width = this.width; - const length = this.length; - const bitarray = this.bitarray; - - for (let w = 0; w < width; w++) { - const bitmask = this.bitmask[w]; - if (!bitmask || bitarray[w * length + index] !== bitmask) return false; - } - return true; - } - - selectOne(dim, index) { - const col = dim >>> 5; - const before = this.bitarray[col * this.length + index]; - const after = before | (1 << (dim % 32)); - this.bitarray[col] = after; - } - - deselectOne(dim, index) { - const col = dim >>> 5; - const before = this.bitarray[col * this.length + index]; - const after = before & ~(1 << (dim % 32)); - this.bitarray[col] = after; - } - - selectAll(dim) { - let col = dim >> 5; - const bitmask = this.bitmask[col]; - const bitarray = this.bitarray; - const one = 1 << (dim % 32); - for (let i = col * this.length, len = i + this.length; i < len; i++) { - bitarray[i] |= one; - } - } - - deselectAll(dim) { - let col = dim >> 5; - const bitmask = this.bitmask[col]; - const bitarray = this.bitarray; - const zero = ~(1 << (dim % 32)); - for (let i = col * this.length, len = i + this.length; i < len; i++) { - bitarray[i] &= zero; - } - } - - // indirect functions are used to map between sort and natural order - selectIndirectFromRange(dim, indirect, range) { - const col = dim >>> 5; - const first = range[0]; - const last = range[1]; - const bitarray = this.bitarray; - const one = 1 << (dim % 32); - const offset = col * this.length; - for (let i = first; i < last; i++) { - bitarray[offset + indirect[i]] |= one; - } - } - - deselectIndirectFromRange(dim, indirect, range) { - const col = dim >>> 5; - const first = range[0]; - const last = range[1]; - const bitarray = this.bitarray; - const zero = ~(1 << (dim % 32)); - const offset = col * this.length; - for (let i = first; i < last; i++) { - bitarray[offset + indirect[i]] &= zero; - } - } - - // Fill the array with selected|deselected value based upon the - // current selection state. - fillBySelection(result, selectedValue, deselectedValue) { - // special case (width === 1) for performance - if (this.width === 1) { - const bitmask = this.bitmask[0]; - const bitarray = this.bitarray; - for (let i = 0, len = this.length; i < len; i++) { - result[i] = bitarray[i] === bitmask ? selectedValue : deselectedValue; - } - } else { - for (let i = 0, len = this.length; i < len; i++) { - result[i] = this.isSelected(i) ? selectedValue : deselectedValue; - } - } - return result; - } -} - -class TypedCrossfilter { - constructor(data) { - this.data = data; - - // filters: array of { id, dimension } - this.filters = []; - this.selection = new BitArray(data.length); - } - - size() { - return this.data.length; - } - - all() { - return this.data; - } - - dimension(value, valueArrayType) { - const id = this.selection.allocDimension(); - let dim; - if (valueArrayType === "enum") { - dim = new EnumDimension(value, this, id); - } else { - dim = new ScalarDimension(value, valueArrayType, this, id); - } - this.filters.push({ id, dim }); - dim.filterAll(); - return dim; - } - - _freeDimension(id) { - this.selection.freeDimension(id); - this.filters = this.filters.filter(f => f.id != id); - } - - // return array of all records that are selected/filtered - // by all dimensions. - allFiltered() { - const selection = this.selection; - const res = []; - for (let i = 0, len = this.data.length; i < len; i++) { - if (selection.isSelected(i)) { - res.push(this.data[i]); - } - } - return res; - } - - countFiltered() { - return this.selection.selectionCount; - } - - isElementFiltered(i) { - return this.selection.isSelected(i); - } - - // fill array with one of two values, based upon selection state - fillByIsFiltered(array, selectedValue, deselectedValue) { - return this.selection.fillBySelection( - array, - selectedValue, - deselectedValue - ); - } -} - -// Base dimension type - value must be a scalar type (eg, int, float), -// and value array must be a TypedArray. -// -class ScalarDimension { - constructor(value, valueArrayType, crossfilter, id) { - this.crossfilter = crossfilter; - this.id = id; - - // current selection filter, expressed as PostiveIntervals. - this.currentFilter = []; - - // Create value array - const array = this._createValueArray( - value, - new valueArrayType(this.crossfilter.data.length) - ); - this.value = array; - - // create sort index - this.index = fillRange(new Uint32Array(this.crossfilter.data.length)); - this.index.sort((a, b) => array[a] - array[b]); - } - - _createValueArray(value, array) { - // create dimension value array - const data = this.crossfilter.data; - const len = data.length; - for (let i = 0; i < len; i++) { - array[i] = value(data[i]); - } - return array; - } - - dispose() { - this.crossfilter._freeDimension(this.id); - } - - id() { - return this.id; - } - - _updateFilters(newFilter) { - newFilter = PositiveIntervals.canonicalize(newFilter); - - // special case optimization - select all/none can bypass - // more complex work and just clobber everything. - // - if (newFilter.length === 0) { - this.crossfilter.selection.deselectAll(this.id); - } else if ( - newFilter.length === 1 && - newFilter[0][0] === 0 && - newFilter[0][1] == this.index.length - ) { - this.crossfilter.selection.selectAll(this.id); - } else { - const adds = PositiveIntervals.difference(newFilter, this.currentFilter); - const dels = PositiveIntervals.difference(this.currentFilter, newFilter); - dels.forEach(interval => - this.crossfilter.selection.deselectIndirectFromRange( - this.id, - this.index, - interval - ) - ); - adds.forEach(interval => - this.crossfilter.selection.selectIndirectFromRange( - this.id, - this.index, - interval - ) - ); - } - - this.currentFilter = newFilter; - } - - // filter by value - exact match - filterExact(value) { - const newFilter = [ - lowerBoundIndirect(this.value, this.index, value, 0, this.value.length), - upperBoundIndirect(this.value, this.index, value, 0, this.value.length) - ]; - if (newFilter[0] <= newFilter[1]) { - this._updateFilters([newFilter]); - } else { - this._updateFilters([]); - } - return this; - } - - // filter by a set of values, eg. enum. - filterEnum(values) { - const newFilter = []; - for (let v = 0, len = values.length; v < len; v++) { - const intv = [ - lowerBoundIndirect( - this.value, - this.index, - values[v], - 0, - this.value.length - ), - upperBoundIndirect( - this.value, - this.index, - values[v], - 0, - this.value.length - ) - ]; - if (intv[0] <= intv[1]) newFilter.push(intv); - } - this._updateFilters(newFilter); - return this; - } - - // filter by value range [lo, hi) - // lo: inclusive, hi: exclusive - filterRange(range) { - const newFilter = []; - const intv = [ - lowerBoundIndirect( - this.value, - this.index, - range[0], - 0, - this.value.length - ), - upperBoundIndirect(this.value, this.index, range[1], 0, this.value.length) - ]; - if (intv[0] < intv[1]) newFilter.push(intv); - this._updateFilters(newFilter); - return this; - } - - // select all - equivalent of selecting all in this dimension - filterAll() { - this._updateFilters([[0, this.value.length]]); - return this; - } - - // select none - filterNone() { - this._updateFilters([]); - } - - // return top k records, starting with offset, in descending order. - // Order is this dimension's sort order - top(k, offset = 0) { - const data = this.crossfilter.data; - const selection = this.crossfilter.selection; - const index = this.index; - const len = index.length; - const ret = []; - let i = 0; - let skip = 0; - let found = 0; - - // skip up to offset records - for (i = len - 1; 0 <= i && skip < offset; i--) { - if (selection.isSelected(index[i])) { - skip++; - } - } - - // grab up to k records - for (; 0 <= i && found < k; i--) { - if (selection.isSelected(index[i])) { - ret.push(data[index[i]]); - found++; - } - } - - return ret; - } - - // return bottom k records, starting with offset, in ascending order. - // Order is this dimension's sort order - bottom(k, offset = 0) { - const data = this.crossfilter.data; - const selection = this.crossfilter.selection; - const index = this.index; - const len = index.length; - const ret = []; - let skip = 0; - let found = 0; - let i = 0; - - // skip up to offset records - for (i = 0; i < len && skip < offset; i++) { - if (selection.isSelected(index[i])) { - skip++; - } - } - - // grab up to k records - for (; i < len && found < k; i++) { - if (selection.isSelected(index[i])) { - ret.push(data[index[i]]); - found++; - } - } - - return ret; - } -} - -// Ordered enumeration - supports any sortable enumerable type, eg, -// strings, which can be mapped into an fixed numeric range [0..n). -// -class EnumDimension extends ScalarDimension { - constructor(value, crossfilter, id) { - super(value, Uint32Array, crossfilter, id); - } - - _createValueArray(value, array) { - const data = this.crossfilter.data; - const len = data.length; - - // create enumeration table - mapping between the value - // and the enum. - const s = new Set(); - for (let i = 0; i < len; i++) { - s.add(value(data[i])); - } - this.enumIndex = Array.from(s); - this.enumIndex.sort(); - - // create dimension value array - const enumLen = this.enumIndex.length; - for (let i = 0; i < len; i++) { - const v = value(data[i]); - const e = lowerBound(this.enumIndex, v, 0, enumLen); - array[i] = e; - } - return array; - } - - filterExact(value) { - return super.filterExact( - lowerBound(this.enumIndex, value, 0, this.enumIndex.length) - ); - } - - filterEnum(values) { - return super.filterEnum( - values.map(v => lowerBound(this.enumIndex, v, 0, this.enumIndex.length)) - ); - } - - filterRange(range) { - return super.filterEnum( - range.map(v => lowerBound(this.enumIndex, v, 0, this.enumIndex.length)) - ); - } -} - -// Wrapper for backwards compat with crossfilter. -// -function crossfilter(data) { - return new TypedCrossfilter(data); -} - -crossfilter.PositiveIntervals = PositiveIntervals; -crossfilter.BitArray = BitArray; -crossfilter.TypedCrossfilter = TypedCrossfilter; -crossfilter.ScalarDimension = ScalarDimension; -crossfilter.EnumDimension = EnumDimension; - -module.exports = crossfilter; diff --git a/src/util/typedCrossfilter/bitArray.js b/src/util/typedCrossfilter/bitArray.js new file mode 100644 index 00000000..3a464679 --- /dev/null +++ b/src/util/typedCrossfilter/bitArray.js @@ -0,0 +1,223 @@ +"use strict"; +// jshint esversion: 6 + +// BitArray is a 2D bitarray with size [length, nBitWidth]. +// Each bit is referred to as a `dimension`. Dimensions may be +// dynamically allocated and deallocated. The overall length +// of the BitArray is fixed at creation time (for simplicity). +// +// Organization of the bitarray is dimension-major. As dimensions +// are added, the underlying store is grown 32 bits at a time. +// NOTE: currently does not deallocate / shrink. +// +// Primary operations on the BitArray are: +// - set & clear dimension +// - test dimension +// - various performance or convenience operations to optimize bulk ops +// +// The underlying data structure uses TypedArrays for performance. +// +class BitArray { + constructor(length) { + // Initially allocate a 32 bit wide array. allocDimension() will expand + // as necessary. + // + // Int32Array is (counterintuitively) used to accomadate JS numeric casting + // (to/from primitive number type). + // + + // Fixed for the life of this object. + this.length = length; + + // Bitarray width. width is always greater than 32*dimensionCount. + this.width = 1; // underlying number of 32 bit arrays + this.dimensionCount = 0; // num allocated dimensions + + this.bitmask = new Int32Array(this.width); // dimension allocation mask + this.bitarray = new Int32Array(this.width * this.length); + } + + get selectionCount() { + return this.countAllOnes(); + } + + countAllOnes() { + let count = 0; + for (let i = 0; i < this.width; i++) { + const bitmask = this.bitmask[i]; + for (let j = i * this.length, len = j + this.length; j < len; j++) { + if (this.bitarray[i * this.length + j] === bitmask) count++; + } + } + return count; + } + + // count trailing zeros - hard to do fast in JS! + // https://en.wikipedia.org/wiki/Find_first_set#CTZ + static ctz(v) { + let c = 32; + v &= -v; // isolate lowest non-zero bit + if (v) c--; + if (v & 0x0000ffff) c -= 16; + if (v & 0x00ff00ff) c -= 8; + if (v & 0x0f0f0f0f) c -= 4; + if (v & 0x33333333) c -= 2; + if (v & 0x55555555) c -= 1; + return c; + } + + // find a free dimension. Return undefined if none + _findFreeDimension() { + let dim; + for (let col = 0; col < this.width; col++) { + const bitmask = this.bitmask[col]; + const lowestZeroBit = ~this.bitmask[col] & -~this.bitmask[col]; + if (lowestZeroBit) { + this.bitmask[col] |= lowestZeroBit; + dim = 32 * col + BitArray.ctz(lowestZeroBit); + } + } + return dim; + } + + // allocate and return the dimension ID (bit position) + // + allocDimension() { + let dim = this._findFreeDimension(); + + // if we did not find free dimension, expand the bitarray. + if (dim === undefined) { + this.width++; + + const biggerBitArray = new Int32Array(this.width * this.length); + biggerBitArray.set(this.bitarray); + this.bitarray = biggerBitArray; + + const biggerBitmask = new Int32Array(this.width); + biggerBitmask.set(this.bitmask); + this.bitmask = biggerBitmask; + + dim = this._findFreeDimension(); + } + + this.dimensionCount++; + return dim; + } + + // free a dimension for later use. MUST deselect the dimension, as other + // code assume the column will be zero valued. + // + freeDimension(dim) { + // all selection tests assume unallocated dimensions are zero valued. + this.deselectAll(dim); + const col = dim >>> 5; + this.bitmask[col] &= ~(1 << (dim % 32)); + this.dimensionCount--; + } + + // return true if this index is selected in ALL dimensions. + // + isSelected(index) { + const width = this.width; + const length = this.length; + const bitarray = this.bitarray; + + for (let w = 0; w < width; w++) { + const bitmask = this.bitmask[w]; + if (!bitmask || bitarray[w * length + index] !== bitmask) return false; + } + return true; + } + + // select index on dimension + // + selectOne(dim, index) { + const col = dim >>> 5; + const before = this.bitarray[col * this.length + index]; + const after = before | (1 << (dim % 32)); + this.bitarray[col] = after; + } + + // deselect index on dimension + // + deselectOne(dim, index) { + const col = dim >>> 5; + const before = this.bitarray[col * this.length + index]; + const after = before & ~(1 << (dim % 32)); + this.bitarray[col] = after; + } + + // select all indices on dimension. + // + selectAll(dim) { + let col = dim >> 5; + const bitmask = this.bitmask[col]; + const bitarray = this.bitarray; + const one = 1 << (dim % 32); + for (let i = col * this.length, len = i + this.length; i < len; i++) { + bitarray[i] |= one; + } + } + + // deselect all indices on dimension + // + deselectAll(dim) { + let col = dim >> 5; + const bitmask = this.bitmask[col]; + const bitarray = this.bitarray; + const zero = ~(1 << (dim % 32)); + for (let i = col * this.length, len = i + this.length; i < len; i++) { + bitarray[i] &= zero; + } + } + + // select range of indices on a dimension, indirect through a sort map. + // Indirect functions are used to map between sort and natural order. + // + selectIndirectFromRange(dim, indirect, range) { + const col = dim >>> 5; + const first = range[0]; + const last = range[1]; + const bitarray = this.bitarray; + const one = 1 << (dim % 32); + const offset = col * this.length; + for (let i = first; i < last; i++) { + bitarray[offset + indirect[i]] |= one; + } + } + + // deselect range of indices on a dimension, indirect through a sort map. + // + deselectIndirectFromRange(dim, indirect, range) { + const col = dim >>> 5; + const first = range[0]; + const last = range[1]; + const bitarray = this.bitarray; + const zero = ~(1 << (dim % 32)); + const offset = col * this.length; + for (let i = first; i < last; i++) { + bitarray[offset + indirect[i]] &= zero; + } + } + + // Fill the array with selected|deselected value based upon the + // current selection state. + // + fillBySelection(result, selectedValue, deselectedValue) { + // special case (width === 1) for performance + if (this.width === 1) { + const bitmask = this.bitmask[0]; + const bitarray = this.bitarray; + for (let i = 0, len = this.length; i < len; i++) { + result[i] = bitarray[i] === bitmask ? selectedValue : deselectedValue; + } + } else { + for (let i = 0, len = this.length; i < len; i++) { + result[i] = this.isSelected(i) ? selectedValue : deselectedValue; + } + } + return result; + } +} + +module.exports = BitArray; diff --git a/src/util/typedCrossfilter/index.js b/src/util/typedCrossfilter/index.js new file mode 100644 index 00000000..ae2d03e6 --- /dev/null +++ b/src/util/typedCrossfilter/index.js @@ -0,0 +1,394 @@ +"use strict"; +// jshint esversion: 6 + +/* +Typedarray Crossfilter - a re-implementation of a subset of crossfilter, with +time/space optimizations predicated upon the following assumptions: + - dimensions are uniformly typed, and all values must be of that type + - dimension values must be a primitive type (int, float, string). Arrays + or other complex types not supported. + - dimension creation requires call-provided type declaration + - no support for adding/removing data to an existing crossfilter. If you + want to do that, you have to create the new crossfilter, using the new + data, from scratch. + +The actual backing store for a dimension is a TypedArray, enabling significant +performance improvements over the original crossfilter. + +There are also a handful of new methods, primarily to take advantage of the +performance (eg, crossfilter.fillBySelection) + +Helpful documents (this module tries to follow the original API as much +as is feasable): + https://github.com/square/crossfilter/ + http://square.github.io/crossfilter/ + +There is also a newer, community supported fork of crossfilter, with a +more complex API. In a few cases, elements of that API were incorporated. + https://github.com/square/crossfilter/ + +*/ + +var PositiveIntervals = require("./positiveIntervals"); +var BitArray = require("./bitArray"); +var Util = require("./util"); + +class TypedCrossfilter { + constructor(data) { + this.data = data; + + // filters: array of { id, dimension } + this.filters = []; + this.selection = new BitArray(data.length); + } + + size() { + return this.data.length; + } + + all() { + return this.data; + } + + dimension(value, valueArrayType) { + const id = this.selection.allocDimension(); + let dim; + if (valueArrayType === "enum") { + dim = new EnumDimension(value, this, id); + } else { + dim = new ScalarDimension(value, valueArrayType, this, id); + } + this.filters.push({ id, dim }); + dim.filterAll(); + return dim; + } + + _freeDimension(id) { + this.selection.freeDimension(id); + this.filters = this.filters.filter(f => f.id != id); + } + + // return array of all records that are selected/filtered + // by all dimensions. + allFiltered() { + const selection = this.selection; + const res = []; + for (let i = 0, len = this.data.length; i < len; i++) { + if (selection.isSelected(i)) { + res.push(this.data[i]); + } + } + return res; + } + + countFiltered() { + return this.selection.selectionCount; + } + + isElementFiltered(i) { + return this.selection.isSelected(i); + } + + // fill array with one of two values, based upon selection state + fillByIsFiltered(array, selectedValue, deselectedValue) { + return this.selection.fillBySelection( + array, + selectedValue, + deselectedValue + ); + } +} + +// Base dimension type - value must be a scalar type (eg, int, float), +// and value array must be a TypedArray. +// +class ScalarDimension { + constructor(value, valueArrayType, crossfilter, id) { + this.crossfilter = crossfilter; + this.id = id; + + // current selection filter, expressed as PostiveIntervals. + this.currentFilter = []; + + // Create value array + const array = this._createValueArray( + value, + new valueArrayType(this.crossfilter.data.length) + ); + this.value = array; + + // create sort index + this.index = Util.fillRange(new Uint32Array(this.crossfilter.data.length)); + this.index.sort((a, b) => array[a] - array[b]); + } + + _createValueArray(value, array) { + // create dimension value array + const data = this.crossfilter.data; + const len = data.length; + for (let i = 0; i < len; i++) { + array[i] = value(data[i]); + } + return array; + } + + dispose() { + this.crossfilter._freeDimension(this.id); + } + + id() { + return this.id; + } + + _updateFilters(newFilter) { + newFilter = PositiveIntervals.canonicalize(newFilter); + + // special case optimization - select all/none can bypass + // more complex work and just clobber everything. + // + if (newFilter.length === 0) { + this.crossfilter.selection.deselectAll(this.id); + } else if ( + newFilter.length === 1 && + newFilter[0][0] === 0 && + newFilter[0][1] == this.index.length + ) { + this.crossfilter.selection.selectAll(this.id); + } else { + const adds = PositiveIntervals.difference(newFilter, this.currentFilter); + const dels = PositiveIntervals.difference(this.currentFilter, newFilter); + dels.forEach(interval => + this.crossfilter.selection.deselectIndirectFromRange( + this.id, + this.index, + interval + ) + ); + adds.forEach(interval => + this.crossfilter.selection.selectIndirectFromRange( + this.id, + this.index, + interval + ) + ); + } + + this.currentFilter = newFilter; + } + + // filter by value - exact match + filterExact(value) { + const newFilter = [ + Util.lowerBoundIndirect( + this.value, + this.index, + value, + 0, + this.value.length + ), + Util.upperBoundIndirect( + this.value, + this.index, + value, + 0, + this.value.length + ) + ]; + if (newFilter[0] <= newFilter[1]) { + this._updateFilters([newFilter]); + } else { + this._updateFilters([]); + } + return this; + } + + // filter by a set of values, eg. enum. + filterEnum(values) { + const newFilter = []; + for (let v = 0, len = values.length; v < len; v++) { + const intv = [ + Util.lowerBoundIndirect( + this.value, + this.index, + values[v], + 0, + this.value.length + ), + Util.upperBoundIndirect( + this.value, + this.index, + values[v], + 0, + this.value.length + ) + ]; + if (intv[0] <= intv[1]) newFilter.push(intv); + } + this._updateFilters(newFilter); + return this; + } + + // filter by value range [lo, hi) + // lo: inclusive, hi: exclusive + filterRange(range) { + const newFilter = []; + const intv = [ + Util.lowerBoundIndirect( + this.value, + this.index, + range[0], + 0, + this.value.length + ), + Util.upperBoundIndirect( + this.value, + this.index, + range[1], + 0, + this.value.length + ) + ]; + if (intv[0] < intv[1]) newFilter.push(intv); + this._updateFilters(newFilter); + return this; + } + + // select all - equivalent of selecting all in this dimension + filterAll() { + this._updateFilters([[0, this.value.length]]); + return this; + } + + // select none + filterNone() { + this._updateFilters([]); + } + + // return top k records, starting with offset, in descending order. + // Order is this dimension's sort order + top(k, offset = 0) { + const data = this.crossfilter.data; + const selection = this.crossfilter.selection; + const index = this.index; + const len = index.length; + const ret = []; + let i = 0; + let skip = 0; + let found = 0; + + // skip up to offset records + for (i = len - 1; 0 <= i && skip < offset; i--) { + if (selection.isSelected(index[i])) { + skip++; + } + } + + // grab up to k records + for (; 0 <= i && found < k; i--) { + if (selection.isSelected(index[i])) { + ret.push(data[index[i]]); + found++; + } + } + + return ret; + } + + // return bottom k records, starting with offset, in ascending order. + // Order is this dimension's sort order + bottom(k, offset = 0) { + const data = this.crossfilter.data; + const selection = this.crossfilter.selection; + const index = this.index; + const len = index.length; + const ret = []; + let skip = 0; + let found = 0; + let i = 0; + + // skip up to offset records + for (i = 0; i < len && skip < offset; i++) { + if (selection.isSelected(index[i])) { + skip++; + } + } + + // grab up to k records + for (; i < len && found < k; i++) { + if (selection.isSelected(index[i])) { + ret.push(data[index[i]]); + found++; + } + } + + return ret; + } +} + +// Ordered enumeration - supports any sortable enumerable type, eg, +// strings, which can be mapped into an fixed numeric range [0..n). +// +class EnumDimension extends ScalarDimension { + constructor(value, crossfilter, id) { + super(value, Uint32Array, crossfilter, id); + } + + _createValueArray(value, array) { + const data = this.crossfilter.data; + const len = data.length; + + // create enumeration table - mapping between the value + // and the enum. + const s = new Set(); + for (let i = 0; i < len; i++) { + s.add(value(data[i])); + } + this.enumIndex = Array.from(s); + this.enumIndex.sort(); + + // create dimension value array + const enumLen = this.enumIndex.length; + for (let i = 0; i < len; i++) { + const v = value(data[i]); + const e = Util.lowerBound(this.enumIndex, v, 0, enumLen); + array[i] = e; + } + return array; + } + + filterExact(value) { + return super.filterExact( + Util.lowerBound(this.enumIndex, value, 0, this.enumIndex.length) + ); + } + + filterEnum(values) { + return super.filterEnum( + values.map(v => + Util.lowerBound(this.enumIndex, v, 0, this.enumIndex.length) + ) + ); + } + + filterRange(range) { + return super.filterEnum( + range.map(v => + Util.lowerBound(this.enumIndex, v, 0, this.enumIndex.length) + ) + ); + } +} + +// Wrapper for backwards compat with crossfilter. +// +function crossfilter(data) { + return new TypedCrossfilter(data); +} + +crossfilter.PositiveIntervals = PositiveIntervals; +crossfilter.BitArray = BitArray; +crossfilter.TypedCrossfilter = TypedCrossfilter; +crossfilter.ScalarDimension = ScalarDimension; +crossfilter.EnumDimension = EnumDimension; + +module.exports = crossfilter; diff --git a/src/util/typedCrossfilter/positiveIntervals.js b/src/util/typedCrossfilter/positiveIntervals.js new file mode 100644 index 00000000..6e5d074a --- /dev/null +++ b/src/util/typedCrossfilter/positiveIntervals.js @@ -0,0 +1,132 @@ +"use strict"; +// jshint esversion: 6 + +// Interval operations - very simple version of interval set relationship +// operators. An interval is a multi-interval list of [min, max), +// where min and max are mandatory. Constraints: +// * min <= max, min >= 0 +// * empty interval groups are OK, ie, [] +// * Legal intervals: [], [ [0, 1], ... ] +// * Not legal: [ [] ] +// +// All intervals are represented by simple JS arrays/numbers. +// +// Code assumes intervals have a low cardinality; many operations are done +// with a brute force scan. Little attempt to reduce GC pressure. +// +class PositiveIntervals { + // Canonicalize - ensure that: + // 1. no overlapping intervals + // 2. sorted in order of interval min. + // + static canonicalize(A) { + if (A.length <= 1) return A; + let copy = A.slice(); + copy.sort((a, b) => a[0] - b[0]); + const res = []; + res.push(copy[0]); + for (let i = 1, len = copy.length; i < len; i++) { + if (copy[i][0] > res[res.length - 1][1]) { + // non-overlapping, add to result + res.push(copy[i]); + } else if (copy[i][1] > res[res.length - 1][1]) { + // merge this into previous + res[res.length - 1][1] = copy[i][1]; + } + } + return res; + } + + // Return interval with values belonging to both A and B. Essentially + // a set union operation. + // + static union(A, B) { + return PositiveIntervals.canonicalize([...A, ...B]); + } + + static _flatten(A, B) { + let points = []; /* point, A, start */ + for (let a = 0; a < A.length; a++) { + points.push([A[a][0], true, true]); + points.push([A[a][1], true, false]); + } + for (let b = 0; b < B.length; b++) { + points.push([B[b][0], false, true]); + points.push([B[b][1], false, false]); + } + // Sort order: point, then start + points.sort((a, b) => (a[0] !== b[0] ? a[0] - b[0] : a[2] ? 1 : -1)); + return points; + } + + // A - B, ie, the interval with all values in A that are not in B. Essentially + // a set difference operation. + // + static difference(A, B) { + // Corner cases + if (A.length === 0 || B.length === 0) { + return PositiveIntervals.canonicalize(A); + } + + A = PositiveIntervals.canonicalize(A); + B = PositiveIntervals.canonicalize(B); + + const points = PositiveIntervals._flatten(A, B); + const res = []; + let aDepth = 0; + let depth = 0; + let intervalStart; + let prevPoint; + for (let i = 0; i < points.length; i++) { + const p = points[i]; + const before = depth; + const delta = p[2] ? 1 : -1; + depth += delta; + if (p[1]) aDepth += delta; + + if (i === points.length - 1 || p[0] !== points[i + 1][0]) { + if (aDepth === 1 && depth === 1) { + intervalStart = p[0]; + } else if (intervalStart !== undefined) { + res.push([intervalStart, p[0]]); + intervalStart = undefined; + } + } + prevPoint = p[0]; + } + // guaranteed to be in canonical form + return res; + } + + // Return interval with values belonging to A or B. Essentially a set + // intersection. + // + static intersection(A, B) { + if (A.length === 0 || B.length === 0) { + return []; + } + + A = PositiveIntervals.canonicalize(A); + B = PositiveIntervals.canonicalize(B); + + const points = PositiveIntervals._flatten(A, B); + const res = []; + let depth = 0; + let intervalStart; + for (let i = 0; i < points.length; i++) { + const p = points[i]; + const before = depth; + depth += p[2] ? 1 : -1; + if (depth === 2) { + intervalStart = p[0]; + } else if (intervalStart !== undefined) { + res.push([intervalStart, p[0]]); + intervalStart = undefined; + } + } + // guaranteed to be in canonical form + return res; + } +} + +module.exports = PositiveIntervals; diff --git a/src/util/typedCrossfilter/util.js b/src/util/typedCrossfilter/util.js new file mode 100644 index 00000000..deffcdec --- /dev/null +++ b/src/util/typedCrossfilter/util.js @@ -0,0 +1,106 @@ +"use strict"; +// jshint esversion: 6 + +/* + Utility functions, private to this module. +*/ + +// fill an array or typedarray with a sequential range of numbers, +// starting with `start` +// +function fillRange(arr, start = 0) { + for (let i = 0, len = arr.length; i < len; i++) { + arr[i] = i + start; + } + return arr; +} + +// Search for `value` in the sorted array `arr`, in the range [first, last). +// Return the first (left most) index where arr[index] >= value. +// +// In other words, return array index I where: +// arr[i] < value for all tarr[lo:I] +// arr[i] >= value for all tarr[I:last] +// +// The same semantics/behavior as: +// C++: lower_bound() +// Python: bisect.bisect_left() +// +// XXX: it is likely that there would be minimal performance hit from creating +// a factory version of lowerBound that takes an accessor (rather than having +// a special-cased version for lining the indirection). +// +function lowerBound(valueArray, value, first, last) { + // this is just a binary search + while (first < last) { + const middle = (first + last) >>> 1; + if (valueArray[middle] < value) { + first = middle + 1; + } else { + last = middle; + } + } + return first; +} + +// Inlined performance optimization - used to indirect through a sort map. +// +function lowerBoundIndirect(valueArray, indexArray, value, first, last) { + // this is just a binary search + while (first < last) { + const middle = (first + last) >>> 1; + if (valueArray[indexArray[middle]] < value) { + first = middle + 1; + } else { + last = middle; + } + } + return first; +} + +// Search for `value in the sorted array `arr`, in the range [first, last). +// Return the first value where arr[index] > value. +// +// In other words, return array index I, where: +// arr[i] <= value for all tarr[lo:I] +// arr[i] > value for all tarr[I:last] +// +// The same semantics/behavior as: +// C++: upper_bound() +// Python: bisect.bisect_right() +// +function upperBound(valueArray, value, first, last) { + // this is just a binary search + while (first < last) { + const middle = (first + last) >>> 1; + if (valueArray[middle] > value) { + last = middle; + } else { + first = middle + 1; + } + } + return first; +} + +// Inline performance optimization +// +function upperBoundIndirect(valueArray, indexArray, value, first, last) { + // this is just a binary search + while (first < last) { + const middle = (first + last) >>> 1; + if (valueArray[indexArray[middle]] > value) { + last = middle; + } else { + first = middle + 1; + } + } + return first; +} + +module.exports = { + fillRange, + lowerBound, + lowerBoundIndirect, + upperBound, + upperBoundIndirect +}; From cfd13cad4189a7f7a345aa4e0837448c46c9d2f3 Mon Sep 17 00:00:00 2001 From: bkmartinjr Date: Sun, 27 May 2018 11:49:50 -0700 Subject: [PATCH 5/8] refactor regraph functionality to use crossfilter; add reset graph --- src/actions/index.js | 7 + src/components/continuous/histogramBrush.js | 4 +- src/components/continuous/parallel.js | 6 +- src/components/graph/graph.js | 22 ++- src/middleware/updateCellColors.js | 26 ++- .../updateCellSelectionMiddleware.js | 17 +- src/reducers/controls.js | 187 ++++++++++-------- 7 files changed, 160 insertions(+), 109 deletions(-) diff --git a/src/actions/index.js b/src/actions/index.js index fc6cc398..7d9b7002 100644 --- a/src/actions/index.js +++ b/src/actions/index.js @@ -65,6 +65,12 @@ const regraph = () => { }; }; +const resetGraph = () => { + return (dispatch, getState) => { + dispatch({ type: "reset graph" }); + }; +}; + const initialize = () => { return (dispatch, getState) => { dispatch({ type: "initialize started" }); @@ -218,6 +224,7 @@ export default { initialize, requestCells, regraph, + resetGraph, requestGeneExpressionCounts, requestGeneExpressionCountsPOST, requestSingleGeneExpressionCountsForColoringPOST, diff --git a/src/components/continuous/histogramBrush.js b/src/components/continuous/histogramBrush.js index fb4bd750..1b67e687 100644 --- a/src/components/continuous/histogramBrush.js +++ b/src/components/continuous/histogramBrush.js @@ -26,7 +26,7 @@ import { connect } from "react-redux"; return { colorAccessor: state.controls.colorAccessor, colorScale: state.controls.colorScale, - allCellsMetadata: state.controls.allCellsMetadata + cellsMetadata: state.controls.cellsMetadata }; }) class HistogramBrush extends React.Component { @@ -52,7 +52,7 @@ class HistogramBrush extends React.Component { calcHistogramCache(nextProps) { // recalculate expensive stuff const allValuesForContinuousFieldAsArray = _.map( - nextProps.allCellsMetadata, + nextProps.cellsMetadata, nextProps.metadataField ); diff --git a/src/components/continuous/parallel.js b/src/components/continuous/parallel.js index 5ffa3463..0f53069e 100644 --- a/src/components/continuous/parallel.js +++ b/src/components/continuous/parallel.js @@ -36,7 +36,7 @@ import { margin, width, height, createDimensions } from "./util"; colorAccessor: state.controls.colorAccessor, colorScale: state.controls.colorScale, graphBrushSelection: state.controls.graphBrushSelection, - allCellsMetadata: state.controls.allCellsMetadata, + cellsMetadata: state.controls.cellsMetadata, axesHaveBeenDrawn: state.controls.axesHaveBeenDrawn }; }) @@ -96,7 +96,7 @@ class Parallel extends React.Component { /* https://stackoverflow.com/questions/23123138/perform-debounce-in-react-js */ if ( nextProps.ranges && - nextProps.allCellsMetadata && + nextProps.cellsMetadata && nextProps.axesHaveBeenDrawn ) { if (this.state._drawLinesCanvas) { @@ -106,7 +106,7 @@ class Parallel extends React.Component { this.state.ctx.clearRect(0, 0, width, height); const _drawLinesCanvas = drawLinesCanvas( - nextProps.allCellsMetadata, + nextProps.cellsMetadata, this.state.dimensions, this.state.xscale, this.state.ctx, diff --git a/src/components/graph/graph.js b/src/components/graph/graph.js index 8ca3596c..4156f935 100644 --- a/src/components/graph/graph.js +++ b/src/components/graph/graph.js @@ -23,7 +23,7 @@ import FaSave from "react-icons/lib/fa/download"; @connect(state => { return { - allCellsMetadata: state.controls.allCellsMetadata, + cellsMetadata: state.controls.cellsMetadata, opacityForDeselectedCells: state.controls.opacityForDeselectedCells, responsive: state.responsive, crossfilter: state.controls.crossfilter @@ -131,7 +131,7 @@ class Graph extends React.Component { // we could add some sort of color-specific indicator to the app state. if ( !this.renderCache.colors || - this.props.allCellsMetadata != nextProps.allCellsMetadata + this.props.cellsMetadata != nextProps.cellsMetadata ) { if (!this.renderCache.colors) this.renderCache.colors = new Float32Array(3 * cellCount); @@ -244,6 +244,24 @@ class Graph extends React.Component { alignItems: "baseline" }} > +