first cut at high performance crossfilter

src/actions/index.js

@@ -89,7 +89,7 @@ const initialize = () => {
 //
 function cleanupExpressionResponse(data) {
   const s = store.getState();
-  const metadata = s.controls.currentCellSelectionMap;
+  const metadata = s.controls.allCellsMetadataMap;
   let errorFound = false;
   data.data.cells = _.filter(data.data.cells, cell => {
     if (!errorFound && !metadata[cell.cellname]) {

src/components/continuous/continuous.js

@@ -38,7 +38,6 @@ import { margin, width, height, createDimensions } from "./util";
     colorAccessor: state.controls.colorAccessor,
     colorScale: state.controls.colorScale,
     graphBrushSelection: state.controls.graphBrushSelection,
-    currentCellSelection: state.controls.currentCellSelection,
     axesHaveBeenDrawn: state.controls.axesHaveBeenDrawn
   };
 })

src/components/continuous/histogramBrush.js

@@ -26,7 +26,7 @@ import { connect } from "react-redux";
   return {
     colorAccessor: state.controls.colorAccessor,
     colorScale: state.controls.colorScale,
-    currentCellSelection: state.controls.currentCellSelection
+    allCellsMetadata: state.controls.allCellsMetadata
   };
 })
 class HistogramBrush extends React.Component {
@@ -52,7 +52,7 @@ class HistogramBrush extends React.Component {
   calcHistogramCache(nextProps) {
     // recalculate expensive stuff
     const allValuesForContinuousFieldAsArray = _.map(
-      nextProps.currentCellSelection,
+      nextProps.allCellsMetadata,
       nextProps.metadataField
     );
 

src/components/continuous/parallel.js

@@ -36,7 +36,7 @@ import { margin, width, height, createDimensions } from "./util";
     colorAccessor: state.controls.colorAccessor,
     colorScale: state.controls.colorScale,
     graphBrushSelection: state.controls.graphBrushSelection,
-    currentCellSelection: state.controls.currentCellSelection,
+    allCellsMetadata: state.controls.allCellsMetadata,
     axesHaveBeenDrawn: state.controls.axesHaveBeenDrawn
   };
 })
@@ -96,7 +96,7 @@ class Parallel extends React.Component {
     /* https://stackoverflow.com/questions/23123138/perform-debounce-in-react-js */
     if (
       nextProps.ranges &&
-      nextProps.currentCellSelection &&
+      nextProps.allCellsMetadata &&
       nextProps.axesHaveBeenDrawn
     ) {
       if (this.state._drawLinesCanvas) {
@@ -106,7 +106,7 @@ class Parallel extends React.Component {
       this.state.ctx.clearRect(0, 0, width, height);
 
       const _drawLinesCanvas = drawLinesCanvas(
-        nextProps.currentCellSelection,
+        nextProps.allCellsMetadata,
         this.state.dimensions,
         this.state.xscale,
         this.state.ctx,

src/components/expression/cellSetButtons.js

@@ -9,13 +9,7 @@ import actions from "../../actions";
 @connect()
 class CellSetButton extends React.Component {
   set() {
-    const set = [];
+    const set = _.map(this.props.crossfilter.cells.allFiltered(), "CellName");
-
-    _.each(this.props.currentCellSelection, cell => {
-      if (cell["__selected__"]) {
-        set.push(cell.CellName);
-      }
-    });
 
     this.props.dispatch({
       type:

src/components/expression/expressionButtons.js

@@ -8,8 +8,8 @@ import CellSetButton from "./cellSetButtons";
 
 @connect(state => {
   return {
-    currentCellSelection: state.controls.currentCellSelection,
+    differential: state.differential,
-    differential: state.differential
+    crossfilter: state.controls.crossfilter
   };
 })
 class Expression extends React.Component {
@@ -43,7 +43,9 @@ class Expression extends React.Component {
           <div style={{ marginBottom: 15, width: 300 }}>
             There are currently
             {" " +
-              _.filter(this.props.currentCellSelection, "__selected__").length +
+              (this.props.crossfilter
+                ? this.props.crossfilter.cells.countFiltered()
+                : 0) +
               " "}
             cells selected, click a cell set button to store them.
           </div>

src/components/graph/graph.js

@@ -22,31 +22,11 @@ import FaSave from "react-icons/lib/fa/download";
 /* https://bl.ocks.org/mbostock/9078690 - quadtree for onClick / hover selections */
 
 @connect(state => {
-  const vertices =
-    state.cells.cells && state.cells.cells.data.graph
-      ? state.cells.cells.data.graph
-      : null;
-  const ranges =
-    state.cells.cells && state.cells.cells.data.ranges
-      ? state.cells.cells.data.ranges
-      : null;
-  const metadata =
-    state.cells.cells && state.cells.cells.data.metadata
-      ? state.cells.cells.data.metadata
-      : null;
-
   return {
-    ranges,
+    allCellsMetadata: state.controls.allCellsMetadata,
-    vertices,
-    metadata,
-    colorAccessor: state.controls.colorAccessor,
-    colorScale: state.controls.colorScale,
-    continuousSelection: state.controls.continuousSelection,
-    graphVec: state.controls.graphVec,
-    currentCellSelection: state.controls.currentCellSelection,
-    graphBrushSelection: state.controls.graphBrushSelection,
     opacityForDeselectedCells: state.controls.opacityForDeselectedCells,
-    responsive: state.responsive
+    responsive: state.responsive,
+    crossfilter: state.controls.crossfilter
   };
 })
 class Graph extends React.Component {
@@ -55,6 +35,10 @@ class Graph extends React.Component {
     this.count = 0;
     this.inverse = mat4.identity([]);
     this.graphPaddingTop = 100;
+    this.renderCache = {
+      positions: null,
+      colors: null
+    };
     this.state = {
       drawn: false,
       svg: null,
@@ -104,42 +88,60 @@ class Graph extends React.Component {
   }
 
   componentWillReceiveProps(nextProps) {
-    if (this.state.regl && nextProps.vertices) {
+    if (this.state.regl && nextProps.crossfilter) {
-      /* update regl */
+      /* update the regl state */
-      const vertices = nextProps.currentCellSelection;
+      const crossfilter = nextProps.crossfilter.cells;
-      const vertexCount = vertices.length;
+      const cells = crossfilter.all();
-      const positions = new Float32Array(2 * vertexCount);
+      const cellCount = cells.length;
-      const colors = new Float32Array(3 * vertexCount);
-      const sizes = new Float32Array(vertexCount);
 
-      // d3.scaleLinear().domain([0,1]).range([-1,1])
+      // X/Y positions for each point - a cached value that only
-      const glScaleX = scaleLinear([0, 1], [-1, 1]);
+      // changes if we have loaded entirely new cell data
-      // d3.scaleLinear().domain([0,1]).range([1,-1])
+      //
-      const glScaleY = scaleLinear([0, 1], [1, -1]);
+      if (
+        !this.renderCache.positions ||
+        this.props.crossfilter.cells != nextProps.crossfilter.cells
+      ) {
+        const positions = new Float32Array(2 * cellCount);
 
-      /*
+        // d3.scaleLinear().domain([0,1]).range([-1,1])
-        Construct Vectors
+        const glScaleX = scaleLinear([0, 1], [-1, 1]);
-      */
+        // d3.scaleLinear().domain([0,1]).range([1,-1])
-      const graphVec = nextProps.graphVec;
+        const glScaleY = scaleLinear([0, 1], [1, -1]);
-      for (var i = 0; i < vertexCount; i++) {
-        const cell = vertices[i];
-        const cellIdx = cell.__cellIndex__;
-        const x = glScaleX(graphVec[2 * cellIdx]);
-        const y = glScaleY(graphVec[2 * cellIdx + 1]);
-        positions[2 * i] = x;
-        positions[2 * i + 1] = y;
 
-        colors.set(cell.__colorRGB__, 3 * i);
+        for (let i = 0; i < cellCount; i++) {
-
+          positions[2 * i] = glScaleX(cells[i].__x__);
-        sizes[i] = cell.__selected__
+          positions[2 * i + 1] = glScaleY(cells[i].__y__);
-          ? 4
+        }
-          : 0.2; /* make this a function of the number of total cells, including regraph */
+        this.renderCache.positions = positions;
       }
 
-      this.state.pointBuffer({ data: positions, dimension: 2 });
+      // Colors for each point - a cached value that only changes when
-      this.state.colorBuffer({ data: colors, dimension: 3 });
+      // the cell metadata changes (done by updateCellColors middleware).
+      // NOTE: this is a slightly pessimistic assumption, as the metadata
+      // could have changed for some other reason, but for now color is
+      // the only metadata that changes client-side.  If this is problematic,
+      // we could add some sort of color-specific indicator to the app state.
+      if (
+        !this.renderCache.colors ||
+        this.props.allCellsMetadata != nextProps.allCellsMetadata
+      ) {
+        const colors = new Float32Array(3 * cellCount);
+        for (let i = 0; i < cellCount; i++) {
+          colors.set(cells[i].__colorRGB__, 3 * i);
+        }
+        this.renderCache.colors = colors;
+      }
+
+      const sizes = new Float32Array(cellCount);
+      crossfilter.fillByIsFiltered(sizes, 4, 0.2);
+
+      this.state.pointBuffer({
+        data: this.renderCache.positions,
+        dimension: 2
+      });
+      this.state.colorBuffer({ data: this.renderCache.colors, dimension: 3 });
       this.state.sizeBuffer({ data: sizes, dimension: 1 });
-      this.count = vertexCount;
+      this.count = cellCount;
     }
 
     if (

src/components/scatterplot/scatterplot.js

@@ -39,11 +39,10 @@ import { margin, width, height, createDimensions } from "./util";
     initializeRanges,
     colorAccessor: state.controls.colorAccessor,
     colorScale: state.controls.colorScale,
-    currentCellSelection: state.controls.currentCellSelection,
-    currentCellSelectionMap: state.controls.currentCellSelectionMap,
     scatterplotXXaccessor: state.controls.scatterplotXXaccessor,
     scatterplotYYaccessor: state.controls.scatterplotYYaccessor,
     opacityForDeselectedCells: state.controls.opacityForDeselectedCells,
+    crossfilter: state.controls.crossfilter,
     differential: state.differential,
     expression: state.expression
   };
@@ -124,7 +123,6 @@ class Scatterplot extends React.Component {
       this.state.pointBuffer &&
       this.state.colorBuffer &&
       this.state.sizeBuffer &&
-      this.props.currentCellSelection &&
       this.props.expression.data &&
       this.props.expression.data.genes &&
       this.props.scatterplotXXaccessor &&
@@ -132,8 +130,7 @@ class Scatterplot extends React.Component {
       this.state.xScale &&
       this.state.yScale
     ) {
-      const currentCellSelectionMap = this.props.currentCellSelectionMap;
+      const crossfilter = this.props.crossfilter.cells;
-
       const data = this.props.expression.data;
       const cells = data.cells;
       const genes = data.genes;
@@ -158,9 +155,8 @@ class Scatterplot extends React.Component {
       /*
         Construct Vectors
       */
-      for (var i = 0; i < cellCount; i++) {
+      for (let i = 0; i < cellCount; i++) {
         const cell = cells[i];
-        const cellMetadata = currentCellSelectionMap[cell.cellname];
 
         positions[2 * i] = glScaleX(
           this.state.xScale(cell.e[geneXXaccessorIndex])
@@ -168,14 +164,15 @@ class Scatterplot extends React.Component {
         positions[2 * i + 1] = glScaleY(
           this.state.yScale(cell.e[geneYYaccessorIndex])
         );
-
-        colors.set(cellMetadata.__colorRGB__, 3 * i);
-
-        sizes[i] = cellMetadata.__selected__
-          ? 4
-          : 0.2; /* make this a function of the number of total cells, including regraph */
       }
 
+      for (let i = 0; i < cellCount; i++) {
+        const metadata = this.props.metadata[i];
+        colors.set(metadata.__colorRGB__, 3 * i);
+      }
+
+      crossfilter.fillByIsFiltered(sizes, 4, 0.2);
+
       this.state.pointBuffer({ data: positions, dimension: 2 });
       this.state.colorBuffer({ data: colors, dimension: 3 });
       this.state.sizeBuffer({ data: sizes, dimension: 1 });

src/middleware/updateCellColors.js

@@ -20,7 +20,7 @@ import { parseRGB } from "../util/parseRGB";
   This is nice because we keep a lot of filtering business logic centralized (what it means in practice to be selected)
 */
 
-const updateCellSelectionMiddleware = store => {
+const updateCellColorsMiddleware = store => {
   return next => {
     return action => {
       const s = store.getState();
@@ -37,7 +37,7 @@ const updateCellSelectionMiddleware = store => {
         ); /* if the cells haven't loaded or the action wasn't a color change, bail */
       }
 
-      let currentSelectionWithUpdatedColors = s.controls.currentCellSelection.slice(
+      let allCellsMetadataWithUpdatedColors = s.controls.allCellsMetadata.slice(
         0
       );
       let colorScale;
@@ -46,7 +46,7 @@ const updateCellSelectionMiddleware = store => {
          in plain language...
 
          (a) once the cells have loaded.
-         (b) each time a user changes a color control we need to update currentCellSelection colors
+         (b) each time a user changes a color control we need to update allCellsMetadata colors
 
          This is available to all the draw functions as cell["__color__"] and cell["__colorRGB__"]
       */
@@ -54,8 +54,8 @@ const updateCellSelectionMiddleware = store => {
       if (action.type === "color by categorical metadata") {
         colorScale = d3.scaleOrdinal().range(globals.ordinalColors);
 
-        for (let i = 0; i < currentSelectionWithUpdatedColors.length; i++) {
+        for (let i = 0; i < allCellsMetadataWithUpdatedColors.length; i++) {
-          const cell = currentSelectionWithUpdatedColors[i];
+          const cell = allCellsMetadataWithUpdatedColors[i];
           let c = colorScale(cell[action.colorAccessor]);
           cell.__color__ = c;
           cell.__colorRGB__ = parseRGB(c);
@@ -68,10 +68,10 @@ const updateCellSelectionMiddleware = store => {
           .domain([0, action.rangeMaxForColorAccessor])
           .range([1, 0]);
 
-        _.each(currentSelectionWithUpdatedColors, (cell, i) => {
+        _.each(allCellsMetadataWithUpdatedColors, (cell, i) => {
           let c = d3.interpolateViridis(colorScale(cell[action.colorAccessor]));
-          currentSelectionWithUpdatedColors[i]["__color__"] = c;
+          allCellsMetadataWithUpdatedColors[i]["__color__"] = c;
-          currentSelectionWithUpdatedColors[i]["__colorRGB__"] = parseRGB(c);
+          allCellsMetadataWithUpdatedColors[i]["__colorRGB__"] = parseRGB(c);
         });
       }
 
@@ -113,12 +113,12 @@ const updateCellSelectionMiddleware = store => {
             0
           ]); /* invert viridis... probably pass this scale through to others */
 
-        _.each(currentSelectionWithUpdatedColors, (cell, i) => {
+        _.each(allCellsMetadataWithUpdatedColors, (cell, i) => {
           let c = d3.interpolateViridis(
             colorScale(expressionMap[cell.CellName][indexOfGene])
           );
-          currentSelectionWithUpdatedColors[i]["__color__"] = c;
+          allCellsMetadataWithUpdatedColors[i]["__color__"] = c;
-          currentSelectionWithUpdatedColors[i]["__colorRGB__"] = parseRGB(c);
+          allCellsMetadataWithUpdatedColors[i]["__colorRGB__"] = parseRGB(c);
         });
       }
 
@@ -126,7 +126,7 @@ const updateCellSelectionMiddleware = store => {
         append the result of all the filters to the action the user just triggered
       */
       let modifiedAction = Object.assign({}, action, {
-        currentSelectionWithUpdatedColors,
+        allCellsMetadataWithUpdatedColors,
         colorScale
       });
 
@@ -135,4 +135,4 @@ const updateCellSelectionMiddleware = store => {
   };
 };
 
-export default updateCellSelectionMiddleware;
+export default updateCellColorsMiddleware;

src/middleware/updateCellSelectionMiddleware.js

@@ -48,7 +48,7 @@ const updateCellSelectionMiddleware = store => {
         - make a FRESH copy of all of the cells
         - metadata has cellname and index, and that's all we ever need to reference cell info
       */
-      let newSelection = s.controls.currentCellSelection.slice(0);
+      let newSelection = s.controls.allCellsMetadata.slice(0);
       // _.forEach(newSelection, cell => (cell.__selected__ = true));
       for (let i = 0; i < newSelection.length; i++) {
         newSelection[i].__selected__ = true;
@@ -58,7 +58,7 @@ const updateCellSelectionMiddleware = store => {
          in plain language...
 
          (a) once the cells have loaded.
-         (b) each time a user changes ANY control we need to update currentCellSelection
+         (b) each time a user changes ANY control we need to update allCellsMetadata
          there are two states:
 
          1. control state we already know about (state.foo)
@@ -66,34 +66,6 @@ const updateCellSelectionMiddleware = store => {
 
       */
 
-      if (
-        /* is there a 2d graph brush selection ? */
-        action.type === "graph brush selection change" ||
-        s.controls.graphBrushSelection
-      ) {
-        const graphBrushSelection /* it exists, so is it new or old */ =
-          action.type === "graph brush selection change"
-            ? action.brushCoords
-            : s.controls.graphBrushSelection;
-
-        const graphVec = s.controls.graphVec;
-        for (let i = 0; i < newSelection.length; i++) {
-          const cell = newSelection[i];
-          const cellId = cell.__cellIndex__;
-          const x = graphVec[2 * cellId];
-          const y = graphVec[2 * cellId + 1];
-
-          const pointIsInsideBrushBounds =
-            x >= graphBrushSelection.northwest[0] &&
-            x <= graphBrushSelection.southeast[0] &&
-            y <= graphBrushSelection.northwest[1] &&
-            y >= graphBrushSelection.southeast[1];
-
-          if (!pointIsInsideBrushBounds) {
-            cell.__selected__ = false;
-          }
-        }
-      }
       if (
         (action.type ===
           "continuous selection using parallel coords brushing" &&
@@ -118,139 +90,8 @@ const updateCellSelectionMiddleware = store => {
         });
       }
 
-      /*
-        Continuous histograms ___---^^^^--[------__]__---___
-
-        Create newContinuousUserDefinedRanges
-        Filter based on them
-      */
-
-      let newContinuousUserDefinedRanges =
-        s.controls.continuousUserDefinedRanges;
-
-      /* check if this is the action and take care of that metadata field */
-      if (action.type === "continuous metadata histogram brush") {
-        /*
-          was this a deselect? if so it will be null
-          was it a select? set the new range [20, 50]
-          we overload this because it is less if statements thru the whole system
-          but it's invisible here, thus comment.
-        */
-        newContinuousUserDefinedRanges[action.selection] = action.range;
-      }
-
-      let activeContinuousHistogramFilters = [];
-
-      _.each(newContinuousUserDefinedRanges, (value, key, i) => {
-        if (value !== null) {
-          activeContinuousHistogramFilters.push(key);
-        }
-      });
-
-      /* see if there are others from previous... */
-      if (activeContinuousHistogramFilters.length > 0) {
-        _.each(activeContinuousHistogramFilters, key => {
-          _.each(newSelection, (cell, i) => {
-            if (
-              +cell[key] < newContinuousUserDefinedRanges[key][0] ||
-              +cell[key] > newContinuousUserDefinedRanges[key][1]
-            ) {
-              newSelection[i]["__selected__"] = false;
-            }
-          });
-        });
-      }
-
-      /*
-        1. figure out if the users have unchecked boxes
-        2. put them in an array
-        3. filter on them
-      */
-
-      let newCategoricalAsBooleansMap = s.controls.categoricalAsBooleansMap;
-
-      /*
-        ...spread for merge: https://github.com/reactjs/redux/issues/432
-
-        we do the update here instead of the reducer because we need it for the reactive computation
-      */
-      if (action.type === "categorical metadata filter select") {
-        newCategoricalAsBooleansMap = {
-          ...s.controls.categoricalAsBooleansMap,
-          [action.metadataField]: {
-            ...s.controls.categoricalAsBooleansMap[action.metadataField],
-            [action.value]: true
-          }
-        };
-      } else if (action.type === "categorical metadata filter deselect") {
-        newCategoricalAsBooleansMap = {
-          ...s.controls.categoricalAsBooleansMap,
-          [action.metadataField]: {
-            ...s.controls.categoricalAsBooleansMap[action.metadataField],
-            [action.value]: false
-          }
-        };
-      } else if (action.type === "categorical metadata filter none of these") {
-        const metadataFieldWithAllOfTheseValueSelected = {};
-
-        /* set EVERYTHING to false in this intermediate object */
-        _.each(
-          s.controls.categoricalAsBooleansMap[action.metadataField],
-          (isActive, option) => {
-            metadataFieldWithAllOfTheseValueSelected[option] = false;
-          }
-        );
-
-        newCategoricalAsBooleansMap = {
-          ...s.controls.categoricalAsBooleansMap,
-          [action.metadataField]: metadataFieldWithAllOfTheseValueSelected
-        };
-      } else if (action.type === "categorical metadata filter all of these") {
-        const metadataFieldWithAllOfTheseValueSelected = {};
-
-        /* set EVERYTHING to true in this intermediate object */
-        _.each(
-          s.controls.categoricalAsBooleansMap[action.metadataField],
-          (isActive, option) => {
-            metadataFieldWithAllOfTheseValueSelected[option] = true;
-          }
-        );
-
-        newCategoricalAsBooleansMap = {
-          ...s.controls.categoricalAsBooleansMap,
-          [action.metadataField]: metadataFieldWithAllOfTheseValueSelected
-        };
-      }
-
-      const inactiveCategories = [];
-      _.each(newCategoricalAsBooleansMap, (options, category) => {
-        _.each(options, (isActive, option) => {
-          if (!isActive) {
-            inactiveCategories.push({ category, option });
-          }
-        });
-      });
-
-      if (inactiveCategories.length > 0) {
-        _.each(inactiveCategories, d => {
-          if (
-            s.controls.categoricalAsCellsMap[d.category] &&
-            s.controls.categoricalAsCellsMap[d.category][d.option]
-          ) {
-            _.forEach(
-              s.controls.categoricalAsCellsMap[d.category][d.option],
-              c => {
-                c.__selected__ = false;
-              }
-            );
-          }
-        });
-      }
-
       let modifiedAction = Object.assign({}, action, {
-        newSelection,
+        newSelection
-        newCategoricalAsBooleansMap,
-        newContinuousUserDefinedRanges
       }); /* append the result of all the filters to the action the user just triggered */
 
       return next(modifiedAction);

src/reducers/controls.js

@@ -1,16 +1,78 @@
 // jshint esversion: 6
 import _ from "lodash";
 import { parseRGB } from "../util/parseRGB";
+var crossfilter = require("../util/typedCrossfilter");
+
+// In the case where the REST server does not implement data schema
+// declaration, we attempt to deduce it by sniffing the data.
+//
+function createSchemaByDataSniffing(ranges) {
+  let schema = {};
+  _.forEach(ranges, (value, key) => {
+    schema[key] = {
+      displayname: key,
+      variabletype: value.options ? "categorical" : "continuous"
+    };
+
+    // Metadata field type is inferred by sniffing the data. This has some risks.
+    // Caveats:
+    //  * Values have been converted to native JS objects by the JSON parser.
+    //  * Lots of assumptions about he REST API behaving properly (eg, min/max
+    //    are the same type, etc).
+    let type;
+    if (schema[key].variabletype === "continuous" && value.range) {
+      // Use min/max as a proxy for all data.
+      const min = value.range.min;
+      const max = value.range.max;
+      type =
+        typeof min !== "number" || typeof max !== "number"
+          ? "string"
+          : Number.isSafeInteger(min) && Number.isSafeInteger(max)
+            ? "int"
+            : "float";
+    } else {
+      // use an option value as a proxy for all data
+      const aVal = value.options[0];
+      type =
+        typeof aVal !== "number"
+          ? "string"
+          : Number.isSafeInteger(aVal) ? "int" : "float";
+    }
+    schema[key].type = type;
+  });
+  return schema;
+}
+
+// Deduce the correct crossfilter dimension type from a metadata
+// schema description.
+//
+function deduceDimensionType(attributes, fieldName) {
+  let dimensionType;
+  if (attributes.type === "string") {
+    dimensionType = "enum";
+  } else if (attributes.type === "int") {
+    dimensionType = Int32Array;
+  } else if (attributes.type === "float") {
+    dimensionType = Float32Array;
+  } else {
+    console.error(
+      `Warning - REST API returned unknown metadata schema (${
+        attributes.type
+      }) for field ${fieldName}.`
+    );
+    // skip it - we don't know what to do with this type
+  }
+  return dimensionType;
+}
 
 const Controls = (
   state = {
     _ranges: null /* this comes from initialize, this is universe */,
     allGeneNames: null,
     allCellsOnClient: null /* this comes from cells endpoint, this is world */,
-    currentCellSelection: null /* this comes from user actions, all draw components use this, it is created by middleware */,
+    allCellsMetadata: null /* this comes from user actions, all draw components use this, it is created by middleware */,
-    graphVec: null,
+    crossfilter: null /* the current user selection state */,
     categoricalAsBooleansMap: null,
-    categoricalAsCellsMap: null,
     colorAccessor: null,
     colorScale: null,
     opacityForDeselectedCells: 0.2,
@@ -28,118 +90,157 @@ const Controls = (
     /**********************************
         Keep a copy of 'universe'
   ***********************************/
-    case "initialize success":
+    case "initialize success": {
+      if (!action.data.data.schema) {
+        console.error("Warning - REST API omitted schema description.");
+      }
       return Object.assign({}, state, {
         _ranges: action.data.data.ranges,
-        allGeneNames: action.data.data.genes
+        allGeneNames: action.data.data.genes,
+        schema: action.data.data.schema
       });
-    case "request cells success":
+    }
-      // Store the graph in a linear array for fast access.  Index into
+    case "request cells success": {
-      // the array by "cell index", which is stored as metadata field
+      const allCellsMetadata = action.data.data.metadata.slice(0);
-      // __cellIndex__.
+      const allCellsMetadataMap = _.keyBy(allCellsMetadata, "CellName");
-      //
-      // Code below relies on the REST API guarantee that the graph and
-      // metadata are returned as arrays with the same order and length.
-      //
-      const graphVec = new Float32Array(2 * action.data.data.graph.length);
-      _.each(action.data.data.graph, (g, i) => {
-        graphVec[2 * i] = g[1];
-        graphVec[2 * i + 1] = g[2];
-      });
-
-      const currentCellSelection = action.data.data.metadata.slice(0);
-      const currentCellSelectionMap = _.keyBy(currentCellSelection, "CellName");
 
       /*
       construct a copy of the ranges object that only has categorical
       replace all counts with bool flags
       ie., everything starts out checked
       we mutate this map in the actions below
-    */
+      */
-      const categoricalAsBooleansMap = {},
+      const categoricalAsBooleansMap = {};
-        categoricalAsCellsMap = {};
-      const continuousUserDefinedRanges = {};
       _.each(action.data.data.ranges, (value, key) => {
         if (
           key !== "CellName" &&
           value.options /* it's categorical, it has options instead of ranges */
         ) {
-          const optionsAsBooleans = {},
+          const optionsAsBooleans = {};
-            optionsAsCells = {};
           _.each(value.options, (_value, _key) => {
             optionsAsBooleans[_key] = true;
-            optionsAsCells[_key] = [];
           });
           categoricalAsBooleansMap[key] = optionsAsBooleans;
-          categoricalAsCellsMap[key] = optionsAsCells;
-        } else if (key !== "CellName" && value.range) {
-          continuousUserDefinedRanges[key] = null;
         }
       });
 
-      _.each(currentCellSelection, (cell, idx) => {
+      const graph = action.data.data.graph;
+      _.each(allCellsMetadata, (cell, idx) => {
         cell.__cellIndex__ = idx;
-        cell.__selected__ = true;
         cell.__color__ =
           "rgba(0,0,0,1)"; /* initial color for all cells in all charts */
         cell.__colorRGB__ = parseRGB(cell.__color__);
-
+        cell.__x__ = graph[idx][1];
-        // Add each cell to its categorical metadata set.
+        cell.__y__ = graph[idx][2];
-        _.forEach(cell, (_value, key) => {
-          if (
-            categoricalAsCellsMap[key] &&
-            categoricalAsCellsMap[key][_value]
-          ) {
-            const s = categoricalAsCellsMap[key][_value];
-            if (s) s.push(cell);
-          }
-        });
       });
 
+      // Build the selection crossfilter.
+      //
+      let cellsCrossfilter = crossfilter(allCellsMetadata);
+      let cellsDimensionsMap = {};
+      cellsDimensionsMap.x = cellsCrossfilter.dimension(
+        r => r.__x__,
+        Float32Array
+      );
+      cellsDimensionsMap.y = cellsCrossfilter.dimension(
+        r => r.__y__,
+        Float32Array
+      );
+
+      // Now walk the schema and make an appropriate dimension for each
+      // metadata field.   This is a simplistic mapping, and could be
+      // optmized to use smaller scalars (to save memory) or larger
+      // floating point where precision is needed.
+      //
+      // If we don't have a schema (bad server!), fake it by inferring
+      // important fields from the ranges element.
+      //
+      if (!state.schema) {
+        state.schema = createSchemaByDataSniffing(action.data.data.ranges);
+      }
+      _.forEach(state.schema, (attributes, key) => {
+        if (key !== "CellName") {
+          const dimensionType = deduceDimensionType(attributes, key);
+          if (dimensionType) {
+            cellsDimensionsMap[key] = cellsCrossfilter.dimension(
+              r => r[key],
+              dimensionType
+            );
+          }
+        }
+      });
       return Object.assign({}, state, {
+        // this is only used as a flag that data has loaded.  Could be
+        // removed (other variables would suffice for the same test).
         allCellsOnClient: action.data.data,
-        currentCellSelection,
+        allCellsMetadata,
-        currentCellSelectionMap,
+        allCellsMetadataMap,
-        graphVec,
         categoricalAsBooleansMap,
-        categoricalAsCellsMap,
+        crossfilter: {
-        continuousUserDefinedRanges,
+          cells: cellsCrossfilter,
+          dimensionMap: cellsDimensionsMap
+        },
         graphBrushSelection: null /* if we are getting new cells from the server, the layout (probably? definitely?) just changed, so this is now irrelevant, and we WILL need to call a function to reset state of this kind when cells success happens */
       });
+    }
     /* * * * * * * * * * * * * * * * * *
             User events
   * * * * * * * * * * * * * * * * * */
-    case "parallel coordinates axes have been drawn":
+    case "parallel coordinates axes have been drawn": {
       return Object.assign({}, state, {
         axesHaveBeenDrawn: true
       });
+    }
     case "continuous selection using parallel coords brushing": {
       return Object.assign({}, state, {
         continuousSelection: action.data,
-        currentCellSelection:
+        crossfilter: {
-          action.newSelection /* this comes from middleware */
+          ...state.crossfilter
+        }
       });
     }
-    case "graph brush selection change":
+    case "graph brush selection change": {
+      state.crossfilter.dimensionMap.x.filterRange([
+        action.brushCoords.northwest[0],
+        action.brushCoords.southeast[0]
+      ]);
+      state.crossfilter.dimensionMap.y.filterRange([
+        action.brushCoords.southeast[1],
+        action.brushCoords.northwest[1]
+      ]);
       return Object.assign({}, state, {
-        graphBrushSelection:
+        graphBrushSelection: action.brushCoords,
-          action.brushCoords /* this has already been applied in middleware but store it for next time */,
+        crossfilter: {
-        currentCellSelection:
+          ...state.crossfilter
-          action.newSelection /* this comes from middleware */
+        }
       });
-    case "graph brush deselect":
+    }
+    case "graph brush deselect": {
+      state.crossfilter.dimensionMap.x.filterAll();
+      state.crossfilter.dimensionMap.y.filterAll();
       return Object.assign({}, state, {
         graphBrushSelection: null,
-        currentCellSelection:
+        crossfilter: {
-          action.newSelection /* this comes from middleware */
+          ...state.crossfilter
+        }
       });
-    case "continuous metadata histogram brush":
+    }
+    case "continuous metadata histogram brush": {
+      // action.selection: metadata name being selected
+      // action.range: filter range, or null if deselected
+      if (!action.range) {
+        state.crossfilter.dimensionMap[action.selection].filterAll();
+      } else {
+        state.crossfilter.dimensionMap[action.selection].filterRange(
+          action.range
+        );
+      }
       return Object.assign({}, state, {
-        newContinuousUserDefinedRanges:
+        crossfilter: {
-          action.newContinuousUserDefinedRanges /* this has already been applied in middleware but store it for next time */,
+          ...state.crossfilter
-        currentCellSelection:
+        }
-          action.newSelection /* this comes from middleware */
       });
+    }
     case "change opacity deselected cells in 2d graph background":
       return Object.assign({}, state, {
         opacityForDeselectedCells: action.data
@@ -147,57 +248,113 @@ const Controls = (
     /*******************************
         Categorical metadata
   *******************************/
-    case "categorical metadata filter select":
+    case "categorical metadata filter select": {
+      const newCategoricalAsBooleansMap = {
+        ...state.categoricalAsBooleansMap,
+        [action.metadataField]: {
+          ...state.categoricalAsBooleansMap[action.metadataField],
+          [action.value]: true
+        }
+      };
+      // update the filter for the one category that changed state
+      state.crossfilter.dimensionMap[action.metadataField].filterEnum(
+        _.filter(
+          _.map(
+            newCategoricalAsBooleansMap[action.metadataField],
+            (val, key) => (val ? key : false)
+          )
+        )
+      );
       return Object.assign({}, state, {
-        categoricalAsBooleansMap:
+        categoricalAsBooleansMap: newCategoricalAsBooleansMap,
-          action.newCategoricalAsBooleansMap /* this comes from middleware */,
+        crossfilter: {
-        currentCellSelection:
+          ...state.crossfilter
-          action.newSelection /* this comes from middleware */
+        }
       });
-    case "categorical metadata filter deselect":
+    }
+    case "categorical metadata filter deselect": {
+      const newCategoricalAsBooleansMap = {
+        ...state.categoricalAsBooleansMap,
+        [action.metadataField]: {
+          ...state.categoricalAsBooleansMap[action.metadataField],
+          [action.value]: false
+        }
+      };
+      // update the filter for the one category that changed state
+      state.crossfilter.dimensionMap[action.metadataField].filterEnum(
+        _.filter(
+          _.map(
+            newCategoricalAsBooleansMap[action.metadataField],
+            (val, key) => (val ? key : false)
+          )
+        )
+      );
       return Object.assign({}, state, {
-        categoricalAsBooleansMap:
+        categoricalAsBooleansMap: newCategoricalAsBooleansMap,
-          action.newCategoricalAsBooleansMap /* this comes from middleware */,
+        crossfilter: {
-        currentCellSelection:
+          ...state.crossfilter
-          action.newSelection /* this comes from middleware */
+        }
       });
-    case "categorical metadata filter none of these":
+    }
+    case "categorical metadata filter none of these": {
+      const newCategoricalAsBooleansMap = {
+        ...state.categoricalAsBooleansMap
+      };
+      _.forEach(
+        newCategoricalAsBooleansMap[action.metadataField],
+        (v, k, c) => {
+          c[k] = false;
+        }
+      );
+      state.crossfilter.dimensionMap[action.metadataField].filterNone();
       return Object.assign({}, state, {
-        categoricalAsBooleansMap:
+        categoricalAsBooleansMap: newCategoricalAsBooleansMap,
-          action.newCategoricalAsBooleansMap /* this comes from middleware */,
+        crossfilter: {
-        currentCellSelection:
+          ...state.crossfilter
-          action.newSelection /* this comes from middleware */
+        }
       });
-    case "categorical metadata filter all of these":
+    }
+    case "categorical metadata filter all of these": {
+      const newCategoricalAsBooleansMap = {
+        ...state.categoricalAsBooleansMap
+      };
+      _.forEach(
+        newCategoricalAsBooleansMap[action.metadataField],
+        (v, k, c) => {
+          c[k] = true;
+        }
+      );
+      state.crossfilter.dimensionMap[action.metadataField].filterAll();
       return Object.assign({}, state, {
-        categoricalAsBooleansMap:
+        categoricalAsBooleansMap: newCategoricalAsBooleansMap,
-          action.newCategoricalAsBooleansMap /* this comes from middleware */,
+        crossfilter: {
-        currentCellSelection:
+          ...state.crossfilter
-          action.newSelection /* this comes from middleware */
+        }
       });
+    }
     /*******************************
             Color Scale
   *******************************/
     case "color by continuous metadata":
       return Object.assign({}, state, {
         colorAccessor: action.colorAccessor,
-        currentCellSelection:
+        allCellsMetadata:
-          action.currentSelectionWithUpdatedColors /* this comes from middleware */,
+          action.allCellsMetadataWithUpdatedColors /* this comes from middleware */,
         colorScale: action.colorScale
       });
     case "color by expression":
       return Object.assign({}, state, {
         colorAccessor: action.gene,
-        currentCellSelection:
+        allCellsMetadata:
-          action.currentSelectionWithUpdatedColors /* this comes from middleware */,
+          action.allCellsMetadataWithUpdatedColors /* this comes from middleware */,
         colorScale: action.colorScale
       });
     case "color by categorical metadata":
       return Object.assign({}, state, {
         colorAccessor:
           action.colorAccessor /* pass the scale through additionally, and it's a legend! */,
-        currentCellSelection:
+        allCellsMetadata:
-          action.currentSelectionWithUpdatedColors /* this comes from middleware */,
+          action.allCellsMetadataWithUpdatedColors /* this comes from middleware */,
         colorScale: action.colorScale
       });
     case "store current cell selection as differential set 1":

src/reducers/index.js

@@ -1,7 +1,7 @@
 // jshint esversion: 6
 import { combineReducers, createStore, applyMiddleware } from "redux";
 import updateURLMiddleware from "../middleware/updateURLMiddleware";
-import updateCellSelectionMiddleware from "../middleware/updateCellSelectionMiddleware";
+// import updateCellSelectionMiddleware from "../middleware/updateCellSelectionMiddleware";
 import updateCellColors from "../middleware/updateCellColors";
 
 import thunk from "redux-thunk";
@@ -28,7 +28,7 @@ let store = createStore(
   applyMiddleware(
     thunk,
     updateURLMiddleware,
-    updateCellSelectionMiddleware,
+    // updateCellSelectionMiddleware,
     updateCellColors
   )
 );

src/util/typedCrossfilter.js

@@ -0,0 +1,785 @@
+"use strict";
+// jshint esversion: 6
+
+/*
+Typedarray Crossfilter - a re-implementation of a subset of crossfilter, with
+major time/space optimizations predicated upon the following assumptions:
+  - dimensions are uniformly typed, and all values must be of that type
+  - dimension values must be a primitive type (int, float, string).  Arrays
+    or other complex types not supported.
+  - dimension creation requires call-provided type declaration
+  - no support for adding/removing data to an existing crossfilter.  If you
+    want to do that, you have to create the new crossfilter, using the new
+    data, from scratch.
+
+The actual backing store for a dimension is a TypedArray, enabling significant
+performance improvements over the original crossfilter.
+
+There are also a handful of new methods, primarily to take advantage of the
+performance (eg, crossfilter.fillBySelection)
+
+Helpful documents (this code tries to follow the original API as much
+as is feasable):
+    https://github.com/square/crossfilter/
+    http://square.github.io/crossfilter/
+
+There is also a newer, community supported fork of crossfilter, with a
+more complex API.  In a few cases, elements of that API were incorporated.
+    https://github.com/square/crossfilter/
+
+*/
+
+/*
+    Utility functions, private to this module
+*/
+
+// fill an array or typedarray with a sequential range of numbers,
+// starting with `start`
+//
+function fillRange(arr, start = 0) {
+  for (let i = 0, len = arr.length; i < len; i++) {
+    arr[i] = i + start;
+  }
+  return arr;
+}
+
+// Search for `value` in the sorted array `tarr`, in the range [first, last).
+// Return the first (left most) index where tarr[index] >= value.
+//
+// In other words, return array index I where:
+//  tarr[i] < value for all tarr[lo:I]
+//  tarr[i] >= value for all tarr[I:last]
+//
+// Essentially the same thing as:
+//    C++: lower_bound()
+//    Python: bisect.bisect_left()
+//
+function lowerBound(valueArray, value, first, last) {
+  // this is just a binary search
+  while (first < last) {
+    const middle = (first + last) >>> 1;
+    if (valueArray[middle] < value) {
+      first = middle + 1;
+    } else {
+      last = middle;
+    }
+  }
+  return first;
+}
+
+// XXX: it is likely that there would be minimal performance hit from creating
+// a factory version of lowerBound that takes an accessor (rather than having
+// a special-cased version for lining the indirection).
+//
+// Benchmarking shows this manual inlining is up to 4X faster than an accessor.
+// The real issue is how often we call it.
+//
+function lowerBoundIndirect(valueArray, indexArray, value, first, last) {
+  // this is just a binary search
+  while (first < last) {
+    const middle = (first + last) >>> 1;
+    if (valueArray[indexArray[middle]] < value) {
+      first = middle + 1;
+    } else {
+      last = middle;
+    }
+  }
+  return first;
+}
+
+// Search for `value in the sorted array `tarr`, in the range [first, last).
+// Return the first value where tarr[index] > value.
+//
+// In other words, return array index I, where:
+//  tarr[i] <= value for all tarr[lo:I]
+//  tarr[i] > value for all tarr[I:last]
+//
+// Essentially the same thing as:
+//    C++: upper_bound()
+//    Python: bisect.bisect_right()
+//
+function upperBound(valueArray, value, first, last) {
+  // this is just a binary search
+  while (first < last) {
+    const middle = (first + last) >>> 1;
+    if (valueArray[middle] > value) {
+      last = middle;
+    } else {
+      first = middle + 1;
+    }
+  }
+  return first;
+}
+
+function upperBoundIndirect(valueArray, indexArray, value, first, last) {
+  // this is just a binary search
+  while (first < last) {
+    const middle = (first + last) >>> 1;
+    if (valueArray[indexArray[middle]] > value) {
+      last = middle;
+    } else {
+      first = middle + 1;
+    }
+  }
+  return first;
+}
+
+// Interval operations - very simple version of interval set relationship
+// operators.   An interval is a multi-interval list of [min, max],
+// where min and max are mandatory.   Constraints:
+//    * min <= max
+//    * Legal intervals:   [], [ [0, 1], ... ]
+//    * all min and max values must be >= 0
+//    * Not legal:   [ [] ]
+//
+// Code assumes intervals have a low cardinality; many operations are done
+// with a brute force scan.
+//
+class PositiveIntervals {
+  // Canonicalize - ensure that:
+  //  1. no overlapping intervals
+  //  2. sorted in order of interval min.
+  //
+  static canonicalize(A) {
+    if (A.length <= 1) return A;
+    let copy = A.slice();
+    copy.sort((a, b) => a[0] - b[0]);
+    const res = [];
+    res.push(copy[0]);
+    for (let i = 1, len = copy.length; i < len; i++) {
+      if (copy[i][0] > res[res.length - 1][1]) {
+        // non-overlapping, add to result
+        res.push(copy[i]);
+      } else if (copy[i][1] > res[res.length - 1][1]) {
+        // merge this into previous
+        res[res.length - 1][1] = copy[i][1];
+      }
+    }
+    return res;
+  }
+
+  // Return interval with values belonging to both A and B.
+  //
+  static union(A, B) {
+    return PositiveIntervals.canonicalize([...A, ...B]);
+  }
+
+  static _flatten(A, B) {
+    let points = []; /* point, A, start */
+    for (let a = 0; a < A.length; a++) {
+      points.push([A[a][0], true, true]);
+      points.push([A[a][1], true, false]);
+    }
+    for (let b = 0; b < B.length; b++) {
+      points.push([B[b][0], false, true]);
+      points.push([B[b][1], false, false]);
+    }
+    // Sort order: point, then start
+    points.sort((a, b) => (a[0] !== b[0] ? a[0] - b[0] : a[2] ? 1 : -1));
+    return points;
+  }
+
+  // A - B, ie, the interval with all values in A that are not in B.
+  //
+  static difference(A, B) {
+    // Corner cases
+    if (A.length === 0 || B.length === 0) {
+      return PositiveIntervals.canonicalize(A);
+    }
+
+    A = PositiveIntervals.canonicalize(A);
+    B = PositiveIntervals.canonicalize(B);
+
+    const points = PositiveIntervals._flatten(A, B);
+    const res = [];
+    let aDepth = 0;
+    let depth = 0;
+    let intervalStart;
+    let prevPoint;
+    for (let i = 0; i < points.length; i++) {
+      const p = points[i];
+      const before = depth;
+      const delta = p[2] ? 1 : -1;
+      depth += delta;
+      if (p[1]) aDepth += delta;
+
+      if (i === points.length - 1 || p[0] !== points[i + 1][0]) {
+        if (aDepth === 1 && depth === 1) {
+          intervalStart = p[0];
+        } else if (intervalStart !== undefined) {
+          res.push([intervalStart, p[0]]);
+          intervalStart = undefined;
+        }
+      }
+      prevPoint = p[0];
+    }
+    // guaranteed to be in canonical form
+    return res;
+  }
+
+  // Return interval with values belonging to A or B.
+  //
+  static intersection(A, B) {
+    if (A.length === 0 || B.length === 0) {
+      return [];
+    }
+
+    A = PositiveIntervals.canonicalize(A);
+    B = PositiveIntervals.canonicalize(B);
+
+    const points = PositiveIntervals._flatten(A, B);
+    const res = [];
+    let depth = 0;
+    let intervalStart;
+    for (let i = 0; i < points.length; i++) {
+      const p = points[i];
+      const before = depth;
+      depth += p[2] ? 1 : -1;
+      if (depth === 2) {
+        intervalStart = p[0];
+      } else if (intervalStart !== undefined) {
+        res.push([intervalStart, p[0]]);
+        intervalStart = undefined;
+      }
+    }
+    // guaranteed to be in canonical form
+    return res;
+  }
+}
+
+// BitArray is a 2D bitarray with size [length, nBitWidth].
+// Each bit is referred to as a `dimension`.  Dimensions may be
+// dynamically allocated and deallocated.  The overall length
+// of the BitArray is fixed at creation time (for simplicity).
+//
+// Organization of the bitarray is dimension-major.
+//
+// Primary operations on the BitArray are:
+//    - set & clear dimension
+//    - test dimension
+//    - various performance or convenience test operations
+//
+// The underlying data structure uses TypedArrays for performance.
+//
+class BitArray {
+  constructor(length) {
+    // Initially allocate a 32 bit wide array.  allocDimension() will expand
+    // as necessary.
+    //
+    // Int32Array is (counterintuitively) used to accomadate JS numeric casting
+    // (to/from primitive number type).
+    //
+
+    // Fixed for the life of this object.
+    this.length = length;
+
+    // Bitarray width.  width is always greater than 32*dimensionCount.
+    this.width = 1; // underlying number of 32 bit arrays
+    this.dimensionCount = 0; // num allocated dimensions
+
+    this.bitmask = new Int32Array(this.width); // dimension allocation mask
+    this.bitarray = new Int32Array(this.width * this.length);
+
+    console.log("a", wasmHelpersModule);
+  }
+
+  get selectionCount() {
+    return this.countAllOnes();
+  }
+
+  countAllOnes() {
+    let count = 0;
+    for (let i = 0; i < this.width; i++) {
+      const bitmask = this.bitmask[i];
+      for (let j = i * this.length, len = j + this.length; j < len; j++) {
+        if (this.bitarray[i * this.length + j] === bitmask) count++;
+      }
+    }
+    return count;
+  }
+
+  // count trailing zeros
+  static ctz(v) {
+    let c = 32;
+    v &= -v; // isolate lowest non-zero bit
+    if (v) c--;
+    if (v & 0x0000ffff) c -= 16;
+    if (v & 0x00ff00ff) c -= 8;
+    if (v & 0x0f0f0f0f) c -= 4;
+    if (v & 0x33333333) c -= 2;
+    if (v & 0x55555555) c -= 1;
+    return c;
+  }
+
+  // find a free dimension.  Return undefined if none
+  _findFreeDimension() {
+    let dim;
+    for (let col = 0; col < this.width; col++) {
+      const bitmask = this.bitmask[col];
+      const lowestZeroBit = ~this.bitmask[col] & -~this.bitmask[col];
+      if (lowestZeroBit) {
+        this.bitmask[col] |= lowestZeroBit;
+        dim = 32 * col + BitArray.ctz(lowestZeroBit);
+      }
+    }
+    return dim;
+  }
+
+  // allocate and return the dimension ID (bit position)
+  allocDimension() {
+    let dim = this._findFreeDimension();
+
+    // if we did not find free dimension, expand the bitarray.
+    if (dim === undefined) {
+      this.width++;
+
+      const biggerBitArray = new Int32Array(this.width * this.length);
+      biggerBitArray.set(this.bitarray);
+      this.bitarray = biggerBitArray;
+
+      const biggerBitmask = new Int32Array(this.width);
+      biggerBitmask.set(this.bitmask);
+      this.bitmask = biggerBitmask;
+
+      dim = this._findFreeDimension();
+    }
+
+    this.dimensionCount++;
+    return dim;
+  }
+
+  freeDimension(dim) {
+    // all selection tests assume unallocated dimensions are zero valued.
+    this.deselectAll(dim);
+    const col = dim >>> 5;
+    this.bitmask[col] &= ~(1 << (dim % 32));
+    this.dimensionCount--;
+  }
+
+  isSelected(index) {
+    const width = this.width;
+    const length = this.length;
+    const bitarray = this.bitarray;
+
+    for (let w = 0; w < width; w++) {
+      const bitmask = this.bitmask[w];
+      if (!bitmask || bitarray[w * length + index] !== bitmask) return false;
+    }
+    return true;
+  }
+
+  selectOne(dim, index) {
+    const col = dim >>> 5;
+    const before = this.bitarray[col * this.length + index];
+    const after = before | (1 << (dim % 32));
+    this.bitarray[col] = after;
+  }
+
+  deselectOne(dim, index) {
+    const col = dim >>> 5;
+    const before = this.bitarray[col * this.length + index];
+    const after = before & ~(1 << (dim % 32));
+    this.bitarray[col] = after;
+  }
+
+  selectAll(dim) {
+    let col = dim >> 5;
+    const bitmask = this.bitmask[col];
+    const bitarray = this.bitarray;
+    const one = 1 << (dim % 32);
+    for (let i = col * this.length, len = i + this.length; i < len; i++) {
+      bitarray[i] |= one;
+    }
+  }
+
+  deselectAll(dim) {
+    let col = dim >> 5;
+    const bitmask = this.bitmask[col];
+    const bitarray = this.bitarray;
+    const zero = ~(1 << (dim % 32));
+    for (let i = col * this.length, len = i + this.length; i < len; i++) {
+      bitarray[i] &= zero;
+    }
+  }
+
+  // indirect functions are used to map between sort and natural order
+  selectIndirectFromRange(dim, indirect, range) {
+    const col = dim >>> 5;
+    const first = range[0];
+    const last = range[1];
+    const bitarray = this.bitarray;
+    const one = 1 << (dim % 32);
+    const offset = col * this.length;
+    for (let i = first; i < last; i++) {
+      bitarray[offset + indirect[i]] |= one;
+    }
+  }
+
+  deselectIndirectFromRange(dim, indirect, range) {
+    const col = dim >>> 5;
+    const first = range[0];
+    const last = range[1];
+    const bitarray = this.bitarray;
+    const zero = ~(1 << (dim % 32));
+    const offset = col * this.length;
+    for (let i = first; i < last; i++) {
+      bitarray[offset + indirect[i]] &= zero;
+    }
+  }
+
+  // Fill the array with selected|deselected value based upon the
+  // current selection state.
+  fillBySelection(result, selectedValue, deselectedValue) {
+    // special case (width === 1) for performance
+    if (this.width === 1) {
+      const bitmask = this.bitmask[0];
+      const bitarray = this.bitarray;
+      for (let i = 0, len = this.length; i < len; i++) {
+        result[i] = bitarray[i] === bitmask ? selectedValue : deselectedValue;
+      }
+    } else {
+      for (let i = 0, len = this.length; i < len; i++) {
+        result[i] = this.isSelected(i) ? selectedValue : deselectedValue;
+      }
+    }
+    return result;
+  }
+}
+
+class TypedCrossfilter {
+  constructor(data) {
+    this.data = data;
+
+    // filters: array of { id, dimension }
+    this.filters = [];
+    this.selection = new BitArray(data.length);
+  }
+
+  size() {
+    return this.data.length;
+  }
+
+  all() {
+    return this.data;
+  }
+
+  dimension(value, valueArrayType) {
+    const id = this.selection.allocDimension();
+    let dim;
+    if (valueArrayType === "enum") {
+      dim = new EnumDimension(value, this, id);
+    } else {
+      dim = new ScalarDimension(value, valueArrayType, this, id);
+    }
+    this.filters.push({ id, dim });
+    dim.filterAll();
+    return dim;
+  }
+
+  _freeDimension(id) {
+    this.selection.freeDimension(id);
+    this.filters = this.filters.filter(f => f.id != id);
+  }
+
+  // return array of all records that are selected/filtered
+  // by all dimensions.
+  allFiltered() {
+    const selection = this.selection;
+    const res = [];
+    for (let i = 0, len = this.data.length; i < len; i++) {
+      if (selection.isSelected(i)) {
+        res.push(this.data[i]);
+      }
+    }
+    return res;
+  }
+
+  countFiltered() {
+    return this.selection.selectionCount;
+  }
+
+  isElementFiltered(i) {
+    return this.selection.isSelected(i);
+  }
+
+  // fill array with one of two values, based upon selection state
+  fillByIsFiltered(array, selectedValue, deselectedValue) {
+    return this.selection.fillBySelection(
+      array,
+      selectedValue,
+      deselectedValue
+    );
+  }
+}
+
+// Base dimension type - value must be a scalar type (eg, int, float),
+// and value array must be a TypedArray.
+//
+class ScalarDimension {
+  constructor(value, valueArrayType, crossfilter, id) {
+    this.crossfilter = crossfilter;
+    this.id = id;
+
+    // current selection filter, expressed as PostiveIntervals.
+    this.currentFilter = [];
+
+    // Create value array
+    const array = this._createValueArray(
+      value,
+      new valueArrayType(this.crossfilter.data.length)
+    );
+    this.value = array;
+
+    // create sort index
+    this.index = fillRange(new Uint32Array(this.crossfilter.data.length));
+    this.index.sort((a, b) => array[a] - array[b]);
+  }
+
+  _createValueArray(value, array) {
+    // create dimension value array
+    const data = this.crossfilter.data;
+    const len = data.length;
+    for (let i = 0; i < len; i++) {
+      array[i] = value(data[i]);
+    }
+    return array;
+  }
+
+  dispose() {
+    this.crossfilter._freeDimension(this.id);
+  }
+
+  id() {
+    return this.id;
+  }
+
+  _updateFilters(newFilter) {
+    newFilter = PositiveIntervals.canonicalize(newFilter);
+
+    // special case optimization - select all/none can bypass
+    // more complex work and just clobber everything.
+    //
+    if (newFilter.length === 0) {
+      this.crossfilter.selection.deselectAll(this.id);
+    } else if (
+      newFilter.length === 1 &&
+      newFilter[0][0] === 0 &&
+      newFilter[0][1] == this.index.length
+    ) {
+      this.crossfilter.selection.selectAll(this.id);
+    } else {
+      const adds = PositiveIntervals.difference(newFilter, this.currentFilter);
+      const dels = PositiveIntervals.difference(this.currentFilter, newFilter);
+      dels.forEach(interval =>
+        this.crossfilter.selection.deselectIndirectFromRange(
+          this.id,
+          this.index,
+          interval
+        )
+      );
+      adds.forEach(interval =>
+        this.crossfilter.selection.selectIndirectFromRange(
+          this.id,
+          this.index,
+          interval
+        )
+      );
+    }
+
+    this.currentFilter = newFilter;
+  }
+
+  // filter by value - exact match
+  filterExact(value) {
+    const newFilter = [
+      lowerBoundIndirect(this.value, this.index, value, 0, this.value.length),
+      upperBoundIndirect(this.value, this.index, value, 0, this.value.length)
+    ];
+    if (newFilter[0] <= newFilter[1]) {
+      this._updateFilters([newFilter]);
+    } else {
+      this._updateFilters([]);
+    }
+    return this;
+  }
+
+  // filter by a set of values, eg. enum.
+  filterEnum(values) {
+    const newFilter = [];
+    for (let v = 0, len = values.length; v < len; v++) {
+      const intv = [
+        lowerBoundIndirect(
+          this.value,
+          this.index,
+          values[v],
+          0,
+          this.value.length
+        ),
+        upperBoundIndirect(
+          this.value,
+          this.index,
+          values[v],
+          0,
+          this.value.length
+        )
+      ];
+      if (intv[0] <= intv[1]) newFilter.push(intv);
+    }
+    this._updateFilters(newFilter);
+    return this;
+  }
+
+  // filter by value range [lo, hi)
+  // lo: inclusive, hi: exclusive
+  filterRange(range) {
+    const newFilter = [];
+    const intv = [
+      lowerBoundIndirect(
+        this.value,
+        this.index,
+        range[0],
+        0,
+        this.value.length
+      ),
+      upperBoundIndirect(this.value, this.index, range[1], 0, this.value.length)
+    ];
+    if (intv[0] < intv[1]) newFilter.push(intv);
+    this._updateFilters(newFilter);
+    return this;
+  }
+
+  // select all - equivalent of selecting all in this dimension
+  filterAll() {
+    this._updateFilters([[0, this.value.length]]);
+    return this;
+  }
+
+  // select none
+  filterNone() {
+    this._updateFilters([]);
+  }
+
+  // return top k records, starting with offset, in descending order.
+  // Order is this dimension's sort order
+  top(k, offset = 0) {
+    const data = this.crossfilter.data;
+    const selection = this.crossfilter.selection;
+    const index = this.index;
+    const len = index.length;
+    const ret = [];
+    let i = 0;
+    let skip = 0;
+    let found = 0;
+
+    // skip up to offset records
+    for (i = len - 1; 0 <= i && skip < offset; i--) {
+      if (selection.isSelected(index[i])) {
+        skip++;
+      }
+    }
+
+    // grab up to k records
+    for (; 0 <= i && found < k; i--) {
+      if (selection.isSelected(index[i])) {
+        ret.push(data[index[i]]);
+        found++;
+      }
+    }
+
+    return ret;
+  }
+
+  // return bottom k records, starting with offset, in ascending order.
+  // Order is this dimension's sort order
+  bottom(k, offset = 0) {
+    const data = this.crossfilter.data;
+    const selection = this.crossfilter.selection;
+    const index = this.index;
+    const len = index.length;
+    const ret = [];
+    let skip = 0;
+    let found = 0;
+    let i = 0;
+
+    // skip up to offset records
+    for (i = 0; i < len && skip < offset; i++) {
+      if (selection.isSelected(index[i])) {
+        skip++;
+      }
+    }
+
+    // grab up to k records
+    for (; i < len && found < k; i++) {
+      if (selection.isSelected(index[i])) {
+        ret.push(data[index[i]]);
+        found++;
+      }
+    }
+
+    return ret;
+  }
+}
+
+// Ordered enumeration - supports any sortable enumerable type, eg,
+// strings, which can be mapped into an fixed numeric range [0..n).
+//
+class EnumDimension extends ScalarDimension {
+  constructor(value, crossfilter, id) {
+    super(value, Uint32Array, crossfilter, id);
+  }
+
+  _createValueArray(value, array) {
+    const data = this.crossfilter.data;
+    const len = data.length;
+
+    // create enumeration table - mapping between the value
+    // and the enum.
+    const s = new Set();
+    for (let i = 0; i < len; i++) {
+      s.add(value(data[i]));
+    }
+    this.enumIndex = Array.from(s);
+    this.enumIndex.sort();
+
+    // create dimension value array
+    const enumLen = this.enumIndex.length;
+    for (let i = 0; i < len; i++) {
+      const v = value(data[i]);
+      const e = lowerBound(this.enumIndex, v, 0, enumLen);
+      array[i] = e;
+    }
+    return array;
+  }
+
+  filterExact(value) {
+    return super.filterExact(
+      lowerBound(this.enumIndex, value, 0, this.enumIndex.length)
+    );
+  }
+
+  filterEnum(values) {
+    return super.filterEnum(
+      values.map(v => lowerBound(this.enumIndex, v, 0, this.enumIndex.length))
+    );
+  }
+
+  filterRange(range) {
+    return super.filterEnum(
+      range.map(v => lowerBound(this.enumIndex, v, 0, this.enumIndex.length))
+    );
+  }
+}
+
+// Wrapper for backwards compat with crossfilter.
+//
+function crossfilter(data) {
+  return new TypedCrossfilter(data);
+}
+
+crossfilter.PositiveIntervals = PositiveIntervals;
+crossfilter.BitArray = BitArray;
+crossfilter.TypedCrossfilter = TypedCrossfilter;
+crossfilter.ScalarDimension = ScalarDimension;
+crossfilter.EnumDimension = EnumDimension;
+
+module.exports = crossfilter;