Merge pull request #71 from chanzuckerberg/bkmartinjr-staterefactor

introduce new selection state machinery

src/actions/index.js

@@ -65,6 +65,12 @@ const regraph = () => {
   };
 };
 
+const resetGraph = () => {
+  return (dispatch, getState) => {
+    dispatch({ type: "reset graph" });
+  };
+};
+
 const initialize = () => {
   return (dispatch, getState) => {
     dispatch({ type: "initialize started" });
@@ -89,7 +95,7 @@ const initialize = () => {
 //
 function cleanupExpressionResponse(data) {
   const s = store.getState();
-  const metadata = s.controls.currentCellSelectionMap;
+  const metadata = s.controls.allCellsMetadataMap;
   let errorFound = false;
   data.data.cells = _.filter(data.data.cells, cell => {
     if (!errorFound && !metadata[cell.cellname]) {
@@ -218,6 +224,7 @@ export default {
   initialize,
   requestCells,
   regraph,
+  resetGraph,
   requestGeneExpressionCounts,
   requestGeneExpressionCountsPOST,
   requestSingleGeneExpressionCountsForColoringPOST,

src/components/continuous/continuous.js

@@ -38,7 +38,6 @@ import { margin, width, height, createDimensions } from "./util";
     colorAccessor: state.controls.colorAccessor,
     colorScale: state.controls.colorScale,
     graphBrushSelection: state.controls.graphBrushSelection,
-    currentCellSelection: state.controls.currentCellSelection,
     axesHaveBeenDrawn: state.controls.axesHaveBeenDrawn
   };
 })

src/components/continuous/histogramBrush.js

@@ -26,7 +26,7 @@ import { connect } from "react-redux";
   return {
     colorAccessor: state.controls.colorAccessor,
     colorScale: state.controls.colorScale,
-    currentCellSelection: state.controls.currentCellSelection
+    cellsMetadata: state.controls.cellsMetadata
   };
 })
 class HistogramBrush extends React.Component {
@@ -52,7 +52,7 @@ class HistogramBrush extends React.Component {
   calcHistogramCache(nextProps) {
     // recalculate expensive stuff
     const allValuesForContinuousFieldAsArray = _.map(
-      nextProps.currentCellSelection,
+      nextProps.cellsMetadata,
       nextProps.metadataField
     );
 

src/components/continuous/parallel.js

@@ -36,7 +36,7 @@ import { margin, width, height, createDimensions } from "./util";
     colorAccessor: state.controls.colorAccessor,
     colorScale: state.controls.colorScale,
     graphBrushSelection: state.controls.graphBrushSelection,
-    currentCellSelection: state.controls.currentCellSelection,
+    cellsMetadata: state.controls.cellsMetadata,
     axesHaveBeenDrawn: state.controls.axesHaveBeenDrawn
   };
 })
@@ -96,7 +96,7 @@ class Parallel extends React.Component {
     /* https://stackoverflow.com/questions/23123138/perform-debounce-in-react-js */
     if (
       nextProps.ranges &&
-      nextProps.currentCellSelection &&
+      nextProps.cellsMetadata &&
       nextProps.axesHaveBeenDrawn
     ) {
       if (this.state._drawLinesCanvas) {
@@ -106,7 +106,7 @@ class Parallel extends React.Component {
       this.state.ctx.clearRect(0, 0, width, height);
 
       const _drawLinesCanvas = drawLinesCanvas(
-        nextProps.currentCellSelection,
+        nextProps.cellsMetadata,
         this.state.dimensions,
         this.state.xscale,
         this.state.ctx,

src/components/expression/cellSetButtons.js

@@ -9,13 +9,7 @@ import actions from "../../actions";
 @connect()
 class CellSetButton extends React.Component {
   set() {
-    const set = [];
+    const set = _.map(this.props.crossfilter.cells.allFiltered(), "CellName");
-
-    _.each(this.props.currentCellSelection, cell => {
-      if (cell["__selected__"]) {
-        set.push(cell.CellName);
-      }
-    });
 
     this.props.dispatch({
       type:

src/components/expression/expressionButtons.js

@@ -8,8 +8,8 @@ import CellSetButton from "./cellSetButtons";
 
 @connect(state => {
   return {
-    currentCellSelection: state.controls.currentCellSelection,
+    differential: state.differential,
-    differential: state.differential
+    crossfilter: state.controls.crossfilter
   };
 })
 class Expression extends React.Component {
@@ -43,7 +43,9 @@ class Expression extends React.Component {
           <div style={{ marginBottom: 15, width: 300 }}>
             There are currently
             {" " +
-              _.filter(this.props.currentCellSelection, "__selected__").length +
+              (this.props.crossfilter
+                ? this.props.crossfilter.cells.countFiltered()
+                : 0) +
               " "}
             cells selected, click a cell set button to store them.
           </div>

src/components/graph/graph.js

@@ -22,31 +22,11 @@ import FaSave from "react-icons/lib/fa/download";
 /* https://bl.ocks.org/mbostock/9078690 - quadtree for onClick / hover selections */
 
 @connect(state => {
-  const vertices =
-    state.cells.cells && state.cells.cells.data.graph
-      ? state.cells.cells.data.graph
-      : null;
-  const ranges =
-    state.cells.cells && state.cells.cells.data.ranges
-      ? state.cells.cells.data.ranges
-      : null;
-  const metadata =
-    state.cells.cells && state.cells.cells.data.metadata
-      ? state.cells.cells.data.metadata
-      : null;
-
   return {
-    ranges,
+    cellsMetadata: state.controls.cellsMetadata,
-    vertices,
-    metadata,
-    colorAccessor: state.controls.colorAccessor,
-    colorScale: state.controls.colorScale,
-    continuousSelection: state.controls.continuousSelection,
-    graphVec: state.controls.graphVec,
-    currentCellSelection: state.controls.currentCellSelection,
-    graphBrushSelection: state.controls.graphBrushSelection,
     opacityForDeselectedCells: state.controls.opacityForDeselectedCells,
-    responsive: state.responsive
+    responsive: state.responsive,
+    crossfilter: state.controls.crossfilter
   };
 })
 class Graph extends React.Component {
@@ -55,6 +35,10 @@ class Graph extends React.Component {
     this.count = 0;
     this.inverse = mat4.identity([]);
     this.graphPaddingTop = 100;
+    this.renderCache = {
+      positions: null,
+      colors: null
+    };
     this.state = {
       drawn: false,
       svg: null,
@@ -104,42 +88,74 @@ class Graph extends React.Component {
   }
 
   componentWillReceiveProps(nextProps) {
-    if (this.state.regl && nextProps.vertices) {
+    if (this.state.regl && nextProps.crossfilter) {
-      /* update regl */
+      /* update the regl state */
-      const vertices = nextProps.currentCellSelection;
+      const crossfilter = nextProps.crossfilter.cells;
-      const vertexCount = vertices.length;
+      const cells = crossfilter.all();
-      const positions = new Float32Array(2 * vertexCount);
+      const cellCount = cells.length;
-      const colors = new Float32Array(3 * vertexCount);
-      const sizes = new Float32Array(vertexCount);
 
-      // d3.scaleLinear().domain([0,1]).range([-1,1])
+      // X/Y positions for each point - a cached value that only
-      const glScaleX = scaleLinear([0, 1], [-1, 1]);
+      // changes if we have loaded entirely new cell data
-      // d3.scaleLinear().domain([0,1]).range([1,-1])
+      //
-      const glScaleY = scaleLinear([0, 1], [1, -1]);
+      if (
+        !this.renderCache.positions ||
+        this.props.crossfilter.cells != nextProps.crossfilter.cells
+      ) {
+        if (!this.renderCache.positions)
+          this.renderCache.positions = new Float32Array(2 * cellCount);
 
-      /*
+        // d3.scaleLinear().domain([0,1]).range([-1,1])
-        Construct Vectors
+        const glScaleX = scaleLinear([0, 1], [-1, 1]);
-      */
+        // d3.scaleLinear().domain([0,1]).range([1,-1])
-      const graphVec = nextProps.graphVec;
+        const glScaleY = scaleLinear([0, 1], [1, -1]);
-      for (var i = 0; i < vertexCount; i++) {
-        const cell = vertices[i];
-        const cellIdx = cell.__cellIndex__;
-        const x = glScaleX(graphVec[2 * cellIdx]);
-        const y = glScaleY(graphVec[2 * cellIdx + 1]);
-        positions[2 * i] = x;
-        positions[2 * i + 1] = y;
 
-        colors.set(cell.__colorRGB__, 3 * i);
+        for (
-
+          let i = 0, positions = this.renderCache.positions;
-        sizes[i] = cell.__selected__
+          i < cellCount;
-          ? 4
+          i++
-          : 0.2; /* make this a function of the number of total cells, including regraph */
+        ) {
+          positions[2 * i] = glScaleX(cells[i].__x__);
+          positions[2 * i + 1] = glScaleY(cells[i].__y__);
+        }
+        this.state.pointBuffer({
+          data: this.renderCache.positions,
+          dimension: 2
+        });
       }
 
-      this.state.pointBuffer({ data: positions, dimension: 2 });
+      // Colors for each point - a cached value that only changes when
-      this.state.colorBuffer({ data: colors, dimension: 3 });
+      // the cell metadata changes (done by updateCellColors middleware).
-      this.state.sizeBuffer({ data: sizes, dimension: 1 });
+      // NOTE: this is a slightly pessimistic assumption, as the metadata
-      this.count = vertexCount;
+      // could have changed for some other reason, but for now color is
+      // the only metadata that changes client-side.  If this is problematic,
+      // we could add some sort of color-specific indicator to the app state.
+      if (
+        !this.renderCache.colors ||
+        this.props.cellsMetadata != nextProps.cellsMetadata
+      ) {
+        if (!this.renderCache.colors)
+          this.renderCache.colors = new Float32Array(3 * cellCount);
+        for (let i = 0, colors = this.renderCache.colors; i < cellCount; i++) {
+          colors.set(cells[i].__colorRGB__, 3 * i);
+        }
+        this.state.colorBuffer({ data: this.renderCache.colors, dimension: 3 });
+      }
+
+      // Sizes for each point - this is presumed to change each time the
+      // component receives new props.   Almost always a true assumption, as
+      // most property upates are due to changes driving a crossfilter
+      // selection set change.
+      //
+      if (
+        !this.renderCache.sizes ||
+        this.props.crossfilter.cells != nextProps.crossfilter.cells
+      ) {
+        this.renderCache.sizes = new Float32Array(cellCount);
+      }
+      crossfilter.fillByIsFiltered(this.renderCache.sizes, 4, 0.2);
+      this.state.sizeBuffer({ data: this.renderCache.sizes, dimension: 1 });
+
+      this.count = cellCount;
     }
 
     if (
@@ -228,6 +244,24 @@ class Graph extends React.Component {
               alignItems: "baseline"
             }}
           >
+            <button
+              onClick={() => {
+                this.props.dispatch(actions.resetGraph());
+              }}
+              style={{
+                fontSize: 14,
+                fontWeight: 700,
+                color: "white",
+                padding: "10px 20px",
+                marginRight: 10,
+                borderRadius: 2,
+                backgroundColor: globals.brightBlue,
+                border: "none",
+                cursor: "pointer"
+              }}
+            >
+              reset graph
+            </button>
             <button
               onClick={() => {
                 this.props.dispatch(actions.regraph());

src/components/scatterplot/scatterplot.js

@@ -39,11 +39,10 @@ import { margin, width, height, createDimensions } from "./util";
     initializeRanges,
     colorAccessor: state.controls.colorAccessor,
     colorScale: state.controls.colorScale,
-    currentCellSelection: state.controls.currentCellSelection,
-    currentCellSelectionMap: state.controls.currentCellSelectionMap,
     scatterplotXXaccessor: state.controls.scatterplotXXaccessor,
     scatterplotYYaccessor: state.controls.scatterplotYYaccessor,
     opacityForDeselectedCells: state.controls.opacityForDeselectedCells,
+    crossfilter: state.controls.crossfilter,
     differential: state.differential,
     expression: state.expression
   };
@@ -124,7 +123,6 @@ class Scatterplot extends React.Component {
       this.state.pointBuffer &&
       this.state.colorBuffer &&
       this.state.sizeBuffer &&
-      this.props.currentCellSelection &&
       this.props.expression.data &&
       this.props.expression.data.genes &&
       this.props.scatterplotXXaccessor &&
@@ -132,8 +130,7 @@ class Scatterplot extends React.Component {
       this.state.xScale &&
       this.state.yScale
     ) {
-      const currentCellSelectionMap = this.props.currentCellSelectionMap;
+      const crossfilter = this.props.crossfilter.cells;
-
       const data = this.props.expression.data;
       const cells = data.cells;
       const genes = data.genes;
@@ -158,9 +155,8 @@ class Scatterplot extends React.Component {
       /*
         Construct Vectors
       */
-      for (var i = 0; i < cellCount; i++) {
+      for (let i = 0; i < cellCount; i++) {
         const cell = cells[i];
-        const cellMetadata = currentCellSelectionMap[cell.cellname];
 
         positions[2 * i] = glScaleX(
           this.state.xScale(cell.e[geneXXaccessorIndex])
@@ -168,14 +164,15 @@ class Scatterplot extends React.Component {
         positions[2 * i + 1] = glScaleY(
           this.state.yScale(cell.e[geneYYaccessorIndex])
         );
-
-        colors.set(cellMetadata.__colorRGB__, 3 * i);
-
-        sizes[i] = cellMetadata.__selected__
-          ? 4
-          : 0.2; /* make this a function of the number of total cells, including regraph */
       }
 
+      for (let i = 0; i < cellCount; i++) {
+        const metadata = this.props.metadata[i];
+        colors.set(metadata.__colorRGB__, 3 * i);
+      }
+
+      crossfilter.fillByIsFiltered(sizes, 4, 0.2);
+
       this.state.pointBuffer({ data: positions, dimension: 2 });
       this.state.colorBuffer({ data: colors, dimension: 3 });
       this.state.sizeBuffer({ data: sizes, dimension: 1 });

src/middleware/updateCellColors.js

@@ -20,7 +20,7 @@ import { parseRGB } from "../util/parseRGB";
   This is nice because we keep a lot of filtering business logic centralized (what it means in practice to be selected)
 */
 
-const updateCellSelectionMiddleware = store => {
+const updateCellColorsMiddleware = store => {
   return next => {
     return action => {
       const s = store.getState();
@@ -31,22 +31,20 @@ const updateCellSelectionMiddleware = store => {
         action.type === "color by continuous metadata" ||
         action.type === "color by categorical metadata";
 
-      if (!filterJustChanged || !s.controls.allCellsOnClient) {
+      if (!filterJustChanged || !s.controls.cellsMetadata) {
         return next(
           action
         ); /* if the cells haven't loaded or the action wasn't a color change, bail */
       }
 
-      let currentSelectionWithUpdatedColors = s.controls.currentCellSelection.slice(
+      let cellsMetadataWithUpdatedColors = s.controls.cellsMetadata.slice(0);
-        0
-      );
       let colorScale;
 
       /*
          in plain language...
 
          (a) once the cells have loaded.
-         (b) each time a user changes a color control we need to update currentCellSelection colors
+         (b) each time a user changes a color control we need to update cellsMetadata colors
 
          This is available to all the draw functions as cell["__color__"] and cell["__colorRGB__"]
       */
@@ -54,8 +52,8 @@ const updateCellSelectionMiddleware = store => {
       if (action.type === "color by categorical metadata") {
         colorScale = d3.scaleOrdinal().range(globals.ordinalColors);
 
-        for (let i = 0; i < currentSelectionWithUpdatedColors.length; i++) {
+        for (let i = 0; i < cellsMetadataWithUpdatedColors.length; i++) {
-          const cell = currentSelectionWithUpdatedColors[i];
+          const cell = cellsMetadataWithUpdatedColors[i];
           let c = colorScale(cell[action.colorAccessor]);
           cell.__color__ = c;
           cell.__colorRGB__ = parseRGB(c);
@@ -68,10 +66,10 @@ const updateCellSelectionMiddleware = store => {
           .domain([0, action.rangeMaxForColorAccessor])
           .range([1, 0]);
 
-        _.each(currentSelectionWithUpdatedColors, (cell, i) => {
+        _.each(cellsMetadataWithUpdatedColors, (cell, i) => {
           let c = d3.interpolateViridis(colorScale(cell[action.colorAccessor]));
-          currentSelectionWithUpdatedColors[i]["__color__"] = c;
+          cellsMetadataWithUpdatedColors[i]["__color__"] = c;
-          currentSelectionWithUpdatedColors[i]["__colorRGB__"] = parseRGB(c);
+          cellsMetadataWithUpdatedColors[i]["__colorRGB__"] = parseRGB(c);
         });
       }
 
@@ -113,12 +111,12 @@ const updateCellSelectionMiddleware = store => {
             0
           ]); /* invert viridis... probably pass this scale through to others */
 
-        _.each(currentSelectionWithUpdatedColors, (cell, i) => {
+        _.each(cellsMetadataWithUpdatedColors, (cell, i) => {
           let c = d3.interpolateViridis(
             colorScale(expressionMap[cell.CellName][indexOfGene])
           );
-          currentSelectionWithUpdatedColors[i]["__color__"] = c;
+          cellsMetadataWithUpdatedColors[i]["__color__"] = c;
-          currentSelectionWithUpdatedColors[i]["__colorRGB__"] = parseRGB(c);
+          cellsMetadataWithUpdatedColors[i]["__colorRGB__"] = parseRGB(c);
         });
       }
 
@@ -126,7 +124,7 @@ const updateCellSelectionMiddleware = store => {
         append the result of all the filters to the action the user just triggered
       */
       let modifiedAction = Object.assign({}, action, {
-        currentSelectionWithUpdatedColors,
+        cellsMetadataWithUpdatedColors,
         colorScale
       });
 
@@ -135,4 +133,4 @@ const updateCellSelectionMiddleware = store => {
   };
 };
 
-export default updateCellSelectionMiddleware;
+export default updateCellColorsMiddleware;

src/middleware/updateCellSelectionMiddleware.js

@@ -2,6 +2,13 @@
 import uri from "urijs";
 import * as globals from "../globals";
 
+/*
+XXX: this file should be obsolete.  We just need to complete the refactoring
+of parallel.js and it can be removed entirely.
+
+It is currently not in use - the middleware constructor does not include include it
+*/
+
 /*
   https://medium.com/@jacobp100/you-arent-using-redux-middleware-enough-94ffe991e6
   storeInstance => functionToCallWithAnActionThatWillSendItToTheNextMiddleware => actionThatDispatchWasCalledWith => valueToUseAsTheReturnValueOfTheDispatchCall
@@ -34,11 +41,7 @@ const updateCellSelectionMiddleware = store => {
         action.type === "categorical metadata filter none of these" ||
         action.type === "categorical metadata filter all of these";
 
-      if (
+      if (!filterJustChanged || !s.controls.cellsMetadata) {
-        !filterJustChanged ||
-        !s.controls.allCellsOnClient
-        /* graph is set at the same time as allCells, so we assume it exists */
-      ) {
         return next(
           action
         ); /* if the cells haven't loaded or the action wasn't a filter, bail */
@@ -48,7 +51,7 @@ const updateCellSelectionMiddleware = store => {
         - make a FRESH copy of all of the cells
         - metadata has cellname and index, and that's all we ever need to reference cell info
       */
-      let newSelection = s.controls.currentCellSelection.slice(0);
+      let newSelection = s.controls.cellsMetadata.slice(0);
       // _.forEach(newSelection, cell => (cell.__selected__ = true));
       for (let i = 0; i < newSelection.length; i++) {
         newSelection[i].__selected__ = true;
@@ -58,7 +61,7 @@ const updateCellSelectionMiddleware = store => {
          in plain language...
 
          (a) once the cells have loaded.
-         (b) each time a user changes ANY control we need to update currentCellSelection
+         (b) each time a user changes ANY control we need to update cellsMetadata
          there are two states:
 
          1. control state we already know about (state.foo)
@@ -66,34 +69,6 @@ const updateCellSelectionMiddleware = store => {
 
       */
 
-      if (
-        /* is there a 2d graph brush selection ? */
-        action.type === "graph brush selection change" ||
-        s.controls.graphBrushSelection
-      ) {
-        const graphBrushSelection /* it exists, so is it new or old */ =
-          action.type === "graph brush selection change"
-            ? action.brushCoords
-            : s.controls.graphBrushSelection;
-
-        const graphVec = s.controls.graphVec;
-        for (let i = 0; i < newSelection.length; i++) {
-          const cell = newSelection[i];
-          const cellId = cell.__cellIndex__;
-          const x = graphVec[2 * cellId];
-          const y = graphVec[2 * cellId + 1];
-
-          const pointIsInsideBrushBounds =
-            x >= graphBrushSelection.northwest[0] &&
-            x <= graphBrushSelection.southeast[0] &&
-            y <= graphBrushSelection.northwest[1] &&
-            y >= graphBrushSelection.southeast[1];
-
-          if (!pointIsInsideBrushBounds) {
-            cell.__selected__ = false;
-          }
-        }
-      }
       if (
         (action.type ===
           "continuous selection using parallel coords brushing" &&
@@ -118,139 +93,8 @@ const updateCellSelectionMiddleware = store => {
         });
       }
 
-      /*
-        Continuous histograms ___---^^^^--[------__]__---___
-
-        Create newContinuousUserDefinedRanges
-        Filter based on them
-      */
-
-      let newContinuousUserDefinedRanges =
-        s.controls.continuousUserDefinedRanges;
-
-      /* check if this is the action and take care of that metadata field */
-      if (action.type === "continuous metadata histogram brush") {
-        /*
-          was this a deselect? if so it will be null
-          was it a select? set the new range [20, 50]
-          we overload this because it is less if statements thru the whole system
-          but it's invisible here, thus comment.
-        */
-        newContinuousUserDefinedRanges[action.selection] = action.range;
-      }
-
-      let activeContinuousHistogramFilters = [];
-
-      _.each(newContinuousUserDefinedRanges, (value, key, i) => {
-        if (value !== null) {
-          activeContinuousHistogramFilters.push(key);
-        }
-      });
-
-      /* see if there are others from previous... */
-      if (activeContinuousHistogramFilters.length > 0) {
-        _.each(activeContinuousHistogramFilters, key => {
-          _.each(newSelection, (cell, i) => {
-            if (
-              +cell[key] < newContinuousUserDefinedRanges[key][0] ||
-              +cell[key] > newContinuousUserDefinedRanges[key][1]
-            ) {
-              newSelection[i]["__selected__"] = false;
-            }
-          });
-        });
-      }
-
-      /*
-        1. figure out if the users have unchecked boxes
-        2. put them in an array
-        3. filter on them
-      */
-
-      let newCategoricalAsBooleansMap = s.controls.categoricalAsBooleansMap;
-
-      /*
-        ...spread for merge: https://github.com/reactjs/redux/issues/432
-
-        we do the update here instead of the reducer because we need it for the reactive computation
-      */
-      if (action.type === "categorical metadata filter select") {
-        newCategoricalAsBooleansMap = {
-          ...s.controls.categoricalAsBooleansMap,
-          [action.metadataField]: {
-            ...s.controls.categoricalAsBooleansMap[action.metadataField],
-            [action.value]: true
-          }
-        };
-      } else if (action.type === "categorical metadata filter deselect") {
-        newCategoricalAsBooleansMap = {
-          ...s.controls.categoricalAsBooleansMap,
-          [action.metadataField]: {
-            ...s.controls.categoricalAsBooleansMap[action.metadataField],
-            [action.value]: false
-          }
-        };
-      } else if (action.type === "categorical metadata filter none of these") {
-        const metadataFieldWithAllOfTheseValueSelected = {};
-
-        /* set EVERYTHING to false in this intermediate object */
-        _.each(
-          s.controls.categoricalAsBooleansMap[action.metadataField],
-          (isActive, option) => {
-            metadataFieldWithAllOfTheseValueSelected[option] = false;
-          }
-        );
-
-        newCategoricalAsBooleansMap = {
-          ...s.controls.categoricalAsBooleansMap,
-          [action.metadataField]: metadataFieldWithAllOfTheseValueSelected
-        };
-      } else if (action.type === "categorical metadata filter all of these") {
-        const metadataFieldWithAllOfTheseValueSelected = {};
-
-        /* set EVERYTHING to true in this intermediate object */
-        _.each(
-          s.controls.categoricalAsBooleansMap[action.metadataField],
-          (isActive, option) => {
-            metadataFieldWithAllOfTheseValueSelected[option] = true;
-          }
-        );
-
-        newCategoricalAsBooleansMap = {
-          ...s.controls.categoricalAsBooleansMap,
-          [action.metadataField]: metadataFieldWithAllOfTheseValueSelected
-        };
-      }
-
-      const inactiveCategories = [];
-      _.each(newCategoricalAsBooleansMap, (options, category) => {
-        _.each(options, (isActive, option) => {
-          if (!isActive) {
-            inactiveCategories.push({ category, option });
-          }
-        });
-      });
-
-      if (inactiveCategories.length > 0) {
-        _.each(inactiveCategories, d => {
-          if (
-            s.controls.categoricalAsCellsMap[d.category] &&
-            s.controls.categoricalAsCellsMap[d.category][d.option]
-          ) {
-            _.forEach(
-              s.controls.categoricalAsCellsMap[d.category][d.option],
-              c => {
-                c.__selected__ = false;
-              }
-            );
-          }
-        });
-      }
-
       let modifiedAction = Object.assign({}, action, {
-        newSelection,
+        newSelection
-        newCategoricalAsBooleansMap,
-        newContinuousUserDefinedRanges
       }); /* append the result of all the filters to the action the user just triggered */
 
       return next(modifiedAction);

src/reducers/controls.js

@@ -1,16 +1,114 @@
 // jshint esversion: 6
 import _ from "lodash";
 import { parseRGB } from "../util/parseRGB";
+import { createSchemaByDataSniffing } from "../util/schema";
+var crossfilter = require("../util/typedCrossfilter");
+
+// Deduce the correct crossfilter dimension type from a metadata
+// schema description.
+//
+function deduceDimensionType(attributes, fieldName) {
+  let dimensionType;
+  if (attributes.type === "string") {
+    dimensionType = "enum";
+  } else if (attributes.type === "int") {
+    dimensionType = Int32Array;
+  } else if (attributes.type === "float") {
+    dimensionType = Float32Array;
+  } else {
+    console.error(
+      `Warning - REST API returned unknown metadata schema (${
+        attributes.type
+      }) for field ${fieldName}.`
+    );
+    // skip it - we don't know what to do with this type
+  }
+  return dimensionType;
+}
+
+// Create view state from /cells data response.  Used both during a data
+// load and during a graph reset.
+//
+function createViewState(schema, data) {
+  const cellsMetadata = data.metadata.slice(0);
+
+  /*
+  construct a copy of the ranges object that only has categorical
+  replace all counts with bool flags
+  ie., everything starts out checked
+  we mutate this map in the actions below
+  */
+  const categoricalAsBooleansMap = {};
+  _.each(data.ranges, (value, key) => {
+    if (
+      key !== "CellName" &&
+      value.options /* it's categorical, it has options instead of ranges */
+    ) {
+      const optionsAsBooleans = {};
+      _.each(value.options, (_value, _key) => {
+        optionsAsBooleans[_key] = true;
+      });
+      categoricalAsBooleansMap[key] = optionsAsBooleans;
+    }
+  });
+
+  const graph = data.graph;
+  _.each(cellsMetadata, (cell, idx) => {
+    cell.__cellIndex__ = idx;
+    cell.__color__ =
+      "rgba(0,0,0,1)"; /* initial color for all cells in all charts */
+    cell.__colorRGB__ = parseRGB(cell.__color__);
+    cell.__x__ = graph[idx][1];
+    cell.__y__ = graph[idx][2];
+  });
+
+  // Build the selection crossfilter.
+  //
+  let cellsCrossfilter = crossfilter(cellsMetadata);
+  let cellsDimensionsMap = {};
+  cellsDimensionsMap.x = cellsCrossfilter.dimension(r => r.__x__, Float32Array);
+  cellsDimensionsMap.y = cellsCrossfilter.dimension(r => r.__y__, Float32Array);
+
+  // Now walk the schema and make an appropriate dimension for each
+  // metadata field.   This is a simplistic mapping, and could be
+  // optmized to use smaller scalars (to save memory) or larger
+  // floating point where precision is needed.
+  //
+  _.forEach(schema, (attributes, key) => {
+    if (key !== "CellName") {
+      const dimensionType = deduceDimensionType(attributes, key);
+      if (dimensionType) {
+        cellsDimensionsMap[key] = cellsCrossfilter.dimension(
+          r => r[key],
+          dimensionType
+        );
+      }
+    }
+  });
+
+  return {
+    cellsMetadata,
+    crossfilter: {
+      cells: cellsCrossfilter,
+      dimensionMap: cellsDimensionsMap
+    },
+    categoricalAsBooleansMap
+  };
+}
 
 const Controls = (
   state = {
+    /* Universe - all cells known to us.  Set once, during initial load */
     _ranges: null /* this comes from initialize, this is universe */,
     allGeneNames: null,
-    allCellsOnClient: null /* this comes from cells endpoint, this is world */,
+    allCells: null /* this comes from cells endpoint, this is universe */,
-    currentCellSelection: null /* this comes from user actions, all draw components use this, it is created by middleware */,
+    allCellsMetadata: null /* this comes from cells endpoint, and is just the metadata for universe */,
-    graphVec: null,
+
+    /* View / World - all cells currently being displayed.  May be a subset of Universe. */
+    cellsMetadata: null,
+    crossfilter: null /* the current user selection state */,
     categoricalAsBooleansMap: null,
-    categoricalAsCellsMap: null,
+
     colorAccessor: null,
     colorScale: null,
     opacityForDeselectedCells: 0.2,
@@ -28,118 +126,107 @@ const Controls = (
     /**********************************
         Keep a copy of 'universe'
   ***********************************/
-    case "initialize success":
+    case "initialize success": {
+      if (!action.data.data.schema) {
+        console.error("Warning - REST API omitted schema description.");
+      }
       return Object.assign({}, state, {
         _ranges: action.data.data.ranges,
-        allGeneNames: action.data.data.genes
+        allGeneNames: action.data.data.genes,
+        schema: action.data.data.schema
       });
-    case "request cells success":
+    }
-      // Store the graph in a linear array for fast access.  Index into
+    case "request cells success": {
-      // the array by "cell index", which is stored as metadata field
+      // If we don't have a schema (bad server!), fake it by inferring
-      // __cellIndex__.
+      // important fields from the ranges element.
       //
-      // Code below relies on the REST API guarantee that the graph and
+      if (!state.schema) {
-      // metadata are returned as arrays with the same order and length.
+        state.schema = createSchemaByDataSniffing(action.data.data.ranges);
-      //
+      }
-      const graphVec = new Float32Array(2 * action.data.data.graph.length);
-      _.each(action.data.data.graph, (g, i) => {
-        graphVec[2 * i] = g[1];
-        graphVec[2 * i + 1] = g[2];
-      });
-
-      const currentCellSelection = action.data.data.metadata.slice(0);
-      const currentCellSelectionMap = _.keyBy(currentCellSelection, "CellName");
-
-      /*
-      construct a copy of the ranges object that only has categorical
-      replace all counts with bool flags
-      ie., everything starts out checked
-      we mutate this map in the actions below
-    */
-      const categoricalAsBooleansMap = {},
-        categoricalAsCellsMap = {};
-      const continuousUserDefinedRanges = {};
-      _.each(action.data.data.ranges, (value, key) => {
-        if (
-          key !== "CellName" &&
-          value.options /* it's categorical, it has options instead of ranges */
-        ) {
-          const optionsAsBooleans = {},
-            optionsAsCells = {};
-          _.each(value.options, (_value, _key) => {
-            optionsAsBooleans[_key] = true;
-            optionsAsCells[_key] = [];
-          });
-          categoricalAsBooleansMap[key] = optionsAsBooleans;
-          categoricalAsCellsMap[key] = optionsAsCells;
-        } else if (key !== "CellName" && value.range) {
-          continuousUserDefinedRanges[key] = null;
-        }
-      });
-
-      _.each(currentCellSelection, (cell, idx) => {
-        cell.__cellIndex__ = idx;
-        cell.__selected__ = true;
-        cell.__color__ =
-          "rgba(0,0,0,1)"; /* initial color for all cells in all charts */
-        cell.__colorRGB__ = parseRGB(cell.__color__);
-
-        // Add each cell to its categorical metadata set.
-        _.forEach(cell, (_value, key) => {
-          if (
-            categoricalAsCellsMap[key] &&
-            categoricalAsCellsMap[key][_value]
-          ) {
-            const s = categoricalAsCellsMap[key][_value];
-            if (s) s.push(cell);
-          }
-        });
-      });
 
+      /* Set viewable world to the provided cell data */
+      const viewState = createViewState(state.schema, action.data.data);
       return Object.assign({}, state, {
-        allCellsOnClient: action.data.data,
+        /* Universe - initialize once */
-        currentCellSelection,
+        allCells: state.allCells ? state.allCells : action.data,
-        currentCellSelectionMap,
+        allCellsMetadata: state.allCellsMetadata
-        graphVec,
+          ? state.allCellsMetadata
-        categoricalAsBooleansMap,
+          : viewState.cellsMetadata,
-        categoricalAsCellsMap,
+        allCellsMetadataMap: state.allCellsMetadataMap
-        continuousUserDefinedRanges,
+          ? state.allCellsMetadataMap
+          : _.keyBy(viewState.cellsMetadata, "CellName"),
+
+        /* World */
+        ...viewState,
+
         graphBrushSelection: null /* if we are getting new cells from the server, the layout (probably? definitely?) just changed, so this is now irrelevant, and we WILL need to call a function to reset state of this kind when cells success happens */
       });
+    }
     /* * * * * * * * * * * * * * * * * *
             User events
   * * * * * * * * * * * * * * * * * */
-    case "parallel coordinates axes have been drawn":
+    case "reset graph": {
+      /* Reset viewable world to the entire Universe */
+      const viewState = createViewState(state.schema, state.allCells.data);
+      return Object.assign({}, state, {
+        ...viewState
+      });
+    }
+    case "parallel coordinates axes have been drawn": {
       return Object.assign({}, state, {
         axesHaveBeenDrawn: true
       });
+    }
     case "continuous selection using parallel coords brushing": {
       return Object.assign({}, state, {
         continuousSelection: action.data,
-        currentCellSelection:
+        crossfilter: {
-          action.newSelection /* this comes from middleware */
+          ...state.crossfilter
+        }
       });
     }
-    case "graph brush selection change":
+    case "graph brush selection change": {
+      state.crossfilter.dimensionMap.x.filterRange([
+        action.brushCoords.northwest[0],
+        action.brushCoords.southeast[0]
+      ]);
+      state.crossfilter.dimensionMap.y.filterRange([
+        action.brushCoords.southeast[1],
+        action.brushCoords.northwest[1]
+      ]);
       return Object.assign({}, state, {
-        graphBrushSelection:
+        graphBrushSelection: action.brushCoords,
-          action.brushCoords /* this has already been applied in middleware but store it for next time */,
+        crossfilter: {
-        currentCellSelection:
+          ...state.crossfilter
-          action.newSelection /* this comes from middleware */
+        }
       });
-    case "graph brush deselect":
+    }
+    case "graph brush deselect": {
+      state.crossfilter.dimensionMap.x.filterAll();
+      state.crossfilter.dimensionMap.y.filterAll();
       return Object.assign({}, state, {
         graphBrushSelection: null,
-        currentCellSelection:
+        crossfilter: {
-          action.newSelection /* this comes from middleware */
+          ...state.crossfilter
+        }
       });
-    case "continuous metadata histogram brush":
+    }
+    case "continuous metadata histogram brush": {
+      // action.selection: metadata name being selected
+      // action.range: filter range, or null if deselected
+      if (!action.range) {
+        state.crossfilter.dimensionMap[action.selection].filterAll();
+      } else {
+        state.crossfilter.dimensionMap[action.selection].filterRange(
+          action.range
+        );
+      }
       return Object.assign({}, state, {
-        newContinuousUserDefinedRanges:
+        crossfilter: {
-          action.newContinuousUserDefinedRanges /* this has already been applied in middleware but store it for next time */,
+          ...state.crossfilter
-        currentCellSelection:
+        }
-          action.newSelection /* this comes from middleware */
       });
+    }
     case "change opacity deselected cells in 2d graph background":
       return Object.assign({}, state, {
         opacityForDeselectedCells: action.data
@@ -147,57 +234,113 @@ const Controls = (
     /*******************************
         Categorical metadata
   *******************************/
-    case "categorical metadata filter select":
+    case "categorical metadata filter select": {
+      const newCategoricalAsBooleansMap = {
+        ...state.categoricalAsBooleansMap,
+        [action.metadataField]: {
+          ...state.categoricalAsBooleansMap[action.metadataField],
+          [action.value]: true
+        }
+      };
+      // update the filter for the one category that changed state
+      state.crossfilter.dimensionMap[action.metadataField].filterEnum(
+        _.filter(
+          _.map(
+            newCategoricalAsBooleansMap[action.metadataField],
+            (val, key) => (val ? key : false)
+          )
+        )
+      );
       return Object.assign({}, state, {
-        categoricalAsBooleansMap:
+        categoricalAsBooleansMap: newCategoricalAsBooleansMap,
-          action.newCategoricalAsBooleansMap /* this comes from middleware */,
+        crossfilter: {
-        currentCellSelection:
+          ...state.crossfilter
-          action.newSelection /* this comes from middleware */
+        }
       });
-    case "categorical metadata filter deselect":
+    }
+    case "categorical metadata filter deselect": {
+      const newCategoricalAsBooleansMap = {
+        ...state.categoricalAsBooleansMap,
+        [action.metadataField]: {
+          ...state.categoricalAsBooleansMap[action.metadataField],
+          [action.value]: false
+        }
+      };
+      // update the filter for the one category that changed state
+      state.crossfilter.dimensionMap[action.metadataField].filterEnum(
+        _.filter(
+          _.map(
+            newCategoricalAsBooleansMap[action.metadataField],
+            (val, key) => (val ? key : false)
+          )
+        )
+      );
       return Object.assign({}, state, {
-        categoricalAsBooleansMap:
+        categoricalAsBooleansMap: newCategoricalAsBooleansMap,
-          action.newCategoricalAsBooleansMap /* this comes from middleware */,
+        crossfilter: {
-        currentCellSelection:
+          ...state.crossfilter
-          action.newSelection /* this comes from middleware */
+        }
       });
-    case "categorical metadata filter none of these":
+    }
+    case "categorical metadata filter none of these": {
+      const newCategoricalAsBooleansMap = {
+        ...state.categoricalAsBooleansMap
+      };
+      _.forEach(
+        newCategoricalAsBooleansMap[action.metadataField],
+        (v, k, c) => {
+          c[k] = false;
+        }
+      );
+      state.crossfilter.dimensionMap[action.metadataField].filterNone();
       return Object.assign({}, state, {
-        categoricalAsBooleansMap:
+        categoricalAsBooleansMap: newCategoricalAsBooleansMap,
-          action.newCategoricalAsBooleansMap /* this comes from middleware */,
+        crossfilter: {
-        currentCellSelection:
+          ...state.crossfilter
-          action.newSelection /* this comes from middleware */
+        }
       });
-    case "categorical metadata filter all of these":
+    }
+    case "categorical metadata filter all of these": {
+      const newCategoricalAsBooleansMap = {
+        ...state.categoricalAsBooleansMap
+      };
+      _.forEach(
+        newCategoricalAsBooleansMap[action.metadataField],
+        (v, k, c) => {
+          c[k] = true;
+        }
+      );
+      state.crossfilter.dimensionMap[action.metadataField].filterAll();
       return Object.assign({}, state, {
-        categoricalAsBooleansMap:
+        categoricalAsBooleansMap: newCategoricalAsBooleansMap,
-          action.newCategoricalAsBooleansMap /* this comes from middleware */,
+        crossfilter: {
-        currentCellSelection:
+          ...state.crossfilter
-          action.newSelection /* this comes from middleware */
+        }
       });
+    }
     /*******************************
             Color Scale
   *******************************/
     case "color by continuous metadata":
       return Object.assign({}, state, {
         colorAccessor: action.colorAccessor,
-        currentCellSelection:
+        cellsMetadata:
-          action.currentSelectionWithUpdatedColors /* this comes from middleware */,
+          action.cellsMetadataWithUpdatedColors /* this comes from middleware */,
         colorScale: action.colorScale
       });
     case "color by expression":
       return Object.assign({}, state, {
         colorAccessor: action.gene,
-        currentCellSelection:
+        cellsMetadata:
-          action.currentSelectionWithUpdatedColors /* this comes from middleware */,
+          action.cellsMetadataWithUpdatedColors /* this comes from middleware */,
         colorScale: action.colorScale
       });
     case "color by categorical metadata":
       return Object.assign({}, state, {
         colorAccessor:
           action.colorAccessor /* pass the scale through additionally, and it's a legend! */,
-        currentCellSelection:
+        cellsMetadata:
-          action.currentSelectionWithUpdatedColors /* this comes from middleware */,
+          action.cellsMetadataWithUpdatedColors /* this comes from middleware */,
         colorScale: action.colorScale
       });
     case "store current cell selection as differential set 1":

src/reducers/index.js

@@ -1,7 +1,7 @@
 // jshint esversion: 6
 import { combineReducers, createStore, applyMiddleware } from "redux";
 import updateURLMiddleware from "../middleware/updateURLMiddleware";
-import updateCellSelectionMiddleware from "../middleware/updateCellSelectionMiddleware";
+// import updateCellSelectionMiddleware from "../middleware/updateCellSelectionMiddleware";
 import updateCellColors from "../middleware/updateCellColors";
 
 import thunk from "redux-thunk";
@@ -28,7 +28,7 @@ let store = createStore(
   applyMiddleware(
     thunk,
     updateURLMiddleware,
-    updateCellSelectionMiddleware,
+    // updateCellSelectionMiddleware,
     updateCellColors
   )
 );

src/util/schema.js

@@ -0,0 +1,41 @@
+// jshint esversion: 6
+
+// In the case where the REST server does not implement data schema
+// declaration, we attempt to deduce it by sniffing the data.
+//
+export function createSchemaByDataSniffing(ranges) {
+  let schema = {};
+  _.forEach(ranges, (value, key) => {
+    schema[key] = {
+      displayname: key,
+      variabletype: value.options ? "categorical" : "continuous"
+    };
+
+    // Metadata field type is inferred by sniffing the data. This has some risks.
+    // Caveats:
+    //  * Values have been converted to native JS objects by the JSON parser.
+    //  * Lots of assumptions about he REST API behaving properly (eg, min/max
+    //    are the same type, etc).
+    let type;
+    if (schema[key].variabletype === "continuous" && value.range) {
+      // Use min/max as a proxy for all data.
+      const min = value.range.min;
+      const max = value.range.max;
+      type =
+        typeof min !== "number" || typeof max !== "number"
+          ? "string"
+          : Number.isSafeInteger(min) && Number.isSafeInteger(max)
+            ? "int"
+            : "float";
+    } else {
+      // use an option value as a proxy for all data
+      const aVal = value.options[0];
+      type =
+        typeof aVal !== "number"
+          ? "string"
+          : Number.isSafeInteger(aVal) ? "int" : "float";
+    }
+    schema[key].type = type;
+  });
+  return schema;
+}

src/util/typedCrossfilter/bitArray.js

@@ -0,0 +1,228 @@
+"use strict";
+// jshint esversion: 6
+
+// BitArray is a 2D bitarray with size [length, nBitWidth].
+// Each bit is referred to as a `dimension`.  Dimensions may be
+// dynamically allocated and deallocated.  The overall length
+// of the BitArray is fixed at creation time (for simplicity).
+//
+// Organization of the bitarray is dimension-major.  As dimensions
+// are added, the underlying store is grown 32 bits at a time.
+// NOTE: currently does not deallocate / shrink.
+//
+// Primary operations on the BitArray are:
+//    - set & clear dimension
+//    - test dimension
+//    - various performance or convenience operations to optimize bulk ops
+//
+// The underlying data structure uses TypedArrays for performance.
+//
+class BitArray {
+  constructor(length) {
+    // Initially allocate a 32 bit wide array.  allocDimension() will expand
+    // as necessary.
+    //
+    // Int32Array is (counterintuitively) used to accomadate JS numeric casting
+    // (to/from primitive number type).
+    //
+
+    // Fixed for the life of this object.
+    this.length = length;
+
+    // Bitarray width.  width is always greater than 32*dimensionCount.
+    this.width = 1; // underlying number of 32 bit arrays
+    this.dimensionCount = 0; // num allocated dimensions
+
+    this.bitmask = new Int32Array(this.width); // dimension allocation mask
+    this.bitarray = new Int32Array(this.width * this.length);
+  }
+
+  // Return the number of records that are selected, ie, have a one bit in
+  // all allocated dimensions.
+  //
+  get selectionCount() {
+    return this.countAllOnes();
+  }
+
+  // Count all records that have a 'one' bit in allocated dimensions.
+  //
+  countAllOnes() {
+    let count = 0;
+    for (let i = 0; i < this.width; i++) {
+      const bitmask = this.bitmask[i];
+      for (let j = i * this.length, len = j + this.length; j < len; j++) {
+        if (this.bitarray[i * this.length + j] === bitmask) count++;
+      }
+    }
+    return count;
+  }
+
+  // count trailing zeros - hard to do fast in JS!
+  // https://en.wikipedia.org/wiki/Find_first_set#CTZ
+  static ctz(v) {
+    let c = 32;
+    v &= -v; // isolate lowest non-zero bit
+    if (v) c--;
+    if (v & 0x0000ffff) c -= 16;
+    if (v & 0x00ff00ff) c -= 8;
+    if (v & 0x0f0f0f0f) c -= 4;
+    if (v & 0x33333333) c -= 2;
+    if (v & 0x55555555) c -= 1;
+    return c;
+  }
+
+  // find a free dimension.  Return undefined if none
+  _findFreeDimension() {
+    let dim;
+    for (let col = 0; col < this.width; col++) {
+      const bitmask = this.bitmask[col];
+      const lowestZeroBit = ~this.bitmask[col] & -~this.bitmask[col];
+      if (lowestZeroBit) {
+        this.bitmask[col] |= lowestZeroBit;
+        dim = 32 * col + BitArray.ctz(lowestZeroBit);
+      }
+    }
+    return dim;
+  }
+
+  // allocate and return the dimension ID (bit position)
+  //
+  allocDimension() {
+    let dim = this._findFreeDimension();
+
+    // if we did not find free dimension, expand the bitarray.
+    if (dim === undefined) {
+      this.width++;
+
+      const biggerBitArray = new Int32Array(this.width * this.length);
+      biggerBitArray.set(this.bitarray);
+      this.bitarray = biggerBitArray;
+
+      const biggerBitmask = new Int32Array(this.width);
+      biggerBitmask.set(this.bitmask);
+      this.bitmask = biggerBitmask;
+
+      dim = this._findFreeDimension();
+    }
+
+    this.dimensionCount++;
+    return dim;
+  }
+
+  // free a dimension for later use.  MUST deselect the dimension, as other
+  // code assume the column will be zero valued.
+  //
+  freeDimension(dim) {
+    // all selection tests assume unallocated dimensions are zero valued.
+    this.deselectAll(dim);
+    const col = dim >>> 5;
+    this.bitmask[col] &= ~(1 << (dim % 32));
+    this.dimensionCount--;
+  }
+
+  // return true if this index is selected in ALL dimensions.
+  //
+  isSelected(index) {
+    const width = this.width;
+    const length = this.length;
+    const bitarray = this.bitarray;
+
+    for (let w = 0; w < width; w++) {
+      const bitmask = this.bitmask[w];
+      if (!bitmask || bitarray[w * length + index] !== bitmask) return false;
+    }
+    return true;
+  }
+
+  // select index on dimension
+  //
+  selectOne(dim, index) {
+    const col = dim >>> 5;
+    const before = this.bitarray[col * this.length + index];
+    const after = before | (1 << (dim % 32));
+    this.bitarray[col] = after;
+  }
+
+  // deselect index on dimension
+  //
+  deselectOne(dim, index) {
+    const col = dim >>> 5;
+    const before = this.bitarray[col * this.length + index];
+    const after = before & ~(1 << (dim % 32));
+    this.bitarray[col] = after;
+  }
+
+  // select all indices on dimension.
+  //
+  selectAll(dim) {
+    let col = dim >> 5;
+    const bitmask = this.bitmask[col];
+    const bitarray = this.bitarray;
+    const one = 1 << (dim % 32);
+    for (let i = col * this.length, len = i + this.length; i < len; i++) {
+      bitarray[i] |= one;
+    }
+  }
+
+  // deselect all indices on dimension
+  //
+  deselectAll(dim) {
+    let col = dim >> 5;
+    const bitmask = this.bitmask[col];
+    const bitarray = this.bitarray;
+    const zero = ~(1 << (dim % 32));
+    for (let i = col * this.length, len = i + this.length; i < len; i++) {
+      bitarray[i] &= zero;
+    }
+  }
+
+  // select range of indices on a dimension, indirect through a sort map.
+  // Indirect functions are used to map between sort and natural order.
+  //
+  selectIndirectFromRange(dim, indirect, range) {
+    const col = dim >>> 5;
+    const first = range[0];
+    const last = range[1];
+    const bitarray = this.bitarray;
+    const one = 1 << (dim % 32);
+    const offset = col * this.length;
+    for (let i = first; i < last; i++) {
+      bitarray[offset + indirect[i]] |= one;
+    }
+  }
+
+  // deselect range of indices on a dimension, indirect through a sort map.
+  //
+  deselectIndirectFromRange(dim, indirect, range) {
+    const col = dim >>> 5;
+    const first = range[0];
+    const last = range[1];
+    const bitarray = this.bitarray;
+    const zero = ~(1 << (dim % 32));
+    const offset = col * this.length;
+    for (let i = first; i < last; i++) {
+      bitarray[offset + indirect[i]] &= zero;
+    }
+  }
+
+  // Fill the array with selected|deselected value based upon the
+  // current selection state.
+  //
+  fillBySelection(result, selectedValue, deselectedValue) {
+    // special case (width === 1) for performance
+    if (this.width === 1) {
+      const bitmask = this.bitmask[0];
+      const bitarray = this.bitarray;
+      for (let i = 0, len = this.length; i < len; i++) {
+        result[i] = bitarray[i] === bitmask ? selectedValue : deselectedValue;
+      }
+    } else {
+      for (let i = 0, len = this.length; i < len; i++) {
+        result[i] = this.isSelected(i) ? selectedValue : deselectedValue;
+      }
+    }
+    return result;
+  }
+}
+
+module.exports = BitArray;

src/util/typedCrossfilter/index.js

@@ -0,0 +1,394 @@
+"use strict";
+// jshint esversion: 6
+
+/*
+Typedarray Crossfilter - a re-implementation of a subset of crossfilter, with
+time/space optimizations predicated upon the following assumptions:
+  - dimensions are uniformly typed, and all values must be of that type
+  - dimension values must be a primitive type (int, float, string).  Arrays
+    or other complex types not supported.
+  - dimension creation requires call-provided type declaration
+  - no support for adding/removing data to an existing crossfilter.  If you
+    want to do that, you have to create the new crossfilter, using the new
+    data, from scratch.
+
+The actual backing store for a dimension is a TypedArray, enabling significant
+performance improvements over the original crossfilter.
+
+There are also a handful of new methods, primarily to take advantage of the
+performance (eg, crossfilter.fillBySelection)
+
+Helpful documents (this module tries to follow the original API as much
+as is feasable):
+    https://github.com/square/crossfilter/
+    http://square.github.io/crossfilter/
+
+There is also a newer, community supported fork of crossfilter, with a
+more complex API.  In a few cases, elements of that API were incorporated.
+    https://github.com/square/crossfilter/
+
+*/
+
+var PositiveIntervals = require("./positiveIntervals");
+var BitArray = require("./bitArray");
+var Util = require("./util");
+
+class TypedCrossfilter {
+  constructor(data) {
+    this.data = data;
+
+    // filters: array of { id, dimension }
+    this.filters = [];
+    this.selection = new BitArray(data.length);
+  }
+
+  size() {
+    return this.data.length;
+  }
+
+  all() {
+    return this.data;
+  }
+
+  dimension(value, valueArrayType) {
+    const id = this.selection.allocDimension();
+    let dim;
+    if (valueArrayType === "enum") {
+      dim = new EnumDimension(value, this, id);
+    } else {
+      dim = new ScalarDimension(value, valueArrayType, this, id);
+    }
+    this.filters.push({ id, dim });
+    dim.filterAll();
+    return dim;
+  }
+
+  _freeDimension(id) {
+    this.selection.freeDimension(id);
+    this.filters = this.filters.filter(f => f.id != id);
+  }
+
+  // return array of all records that are selected/filtered
+  // by all dimensions.
+  allFiltered() {
+    const selection = this.selection;
+    const res = [];
+    for (let i = 0, len = this.data.length; i < len; i++) {
+      if (selection.isSelected(i)) {
+        res.push(this.data[i]);
+      }
+    }
+    return res;
+  }
+
+  countFiltered() {
+    return this.selection.selectionCount;
+  }
+
+  isElementFiltered(i) {
+    return this.selection.isSelected(i);
+  }
+
+  // fill array with one of two values, based upon selection state
+  fillByIsFiltered(array, selectedValue, deselectedValue) {
+    return this.selection.fillBySelection(
+      array,
+      selectedValue,
+      deselectedValue
+    );
+  }
+}
+
+// Base dimension type - value must be a scalar type (eg, int, float),
+// and value array must be a TypedArray.
+//
+class ScalarDimension {
+  constructor(value, valueArrayType, crossfilter, id) {
+    this.crossfilter = crossfilter;
+    this.id = id;
+
+    // current selection filter, expressed as PostiveIntervals.
+    this.currentFilter = [];
+
+    // Create value array
+    const array = this._createValueArray(
+      value,
+      new valueArrayType(this.crossfilter.data.length)
+    );
+    this.value = array;
+
+    // create sort index
+    this.index = Util.fillRange(new Uint32Array(this.crossfilter.data.length));
+    this.index.sort((a, b) => array[a] - array[b]);
+  }
+
+  _createValueArray(value, array) {
+    // create dimension value array
+    const data = this.crossfilter.data;
+    const len = data.length;
+    for (let i = 0; i < len; i++) {
+      array[i] = value(data[i]);
+    }
+    return array;
+  }
+
+  dispose() {
+    this.crossfilter._freeDimension(this.id);
+  }
+
+  id() {
+    return this.id;
+  }
+
+  _updateFilters(newFilter) {
+    newFilter = PositiveIntervals.canonicalize(newFilter);
+
+    // special case optimization - select all/none can bypass
+    // more complex work and just clobber everything.
+    //
+    if (newFilter.length === 0) {
+      this.crossfilter.selection.deselectAll(this.id);
+    } else if (
+      newFilter.length === 1 &&
+      newFilter[0][0] === 0 &&
+      newFilter[0][1] == this.index.length
+    ) {
+      this.crossfilter.selection.selectAll(this.id);
+    } else {
+      const adds = PositiveIntervals.difference(newFilter, this.currentFilter);
+      const dels = PositiveIntervals.difference(this.currentFilter, newFilter);
+      dels.forEach(interval =>
+        this.crossfilter.selection.deselectIndirectFromRange(
+          this.id,
+          this.index,
+          interval
+        )
+      );
+      adds.forEach(interval =>
+        this.crossfilter.selection.selectIndirectFromRange(
+          this.id,
+          this.index,
+          interval
+        )
+      );
+    }
+
+    this.currentFilter = newFilter;
+  }
+
+  // filter by value - exact match
+  filterExact(value) {
+    const newFilter = [
+      Util.lowerBoundIndirect(
+        this.value,
+        this.index,
+        value,
+        0,
+        this.value.length
+      ),
+      Util.upperBoundIndirect(
+        this.value,
+        this.index,
+        value,
+        0,
+        this.value.length
+      )
+    ];
+    if (newFilter[0] <= newFilter[1]) {
+      this._updateFilters([newFilter]);
+    } else {
+      this._updateFilters([]);
+    }
+    return this;
+  }
+
+  // filter by a set of values, eg. enum.
+  filterEnum(values) {
+    const newFilter = [];
+    for (let v = 0, len = values.length; v < len; v++) {
+      const intv = [
+        Util.lowerBoundIndirect(
+          this.value,
+          this.index,
+          values[v],
+          0,
+          this.value.length
+        ),
+        Util.upperBoundIndirect(
+          this.value,
+          this.index,
+          values[v],
+          0,
+          this.value.length
+        )
+      ];
+      if (intv[0] <= intv[1]) newFilter.push(intv);
+    }
+    this._updateFilters(newFilter);
+    return this;
+  }
+
+  // filter by value range [lo, hi)
+  // lo: inclusive, hi: exclusive
+  filterRange(range) {
+    const newFilter = [];
+    const intv = [
+      Util.lowerBoundIndirect(
+        this.value,
+        this.index,
+        range[0],
+        0,
+        this.value.length
+      ),
+      Util.upperBoundIndirect(
+        this.value,
+        this.index,
+        range[1],
+        0,
+        this.value.length
+      )
+    ];
+    if (intv[0] < intv[1]) newFilter.push(intv);
+    this._updateFilters(newFilter);
+    return this;
+  }
+
+  // select all - equivalent of selecting all in this dimension
+  filterAll() {
+    this._updateFilters([[0, this.value.length]]);
+    return this;
+  }
+
+  // select none
+  filterNone() {
+    this._updateFilters([]);
+  }
+
+  // return top k records, starting with offset, in descending order.
+  // Order is this dimension's sort order
+  top(k, offset = 0) {
+    const data = this.crossfilter.data;
+    const selection = this.crossfilter.selection;
+    const index = this.index;
+    const len = index.length;
+    const ret = [];
+    let i = 0;
+    let skip = 0;
+    let found = 0;
+
+    // skip up to offset records
+    for (i = len - 1; 0 <= i && skip < offset; i--) {
+      if (selection.isSelected(index[i])) {
+        skip++;
+      }
+    }
+
+    // grab up to k records
+    for (; 0 <= i && found < k; i--) {
+      if (selection.isSelected(index[i])) {
+        ret.push(data[index[i]]);
+        found++;
+      }
+    }
+
+    return ret;
+  }
+
+  // return bottom k records, starting with offset, in ascending order.
+  // Order is this dimension's sort order
+  bottom(k, offset = 0) {
+    const data = this.crossfilter.data;
+    const selection = this.crossfilter.selection;
+    const index = this.index;
+    const len = index.length;
+    const ret = [];
+    let skip = 0;
+    let found = 0;
+    let i = 0;
+
+    // skip up to offset records
+    for (i = 0; i < len && skip < offset; i++) {
+      if (selection.isSelected(index[i])) {
+        skip++;
+      }
+    }
+
+    // grab up to k records
+    for (; i < len && found < k; i++) {
+      if (selection.isSelected(index[i])) {
+        ret.push(data[index[i]]);
+        found++;
+      }
+    }
+
+    return ret;
+  }
+}
+
+// Ordered enumeration - supports any sortable enumerable type, eg,
+// strings, which can be mapped into an fixed numeric range [0..n).
+//
+class EnumDimension extends ScalarDimension {
+  constructor(value, crossfilter, id) {
+    super(value, Uint32Array, crossfilter, id);
+  }
+
+  _createValueArray(value, array) {
+    const data = this.crossfilter.data;
+    const len = data.length;
+
+    // create enumeration table - mapping between the value
+    // and the enum.
+    const s = new Set();
+    for (let i = 0; i < len; i++) {
+      s.add(value(data[i]));
+    }
+    this.enumIndex = Array.from(s);
+    this.enumIndex.sort();
+
+    // create dimension value array
+    const enumLen = this.enumIndex.length;
+    for (let i = 0; i < len; i++) {
+      const v = value(data[i]);
+      const e = Util.lowerBound(this.enumIndex, v, 0, enumLen);
+      array[i] = e;
+    }
+    return array;
+  }
+
+  filterExact(value) {
+    return super.filterExact(
+      Util.lowerBound(this.enumIndex, value, 0, this.enumIndex.length)
+    );
+  }
+
+  filterEnum(values) {
+    return super.filterEnum(
+      values.map(v =>
+        Util.lowerBound(this.enumIndex, v, 0, this.enumIndex.length)
+      )
+    );
+  }
+
+  filterRange(range) {
+    return super.filterEnum(
+      range.map(v =>
+        Util.lowerBound(this.enumIndex, v, 0, this.enumIndex.length)
+      )
+    );
+  }
+}
+
+// Wrapper for backwards compat with crossfilter.
+//
+function crossfilter(data) {
+  return new TypedCrossfilter(data);
+}
+
+crossfilter.PositiveIntervals = PositiveIntervals;
+crossfilter.BitArray = BitArray;
+crossfilter.TypedCrossfilter = TypedCrossfilter;
+crossfilter.ScalarDimension = ScalarDimension;
+crossfilter.EnumDimension = EnumDimension;
+
+module.exports = crossfilter;

src/util/typedCrossfilter/positiveIntervals.js

@@ -0,0 +1,132 @@
+"use strict";
+// jshint esversion: 6
+
+// Interval operations - very simple version of interval set relationship
+// operators.   An interval is a multi-interval list of [min, max),
+// where min and max are mandatory.   Constraints:
+//    * min <= max, min >= 0
+//    * empty interval groups are OK, ie, []
+//    * Legal intervals:   [], [ [0, 1], ... ]
+//    * Not legal:   [ [] ]
+//
+// All intervals are represented by simple JS arrays/numbers.
+//
+// Code assumes intervals have a low cardinality; many operations are done
+// with a brute force scan.   Little attempt to reduce GC pressure.
+//
+class PositiveIntervals {
+  // Canonicalize - ensure that:
+  //  1. no overlapping intervals
+  //  2. sorted in order of interval min.
+  //
+  static canonicalize(A) {
+    if (A.length <= 1) return A;
+    let copy = A.slice();
+    copy.sort((a, b) => a[0] - b[0]);
+    const res = [];
+    res.push(copy[0]);
+    for (let i = 1, len = copy.length; i < len; i++) {
+      if (copy[i][0] > res[res.length - 1][1]) {
+        // non-overlapping, add to result
+        res.push(copy[i]);
+      } else if (copy[i][1] > res[res.length - 1][1]) {
+        // merge this into previous
+        res[res.length - 1][1] = copy[i][1];
+      }
+    }
+    return res;
+  }
+
+  // Return interval with values belonging to both A and B. Essentially
+  // a set union operation.
+  //
+  static union(A, B) {
+    return PositiveIntervals.canonicalize([...A, ...B]);
+  }
+
+  static _flatten(A, B) {
+    let points = []; /* point, A, start */
+    for (let a = 0; a < A.length; a++) {
+      points.push([A[a][0], true, true]);
+      points.push([A[a][1], true, false]);
+    }
+    for (let b = 0; b < B.length; b++) {
+      points.push([B[b][0], false, true]);
+      points.push([B[b][1], false, false]);
+    }
+    // Sort order: point, then start
+    points.sort((a, b) => (a[0] !== b[0] ? a[0] - b[0] : a[2] ? 1 : -1));
+    return points;
+  }
+
+  // A - B, ie, the interval with all values in A that are not in B.  Essentially
+  // a set difference operation.
+  //
+  static difference(A, B) {
+    // Corner cases
+    if (A.length === 0 || B.length === 0) {
+      return PositiveIntervals.canonicalize(A);
+    }
+
+    A = PositiveIntervals.canonicalize(A);
+    B = PositiveIntervals.canonicalize(B);
+
+    const points = PositiveIntervals._flatten(A, B);
+    const res = [];
+    let aDepth = 0;
+    let depth = 0;
+    let intervalStart;
+    let prevPoint;
+    for (let i = 0; i < points.length; i++) {
+      const p = points[i];
+      const before = depth;
+      const delta = p[2] ? 1 : -1;
+      depth += delta;
+      if (p[1]) aDepth += delta;
+
+      if (i === points.length - 1 || p[0] !== points[i + 1][0]) {
+        if (aDepth === 1 && depth === 1) {
+          intervalStart = p[0];
+        } else if (intervalStart !== undefined) {
+          res.push([intervalStart, p[0]]);
+          intervalStart = undefined;
+        }
+      }
+      prevPoint = p[0];
+    }
+    // guaranteed to be in canonical form
+    return res;
+  }
+
+  // Return interval with values belonging to A or B.  Essentially a set
+  // intersection.
+  //
+  static intersection(A, B) {
+    if (A.length === 0 || B.length === 0) {
+      return [];
+    }
+
+    A = PositiveIntervals.canonicalize(A);
+    B = PositiveIntervals.canonicalize(B);
+
+    const points = PositiveIntervals._flatten(A, B);
+    const res = [];
+    let depth = 0;
+    let intervalStart;
+    for (let i = 0; i < points.length; i++) {
+      const p = points[i];
+      const before = depth;
+      depth += p[2] ? 1 : -1;
+      if (depth === 2) {
+        intervalStart = p[0];
+      } else if (intervalStart !== undefined) {
+        res.push([intervalStart, p[0]]);
+        intervalStart = undefined;
+      }
+    }
+    // guaranteed to be in canonical form
+    return res;
+  }
+}
+
+module.exports = PositiveIntervals;

src/util/typedCrossfilter/util.js

@@ -0,0 +1,106 @@
+"use strict";
+// jshint esversion: 6
+
+/*
+    Utility functions, private to this module.
+*/
+
+// fill an array or typedarray with a sequential range of numbers,
+// starting with `start`
+//
+function fillRange(arr, start = 0) {
+  for (let i = 0, len = arr.length; i < len; i++) {
+    arr[i] = i + start;
+  }
+  return arr;
+}
+
+// Search for `value` in the sorted array `arr`, in the range [first, last).
+// Return the first (left most) index where arr[index] >= value.
+//
+// In other words, return array index I where:
+//  arr[i] < value for all tarr[lo:I]
+//  arr[i] >= value for all tarr[I:last]
+//
+// The same semantics/behavior as:
+//    C++: lower_bound()
+//    Python: bisect.bisect_left()
+//
+// XXX: it is likely that there would be minimal performance hit from creating
+// a factory version of lowerBound that takes an accessor (rather than having
+// a special-cased version for lining the indirection).
+//
+function lowerBound(valueArray, value, first, last) {
+  // this is just a binary search
+  while (first < last) {
+    const middle = (first + last) >>> 1;
+    if (valueArray[middle] < value) {
+      first = middle + 1;
+    } else {
+      last = middle;
+    }
+  }
+  return first;
+}
+
+// Inlined performance optimization - used to indirect through a sort map.
+//
+function lowerBoundIndirect(valueArray, indexArray, value, first, last) {
+  // this is just a binary search
+  while (first < last) {
+    const middle = (first + last) >>> 1;
+    if (valueArray[indexArray[middle]] < value) {
+      first = middle + 1;
+    } else {
+      last = middle;
+    }
+  }
+  return first;
+}
+
+// Search for `value in the sorted array `arr`, in the range [first, last).
+// Return the first value where arr[index] > value.
+//
+// In other words, return array index I, where:
+//  arr[i] <= value for all tarr[lo:I]
+//  arr[i] > value for all tarr[I:last]
+//
+// The same semantics/behavior as:
+//    C++: upper_bound()
+//    Python: bisect.bisect_right()
+//
+function upperBound(valueArray, value, first, last) {
+  // this is just a binary search
+  while (first < last) {
+    const middle = (first + last) >>> 1;
+    if (valueArray[middle] > value) {
+      last = middle;
+    } else {
+      first = middle + 1;
+    }
+  }
+  return first;
+}
+
+// Inline performance optimization
+//
+function upperBoundIndirect(valueArray, indexArray, value, first, last) {
+  // this is just a binary search
+  while (first < last) {
+    const middle = (first + last) >>> 1;
+    if (valueArray[indexArray[middle]] > value) {
+      last = middle;
+    } else {
+      first = middle + 1;
+    }
+  }
+  return first;
+}
+
+module.exports = {
+  fillRange,
+  lowerBound,
+  lowerBoundIndirect,
+  upperBound,
+  upperBoundIndirect
+};