refactoring - immutable crossfilter (#647)

* immutable crossfilter

* PR review changes

client/__tests__/util/stateManager/world.test.js

@@ -3,6 +3,7 @@ import * as Universe from "../../../src/util/stateManager/universe";
 import * as World from "../../../src/util/stateManager/world";
 import * as Dataframe from "../../../src/util/dataframe";
 import Crossfilter from "../../../src/util/typedCrossfilter";
+import { DimTypes } from "../../../src/util/typedCrossfilter/crossfilter";
 import * as REST from "./sampleResponses";
 import {
   obsAnnoDimensionName,
@@ -26,15 +27,15 @@ const defaultBigBang = () => {
   /* create world */
   const world = World.createWorldFromEntireUniverse(universe);
   /* create crossfilter */
-  const crossfilter = Crossfilter(world.obsAnnotations);
+  const crossfilter = World.createObsDimensions(
-  /* create dimension map */
+    new Crossfilter(world.obsAnnotations),
-  const dimensionMap = World.createObsDimensionMap(crossfilter, world);
+    world
+  );
 
   return {
     universe,
     world,
-    crossfilter,
+    crossfilter
-    dimensionMap
   };
 };
 
@@ -71,13 +72,16 @@ describe("createWorldFromCurrentSelection", () => {
     const {
       universe,
       world: originalWorld,
-      crossfilter,
+      crossfilter: originalCrossfilter
-      dimensionMap
     } = defaultBigBang();
 
     /* mock a selection */
-    dimensionMap[obsAnnoDimensionName("field1")].filterRange([0, 5]);
+    let crossfilter = originalCrossfilter
-    dimensionMap[obsAnnoDimensionName("field3")].filterExact(false);
+      .select(obsAnnoDimensionName("field1"), { mode: "range", lo: 0, hi: 5 })
+      .select(obsAnnoDimensionName("field3"), {
+        mode: "exact",
+        values: [false]
+      });
 
     /* create the world from the selection */
     const world = World.createWorldFromCurrentSelection(
@@ -86,7 +90,7 @@ describe("createWorldFromCurrentSelection", () => {
       crossfilter
     );
     expect(world).toBeDefined();
-    expect(world.nObs).toEqual(crossfilter.countFiltered());
+    expect(world.nObs).toEqual(crossfilter.countSelected());
 
     /*
     calculate expected values and match against result
@@ -136,43 +140,32 @@ describe("createObsDimensionMap", () => {
     - check that dimension typing is sane
     */
 
-    const { dimensionMap } = defaultBigBang();
+    const { crossfilter } = defaultBigBang();
     const annotationNames = _.map(
       REST.schema.schema.annotations.obs,
       c => c.name
     );
     const schemaByObsName = _.keyBy(REST.schema.schema.annotations.obs, "name");
-    expect(dimensionMap).toBeDefined();
+    expect(crossfilter).toBeDefined();
     annotationNames.forEach(name => {
-      const dim = dimensionMap[obsAnnoDimensionName(name)];
+      const dim = crossfilter.dimensions[obsAnnoDimensionName(name)];
       if (name === "name") {
         expect(dim).toBeUndefined();
       } else {
         const { type } = schemaByObsName[name];
         if (type === "string" || type === "boolean" || type === "categorical") {
-          expect(dim).toBeInstanceOf(Crossfilter.EnumDimension);
+          expect(dim.dim).toBeInstanceOf(DimTypes.enum);
         } else {
-          expect(dim).toBeInstanceOf(Crossfilter.ScalarDimension);
+          expect(dim.dim).toBeInstanceOf(DimTypes.scalar);
         }
       }
     });
-    expect(dimensionMap[layoutDimensionName("XY")]).toBeInstanceOf(
+    expect(
-      Crossfilter.SpatialDimension
+      crossfilter.dimensions[layoutDimensionName("XY")].dim
-    );
+    ).toBeInstanceOf(DimTypes.spatial);
   });
 });
 
-describe("createVarDataDimension", () => {
-  /* create default universe */
-  const { world, crossfilter } = defaultBigBang();
-  world.varData = world.varData.withCol(
-    "GENE",
-    Float32Array.from(_.range(world.nObs))
-  );
-  const result = World.createVarDataDimension(world, crossfilter, "GENE");
-  expect(result).toBeInstanceOf(Crossfilter.ScalarDimension);
-});
-
 describe("worldEqUniverse", () => {
   const { universe, world } = defaultBigBang();
   const result = World.worldEqUniverse(world, universe);

client/__tests__/util/typedCrossfilter/crossfilter.test.js

@@ -0,0 +1,330 @@
+import _ from "lodash";
+import { polygonContains } from "d3";
+
+import Crossfilter from "../../../src/util/typedCrossfilter";
+
+const someData = [
+  {
+    date: "2011-11-14T16:17:54Z",
+    quantity: 2,
+    total: 190,
+    tip: 100,
+    type: "tab",
+    productIDs: ["001"],
+    coords: [0, 0]
+  },
+  {
+    date: "2011-11-14T16:20:19Z",
+    quantity: 2,
+    total: 190,
+    tip: 100,
+    type: "tab",
+    productIDs: ["001", "005"],
+    coords: [0.4, 0.4]
+  },
+  {
+    date: "2011-11-14T16:28:54Z",
+    quantity: 1,
+    total: 300,
+    tip: 200,
+    type: "visa",
+    productIDs: ["004", "005"],
+    coords: [0.3, 0.1]
+  },
+  {
+    date: "2011-11-14T16:30:43Z",
+    quantity: 2,
+    total: 90,
+    tip: 0,
+    type: "tab",
+    productIDs: ["001", "002"],
+    coords: [0.392, 0.1]
+  },
+  {
+    date: "2011-11-14T16:48:46Z",
+    quantity: 2,
+    total: 90,
+    tip: 0,
+    type: "tab",
+    productIDs: ["005"],
+    coords: [0.7, 0.0482]
+  },
+  {
+    date: "2011-11-14T16:53:41Z",
+    quantity: 2,
+    total: 90,
+    tip: 0,
+    type: "tab",
+    productIDs: ["001", "004", "005"],
+    coords: [0.9999, 1.0]
+  },
+  {
+    date: "2011-11-14T16:54:06Z",
+    quantity: 1,
+    total: 100,
+    tip: 0,
+    type: "cash",
+    productIDs: ["001", "002", "003", "004", "005"],
+    coords: [0.384, 0.6938]
+  },
+  {
+    date: "2011-11-14T16:58:03Z",
+    quantity: 2,
+    total: 90,
+    tip: 0,
+    type: "tab",
+    productIDs: ["001"],
+    coords: [0.4822, 0.482]
+  },
+  {
+    date: "2011-11-14T17:07:21Z",
+    quantity: 2,
+    total: 90,
+    tip: 0,
+    type: "tab",
+    productIDs: ["004", "005"],
+    coords: [0.2234, 0]
+  },
+  {
+    date: "2011-11-14T17:22:59Z",
+    quantity: 2,
+    total: 90,
+    tip: 0,
+    type: "tab",
+    productIDs: ["001", "002", "004", "005"],
+    coords: [0.382, 0.38485]
+  },
+  {
+    date: "2011-11-14T17:25:45Z",
+    quantity: 2,
+    total: 200,
+    tip: 0,
+    type: "cash",
+    productIDs: ["002"],
+    coords: [0.998, 0.8472]
+  },
+  {
+    date: "2011-11-14T17:29:52Z",
+    quantity: 1,
+    total: 200,
+    tip: 100,
+    type: "visa",
+    productIDs: ["004"],
+    coords: [0.8273, 0.3384]
+  }
+];
+
+let payments = null;
+beforeEach(() => {
+  payments = new Crossfilter(someData);
+});
+
+describe("ImmutableTypedCrossfilter", () => {
+  test("create crossfilter", () => {
+    expect(payments).toBeDefined();
+    expect(payments.size()).toEqual(someData.length);
+    expect(payments.all()).toEqual(someData);
+
+    const p = payments
+      .addDimension("quantity", "scalar", (i, d) => d[i].quantity, Int32Array)
+      .select("quantity", { mode: "all" });
+    expect(p).toBeDefined();
+    expect(p.all()).toEqual(someData);
+    expect(p.size()).toEqual(someData.length);
+    expect(p.isElementSelected(0)).toBeTruthy();
+    expect(p.countSelected()).toEqual(someData.length);
+    expect(p.allSelected()).toEqual(someData);
+  });
+
+  test("immutability", () => {
+    /*
+    the following should return a new crossfilter:
+    - addDimension()
+    - delDimension()
+    - select
+    */
+    const p2 = payments.addDimension(
+      "quantity",
+      "scalar",
+      (i, data) => data[i].quantity,
+      Int32Array
+    );
+
+    expect(payments).not.toBe(p2);
+    const p3 = p2.select("quantity", { mode: "all" });
+    expect(p3).not.toBe(p2);
+
+    const p4 = p3.delDimension("quantity");
+    expect(p4).not.toBe(p3);
+  });
+
+  test("select all and none", () => {
+    let p = payments
+      .addDimension("quantity", "scalar", (i, d) => d[i].quantity, Int32Array)
+      .addDimension("tip", "scalar", (i, d) => d[i].tip, Float32Array)
+      .addDimension("total", "scalar", (i, d) => d[i].total, Float32Array)
+      .addDimension("type", "enum", (i, d) => d[i].type);
+    expect(p).toBeDefined();
+
+    /* expect all records to be selected - default init state */
+    expect(p.allSelected()).toEqual(someData);
+    expect(p.countSelected()).toEqual(someData.length);
+    expect(p.allSelectedMask()).toEqual(
+      new Uint8Array(someData.length).fill(1)
+    );
+    expect(p.fillByIsSelected(new Uint8Array(someData.length), 99, 0)).toEqual(
+      new Uint8Array(someData.length).fill(99)
+    );
+    for (let i = 0; i < someData.length; i += 1) {
+      expect(p.isElementSelected(i)).toBeTruthy();
+    }
+
+    /* expect a selectAll on one dimension to change nothing */
+    p = p.select("tip", { mode: "all" });
+    expect(p.allSelected()).toEqual(someData);
+
+    /* ditto */
+    p = p.select("quantity", { mode: "all" });
+    expect(p.allSelected()).toEqual(someData);
+
+    /* select none on one dimension */
+    p = p.select("type", { mode: "none" });
+    expect(p.allSelected()).toEqual([]);
+    expect(p.countSelected()).toEqual(0);
+    expect(p.allSelectedMask()).toEqual(
+      new Uint8Array(someData.length).fill(0)
+    );
+    expect(p.fillByIsSelected(new Uint8Array(someData.length), 99, 0)).toEqual(
+      new Uint8Array(someData.length).fill(0)
+    );
+    for (let i = 0; i < someData.length; i += 1) {
+      expect(p.isElementSelected(i)).toBeFalsy();
+    }
+
+    p = p.select("quantity", { mode: "none" });
+    expect(p.allSelected()).toEqual([]);
+
+    // invert the first none; should have no effect because type is
+    // still not filtered.
+    p = p.select("quantity", { mode: "all" });
+    expect(p.allSelected()).toEqual([]);
+
+    /* select all of type; should select all records */
+    p = p.select("type", { mode: "all" });
+    expect(p.allSelected()).toEqual(someData);
+  });
+
+  describe("scalar dimension", () => {
+    let p;
+    beforeEach(() => {
+      p = payments
+        .addDimension("quantity", "scalar", (i, d) => d[i].quantity, Int32Array)
+        .addDimension("tip", "scalar", (i, d) => d[i].tip, Float32Array)
+        .select("tip", { mode: "all" });
+    });
+
+    /*
+    select modes:  all, none, exact, range
+    */
+    test("all", () => {
+      expect(p.select("quantity", { mode: "all" }).countSelected()).toEqual(
+        someData.length
+      );
+    });
+    test("none", () => {
+      expect(p.select("quantity", { mode: "none" }).countSelected()).toEqual(0);
+    });
+    test.each([[[]], [[2]], [[2, 1]], [[9, 82]], [[0, 1]]])("exact: %p", v =>
+      expect(
+        p.select("quantity", { mode: "exact", values: v }).countSelected()
+      ).toEqual(_.filter(someData, d => v.includes(d.quantity)).length)
+    );
+    test.each([[0, 1], [1, 2], [0, 99], [99, 100000]])("range %p", (lo, hi) =>
+      expect(
+        p.select("quantity", { mode: "range", lo, hi }).countSelected()
+      ).toEqual(
+        _.filter(someData, d => d.quantity >= lo && d.quantity < hi).length
+      )
+    );
+    test("bad mode", () => {
+      expect(() => p.select("type", { mode: "bad mode" })).toThrow(Error);
+    });
+  });
+
+  describe("enum dimension", () => {
+    let p;
+    beforeEach(() => {
+      p = payments.addDimension("type", "enum", (i, d) => d[i].type);
+    });
+
+    test("all", () => {
+      expect(p.select("type", { mode: "all" }).countSelected()).toEqual(
+        someData.length
+      );
+    });
+    test("none", () => {
+      expect(p.select("type", { mode: "none" }).countSelected()).toEqual(0);
+    });
+    test.each([
+      [[]],
+      [["tab"]],
+      [["visa"]],
+      [["visa", "tab"]],
+      [["cash", "tab", "visa"]]
+    ])("exact: %p", v =>
+      expect(
+        p.select("type", { mode: "exact", values: v }).countSelected()
+      ).toEqual(_.filter(someData, d => v.includes(d.type)).length)
+    );
+    test("range", () => {
+      expect(() => p.select("type", { mode: "range", lo: 0, hi: 9 })).toThrow(
+        Error
+      );
+    });
+    test("bad mode", () => {
+      expect(() => p.select("type", { mode: "bad mode" })).toThrow(Error);
+    });
+  });
+
+  describe("spatial dimension", () => {
+    let p;
+    beforeEach(() => {
+      const X = someData.map(r => r.coords[0]);
+      const Y = someData.map(r => r.coords[1]);
+      p = payments.addDimension("coords", "spatial", X, Y);
+    });
+
+    test("all", () => {
+      expect(p.select("coords", { mode: "all" }).countSelected()).toEqual(
+        someData.length
+      );
+    });
+    test("none", () => {
+      expect(p.select("coords", { mode: "none" }).countSelected()).toEqual(0);
+    });
+    test.each([[0, 0, 1, 1], [0, 0, 0.5, 0.5], [0.5, 0.5, 1, 1]])(
+      "within-rect %d %d %d %d",
+      (x0, y0, x1, y1) => {
+        expect(
+          p
+            .select("coords", { mode: "within-rect", x0, y0, x1, y1 })
+            .allSelected()
+        ).toEqual(
+          _.filter(someData, d => {
+            const [x, y] = d.coords;
+            return x0 <= x && x < x1 && y0 <= y && y < y1;
+          })
+        );
+      }
+    );
+
+    test.each([
+      [[[0, 0], [0, 1], [1, 1], [1, 0]]],
+      [[[0, 0], [0, 0.5], [0.5, 0.5], [0.5, 0]]]
+    ])("within-polygon %p", polygon => {
+      expect(
+        p.select("coords", { mode: "within-polygon", polygon }).allSelected()
+      ).toEqual(_.filter(someData, d => polygonContains(polygon, d.coords)));
+    });
+  });
+});

client/__tests__/util/typedCrossfilter/typedCrossfilter.test.js

@@ -1,590 +0,0 @@
-// jshint esversion: 6
-import _ from "lodash";
-import crossfilter from "../../../src/util/typedCrossfilter";
-
-const someData = [
-  {
-    date: "2011-11-14T16:17:54Z",
-    quantity: 2,
-    total: 190,
-    tip: 100,
-    type: "tab",
-    productIDs: ["001"]
-  },
-  {
-    date: "2011-11-14T16:20:19Z",
-    quantity: 2,
-    total: 190,
-    tip: 100,
-    type: "tab",
-    productIDs: ["001", "005"]
-  },
-  {
-    date: "2011-11-14T16:28:54Z",
-    quantity: 1,
-    total: 300,
-    tip: 200,
-    type: "visa",
-    productIDs: ["004", "005"]
-  },
-  {
-    date: "2011-11-14T16:30:43Z",
-    quantity: 2,
-    total: 90,
-    tip: 0,
-    type: "tab",
-    productIDs: ["001", "002"]
-  },
-  {
-    date: "2011-11-14T16:48:46Z",
-    quantity: 2,
-    total: 90,
-    tip: 0,
-    type: "tab",
-    productIDs: ["005"]
-  },
-  {
-    date: "2011-11-14T16:53:41Z",
-    quantity: 2,
-    total: 90,
-    tip: 0,
-    type: "tab",
-    productIDs: ["001", "004", "005"]
-  },
-  {
-    date: "2011-11-14T16:54:06Z",
-    quantity: 1,
-    total: 100,
-    tip: 0,
-    type: "cash",
-    productIDs: ["001", "002", "003", "004", "005"]
-  },
-  {
-    date: "2011-11-14T16:58:03Z",
-    quantity: 2,
-    total: 90,
-    tip: 0,
-    type: "tab",
-    productIDs: ["001"]
-  },
-  {
-    date: "2011-11-14T17:07:21Z",
-    quantity: 2,
-    total: 90,
-    tip: 0,
-    type: "tab",
-    productIDs: ["004", "005"]
-  },
-  {
-    date: "2011-11-14T17:22:59Z",
-    quantity: 2,
-    total: 90,
-    tip: 0,
-    type: "tab",
-    productIDs: ["001", "002", "004", "005"]
-  },
-  {
-    date: "2011-11-14T17:25:45Z",
-    quantity: 2,
-    total: 200,
-    tip: 0,
-    type: "cash",
-    productIDs: ["002"]
-  },
-  {
-    date: "2011-11-14T17:29:52Z",
-    quantity: 1,
-    total: 200,
-    tip: 100,
-    type: "visa",
-    productIDs: ["004"]
-  }
-];
-
-function groupReduce(data, valueMap, valueReduce, valueInit) {
-  return _.reduce(
-    data,
-    (acc, value) => {
-      const k = valueMap(value);
-      let r = _.find(acc, o => o.key === k);
-      if (!r) {
-        r = { key: k, value: valueInit() };
-        acc.push(r);
-      }
-      r.value = valueReduce(r.value, value);
-      return acc;
-    },
-    []
-  ).sort((a, b) => (a.key < b.key ? -1 : a.key > b.key ? 1 : 0));
-}
-
-function groupCount(data, map) {
-  return groupReduce(data, map, p => p + 1, () => 0);
-}
-
-function groupSum(data, map) {
-  return groupReduce(
-    data,
-    map,
-    (p, v) => {
-      p += map(v);
-      return p;
-    },
-    () => 0
-  );
-}
-
-let payments = null;
-beforeEach(() => {
-  payments = crossfilter(someData);
-});
-
-describe("typedCrossfilter", () => {
-  test("alloc and free", () => {
-    expect(payments).toBeDefined();
-    expect(payments.size()).toEqual(someData.length);
-    expect(payments.all()).toEqual(someData);
-
-    const quantity = payments.dimension(
-      crossfilter.ScalarDimension,
-      (i, data) => data[i].quantity,
-      Int32Array
-    );
-    expect(quantity).toBeDefined();
-    expect(quantity.id()).toBeDefined();
-
-    quantity.dispose();
-    expect(payments.size()).toEqual(someData.length);
-    expect(payments.all()).toEqual(someData);
-  });
-
-  test("filterAll and filterNone", () => {
-    expect(payments).toBeDefined();
-    const quantity = payments.dimension(
-      crossfilter.ScalarDimension,
-      (i, data) => data[i].quantity,
-      Int32Array
-    );
-    const tip = payments.dimension(
-      crossfilter.ScalarDimension,
-      (i, data) => data[i].tip,
-      Float32Array
-    );
-    const total = payments.dimension(
-      crossfilter.ScalarDimension,
-      (i, data) => data[i].total,
-      Float32Array
-    );
-    const type = payments.dimension(
-      crossfilter.EnumDimension,
-      (i, data) => data[i].type
-    );
-
-    expect(quantity).toBeDefined();
-    expect(tip).toBeDefined();
-    expect(total).toBeDefined();
-    expect(type).toBeDefined();
-
-    // initially, all should be filtered
-    expect(payments.allFiltered()).toHaveLength(payments.size());
-    expect(payments.allFiltered()).toEqual(payments.all());
-    expect(payments.countFiltered()).toEqual(someData.length);
-
-    // filterAll
-    tip.filterAll(); // should change nothing
-    expect(payments.allFiltered()).toEqual(payments.all());
-    expect(payments.countFiltered()).toEqual(someData.length);
-
-    // ditto
-    total.filterAll();
-    expect(payments.allFiltered()).toEqual(payments.all());
-    expect(payments.countFiltered()).toEqual(someData.length);
-
-    // filterNone
-    type.filterNone();
-    expect(payments.allFiltered()).toEqual([]);
-    expect(payments.countFiltered()).toEqual(0);
-
-    quantity.filterNone();
-    expect(payments.allFiltered()).toEqual([]);
-    expect(payments.countFiltered()).toEqual(0);
-
-    // invert the first none; should have no effect because type is
-    // still not filtered
-    quantity.filterAll();
-    expect(payments.allFiltered()).toEqual([]);
-    expect(payments.countFiltered()).toEqual(0);
-
-    // filter all of type; should select all
-    type.filterAll();
-    expect(payments.allFiltered()).toEqual(payments.all());
-    expect(payments.countFiltered()).toEqual(payments.size());
-  });
-
-  test("filterExact", () => {
-    expect(payments).toBeDefined();
-    const quantity = payments.dimension(
-      crossfilter.ScalarDimension,
-      (i, data) => data[i].quantity,
-      Int32Array
-    );
-    const tip = payments.dimension(
-      crossfilter.ScalarDimension,
-      (i, data) => data[i].tip,
-      Float32Array
-    );
-    const type = payments.dimension(
-      crossfilter.EnumDimension,
-      (i, data) => data[i].type
-    );
-
-    quantity.filterExact(1);
-    expect(payments.countFiltered()).toEqual(
-      _.countBy(someData, "quantity")[1]
-    );
-    expect(payments.allFiltered()).toEqual(_.filter(someData, { quantity: 1 }));
-
-    tip.filterExact(0);
-    expect(payments.allFiltered()).toEqual(
-      _.filter(someData, { tip: 0, quantity: 1 })
-    );
-
-    type.filterExact("cash");
-    expect(payments.allFiltered()).toEqual(
-      _.filter(someData, { tip: 0, quantity: 1, type: "cash" })
-    );
-  });
-
-  test("filterRange", () => {
-    expect(payments).toBeDefined();
-    const quantity = payments.dimension(
-      crossfilter.ScalarDimension,
-      (i, data) => data[i].quantity,
-      Int32Array
-    );
-    const tip = payments.dimension(
-      crossfilter.ScalarDimension,
-      (i, data) => data[i].tip,
-      Float32Array
-    );
-    const total = payments.dimension(
-      crossfilter.ScalarDimension,
-      (i, data) => data[i].total,
-      Float32Array
-    );
-    const type = payments.dimension(
-      crossfilter.EnumDimension,
-      (i, data) => data[i].type
-    );
-
-    tip.filterRange([0, 91]);
-    expect(payments.allFiltered()).toEqual(
-      _(someData)
-        .filter(r => r.tip >= 0 && r.tip < 91)
-        .value()
-    );
-
-    tip.filterRange([0, 90]);
-    expect(payments.allFiltered()).toEqual(
-      _(someData)
-        .filter(r => r.tip >= 0 && r.tip < 90)
-        .value()
-    );
-
-    tip.filterRange([1, 90]);
-    expect(payments.allFiltered()).toEqual(
-      _(someData)
-        .filter(r => r.tip >= 1 && r.tip < 91)
-        .value()
-    );
-  });
-
-  test("filterEnum", () => {
-    expect(payments).toBeDefined();
-    const quantity = payments.dimension(
-      crossfilter.ScalarDimension,
-      (i, data) => data[i].quantity,
-      Int32Array
-    );
-    const tip = payments.dimension(
-      crossfilter.ScalarDimension,
-      (i, data) => data[i].tip,
-      Float32Array
-    );
-    const total = payments.dimension(
-      crossfilter.ScalarDimension,
-      (i, data) => data[i].total,
-      Float32Array
-    );
-    const type = payments.dimension(
-      crossfilter.EnumDimension,
-      (i, data) => data[i].type
-    );
-
-    type.filterEnum(["tab", "cash"]);
-    expect(payments.allFiltered()).toEqual(
-      _(someData)
-        .filter(r => r.type === "cash" || r.type === "tab")
-        .value()
-    );
-
-    tip.filterEnum([0, 100]);
-    expect(payments.allFiltered()).toEqual(
-      _(someData)
-        .filter(r => r.type === "cash" || r.type === "tab")
-        .filter(r => r.tip === 0 || r.tip === 100)
-        .value()
-    );
-  });
-
-  test("more than 32 dimensions", () => {
-    expect(payments).toBeDefined();
-    const quantity = payments.dimension(
-      crossfilter.ScalarDimension,
-      (i, data) => data[i].quantity,
-      Int32Array
-    );
-    const tip = payments.dimension(
-      crossfilter.ScalarDimension,
-      (i, data) => data[i].tip,
-      Float32Array
-    );
-    const total = payments.dimension(
-      crossfilter.ScalarDimension,
-      (i, data) => data[i].total,
-      Float32Array
-    );
-    const type = payments.dimension(
-      crossfilter.EnumDimension,
-      (i, data) => data[i].type
-    );
-
-    // Create a bunch of fake dimensions to ensure we can handle > 32
-    let dimMap = {};
-    for (let i = 0; i < 65; i++) {
-      dimMap[i] = payments.dimension(
-        crossfilter.ScalarDimension,
-        () => Math.random(),
-        Float32Array
-      );
-      expect(dimMap[i]).toBeDefined();
-      expect(dimMap[i].id()).toBeDefined();
-    }
-
-    // everything should start as selected/filtered
-    expect(payments.countFiltered()).toEqual(someData.length);
-
-    dimMap[0].filterAll();
-    dimMap[64].filterAll();
-    expect(payments.countFiltered()).toEqual(someData.length);
-
-    dimMap[33].filterNone();
-    expect(payments.allFiltered()).toEqual([]);
-
-    dimMap[33].filterAll();
-    expect(payments.allFiltered()).toEqual(someData);
-  });
-
-  test("group, default mapping, default reducer, no filter", () => {
-    expect(payments).toBeDefined();
-
-    const quantity = payments.dimension(
-      crossfilter.ScalarDimension,
-      (i, data) => data[i].quantity,
-      Int32Array
-    );
-    const tip = payments.dimension(
-      crossfilter.ScalarDimension,
-      (i, data) => data[i].tip,
-      Int32Array
-    );
-    const type = payments.dimension(
-      crossfilter.EnumDimension,
-      (i, data) => data[i].type
-    );
-    const total = payments.dimension(
-      crossfilter.ScalarDimension,
-      (i, data) => data[i].total,
-      Int32Array
-    );
-
-    _.each(
-      {
-        tip: tip.group(r => r),
-        type: type.group(),
-        total: total.group(),
-        quantity: quantity.group()
-      },
-      (grp, k) => {
-        const whatWeExpect = groupCount(someData, v => v[k]);
-        expect(grp.all()).toEqual(whatWeExpect);
-        expect(grp.size()).toEqual(whatWeExpect.length);
-        expect(grp.dispose()).toEqual(grp);
-      }
-    );
-  });
-
-  test("group, custom map, default reducer, no filters", () => {
-    expect(payments).toBeDefined();
-
-    // custom mapping in groups only works for scalar types.  Enums do not
-    // currently implement it.
-
-    const tip = payments.dimension(
-      crossfilter.ScalarDimension,
-      (i, data) => data[i].tip,
-      Int32Array
-    );
-    const totalX10 = payments.dimension(
-      crossfilter.ScalarDimension,
-      (i, data) => data[i].total * 10,
-      Int32Array
-    );
-    const type = payments.dimension(
-      crossfilter.EnumDimension,
-      (i, data) => data[i].type
-    );
-
-    const paymentsByTip_A = tip.group();
-    const paymentsByTip_B = tip.group(r => 10 * r);
-    const paymentsByType = type.group(); // identity only
-    const paymentsByTotalX10_A = totalX10.group();
-    const paymentsByTotalX10_B = totalX10.group(r => r / 10);
-
-    expect(paymentsByTip_A.all()).toEqual(groupCount(someData, v => v.tip));
-    expect(paymentsByTip_B.all()).toEqual(
-      groupCount(someData, v => 10 * v.tip)
-    );
-    expect(paymentsByType.all()).toEqual(groupCount(someData, v => v.type));
-    expect(paymentsByTotalX10_A.all()).toEqual(
-      groupCount(someData, v => 10 * v.total)
-    );
-    expect(paymentsByTotalX10_B.all()).toEqual(
-      groupCount(someData, v => (10 * v.total) / 10)
-    );
-
-    for (let i of [
-      paymentsByTip_A,
-      paymentsByTip_B,
-      paymentsByType,
-      paymentsByTotalX10_A,
-      paymentsByTotalX10_B,
-      tip,
-      totalX10,
-      type
-    ]) {
-      expect(i.dispose()).toEqual(i);
-    }
-  });
-
-  test("group, default map, custom reducer, no filters", () => {
-    expect(payments).toBeDefined();
-
-    const total = payments.dimension(
-      crossfilter.ScalarDimension,
-      (i, data) => data[i].total,
-      Float32Array
-    );
-    const type = payments.dimension(
-      crossfilter.EnumDimension,
-      (i, data) => data[i].type
-    );
-
-    const paymentsByTotal = total.group();
-    const paymentsByType = type.group();
-
-    // reduceCount
-    expect(paymentsByTotal.reduceCount()).toEqual(paymentsByTotal);
-    expect(paymentsByTotal.all()).toEqual(groupCount(someData, v => v.total));
-
-    // reduceSum
-    expect(paymentsByTotal.reduceSum(v => v.total)).toEqual(paymentsByTotal);
-    expect(paymentsByTotal.all()).toEqual(groupSum(someData, v => v.total));
-
-    // use custom reducers (my reducers) - count by three, init 1
-    expect(
-      paymentsByTotal.reduce((p, v) => (p += 3), (p, v) => (p -= 3), () => 1)
-    ).toEqual(paymentsByTotal);
-    expect(paymentsByTotal.all()).toEqual(
-      groupReduce(someData, v => v.total, (p, v) => p + 3, () => 1)
-    );
-
-    for (let i of [paymentsByTotal, paymentsByType, type]) {
-      expect(i.dispose()).toEqual(i);
-    }
-  });
-
-  test("group, default map, default reducer, filters", () => {
-    // From the docs:
-    // Note: a grouping intersects the crossfilter's current filters, except for the
-    // associated dimension's filter. Thus, group methods consider only records that
-    // satisfy every filter except this dimension's filter. So, if the crossfilter of
-    // payments is filtered by type and total, then group by total only observes the
-    // filter by type.
-
-    expect(payments).toBeDefined();
-
-    const tip = payments.dimension(
-      crossfilter.ScalarDimension,
-      (i, data) => data[i].tip,
-      Int32Array
-    );
-    const total = payments.dimension(
-      crossfilter.ScalarDimension,
-      (i, data) => data[i].total,
-      Int32Array
-    );
-    const type = payments.dimension(
-      crossfilter.EnumDimension,
-      (i, data) => data[i].type
-    );
-
-    const paymentsByTip = tip.group();
-    const paymentsByTotal = total.group();
-    const paymentsByType = type.group();
-
-    // 1. confirm that changing the filter on a dimension does NOT change that
-    // dimensions groups.
-    {
-      tip.filterAll(), total.filterAll(), type.filterAll();
-      let before = _.cloneDeep(paymentsByTip.all());
-      tip.filterExact(0);
-      expect(paymentsByTip.all()).toEqual(before);
-    }
-
-    // 2. confirm that changing a filter on a different dimension DOES change
-    // all other groups.
-    {
-      tip.filterAll(), total.filterAll(), type.filterAll();
-      const before = _.cloneDeep([paymentsByTotal.all(), paymentsByType.all()]);
-      tip.filterExact(0);
-      const after = [paymentsByTotal.all(), paymentsByType.all()];
-      expect(after).not.toEqual(before);
-      expect(after).toEqual([
-        groupReduce(
-          someData,
-          v => v.total,
-          (p, v) => (v.tip !== 0 ? p : p + 1),
-          () => 0
-        ),
-        groupReduce(
-          someData,
-          v => v.type,
-          (p, v) => (v.tip !== 0 ? p : p + 1),
-          () => 0
-        )
-      ]);
-    }
-
-    for (let i of [
-      paymentsByTip,
-      paymentsByTotal,
-      paymentsByType,
-      tip,
-      total,
-      type
-    ]) {
-      expect(i.dispose()).toEqual(i);
-    }
-  });
-});

client/src/components/continuous/continuous.js

@@ -11,8 +11,7 @@ import HistogramBrush from "../brushableHistogram";
 @connect(state => ({
   obsAnnotations: _.get(state.controls.world, "obsAnnotations", null),
   colorAccessor: state.controls.colorAccessor,
-  colorScale: state.controls.colors.scale,
+  colorScale: state.controls.colorScale,
-  selectionUpdate: _.get(state.controls, "crossfilter.updateTime", null),
   schema: _.get(state.controls.world, "schema", null)
 }))
 class Continuous extends React.Component {

client/src/components/geneExpression/expressionButtons.js

@@ -10,8 +10,7 @@ import CellSetButton from "./cellSetButtons";
 @connect(state => ({
   differential: state.differential,
   world: state.controls.world,
-  crossfilter: state.controls.crossfilter,
+  crossfilter: state.controls.crossfilter
-  selectionUpdate: _.get(state.controls, "crossfilter.updateTime", null)
 }))
 class Expression extends React.Component {
   constructor(props) {

client/src/components/graph/graph.js

@@ -32,7 +32,6 @@ import { World } from "../../util/stateManager";
   responsive: state.responsive,
   colorRGB: _.get(state.controls, "colors.rgb", null),
   opacityForDeselectedCells: state.controls.opacityForDeselectedCells,
-  selectionUpdate: _.get(state.controls, "crossfilter.updateTime", null),
   resettingInterface: state.controls.resettingInterface,
   userDefinedGenes: state.controls.userDefinedGenes,
   diffexpGenes: state.controls.diffexpGenes,
@@ -103,13 +102,7 @@ class Graph extends React.Component {
 
   componentDidUpdate(prevProps) {
     const { renderCache } = this;
-    const {
+    const { world, crossfilter, colorRGB, responsive } = this.props;
-      world,
-      crossfilter,
-      colorRGB,
-      responsive,
-      selectionUpdate
-    } = this.props;
     const {
       reglRender,
       mode,
@@ -173,11 +166,11 @@ class Graph extends React.Component {
 
       // Sizes for each point - updates are triggered only when selected
       // obs change
-      if (!renderCache.sizes || selectionUpdate !== prevProps.selectionUpdate) {
+      if (!renderCache.sizes || crossfilter !== prevProps.crossfilter) {
         if (!renderCache.sizes) {
           renderCache.sizes = new Float32Array(nObs);
         }
-        crossfilter.fillByIsFiltered(renderCache.sizes, 4, 0.2);
+        crossfilter.fillByIsSelected(renderCache.sizes, 4, 0.2);
         sizeBuffer({ data: renderCache.sizes, dimension: 1 });
       }
 
@@ -242,7 +235,7 @@ class Graph extends React.Component {
     if (!crossfilter || !world || !universe) {
       return false;
     }
-    const nothingSelected = crossfilter.countFiltered() === crossfilter.size();
+    const nothingSelected = crossfilter.countSelected() === crossfilter.size();
     const nothingColoredBy = !colorAccessor;
     const noGenes = userDefinedGenes.length === 0 && diffexpGenes.length === 0;
     const scatterNotDpl = !scatterplotXXaccessor || !scatterplotYYaccessor;
@@ -450,8 +443,8 @@ class Graph extends React.Component {
                 data-testid="subset-button"
                 disabled={
                   crossfilter &&
-                  (crossfilter.countFiltered() === 0 ||
+                  (crossfilter.countSelected() === 0 ||
-                    crossfilter.countFiltered() === crossfilter.size())
+                    crossfilter.countSelected() === crossfilter.size())
                 }
                 style={{ marginRight: 10 }}
                 onClick={() => {

client/src/components/scatterplot/scatterplot.js

@@ -58,9 +58,7 @@ import finiteExtent from "../../util/finiteExtent";
     expressionX,
     expressionY,
 
-    crossfilter,
+    crossfilter
-    // updated whenever the crossfilter selection is updated
-    selectionUpdate: _.get(state.controls, "crossfilter.updateTime", null)
   };
 })
 class Scatterplot extends React.Component {
@@ -136,8 +134,7 @@ class Scatterplot extends React.Component {
       scatterplotYYaccessor,
       expressionX,
       expressionY,
-      colorRGB,
+      colorRGB
-      selectionUpdate
     } = this.props;
     const {
       reglRender,
@@ -210,11 +207,11 @@ class Scatterplot extends React.Component {
 
       // Sizes for each point - updates are triggered only when selected
       // obs change
-      if (!renderCache.sizes || selectionUpdate !== prevProps.selctionUpdate) {
+      if (!renderCache.sizes || crossfilter !== prevProps.crossfilter) {
         if (!renderCache.sizes) {
           renderCache.sizes = new Float32Array(cellCount);
         }
-        crossfilter.fillByIsFiltered(renderCache.sizes, 4, 0.2);
+        crossfilter.fillByIsSelected(renderCache.sizes, 4, 0.2);
         sizeBuffer({ data: renderCache.sizes, dimension: 1 });
       }
 

client/src/reducers/controls.js

@@ -35,7 +35,6 @@ const Controls = (
     world: null,
     categoricalSelectionState: null,
     crossfilter: null,
-    dimensionMap: null,
     userDefinedGenes: [],
     userDefinedGenesLoading: false,
     diffexpGenes: [],
@@ -95,8 +94,10 @@ const Controls = (
         state,
         world
       );
-      const crossfilter = Crossfilter(world.obsAnnotations);
+      const crossfilter = World.createObsDimensions(
-      const dimensionMap = World.createObsDimensionMap(crossfilter, world);
+        new Crossfilter(world.obsAnnotations),
+        world
+      );
       WorldUtil.clearCaches();
 
       return {
@@ -104,11 +105,10 @@ const Controls = (
         loading: false,
         error: null,
         universe,
-        fullUniverseCache: { world, crossfilter, dimensionMap },
+        fullUniverseCache: { world, crossfilter },
         world,
         categoricalSelectionState,
         crossfilter,
-        dimensionMap,
         colorMode,
         colorAccessor: null,
         colors,
@@ -116,40 +116,35 @@ const Controls = (
       };
     }
     case "reset World to eq Universe": {
+      /*
+      1. Reset world & crossfilter, using previously created objects which were
+         stashed in `fullUniverseCache`
+      2. Add crossfilter dimension for all userDefined and diffexp genes/varData,
+         as they are not part of the cached crossfilter.
+      3. Compute categorical selection summary
+      4. Reset all WorldUtil caches
+      5. Reset color-by
+      */
       const { userDefinedGenes, diffexpGenes, fullUniverseCache } = state;
-      const { world, crossfilter } = fullUniverseCache;
+      const { world } = fullUniverseCache;
-      // reset all crossfilter dimensions
+      const crossfilter = ControlsHelpers.createGeneDimensions(
-      _.forEach(fullUniverseCache.dimensionMap, dim => dim.filterAll());
+        userDefinedGenes,
+        diffexpGenes,
+        world,
+        fullUniverseCache.crossfilter
+      );
       const colorMode = null;
       const colors = createColors(world, colorMode);
       const categoricalSelectionState = ControlsHelpers.createCategoricalSelectionState(
         state,
         world
       );
-
-      /* free dimensions not in cache (otherwise they leak) */
-      _.forEach(state.dimensionMap, (dim, dimName) => {
-        if (!fullUniverseCache.dimensionMap[dimName]) {
-          dim.dispose();
-        }
-      });
-      const dimensionMap = {
-        ...fullUniverseCache.dimensionMap,
-        ...ControlsHelpers.createGenesDimMap(
-          userDefinedGenes,
-          diffexpGenes,
-          world,
-          crossfilter
-        )
-      };
       WorldUtil.clearCaches();
-
       return {
         ...state,
         world,
         categoricalSelectionState,
         crossfilter,
-        dimensionMap,
         colorMode,
         colorAccessor: null,
         colors,
@@ -175,16 +170,15 @@ const Controls = (
         state,
         world
       );
-      const crossfilter = Crossfilter(world.obsAnnotations);
+      let crossfilter = new Crossfilter(world.obsAnnotations);
-      const dimensionMap = {
+      crossfilter = World.createObsDimensions(crossfilter, world);
-        ...World.createObsDimensionMap(crossfilter, world),
+      crossfilter = ControlsHelpers.createGeneDimensions(
-        ...ControlsHelpers.createGenesDimMap(
+        userDefinedGenes,
-          userDefinedGenes,
+        diffexpGenes,
-          diffexpGenes,
+        world,
-          world,
+        crossfilter
-          crossfilter
+      );
-        )
+
-      };
       WorldUtil.clearCaches();
 
       return {
@@ -194,8 +188,7 @@ const Controls = (
         world,
         colors,
         categoricalSelectionState,
-        crossfilter,
+        crossfilter
-        dimensionMap
       };
     }
     case "expression load success": {
@@ -276,56 +269,57 @@ const Controls = (
       };
     }
     case "request user defined gene success": {
-      const { world, crossfilter, dimensionMap, userDefinedGenes } = state;
+      const { world, crossfilter: oldCrossfilter, userDefinedGenes } = state;
       const _userDefinedGenes = userDefinedGenes.slice();
       const gene = action.data.genes[0];
 
-      dimensionMap[
+      const crossfilter = oldCrossfilter.addDimension(
-        userDefinedDimensionName(gene)
+        userDefinedDimensionName(gene),
-      ] = World.createVarDataDimension(world, crossfilter, gene);
+        "scalar",
-
+        world.varData.col(gene).asArray(),
+        Float32Array
+      );
       return {
         ...state,
-        dimensionMap,
+        crossfilter,
         userDefinedGenes: _userDefinedGenes,
         userDefinedGenesLoading: false
       };
     }
     case "request differential expression success": {
-      const { world, crossfilter, dimensionMap } = state;
+      const { world, crossfilter: oldCrossfilter } = state;
       const _diffexpGenes = [];
 
       action.data.forEach(d => {
         _diffexpGenes.push(world.varAnnotations.at(d[0], "name"));
       });
 
+      let crossfilter = oldCrossfilter;
       _.forEach(_diffexpGenes, gene => {
-        dimensionMap[diffexpDimensionName(gene)] = World.createVarDataDimension(
+        crossfilter = crossfilter.addDimension(
-          world,
+          diffexpDimensionName(gene),
-          crossfilter,
+          "scalar",
-          gene
+          world.varData.col(gene).asArray(),
+          Float32Array
         );
       });
 
       return {
         ...state,
-        dimensionMap,
+        crossfilter,
         diffexpGenes: _diffexpGenes
       };
     }
     case "clear differential expression": {
-      const { world, dimensionMap } = state;
+      const { world } = state;
-      const _dimensionMap = dimensionMap;
+      let { crossfilter } = state;
       _.forEach(action.diffExp, values => {
         const name = world.varAnnotations.at(values[0], "name");
-        // clean up crossfilter dimensions
+        crossfilter = crossfilter.delDimension(diffexpDimensionName(name));
-        const dimension = dimensionMap[diffexpDimensionName(name)];
-        dimension.dispose();
-        delete dimensionMap[diffexpDimensionName(name)];
       });
       return {
         ...state,
-        dimensionMap: _dimensionMap,
+        crossfilter,
         diffexpGenes: []
       };
     }
@@ -342,34 +336,29 @@ const Controls = (
       };
     }
     case "clear user defined gene": {
-      const { userDefinedGenes, dimensionMap } = state;
+      const { userDefinedGenes, crossfilter: oldCrossfilter } = state;
       const newUserDefinedGenes = _.filter(
         userDefinedGenes,
         d => d !== action.data
       );
-
+      const crossfilter = oldCrossfilter.delDimension(
-      const dimension = dimensionMap[userDefinedDimensionName(action.data)];
+        userDefinedDimensionName(action.data)
-      dimension.dispose();
+      );
-      delete dimensionMap[userDefinedDimensionName(action.data)];
-
       return {
         ...state,
-        dimensionMap,
+        crossfilter,
         userDefinedGenes: newUserDefinedGenes
       };
     }
     case "clear all user defined genes": {
-      const { userDefinedGenes, dimensionMap } = state;
+      const { userDefinedGenes } = state;
-
+      let { crossfilter } = state;
       _.forEach(userDefinedGenes, gene => {
-        const dimension = dimensionMap[userDefinedDimensionName(gene)];
+        crossfilter = crossfilter.delDimension(userDefinedDimensionName(gene));
-        dimension.dispose();
-        delete dimensionMap[userDefinedDimensionName(gene)];
       });
-
       return {
         ...state,
-        dimensionMap,
+        crossfilter,
         userDefinedGenes: []
       };
     }
@@ -395,34 +384,49 @@ const Controls = (
              User Events
      *******************************/
     case "graph brush selection change": {
-      state.dimensionMap[layoutDimensionName("XY")].filterWithinRect(
+      const name = layoutDimensionName("XY");
-        action.brushCoords.northwest,
+      const [x0, y0] = action.brushCoords.northwest;
-        action.brushCoords.southeast
+      const [x1, y1] = action.brushCoords.southeast;
-      );
+      const crossfilter = state.crossfilter.select(name, {
+        mode: "within-rect",
+        x0,
+        y0,
+        x1,
+        y1
+      });
       return {
         ...state,
+        crossfilter,
         graphBrushSelection: action.brushCoords
       };
     }
     case "lasso deselect":
     case "graph brush deselect": {
-      state.dimensionMap[layoutDimensionName("XY")].filterAll();
+      const name = layoutDimensionName("XY");
+      const crossfilter = state.crossfilter.select(name, { mode: "all" });
       return {
         ...state,
+        crossfilter,
         graphBrushSelection: null
       };
     }
     case "lasso selection": {
       const { polygon } = action;
-      const dXY = state.dimensionMap[layoutDimensionName("XY")];
+      const name = layoutDimensionName("XY");
+      const { crossfilter: oldCrossfilter } = state;
+      let crossfilter;
       if (polygon.length < 3) {
         // single point or a line is not a polygon, and is therefore a deselect
-        dXY.filterAll();
+        crossfilter = oldCrossfilter.select(name, { mode: "all" });
       } else {
-        dXY.filterWithinPolygon(polygon);
+        crossfilter = oldCrossfilter.select(name, {
+          mode: "within-polygon",
+          polygon
+        });
       }
       return {
-        ...state
+        ...state,
+        crossfilter
       };
     }
     case "continuous metadata histogram brush": {
@@ -430,15 +434,17 @@ const Controls = (
         action.continuousNamespace,
         action.selection
       );
+      let { crossfilter } = state;
 
       // action.selection: metadata name being selected
       // action.range: filter range, or null if deselected
       if (!action.range) {
-        state.dimensionMap[name].filterAll();
+        crossfilter = crossfilter.select(name, { mode: "all" });
       } else {
-        state.dimensionMap[name].filterRange(action.range);
+        const [lo, hi] = action.range;
+        crossfilter = crossfilter.select(name, { mode: "range", lo, hi });
       }
-      return { ...state };
+      return { ...state, crossfilter };
     }
     case "change opacity deselected cells in 2d graph background":
       return {
@@ -476,13 +482,15 @@ const Controls = (
 
       // update the filter to match all selected options
       const cat = newCategoricalSelectionState[action.metadataField];
-      state.dimensionMap[obsAnnoDimensionName(action.metadataField)].filterEnum(
+      const dName = obsAnnoDimensionName(action.metadataField);
-        ControlsHelpers.selectedValuesForCategory(cat)
+      const crossfilter = state.crossfilter.select(dName, {
-      );
+        mode: "exact",
-
+        values: ControlsHelpers.selectedValuesForCategory(cat)
+      });
       return {
         ...state,
-        categoricalSelectionState: newCategoricalSelectionState
+        categoricalSelectionState: newCategoricalSelectionState,
+        crossfilter
       };
     }
     case "categorical metadata filter deselect": {
@@ -500,13 +508,15 @@ const Controls = (
 
       // update the filter to match all selected options
       const cat = newCategoricalSelectionState[action.metadataField];
-      state.dimensionMap[obsAnnoDimensionName(action.metadataField)].filterEnum(
+      const dName = obsAnnoDimensionName(action.metadataField);
-        ControlsHelpers.selectedValuesForCategory(cat)
+      const crossfilter = state.crossfilter.select(dName, {
-      );
+        mode: "exact",
-
+        values: ControlsHelpers.selectedValuesForCategory(cat)
+      });
       return {
         ...state,
-        categoricalSelectionState: newCategoricalSelectionState
+        categoricalSelectionState: newCategoricalSelectionState,
+        crossfilter
       };
     }
     case "categorical metadata filter none of these": {
@@ -520,12 +530,14 @@ const Controls = (
           ).fill(false)
         }
       };
-      state.dimensionMap[
+      const dName = obsAnnoDimensionName(action.metadataField);
-        obsAnnoDimensionName(action.metadataField)
+      const crossfilter = state.crossfilter.select(dName, {
-      ].filterNone();
+        mode: "none"
+      });
       return {
         ...state,
-        categoricalSelectionState: newCategoricalSelectionState
+        categoricalSelectionState: newCategoricalSelectionState,
+        crossfilter
       };
     }
     case "categorical metadata filter all of these": {
@@ -539,12 +551,14 @@ const Controls = (
           ).fill(true)
         }
       };
-      state.dimensionMap[
+      const dName = obsAnnoDimensionName(action.metadataField);
-        obsAnnoDimensionName(action.metadataField)
+      const crossfilter = state.crossfilter.select(dName, {
-      ].filterAll();
+        mode: "all"
+      });
       return {
         ...state,
-        categoricalSelectionState: newCategoricalSelectionState
+        categoricalSelectionState: newCategoricalSelectionState,
+        crossfilter
       };
     }
 

client/src/util/stateManager/controlsHelpers.js

@@ -98,29 +98,35 @@ export function selectedValuesForCategory(categorySelectionState) {
 }
 
 /*
-build a crossfilter dimension map for all gene expression related dimensions.
+build a crossfilter dimensions for all gene expression related dimensions.
 */
-export function createGenesDimMap(
+export function createGeneDimensions(
   userDefinedGenes,
   diffexpGenes,
   world,
   crossfilter
 ) {
-  function _createGenesDimMap(genes, nameCreator) {
+  crossfilter = userDefinedGenes.reduce(
-    return genes.reduce((acc, gene) => {
+    (xflt, gene) =>
-      acc[nameCreator(gene)] = World.createVarDataDimension(
+      xflt.addDimension(
-        world,
+        userDefinedDimensionName(gene),
-        crossfilter,
+        "scalar",
-        gene
+        world.varData.col(gene).asArray(),
-      );
+        Float32Array
-      return acc;
+      ),
-    }, {});
+    crossfilter
-  }
+  );
-
+  crossfilter = diffexpGenes.reduce(
-  return {
+    (xflt, gene) =>
-    ..._createGenesDimMap(userDefinedGenes, userDefinedDimensionName),
+      xflt.addDimension(
-    ..._createGenesDimMap(diffexpGenes, diffexpDimensionName)
+        diffexpDimensionName(gene),
-  };
+        "scalar",
+        world.varData.col(gene).asArray(),
+        Float32Array
+      ),
+    crossfilter
+  );
+  return crossfilter;
 }
 
 export function pruneVarDataCache(varData, needed) {

client/src/util/stateManager/world.js

@@ -1,8 +1,6 @@
 // jshint esversion: 6
 
-import _ from "lodash";
 import { layoutDimensionName, obsAnnoDimensionName } from "../nameCreators";
-import Crossfilter from "../typedCrossfilter";
 import * as Dataframe from "../dataframe";
 
 /*
@@ -98,7 +96,7 @@ export function createWorldFromCurrentSelection(universe, world, crossfilter) {
   newWorld.varAnnotations = universe.varAnnotations;
 
   /* now subset/cut obs */
-  const mask = crossfilter.allFilteredMask();
+  const mask = crossfilter.allSelectedMask();
   newWorld.obsAnnotations = world.obsAnnotations.isubsetMask(mask);
   newWorld.obsLayout = world.obsLayout.isubsetMask(mask);
   newWorld.nObs = newWorld.obsAnnotations.dims[0];
@@ -139,63 +137,32 @@ function deduceDimensionType(attributes, fieldName) {
   return dimensionType;
 }
 
-/*
+export function createObsDimensions(crossfilter, world) {
-  Return a crossfilter dimension for the specified world & named gene.
-
-  NOTE: this assumes that the expression data was already loaded,
-  by calling an appropriate action creator.
-
-  Caller needs to *save* this dimension somewhere for it to be later used.
-  Dimension must be destroyed by calling dimension.dispose()
-  when it is no longer needed
-  (it will not be garbage collected without this call)
-*/
-export function createVarDataDimension(world, crossfilter, name) {
-  return crossfilter.dimension(
-    Crossfilter.ScalarDimension,
-    world.varData.col(name).asArray(),
-    Float32Array
-  );
-}
-
-export function createObsDimensionMap(crossfilter, world) {
   /*
-  create and return a crossfilter dimension for every obs annotation
+  create and return a crossfilter with a dimension for every obs annotation
-  for which we have a supported type.
+  for which we have a supported type, *except* 'name'
   */
   const { schema, obsLayout, obsAnnotations } = world;
+  const annoList = schema.annotations.obs.filter(anno => anno.name !== "name");
+  crossfilter = annoList.reduce((xfltr, anno) => {
+    const dimType = deduceDimensionType(anno, anno.name);
+    const colData = obsAnnotations.col(anno.name).asArray();
+    const name = obsAnnoDimensionName(anno.name);
+    if (dimType === "enum") {
+      return xfltr.addDimension(name, "enum", colData);
+    }
+    if (dimType) {
+      return xfltr.addDimension(name, "scalar", colData, dimType);
+    }
+    return xfltr;
+  }, crossfilter);
 
-  // Create a crossfilter dimension for all obs annotations *except* 'name'
+  return crossfilter.addDimension(
-  const dimensionMap = _(schema.annotations.obs)
+    layoutDimensionName("XY"),
-    .filter(anno => anno.name !== "name")
+    "spatial",
-    .transform((result, anno) => {
-      const dimType = deduceDimensionType(anno, anno.name);
-      const colData = obsAnnotations.col(anno.name).asArray();
-      if (dimType === "enum") {
-        result[obsAnnoDimensionName(anno.name)] = crossfilter.dimension(
-          Crossfilter.EnumDimension,
-          colData
-        );
-      } else if (dimType) {
-        result[obsAnnoDimensionName(anno.name)] = crossfilter.dimension(
-          Crossfilter.ScalarDimension,
-          colData,
-          dimType
-        );
-      } // else ignore the annotation
-    }, {})
-    .value();
-
-  /*
-  Add crossfilter dimensions allowing filtering on layout
-  */
-  dimensionMap[layoutDimensionName("XY")] = crossfilter.dimension(
-    Crossfilter.SpatialDimension,
     obsLayout.col("X").asArray(),
     obsLayout.col("Y").asArray()
   );
-
-  return dimensionMap;
 }
 
 export function worldEqUniverse(world, universe) {
@@ -206,7 +173,7 @@ export function getSelectedByIndex(crossfilter) {
   /*
   return array of obsIndex, containing all selected obs/cells.
   */
-  const selected = crossfilter.allFilteredMask(); // array of bool-ish
+  const selected = crossfilter.allSelectedMask(); // array of bool-ish
   const keys = crossfilter.data.rowIndex.keys(); // row keys, aka universe rowIndex
 
   const set = new Int32Array(selected.length);

client/src/util/typedCrossfilter/bitArray.js

@@ -211,6 +211,19 @@ class BitArray {
     }
   }
 
+  // select range of indices on a dimension
+  //
+  selectFromRange(dim, range) {
+    const col = dim >>> 5;
+    const first = range[0];
+    const last = range[1];
+    const one = 1 << dim % 32;
+    const offset = col * this.length;
+    for (let i = first; i < last; i += 1) {
+      this.bitarray[offset + i] |= one;
+    }
+  }
+
   // select range of indices on a dimension, indirect through a sort map.
   // Indirect functions are used to map between sort and natural order.
   //
@@ -225,6 +238,19 @@ class BitArray {
     }
   }
 
+  // deselect range of indices on a dimension
+  //
+  deselectFromRange(dim, range) {
+    const col = dim >>> 5;
+    const first = range[0];
+    const last = range[1];
+    const zero = ~(1 << dim % 32);
+    const offset = col * this.length;
+    for (let i = first; i < last; i += 1) {
+      this.bitarray[offset + i] &= zero;
+    }
+  }
+
   // deselect range of indices on a dimension, indirect through a sort map.
   //
   deselectIndirectFromRange(dim, indirect, range) {

client/src/util/typedCrossfilter/crossfilter.js

@@ -0,0 +1,594 @@
+import { polygonContains } from "d3";
+
+import PositiveIntervals from "./positiveIntervals";
+import BitArray from "./bitArray";
+import { sort } from "./sort";
+import {
+  makeSortIndex,
+  lowerBound,
+  lowerBoundIndirect,
+  upperBoundIndirect
+} from "./util";
+
+class NotImplementedError extends Error {
+  constructor(...params) {
+    super(...params);
+
+    // Maintains proper stack trace for where our error was thrown (only available on V8)
+    if (Error.captureStackTrace) {
+      Error.captureStackTrace(this, NotImplementedError);
+    }
+  }
+}
+
+export default class ImmutableTypedCrossfilter {
+  constructor(data, dimensions = {}, selectionCache = null) {
+    /*
+    Typically, parameter 'data' is one of:
+      - Array of objects/records
+      - Dataframe (util/dataframe)
+    Other parameters are only used internally.
+
+    Object field description:
+    - data: reference to the array of records in the crossfilter
+    - selectionBitArray: bit array containing the flatted selection state
+      of all dimensions.  This is lazily created and is effectively
+      a perfomance cache.  Methods which return a new crossfilter,
+      such as select(), addDimention() and delDimension(), will pass
+      the cache forward to the new object, as the typical "immutable API"
+      usage pattern is to retain the new crossfilter and discard the old.
+    - dimensions: contains each dimension and its current state:
+        - id: bit offset in the cached bit array
+        - dim: the dimension object
+        - name: the dimension name
+        - selection: the dimension's current selection
+    */
+    this.data = data;
+    this.selectionCache = selectionCache; /* BitArray */
+    this.dimensions = dimensions; /* name: { id, dim, name, selection } */
+  }
+
+  size() {
+    return this.data.length;
+  }
+
+  all() {
+    return this.data;
+  }
+
+  dimensionNames() {
+    /* return array of all dimensions (by name) */
+    return Object.keys(this.dimensions);
+  }
+
+  addDimension(name, type, ...rest) {
+    /*
+    Add a new dimension to this crossfilter, of type DimensionType.
+    Remainder of parameters are dimension-type-specific.
+    */
+    const { data, selectionCache } = this;
+
+    if (this.dimensions[name] !== undefined) {
+      throw new Error(`Adding duplicate dimension name ${name}`);
+    }
+
+    this.selectionCache = null; // pass ownership to new crossfilter
+
+    let id;
+    if (selectionCache) {
+      id = selectionCache.allocDimension();
+      selectionCache.selectAll(id);
+    }
+    const DimensionType = DimTypes[type];
+    const dim = new DimensionType(name, data, ...rest);
+    const dimensions = {
+      ...this.dimensions,
+      [name]: {
+        id,
+        dim,
+        name,
+        selection: dim.select({ mode: "all" })
+      }
+    };
+    return new ImmutableTypedCrossfilter(data, dimensions, selectionCache);
+  }
+
+  delDimension(name) {
+    const { data, selectionCache } = this;
+    const dimensions = { ...this.dimensions };
+    if (dimensions[name] === undefined) {
+      throw new ReferenceError(`Unable to delete unknown dimension ${name}`);
+    }
+
+    const { id } = dimensions[name];
+    delete dimensions[name];
+    this.selectionCache = null; // pass ownership to new crossfilter
+    if (selectionCache) {
+      selectionCache.freeDimension(id);
+    }
+    return new ImmutableTypedCrossfilter(data, dimensions, selectionCache);
+  }
+
+  select(name, spec) {
+    /*
+    select on named dimension, as indicated by `spec`.   Spec is an object
+    specifying the selection, and must contain at least a `mode` field.
+
+    Examples:
+      select("foo", {mode: "all"});
+      select("bar", {mode: "none"});
+      select("mumble", {mode: "exact", values: "blue"});
+      select("mumble", {mode: "exact", values: ["red", "green", "blue"]});
+      select("blort", {mode: "range", lo: 0, hi: 999.99});
+    */
+    const { data, selectionCache } = this;
+    this.selectionCache = null;
+    const dimensions = { ...this.dimensions };
+    const { dim, id, selection: oldSelection } = dimensions[name];
+    const newSelection = dim.select(spec);
+    newSelection.ranges = PositiveIntervals.canonicalize(newSelection.ranges);
+    dimensions[name] = { id, dim, name, selection: newSelection };
+    ImmutableTypedCrossfilter._dimSelnHasUpdated(
+      selectionCache,
+      id,
+      newSelection,
+      oldSelection
+    );
+    return new ImmutableTypedCrossfilter(data, dimensions, selectionCache);
+  }
+
+  static _dimSelnHasUpdated(selectionCache, id, newSeln, oldSeln) {
+    /*
+    Selection has updated from oldSeln to newSeln.   Update the
+    bit array if it exists.  If not, we will lazy create it when
+    needed.
+    */
+    if (selectionCache) {
+      /*
+      if both new and old selection use the same index, we can
+      perform an incremental update. If the index changed, we have
+      to do a suboptimal full deselect/select.
+      */
+      let adds;
+      let dels;
+      if (newSeln.index === oldSeln.index) {
+        adds = PositiveIntervals.difference(newSeln.ranges, oldSeln.ranges);
+        dels = PositiveIntervals.difference(oldSeln.ranges, newSeln.ranges);
+      } else {
+        // console.log("suboptimal selection update - index changed");
+        adds = newSeln.ranges;
+        dels = oldSeln.ranges;
+      }
+
+      /*
+      allow dimensions to return selected ranges in either dimension sort
+      order (indirect via index), or in original record order.
+
+      If sort index exists in the dimension, assume sort ordered ranges.
+      */
+      if (oldSeln.index) {
+        dels.forEach(interval =>
+          selectionCache.deselectIndirectFromRange(id, oldSeln.index, interval)
+        );
+      } else {
+        dels.forEach(interval =>
+          selectionCache.deselectFromRange(id, interval)
+        );
+      }
+
+      if (newSeln.index) {
+        adds.forEach(interval =>
+          selectionCache.selectIndirectFromRange(id, newSeln.index, interval)
+        );
+      } else {
+        adds.forEach(interval => selectionCache.selectFromRange(id, interval));
+      }
+    }
+  }
+
+  _getSelectionCache() {
+    if (!this.selectionCache) {
+      // console.log("...rebuilding crossfilter cache...");
+      const selectionCache = new BitArray(this.data.length);
+      Object.keys(this.dimensions).forEach(name => {
+        const { selection } = this.dimensions[name];
+        const id = selectionCache.allocDimension();
+        this.dimensions[name].id = id;
+        const { ranges, index } = selection;
+        ranges.forEach(range => {
+          if (index) {
+            selectionCache.selectIndirectFromRange(id, index, range);
+          } else {
+            selectionCache.selectFromRange(id, range);
+          }
+        });
+      });
+      this.selectionCache = selectionCache;
+    }
+    return this.selectionCache;
+  }
+
+  allSelected() {
+    /*
+    return array of all records currently selected by all dimensions
+    */
+    const selectionCache = this._getSelectionCache();
+    const { data } = this;
+    if (Array.isArray(data)) {
+      const res = [];
+      for (let i = 0, len = data.length; i < len; i += 1) {
+        if (selectionCache.isSelected(i)) {
+          res.push(data[i]);
+        }
+      }
+      return res;
+    }
+    /* else, Dataframe-like */
+    return data.isubsetMask(this.allSelectedMask());
+  }
+
+  allSelectedMask() {
+    /*
+    return Uint8Array containing selection state (truthy/falsey) for each record.
+    */
+    const selectionCache = this._getSelectionCache();
+    return selectionCache.fillBySelection(
+      new Uint8Array(this.data.length),
+      1,
+      0
+    );
+  }
+
+  countSelected() {
+    /*
+    return number of records selected on all dimensions
+    */
+    const selectionCache = this._getSelectionCache();
+    return selectionCache.selectionCount();
+  }
+
+  isElementSelected(i) {
+    /*
+    return truthy/falsey if this record is selected on all dimensions
+    */
+    const selectionCache = this._getSelectionCache();
+    return selectionCache.isSelected(i);
+  }
+
+  fillByIsSelected(array, selectedValue, deselectedValue) {
+    /*
+    fill array with one of two values, based upon selection state.
+    */
+    const selectionCache = this._getSelectionCache();
+    return selectionCache.fillBySelection(
+      array,
+      selectedValue,
+      deselectedValue
+    );
+  }
+}
+
+/*
+Base dimension object.
+
+A Dimension is an index, accessed via a select() method.  The protocol
+for a dimension:
+  - constructor - first param is name, remainder is whatever params are
+    required to initialize the dimension.
+  - select - one and only param is the selection specifier.  Returns an
+    array of record IDs.
+  - name - the dimension name/label.
+*/
+class _ImmutableBaseDimension {
+  constructor(name) {
+    this.name = name;
+  }
+
+  /* eslint-disable class-methods-use-this */
+  select(spec) {
+    const { mode } = spec;
+    if (mode === undefined) {
+      throw new Error("select spec does not contain 'mode'");
+    }
+    throw new Error(`select mode ${mode} not implemented`);
+  }
+  /* eslint-enable class-methods-use-this */
+}
+
+class ImmutableScalarDimension extends _ImmutableBaseDimension {
+  constructor(name, data, value, ValueArrayType) {
+    super(name);
+
+    // Three modes - caller can provide a pre-created value array,
+    // a map function which will create it, or another array which
+    // will used with an identity map function.
+    let array;
+    if (value instanceof ValueArrayType) {
+      // user has provided the final typed array - just use it
+      if (value.length !== data.length) {
+        throw new RangeError(
+          "ScalarDimension values length must equal crossfilter data record count"
+        );
+      }
+      array = value;
+    } else if (value instanceof Function) {
+      // Create value array from user-provided map function.
+      array = this._createValueArray(
+        data,
+        value,
+        new ValueArrayType(data.length)
+      );
+    } else if (isArrayOrTypedArray(value)) {
+      // Create value array from user-provided array.  Typically used
+      // only by enumerated dimensions
+      array = this._createValueArray(
+        data,
+        i => value[i],
+        new ValueArrayType(data.length)
+      );
+    } else {
+      throw new NotImplementedError(
+        "dimension value must be function or value array type"
+      );
+    }
+    this.value = array;
+
+    // create sort index
+    this.index = makeSortIndex(array);
+  }
+
+  /* eslint-disable class-methods-use-this */
+  _createValueArray(data, mapf, array) {
+    // create dimension value array
+    const len = data.length;
+    const larray = array;
+    for (let i = 0; i < len; i += 1) {
+      larray[i] = mapf(i, data);
+    }
+    return larray;
+  }
+  /* eslint-enable class-methods-use-this */
+
+  select(spec) {
+    const { mode } = spec;
+    const { index } = this;
+    switch (mode) {
+      case "all":
+        return { ranges: [[0, this.value.length]], index };
+      case "none":
+        return { ranges: [], index };
+      case "exact":
+        return this.selectExact(spec);
+      case "range":
+        return this.selectRange(spec);
+      default:
+        return super.select(spec);
+    }
+  }
+
+  selectExact(spec) {
+    const { value, index } = this;
+    let { values } = spec;
+    if (!Array.isArray(values)) {
+      values = [values];
+    }
+    const ranges = [];
+    for (let v = 0, len = values.length; v < len; v += 1) {
+      const r = [
+        lowerBoundIndirect(value, index, values[v], 0, value.length),
+        upperBoundIndirect(value, index, values[v], 0, value.length)
+      ];
+      if (r[0] <= r[1]) {
+        ranges.push(r);
+      }
+    }
+    return { ranges, index };
+  }
+
+  selectRange(spec) {
+    const { value, index } = this;
+    /* [lo, hi) */
+    const { lo, hi } = spec;
+    const ranges = [];
+    const r = [
+      lowerBoundIndirect(value, index, lo, 0, value.length),
+      lowerBoundIndirect(value, index, hi, 0, value.length)
+    ];
+    if (r[0] < r[1]) ranges.push(r);
+    return { ranges, index };
+  }
+}
+
+class ImmutableEnumDimension extends ImmutableScalarDimension {
+  constructor(name, data, value) {
+    super(name, data, value, Uint32Array);
+  }
+
+  _createValueArray(data, mapf, array) {
+    const len = data.length;
+    const larray = array;
+
+    // create enumeration table - mapping between the value
+    // and the enum.
+    const s = new Set();
+    for (let i = 0; i < len; i += 1) {
+      s.add(mapf(i, data));
+    }
+    const enumIndex = sort(Array.from(s));
+    this.enumIndex = enumIndex;
+
+    // create dimension value array
+    const enumLen = enumIndex.length;
+    for (let i = 0; i < len; i += 1) {
+      const v = mapf(i, data);
+      const e = lowerBound(enumIndex, v, 0, enumLen);
+      larray[i] = e;
+    }
+    return larray;
+  }
+
+  selectExact(spec) {
+    const { enumIndex } = this;
+    const { values } = spec;
+    return super.selectExact({
+      mode: spec.mode,
+      values: values.map(v => lowerBound(enumIndex, v, 0, enumIndex.length))
+    });
+  }
+
+  /* eslint-disable class-methods-use-this */
+  selectRange() {
+    throw new Error("range selection unsupported on Enumerated dimension");
+  }
+  /* eslint-enable class-methods-use-this */
+}
+
+class ImmutableSpatialDimension extends _ImmutableBaseDimension {
+  constructor(name, data, X, Y) {
+    super(name);
+
+    if (X.length !== Y.length && X.length !== data.length) {
+      throw new RangeError(
+        "SpatialDimension values must have same dimensionality as crossfilter"
+      );
+    }
+    this.X = X;
+    this.Y = Y;
+
+    this.Xindex = makeSortIndex(X);
+    this.Yindex = makeSortIndex(Y);
+  }
+
+  select(spec) {
+    const { mode } = spec;
+    switch (mode) {
+      case "all":
+        return { ranges: [[0, this.X.length]], index: null };
+      case "none":
+        return { ranges: [], index: null };
+      case "within-rect":
+        return this.selectWithinRect(spec);
+      case "within-polygon":
+        return this.selectWithinPolygon(spec);
+      default:
+        return super.select(spec);
+    }
+  }
+
+  selectWithinRect(spec) {
+    /*
+      { mode: "within-rect", x0: 1, y0: 0, x1: 3, y1: 9 }
+    */
+    const { x0, y0, x1, y1 } = spec;
+    const { X, Y } = this;
+    const ranges = [];
+    let start = -1;
+    for (let i = 0, l = X.length; i < l; i += 1) {
+      const x = X[i];
+      const y = Y[i];
+      const inside = x0 <= x && x < x1 && y0 <= y && y < y1;
+      if (inside && start === -1) start = i;
+      if (!inside && start !== -1) {
+        ranges.push([start, i]);
+        start = -1;
+      }
+    }
+    if (start !== -1) ranges.push([start, X.length]);
+    return { ranges, index: null };
+  }
+
+  /*
+  Relatively brute force filter by polygon.
+
+  Currently uses d3.polygonContains() to test for polygon inclusion, which itself
+  uses a ray casting (crossing number) algorithm.  There are a series of optimizations
+  to make this faster:
+    * first sliced by X or Y, using an index on the axis
+    * then the polygon bounding box is used for trivial rejection
+    * then the polygon test is applied
+  */
+
+  selectWithinPolygon(spec) {
+    /*
+      { mode: "within-polygon", polygon: [ [x0, y0], ... ] }
+    */
+    const { polygon } = spec;
+    const [minX, minY, maxX, maxY] = polygonBoundingBox(polygon);
+    const { X, Y, Xindex, Yindex } = this;
+    const { length } = X;
+    let slice;
+    let index;
+    if (maxY - minY > maxX - minX) {
+      slice = [
+        lowerBoundIndirect(X, Xindex, minX, 0, length),
+        lowerBoundIndirect(X, Xindex, maxX, 0, length)
+      ];
+      index = Xindex;
+    } else {
+      slice = [
+        lowerBoundIndirect(Y, Yindex, minY, 0, length),
+        lowerBoundIndirect(Y, Yindex, maxY, 0, length)
+      ];
+      index = Yindex;
+    }
+
+    const ranges = [];
+    let start = -1;
+    for (let i = slice[0], e = slice[1]; i < e; i += 1) {
+      const rid = index[i];
+      const x = X[rid];
+      const y = Y[rid];
+      const inside =
+        minX <= x &&
+        x < maxX &&
+        minY <= y &&
+        y < maxY &&
+        withinPolygon(polygon, x, y);
+
+      if (inside && start === -1) start = i;
+      if (!inside && start !== -1) {
+        ranges.push([start, i]);
+        start = -1;
+      }
+    }
+    if (start !== -1) ranges.push([start, slice[1]]);
+    return { ranges, index };
+  }
+}
+
+/* Helpers */
+export const DimTypes = {
+  scalar: ImmutableScalarDimension,
+  enum: ImmutableEnumDimension,
+  spatial: ImmutableSpatialDimension
+};
+
+function isArrayOrTypedArray(x) {
+  return (
+    Array.isArray(x) ||
+    (ArrayBuffer.isView(x) &&
+      Object.prototype.toString.call(x) !== "[object DataView]")
+  );
+}
+
+/* return bounding box of the polygon */
+function polygonBoundingBox(polygon) {
+  let minX = Number.MAX_VALUE;
+  let minY = Number.MAX_VALUE;
+  let maxX = Number.MIN_VALUE;
+  let maxY = Number.MIN_VALUE;
+  for (let i = 0, l = polygon.length; i < l; i += 1) {
+    const point = polygon[i];
+    const [x, y] = point;
+    if (x < minX) minX = x;
+    if (y < minY) minY = y;
+    if (x > maxX) maxX = x;
+    if (y > maxY) maxY = y;
+  }
+  return [minX, minY, maxX, maxY];
+}
+
+function withinPolygon(polygon, x, y) {
+  // TODO XXX replace
+  return polygonContains(polygon, [x, y]);
+}

client/src/util/typedCrossfilter/index.js

@@ -1,7 +1,5 @@
-// jshint esversion: 6
-
 /*
-Typedarray Crossfilter - a re-implementation of a subset of crossfilter, with
+Crossfilter - a re-implementation of a subset of crossfilter, with
 time/space optimizations predicated upon the following assumptions:
   - dimensions are uniformly typed, and all values must be of that type
   - dimension values must be a primitive type (int, float, string).  Arrays
@@ -11,14 +9,18 @@ time/space optimizations predicated upon the following assumptions:
     want to do that, you have to create the new crossfilter, using the new
     data, from scratch.
 
-The actual backing store for a dimension is a TypedArray, enabling significant
+In addition, this implementation is easier to use with a "redux" style
+app, as all operations on the crossfilter are immutable (ie, return a
+new crossfilter).
+
+The actual backing store for a dimension is a TypedArray, enabling
 performance improvements over the original crossfilter.
 
 There are also a handful of new methods, primarily to take advantage of the
 performance (eg, crossfilter.fillBySelection)
 
-Helpful documents (this module tries to follow the original API as much
+Helpful documents (this module follows similar concepts as the original,
-as is feasable):
+but deviates from the API):
     https://github.com/square/crossfilter/
     http://square.github.io/crossfilter/
 
@@ -26,766 +28,7 @@ There is also a newer, community supported fork of crossfilter, with a
 more complex API.  In a few cases, elements of that API were incorporated.
     https://github.com/square/crossfilter/
 
+See test cases for some concrete examples.
 */
-// XXX replace
-import { polygonContains } from "d3";
 
-import PositiveIntervals from "./positiveIntervals";
+export { default } from "./crossfilter";
-import BitArray from "./bitArray";
-import {
-  makeSortIndex,
-  lowerBound,
-  lowerBoundIndirect,
-  upperBoundIndirect
-} from "./util";
-
-function isArrayOrTypedArray(x) {
-  return (
-    Array.isArray(x) ||
-    (ArrayBuffer.isView(x) &&
-      Object.prototype.toString.call(x) !== "[object DataView]")
-  );
-}
-
-class NotImplementedError extends Error {
-  constructor(...params) {
-    super(...params);
-
-    // Maintains proper stack trace for where our error was thrown (only available on V8)
-    if (Error.captureStackTrace) {
-      Error.captureStackTrace(this, NotImplementedError);
-    }
-  }
-}
-
-class TypedCrossfilter {
-  constructor(data) {
-    /*
-    Typically, data is one of:
-      - Array of objects/records
-      - Dataframe (util/dataframe)
-    */
-    this.data = data;
-
-    // filters: array of { id, dimension }
-    this.filters = [];
-    this.selection = new BitArray(data.length);
-    this.updateTime = 0;
-  }
-
-  size() {
-    return this.data.length;
-  }
-
-  all() {
-    return this.data;
-  }
-
-  /*
-  Create a crossfilter dimension, upon which filtering (subselection) can
-  be done.  Each dimension is typed, and has a particular set of filtering
-  semantics.
-    * ScalarDimension - backed by TypedArray values, supporting filtering
-      by value (within a value range, or one or more exact values)
-    * EnumDimension - backed by an enumeration (eg, strings, bools), filtering
-      by one or more enum categories.
-    * SpatialDimension - backed by 2D points, filter by containment within
-      various shapes (currently supports within Rectangle and within Polygon).
-  Call this method to create a dimension, passing arguments appropriate for
-  the dimension constructor.
-  */
-  dimension(DimensionType, ...rest) {
-    const id = this.selection.allocDimension();
-    const dim = new DimensionType(this, id, ...rest);
-    this.filters.push({ id, dim });
-    dim.filterAll();
-    return dim;
-  }
-
-  _freeDimension(id) {
-    this.selection.freeDimension(id);
-    this.filters = this.filters.filter(f => f._id !== id);
-  }
-
-  // return array of all records that are selected/filtered
-  // by all dimensions.
-  allFiltered() {
-    const { data, selection } = this;
-    if (Array.isArray(data)) {
-      const res = [];
-      for (let i = 0, len = data.length; i < len; i += 1) {
-        if (selection.isSelected(i)) {
-          res.push(data[i]);
-        }
-      }
-      return res;
-    }
-    /* else, Dataframe-like */
-    return data.isubsetMask(this.allFilteredMask());
-  }
-
-  // return Uint8array containing selection state (truthy/falsey) for each record.
-  //
-  allFilteredMask() {
-    return this.selection.fillBySelection(
-      new Uint8Array(this.data.length),
-      1,
-      0
-    );
-  }
-
-  countFiltered() {
-    return this.selection.selectionCount();
-  }
-
-  isElementFiltered(i) {
-    return this.selection.isSelected(i);
-  }
-
-  // fill array with one of two values, based upon selection state
-  fillByIsFiltered(array, selectedValue, deselectedValue) {
-    return this.selection.fillBySelection(
-      array,
-      selectedValue,
-      deselectedValue
-    );
-  }
-}
-
-// Base dimension type - not exported.
-class _Dimension {
-  constructor(xfltr, id) {
-    this.crossfilter = xfltr;
-    this._id = id;
-    this.groups = [];
-  }
-
-  dispose() {
-    this.crossfilter._freeDimension(this._id);
-    return this;
-  }
-
-  id() {
-    return this._id;
-  }
-
-  _filterUpdate() {
-    this.crossfilter.updateTime += 1;
-  }
-}
-
-// Scalar dimension type - value must be a scalar type (eg, int, float),
-// and value array must be a TypedArray.
-//
-class ScalarDimension extends _Dimension {
-  constructor(xfltr, id, value, ValueArrayType) {
-    super(xfltr, id);
-
-    // current selection filter, expressed as PostiveIntervals.
-    this.currentFilter = [];
-
-    // Two modes - caller can provide a pre-created value array,
-    // or a map function which will create it.
-    let array;
-    if (value instanceof ValueArrayType) {
-      // user has provided the final typed array - just use it
-      if (value.length !== this.crossfilter.data.length) {
-        throw new RangeError(
-          "ScalarDimension values length must equal crossfilter data record count"
-        );
-      }
-      array = value;
-    } else if (value instanceof Function) {
-      // Create value array from user-provided map function.
-      array = this._createValueArray(
-        value,
-        new ValueArrayType(this.crossfilter.data.length)
-      );
-    } else if (isArrayOrTypedArray(value)) {
-      // Create value array from user-provided array.  Typically used
-      // only by enumerated dimensions
-      array = this._createValueArray(
-        i => value[i],
-        new ValueArrayType(this.crossfilter.data.length)
-      );
-    } else {
-      throw new NotImplementedError(
-        "dimension value must be function or value array type"
-      );
-    }
-    this.value = array;
-
-    // create sort index
-    this.index = makeSortIndex(array);
-  }
-
-  _createValueArray(value, array) {
-    // create dimension value array
-    const { data } = this.crossfilter;
-    const len = data.length;
-    const larray = array;
-    for (let i = 0; i < len; i += 1) {
-      larray[i] = value(i, data);
-    }
-    return larray;
-  }
-
-  // Argument is an array of intervals indicating records newly selected/filtered
-  //
-  _updateFilters(newFilter) {
-    const cNewFilter = PositiveIntervals.canonicalize(newFilter);
-
-    const adds = PositiveIntervals.difference(cNewFilter, this.currentFilter);
-    const dels = PositiveIntervals.difference(this.currentFilter, cNewFilter);
-
-    this.crossfilter.filters.forEach(f =>
-      f.dim.groups.forEach(grp => grp._updateReduceDel(this, dels))
-    );
-
-    dels.forEach(interval =>
-      this.crossfilter.selection.deselectIndirectFromRange(
-        this._id,
-        this.index,
-        interval
-      )
-    );
-
-    adds.forEach(interval =>
-      this.crossfilter.selection.selectIndirectFromRange(
-        this._id,
-        this.index,
-        interval
-      )
-    );
-
-    this.crossfilter.filters.forEach(f =>
-      f.dim.groups.forEach(grp => grp._updateReduceAdd(this, adds))
-    );
-
-    this.currentFilter = cNewFilter;
-    this._filterUpdate();
-  }
-
-  // filter by value - exact match
-  filterExact(value) {
-    const newFilter = [
-      lowerBoundIndirect(this.value, this.index, value, 0, this.value.length),
-      upperBoundIndirect(this.value, this.index, value, 0, this.value.length)
-    ];
-    if (newFilter[0] <= newFilter[1]) {
-      this._updateFilters([newFilter]);
-    } else {
-      this._updateFilters([]);
-    }
-    return this;
-  }
-
-  // filter by a set of values, eg. enum.
-  filterEnum(values) {
-    const newFilter = [];
-    for (let v = 0, len = values.length; v < len; v += 1) {
-      const intv = [
-        lowerBoundIndirect(
-          this.value,
-          this.index,
-          values[v],
-          0,
-          this.value.length
-        ),
-        upperBoundIndirect(
-          this.value,
-          this.index,
-          values[v],
-          0,
-          this.value.length
-        )
-      ];
-      if (intv[0] <= intv[1]) newFilter.push(intv);
-    }
-    this._updateFilters(newFilter);
-    return this;
-  }
-
-  // filter by value range [lo, hi)
-  // lo: inclusive, hi: exclusive
-  filterRange(range) {
-    const newFilter = [];
-    const intv = [
-      lowerBoundIndirect(
-        this.value,
-        this.index,
-        range[0],
-        0,
-        this.value.length
-      ),
-      upperBoundIndirect(this.value, this.index, range[1], 0, this.value.length)
-    ];
-    if (intv[0] < intv[1]) newFilter.push(intv);
-    this._updateFilters(newFilter);
-    return this;
-  }
-
-  // select all - equivalent of selecting all in this dimension
-  filterAll() {
-    this._updateFilters([[0, this.value.length]]);
-    return this;
-  }
-
-  // select none
-  filterNone() {
-    this._updateFilters([]);
-  }
-
-  // return top k records, starting with offset, in descending order.
-  // Order is this dimension's sort order
-  top(k, offset = 0) {
-    const { data, selection } = this.crossfilter;
-    const { index } = this;
-    const len = index.length;
-    const ret = [];
-    let i = 0;
-    let skip = 0;
-    let found = 0;
-
-    // skip up to offset records
-    for (i = len - 1; i >= 0 && skip < offset; i -= 1) {
-      if (selection.isSelected(index[i])) {
-        skip += 1;
-      }
-    }
-
-    // grab up to k records
-    for (; i >= 0 && found < k; i -= 1) {
-      if (selection.isSelected(index[i])) {
-        ret.push(data[index[i]]);
-        found += 1;
-      }
-    }
-
-    return ret;
-  }
-
-  // return bottom k records, starting with offset, in ascending order.
-  // Order is this dimension's sort order
-  bottom(k, offset = 0) {
-    const { data, selection } = this.crossfilter;
-    const { index } = this;
-    const len = index.length;
-    const ret = [];
-    let skip = 0;
-    let found = 0;
-    let i = 0;
-
-    // skip up to offset records
-    for (i = 0; i < len && skip < offset; i += 1) {
-      if (selection.isSelected(index[i])) {
-        skip += 1;
-      }
-    }
-
-    // grab up to k records
-    for (; i < len && found < k; i += 1) {
-      if (selection.isSelected(index[i])) {
-        ret.push(data[index[i]]);
-        found += 1;
-      }
-    }
-
-    return ret;
-  }
-
-  group(groupValue) {
-    const grp = new ScalarGroup(groupValue, this.value.constructor, this);
-    this.groups.push(grp);
-    return grp;
-  }
-
-  _freeGroup(group) {
-    this.groups = this.groups.filter(e => e !== group);
-  }
-}
-
-// Ordered enumeration - supports any sortable enumerable type, eg,
-// strings, which can be mapped into an fixed numeric range [0..n).
-//
-class EnumDimension extends ScalarDimension {
-  constructor(xfltr, id, value) {
-    super(xfltr, id, value, Uint32Array);
-  }
-
-  _createValueArray(value, array) {
-    const { data } = this.crossfilter;
-    const len = data.length;
-    const larray = array;
-
-    // create enumeration table - mapping between the value
-    // and the enum.
-    const s = new Set();
-    for (let i = 0; i < len; i += 1) {
-      s.add(value(i, data));
-    }
-    this.enumIndex = Array.from(s);
-    this.enumIndex.sort();
-
-    // create dimension value array
-    const enumLen = this.enumIndex.length;
-    for (let i = 0; i < len; i += 1) {
-      const v = value(i, data);
-      const e = lowerBound(this.enumIndex, v, 0, enumLen);
-      larray[i] = e;
-    }
-    return larray;
-  }
-
-  filterExact(value) {
-    return super.filterExact(
-      lowerBound(this.enumIndex, value, 0, this.enumIndex.length)
-    );
-  }
-
-  filterEnum(values) {
-    return super.filterEnum(
-      values.map(v => lowerBound(this.enumIndex, v, 0, this.enumIndex.length))
-    );
-  }
-
-  filterRange(range) {
-    return super.filterEnum(
-      range.map(v => lowerBound(this.enumIndex, v, 0, this.enumIndex.length))
-    );
-  }
-
-  group(groupValue) {
-    const grp = new EnumGroup(groupValue, this.value.constructor, this);
-    this.groups.push(grp);
-    return grp;
-  }
-}
-
-/*
-Super simple 2D spatial dimension, supporting basic "filter within"
-operations.
-*/
-class SpatialDimension extends _Dimension {
-  constructor(xfltr, id, X, Y) {
-    super(xfltr, id);
-
-    if (X.length !== Y.length && X.length !== this.crossfilter.data.length) {
-      throw new RangeError(
-        "SpatialDimension values must have same dimensionality as crossfilter"
-      );
-    }
-    this.X = X;
-    this.Y = Y;
-
-    this.Xindex = makeSortIndex(X);
-    this.Yindex = makeSortIndex(Y);
-  }
-
-  filterAll() {
-    this.crossfilter.selection.selectAll(this._id);
-    this._filterUpdate();
-  }
-
-  filterNone() {
-    this.crossfilter.selection.deselectAll(this._id);
-    this._filterUpdate();
-  }
-
-  /*
-  this could be smarter, but we don't currently use it...
-  */
-  filterWithinRect(northwest, southeast) {
-    const [x0, y0] = northwest;
-    const [x1, y1] = southeast;
-    const { X, Y } = this;
-    const seln = this.crossfilter.selection;
-    const { _id } = this;
-    seln.deselectAll(_id);
-    for (let i = 0, l = this.X.length; i < l; i += 1) {
-      const x = X[i];
-      const y = Y[i];
-      if (x0 <= x && x < x1 && y0 <= y && y < y1) {
-        seln.selectOne(_id, i);
-      }
-    }
-    this._filterUpdate();
-  }
-
-  /*
-  Relatively brute force filter by polygon.   Polygon is array of points, where
-  each point is [x,y].   Eg, [[x0,y0], [x1,y1], ...].
-
-  Currently uses d3.polygonContains() to test for polygon inclusion, which itself
-  uses a ray casting (crossing number) algorithm.  There are a series of optimizations
-  to make this faster:
-    * first sliced by X or Y, using an index on the axis
-    * then the polygon bounding box is used for trivial rejection
-    * then the polygon test is applied
-  */
-  filterWithinPolygon(polygon) {
-    /* return bounding box of the polygon */
-    function polygonBoundingBox(pg) {
-      let minX = Number.MAX_VALUE;
-      let minY = Number.MAX_VALUE;
-      let maxX = Number.MIN_VALUE;
-      let maxY = Number.MIN_VALUE;
-      for (let i = 0, l = pg.length; i < l; i += 1) {
-        const p = pg[i];
-        const x = p[0];
-        const y = p[1];
-        if (x < minX) minX = x;
-        if (y < minY) minY = y;
-        if (x > maxX) maxX = x;
-        if (y > maxY) maxY = y;
-      }
-      return [minX, minY, maxX, maxY];
-    }
-
-    const [minX, minY, maxX, maxY] = polygonBoundingBox(polygon);
-    const { X, Y } = this;
-    let slice;
-    let index;
-    if (maxY - minY > maxX - minX) {
-      slice = [
-        lowerBoundIndirect(X, this.Xindex, minX, 0, X.length),
-        upperBoundIndirect(X, this.Xindex, maxX, 0, X.length)
-      ];
-      index = this.Xindex;
-    } else {
-      slice = [
-        lowerBoundIndirect(Y, this.Yindex, minY, 0, Y.length),
-        upperBoundIndirect(Y, this.Yindex, maxY, 0, Y.length)
-      ];
-      index = this.Yindex;
-    }
-
-    const seln = this.crossfilter.selection;
-    const { _id } = this;
-    const testWithin = polygonContains; // d3.polygonContains()
-    seln.deselectAll(_id);
-
-    for (let i = slice[0], e = slice[1]; i < e; i += 1) {
-      const rid = index[i];
-      const x = X[rid];
-      const y = Y[rid];
-      if (
-        minX <= x &&
-        x < maxX &&
-        minY <= y &&
-        y < maxY &&
-        testWithin(polygon, [x, y])
-      ) {
-        seln.selectOne(_id, rid);
-      }
-    }
-    this._filterUpdate();
-  }
-}
-
-// Groups!  Map/reduce
-//
-class ScalarGroup {
-  constructor(groupValue, groupValueType, dimension) {
-    // parent dimension
-    this.dimension = dimension;
-
-    // generate group names from dimension values
-    this.mapValue = this.constructor._map(
-      groupValue,
-      groupValueType,
-      dimension
-    );
-
-    // group index is mapping from data record index to group index
-    this.groupIndex = new Uint32Array(dimension.crossfilter.data.length);
-
-    // default to counting
-    this.reduceCount();
-
-    // Creates this.groups
-    this._reduce();
-  }
-
-  // internal support function - map all dimension values to group values.
-  //
-  static _map(groupValue, GroupValueType, dimension) {
-    // groupValue is optional.  Defaults to identity.  Used to perform
-    // initial map operation.
-    //
-    // identity: save some memory...
-    if (groupValue === undefined) return dimension.value;
-
-    const data = dimension.value;
-    const len = data.length;
-    const mapValue = new GroupValueType(dimension.value.length);
-    for (let i = 0; i < len; i += 1) {
-      mapValue[i] = groupValue(data[i]);
-    }
-    return mapValue;
-  }
-
-  // Update the group reduction incrementally.  Called when *any* dimension filter
-  // changes.  Guaranteed to be called AFTER the crossfilter is updated.
-  //
-  // Arguments:
-  //  * dim: the dimension that is changing
-  //  * intv: interval list of newly selected values on `dim` (adds)
-  //
-  _updateReduceAdd(dim, intv) {
-    // ignore updates to self, as we don't reduce inclusive of our filter
-    if (dim === this.dimension || intv.length === 0) return;
-
-    // Each item in the range was just added to `dim`.  It was NOT previously
-    // selected - reduceAdd if it is now selected.
-    const { data, selection } = this.dimension.crossfilter;
-    intv.forEach(rng => {
-      for (let r = rng[0]; r < rng[1]; r += 1) {
-        const i = dim.index[r];
-        if (selection.isSelectedIgnoringDim(i, this.dimension.id())) {
-          const group = this.groups[this.groupIndex[i]];
-          group.value = this.reduceAdd(group.value, data[i]);
-        }
-      }
-    });
-  }
-
-  // Update the group reduction incrementally.  Called when *any* dimension filter
-  // changes.   Guaranteed to be called BEFORE the crossfilter is updated.
-  //
-  // Arguments:
-  //  * dim: the dimension that is changing
-  //  * intv: interval list of previously selected values on `dim` (dels)
-  //
-  _updateReduceDel(dim, intv) {
-    // ignore updates to self, as we don't reduce inclusive of our filter
-    if (dim === this.dimension || intv.length === 0) return;
-
-    // Each item in the range will be remved from `dim`.  reduceRemove if it
-    // is currently selected.
-    const { data, selection } = this.dimension.crossfilter;
-    intv.forEach(rng => {
-      for (let r = rng[0]; r < rng[1]; r += 1) {
-        const i = dim.index[r];
-        if (selection.isSelectedIgnoringDim(i, this.dimension.id())) {
-          const group = this.groups[this.groupIndex[i]];
-          group.value = this.reduceRemove(group.value, data[i]);
-        }
-      }
-    });
-  }
-
-  // Reduce the entire data set, creating both the group index and the
-  // groups data.
-  //
-  _reduce() {
-    const { dimension } = this;
-    const { data } = dimension.crossfilter;
-
-    // Create groups
-    const groupNames = new Set(this.mapValue);
-    this.groups = [];
-    const groupIndexByName = {};
-    groupNames.forEach(name => {
-      this.groups.push({ key: name, value: this.reduceInitial() });
-      groupIndexByName[name] = this.groups.length - 1;
-    });
-
-    // Create groupIndex - index map between data record index and group index
-    for (let i = 0, len = this.mapValue.length; i < len; i += 1) {
-      this.groupIndex[i] = groupIndexByName[this.mapValue[i]];
-    }
-
-    // reduce all filtered records, IGNORING the current dimension's filter
-    const { selection } = dimension.crossfilter;
-    for (let i = 0, len = data.length; i < len; i += 1) {
-      if (selection.isSelectedIgnoringDim(i, dimension.id())) {
-        const group = this.groups[this.groupIndex[i]];
-        group.value = this.reduceAdd(group.value, data[i]);
-      }
-    }
-  }
-
-  dispose() {
-    this.dimension._freeGroup(this);
-    return this;
-  }
-
-  // return number of distinct values in the group, independent of any filters.
-  //
-  size() {
-    return this.groups.length;
-  }
-
-  // Set the reduce functions and return the grouping.
-  //
-  reduce(add, remove, initial) {
-    this.reduceAdd = add;
-    this.reduceRemove = remove;
-    this.reduceInitial = initial;
-    this._reduce();
-    return this;
-  }
-
-  // set the reduce functions to count records.
-  reduceCount() {
-    return this.reduce(p => p + 1, p => p - 1, () => 0);
-  }
-
-  // set the reduce functions to sum records using specified value accessor.
-  //
-  reduceSum(value) {
-    return this.reduce((p, v) => p + value(v), (p, v) => p - value(v), () => 0);
-  }
-
-  all() {
-    const res = [...this.groups];
-    res.sort((a, b) => (a.key < b.key ? -1 : a.key > b.key ? 1 : 0));
-    return res;
-  }
-}
-
-class EnumGroup extends ScalarGroup {
-  static _map(groupValue, groupValueType, dimension) {
-    // groupValue is optional.  Defaults to identity.  Used to perform
-    // initial map operation.
-    //
-    // identity: save some memory
-    if (groupValue === undefined) return dimension.value;
-
-    // non-identity mapping unsupported for EnumDimension/EnumGroup.
-    // XXX: this could be implemented, but would require another index
-    // array to map from the group names/keys back to the dimension values.
-    // With this, we just rely on the dimensions `enumIndex` to map from
-    // enumeration value to the record.
-    throw new NotImplementedError("enumerated group mapping not implemented");
-  }
-
-  all() {
-    const res = [];
-    this.groups.forEach(e =>
-      res.push({
-        // XXX: assumes identity group map - see comment in _map()
-        key: this.dimension.enumIndex[e.key],
-        value: e.value
-      })
-    );
-    res.sort((a, b) => (a.key < b.key ? -1 : a.key > b.key ? 1 : 0));
-    return res;
-  }
-}
-
-// Wrapper for backwards compat with crossfilter.
-//
-function crossfilter(data) {
-  return new TypedCrossfilter(data);
-}
-
-crossfilter.PositiveIntervals = PositiveIntervals;
-crossfilter.BitArray = BitArray;
-crossfilter.TypedCrossfilter = TypedCrossfilter;
-crossfilter.ScalarDimension = ScalarDimension;
-crossfilter.EnumDimension = EnumDimension;
-crossfilter.SpatialDimension = SpatialDimension;
-
-export default crossfilter;