Dataframe (#576)

* initial dataframe commit

* initial dataframe port of core app

* rename variables for clarity

* remove unused import

* comment out unused code

* fix array handling bug in crossfilter dimension creation

* allow creation of empty dataframes

* handle non-existent columns

* handle non-existent columns

* revise tests for new dataframe

* comments for clarity

* comments for clarity

* generate bulk add placeholder with real gene names

* fix bug in gene name adding

* more dataframe unit tests

* fix bug - subset from current world, not universe

* put cut and pasted code into a single function

* improve caching of crossfilter

* remove cascading update bug from graph

* more performance work

* improve state handling for scatterplot

* performance optimization of critical path

* add column summarization

* dataframe utils

* add callOnceLazy

* fix tests

* minor updates found during review

* fix misspelling

* remove RESTv02 from function names

* comment cleanup

* cut/icut col parameter defaults to null

* break up large test

* improve tests and comments on dataframe at/has functions

client/__tests__/util/dataframe/dataframe.test.js

@@ -0,0 +1,377 @@
+import * as Dataframe from "../../../src/util/dataframe";
+
+describe("dataframe constructor", () => {
+  test("empty dataframe", () => {
+    const df = new Dataframe.Dataframe([0, 0], []);
+    expect(df).toBeDefined();
+    expect(df.dims).toEqual([0, 0]);
+    expect(df).toHaveLength(0);
+    expect(df.icol(0)).not.toBeDefined();
+  });
+
+  test("create with default indices", () => {
+    const df = new Dataframe.Dataframe(
+      [3, 2],
+      [new Int32Array(3).fill(0), new Int32Array(3).fill(1)]
+    );
+
+    expect(df).toBeDefined();
+    expect(df.dims).toEqual([3, 2]);
+    expect(df.rowIndex).toBeInstanceOf(Dataframe.IdentityInt32Index);
+    expect(df.colIndex).toBeInstanceOf(Dataframe.IdentityInt32Index);
+    expect(df.at(0, 0)).toEqual(0);
+    expect(df.at(2, 1)).toEqual(1);
+    expect(df.iat(0, 0)).toEqual(0);
+    expect(df.iat(2, 1)).toEqual(1);
+  });
+
+  test("create with labelled indices", () => {
+    const df = new Dataframe.Dataframe(
+      [3, 2],
+      [new Int32Array([0, 1, 2]), new Int32Array([3, 4, 5])],
+      new Dataframe.DenseInt32Index([2, 1, 0]),
+      new Dataframe.KeyIndex(["A", "B"])
+    );
+
+    expect(df).toBeDefined();
+    expect(df.dims).toEqual([3, 2]);
+
+    expect(df.rowIndex).toBeInstanceOf(Dataframe.DenseInt32Index);
+    expect(df.colIndex).toBeInstanceOf(Dataframe.KeyIndex);
+    expect(df.rowIndex.keys()).toEqual(new Int32Array([2, 1, 0]));
+    expect(df.colIndex.keys()).toEqual(["A", "B"]);
+
+    expect(df.at(0, "A")).toEqual(2);
+    expect(df.at(2, "B")).toEqual(3);
+    expect(df.iat(0, 0)).toEqual(0);
+    expect(df.iat(2, 1)).toEqual(5);
+  });
+});
+
+describe("simple data access", () => {
+  const df = new Dataframe.Dataframe(
+    [4, 2],
+    [
+      new Float64Array([0.0, Number.NaN, Number.POSITIVE_INFINITY, 3.14159]),
+      ["red", "blue", "green", "nan"]
+    ],
+    new Dataframe.DenseInt32Index([3, 2, 1, 0]),
+    new Dataframe.KeyIndex(["numbers", "colors"])
+  );
+
+  test("iat", () => {
+    expect(df).toBeDefined();
+
+    // present
+    expect(df.iat(0, 0)).toEqual(0.0);
+    expect(df.iat(0, 1)).toEqual("red");
+    expect(df.iat(1, 0)).toEqual(Number.NaN);
+    expect(df.iat(1, 1)).toEqual("blue");
+    expect(df.iat(2, 0)).toEqual(Number.POSITIVE_INFINITY);
+    expect(df.iat(2, 1)).toEqual("green");
+    expect(df.iat(3, 0)).toEqual(3.14159);
+    expect(df.iat(3, 1)).toEqual("nan");
+
+    // labels out of range have no defined behavior
+  });
+
+  test("at", () => {
+    expect(df).toBeDefined();
+
+    // present
+    expect(df.at(3, "numbers")).toEqual(0.0);
+    expect(df.at(3, "colors")).toEqual("red");
+    expect(df.at(2, "numbers")).toEqual(Number.NaN);
+    expect(df.at(2, "colors")).toEqual("blue");
+    expect(df.at(1, "numbers")).toEqual(Number.POSITIVE_INFINITY);
+    expect(df.at(1, "colors")).toEqual("green");
+    expect(df.at(0, "numbers")).toEqual(3.14159);
+    expect(df.at(0, "colors")).toEqual("nan");
+
+    // labels out of range have no defined behavior
+  });
+
+  test("ihas", () => {
+    expect(df).toBeDefined();
+
+    // present
+    expect(df.ihas(0, 0)).toBeTruthy();
+    expect(df.ihas(1, 1)).toBeTruthy();
+    expect(df.ihas(3, 1)).toBeTruthy();
+
+    // not present
+    expect(df.ihas(-1, -1)).toBeFalsy();
+    expect(df.ihas(0, 99)).toBeFalsy();
+    expect(df.ihas(99, 0)).toBeFalsy();
+    expect(df.ihas(99, 99)).toBeFalsy();
+    expect(df.ihas(-1, 0)).toBeFalsy();
+    expect(df.ihas(0, -1)).toBeFalsy();
+  });
+
+  test("has", () => {
+    expect(df).toBeDefined();
+
+    // present
+    expect(df.has(3, "numbers")).toBeTruthy();
+    expect(df.has(0, "numbers")).toBeTruthy();
+    expect(df.has(3, "colors")).toBeTruthy();
+    expect(df.has(0, "colors")).toBeTruthy();
+
+    // not present
+    expect(df.has(3, "foo")).toBeFalsy();
+    expect(df.has(-1, "numbers")).toBeFalsy();
+    expect(df.has(-1, -1)).toBeFalsy();
+    expect(df.has(null, null)).toBeFalsy();
+    expect(df.has(0, "foo")).toBeFalsy();
+    expect(df.has(99, "numbers")).toBeFalsy();
+    expect(df.has(99, "foo")).toBeFalsy();
+  });
+});
+
+describe("dataframe subsetting", () => {
+  describe("cutByList", () => {
+    const sourceDf = new Dataframe.Dataframe(
+      [3, 4],
+      [
+        new Int32Array([0, 1, 2]),
+        ["A", "B", "C"],
+        new Float32Array([4.4, 5.5, 6.6]),
+        ["red", "green", "blue"]
+      ],
+      null,
+      new Dataframe.KeyIndex(["int32", "string", "float32", "colors"])
+    );
+
+    test("all rows, one column", () => {
+      const dfA = sourceDf.cutByList(null, ["colors"]);
+      expect(dfA).toBeDefined();
+      expect(dfA.dims).toEqual([3, 1]);
+      expect(dfA.iat(0, 0)).toEqual("red");
+      expect(dfA.at(2, "colors")).toEqual("blue");
+      expect(dfA.col("colors").asArray()).toEqual(["red", "green", "blue"]);
+      expect(dfA.icol(0).asArray()).toEqual(["red", "green", "blue"]);
+      expect(dfA.col("colors").asArray()).toEqual(
+        sourceDf.col("colors").asArray()
+      );
+      expect(dfA.rowIndex.keys()).toEqual(sourceDf.rowIndex.keys());
+      expect(dfA.colIndex.keys()).toEqual(["colors"]);
+    });
+
+    test("all rows, two columns", () => {
+      const dfB = sourceDf.cutByList(null, ["colors", "float32"]);
+      expect(dfB).toBeDefined();
+      expect(dfB.dims).toEqual([3, 2]);
+      expect(dfB.iat(0, 0)).toBeCloseTo(4.4);
+      expect(dfB.iat(0, 1)).toEqual("red");
+      expect(dfB.at(2, "colors")).toEqual("blue");
+      expect(dfB.at(2, "float32")).toBeCloseTo(6.6);
+      expect(dfB.col("colors").asArray()).toEqual(["red", "green", "blue"]);
+      expect(dfB.col("float32").asArray()).toEqual(
+        new Float32Array([4.4, 5.5, 6.6])
+      );
+      expect(dfB.icol(0).asArray()).toEqual(dfB.col("float32").asArray());
+      expect(dfB.icol(1).asArray()).toEqual(dfB.col("colors").asArray());
+      expect(dfB.col("colors").asArray()).toEqual(
+        sourceDf.col("colors").asArray()
+      );
+      expect(dfB.col("float32").asArray()).toEqual(
+        sourceDf.col("float32").asArray()
+      );
+      expect(dfB.rowIndex.keys()).toEqual(sourceDf.rowIndex.keys());
+      expect(dfB.colIndex.keys()).toEqual(["float32", "colors"]);
+    });
+
+    test("one row, all columns", () => {
+      const dfC = sourceDf.cutByList([1], null);
+      expect(dfC).toBeDefined();
+      expect(dfC.dims).toEqual([1, 4]);
+      expect(dfC.iat(0, 0)).toEqual(1);
+      expect(dfC.iat(0, 1)).toEqual("B");
+      expect(dfC.iat(0, 2)).toBeCloseTo(5.5);
+      expect(dfC.iat(0, 3)).toEqual("green");
+      expect(dfC.rowIndex.keys()).toEqual(new Int32Array([1]));
+      expect(dfC.colIndex.keys()).toEqual(sourceDf.colIndex.keys());
+    });
+
+    test("two rows, all columns", () => {
+      const dfD = sourceDf.cutByList([0, 2], null);
+      expect(dfD).toBeDefined();
+      expect(dfD.dims).toEqual([2, 4]);
+      expect(dfD.icol(0).asArray()).toEqual(new Int32Array([0, 2]));
+      expect(dfD.icol(1).asArray()).toEqual(["A", "C"]);
+      expect(dfD.icol(2).asArray()).toEqual(new Float32Array([4.4, 6.6]));
+      expect(dfD.icol(3).asArray()).toEqual(["red", "blue"]);
+      expect(dfD.rowIndex.keys()).toEqual(new Int32Array([0, 2]));
+      expect(dfD.colIndex.keys()).toEqual(sourceDf.colIndex.keys());
+    });
+
+    test("all rows, all columns", () => {
+      const dfE = sourceDf.cutByList(null, null);
+      expect(dfE).toBeDefined();
+      expect(dfE.dims).toEqual([3, 4]);
+      expect(dfE.icol(0).asArray()).toEqual(sourceDf.icol(0).asArray());
+      expect(dfE.icol(1).asArray()).toEqual(sourceDf.icol(1).asArray());
+      expect(dfE.icol(2).asArray()).toEqual(sourceDf.icol(2).asArray());
+      expect(dfE.icol(3).asArray()).toEqual(sourceDf.icol(3).asArray());
+      expect(dfE.rowIndex.keys()).toEqual(sourceDf.rowIndex.keys());
+      expect(dfE.colIndex.keys()).toEqual(sourceDf.colIndex.keys());
+    });
+
+    test("two rows, two colums", () => {
+      const dfF = sourceDf.cutByList([0, 2], ["int32", "float32"]);
+      expect(dfF).toBeDefined();
+      expect(dfF.dims).toEqual([2, 2]);
+      expect(dfF.icol(0).asArray()).toEqual(new Int32Array([0, 2]));
+      expect(dfF.icol(1).asArray()).toEqual(new Float32Array([4.4, 6.6]));
+      expect(dfF.rowIndex.keys()).toEqual(new Int32Array([0, 2]));
+      expect(dfF.colIndex.keys()).toEqual(["int32", "float32"]);
+    });
+  });
+
+  test("icutByMask", () => {
+    const sourceDf = new Dataframe.Dataframe(
+      [3, 4],
+      [
+        new Int32Array([0, 1, 2]),
+        ["A", "B", "C"],
+        new Float32Array([4.4, 5.5, 6.6]),
+        ["red", "green", "blue"]
+      ],
+      new Dataframe.DenseInt32Index([2, 4, 6]),
+      new Dataframe.KeyIndex(["int32", "string", "float32", "colors"])
+    );
+
+    const dfA = sourceDf.icutByMask(
+      new Uint8Array([0, 1, 1]),
+      new Uint8Array([1, 0, 0, 1])
+    );
+    expect(dfA.dims).toEqual([2, 2]);
+    expect(dfA.icol(0).asArray()).toEqual(new Int32Array([1, 2]));
+    expect(dfA.icol(1).asArray()).toEqual(["green", "blue"]);
+    expect(dfA.rowIndex.keys()).toEqual(new Int32Array([4, 6]));
+    expect(dfA.colIndex.keys()).toEqual(["int32", "colors"]);
+  });
+});
+
+describe("dataframe factories", () => {
+  test("create", () => {
+    const df = Dataframe.Dataframe.create(
+      [3, 3],
+      [
+        new Array(3).fill(0),
+        new Int16Array(3).fill(99),
+        new Float64Array(3).fill(1.1)
+      ]
+    );
+
+    expect(df).toBeDefined();
+    expect(df.dims).toEqual([3, 3]);
+    expect(df).toHaveLength(3);
+    expect(df.iat(0, 0)).toEqual(0);
+    expect(df.iat(1, 1)).toEqual(99);
+    expect(df.iat(2, 2)).toBeCloseTo(1.1);
+    expect(df.iat(0, 0)).toEqual(df.at(0, 0));
+    expect(df.iat(1, 1)).toEqual(df.at(1, 1));
+    expect(df.iat(2, 2)).toEqual(df.at(2, 2));
+  });
+
+  test("clone", () => {
+    const dfA = new Dataframe.Dataframe(
+      [3, 2],
+      [new Int32Array([0, 1, 2]), new Int32Array([3, 4, 5])],
+      new Dataframe.DenseInt32Index([2, 1, 0]),
+      new Dataframe.KeyIndex(["A", "B"])
+    );
+
+    const dfB = dfA.clone();
+    expect(dfB).not.toBe(dfA);
+    expect(dfB.dims).toEqual(dfA.dims);
+    expect(dfB).toHaveLength(dfA.length);
+    expect(dfB.rowIndex.keys()).toEqual(dfA.rowIndex.keys());
+    expect(dfB.colIndex.keys()).toEqual(dfA.colIndex.keys());
+    for (let i = 0, l = dfB.dims[1]; i < l; i += 1) {
+      expect(dfB.icol(i).asArray()).toEqual(dfA.icol(i).asArray());
+    }
+  });
+});
+
+describe("dataframe col", () => {
+  let df = null;
+  beforeEach(() => {
+    df = new Dataframe.Dataframe(
+      [2, 2],
+      [[true, false], [1, 0]],
+      null,
+      new Dataframe.KeyIndex(["A", "B"])
+    );
+  });
+
+  test("col", () => {
+    expect(df).toBeDefined();
+    expect(df.col("A")).toBe(df.icol(0));
+    expect(df.col("B")).toBe(df.icol(1));
+    expect(df.col("undefined")).toBeUndefined();
+    expect(df.icol("undefined")).toBeUndefined();
+
+    const colA = df.col("A");
+    expect(colA).toBeInstanceOf(Function);
+    expect(colA.asArray).toBeInstanceOf(Function);
+    expect(colA.has).toBeInstanceOf(Function);
+    expect(colA.ihas).toBeInstanceOf(Function);
+    expect(colA.indexOf).toBeInstanceOf(Function);
+    expect(colA.iget).toBeInstanceOf(Function);
+  });
+
+  test("col.asArray", () => {
+    expect(df).toBeDefined();
+    expect(df.col("A").asArray()).toEqual([true, false]);
+    expect(df.icol(0).asArray()).toEqual([true, false]);
+    expect(df.col("B").asArray()).toEqual([1, 0]);
+    expect(df.icol(1).asArray()).toEqual([1, 0]);
+  });
+
+  test("col.has", () => {
+    expect(df).toBeDefined();
+    expect(df.col("A").has(-1)).toBe(false);
+    expect(df.col("A").has(0)).toBe(true);
+    expect(df.col("A").has(1)).toBe(true);
+    expect(df.col("A").has(2)).toBe(false);
+    expect(df.col("B").has(-1)).toBe(false);
+    expect(df.col("B").has(0)).toBe(true);
+    expect(df.col("B").has(1)).toBe(true);
+    expect(df.col("B").has(2)).toBe(false);
+  });
+
+  test("col.ihas", () => {
+    expect(df).toBeDefined();
+    expect(df.col("A").ihas(-1)).toBe(false);
+    expect(df.col("A").ihas(0)).toBe(true);
+    expect(df.col("A").ihas(1)).toBe(true);
+    expect(df.col("A").ihas(2)).toBe(false);
+    expect(df.col("B").ihas(-1)).toBe(false);
+    expect(df.col("B").ihas(0)).toBe(true);
+    expect(df.col("B").ihas(1)).toBe(true);
+    expect(df.col("B").ihas(2)).toBe(false);
+  });
+
+  test("col.iget", () => {
+    expect(df).toBeDefined();
+    expect(df.col("A").iget(0)).toEqual(df.iat(0, 0));
+    expect(df.col("B").iget(1)).toEqual(df.iat(1, 1));
+  });
+
+  test("col.indexOf", () => {
+    expect(df).toBeDefined();
+    expect(df.col("A").indexOf(true)).toEqual(0);
+    expect(df.col("A").indexOf(false)).toEqual(1);
+    expect(df.col("A").indexOf(99)).toBeUndefined();
+    expect(df.col("A").indexOf(undefined)).toBeUndefined();
+    expect(df.col("A").indexOf(1)).toBeUndefined();
+
+    expect(df.col("B").indexOf(1)).toEqual(0);
+    expect(df.col("B").indexOf(0)).toEqual(1);
+    expect(df.col("B").indexOf(99)).toBeUndefined();
+    expect(df.col("B").indexOf(undefined)).toBeUndefined();
+    expect(df.col("B").indexOf(true)).toBeUndefined();
+  });
+});

client/__tests__/util/stateManager/sampleResponses.js

@@ -157,16 +157,6 @@ const anAnnotationsVarFBSResponse = (() => {
   return encodeMatrix(columns, anAnnotationsVarJSONResponse.names);
 })();
 
-const aLayoutJSONResponse = {
-  layout: {
-    ndims: 2,
-    coordinates: _()
-      .range(nObs)
-      .map(idx => [idx, Math.random(), Math.random()])
-      .value()
-  }
-};
-
 const aLayoutFBSResponse = (() => {
   const coords = [
     new Float32Array(nObs).fill(Math.random()),
@@ -190,7 +180,7 @@ const aLayoutFBSResponse = (() => {
 
   NetEncoding.Matrix.startMatrix(builder);
   NetEncoding.Matrix.addNRows(builder, nObs);
-  NetEncoding.Matrix.addNCols(builder, nVar);
+  NetEncoding.Matrix.addNCols(builder, coords.length);
   NetEncoding.Matrix.addColumns(builder, columns);
   const matrix = NetEncoding.Matrix.endMatrix(builder);
   builder.finish(matrix);

client/__tests__/util/stateManager/summarizeAnnotations.test.js

@@ -1,4 +1,9 @@
 import summarizeAnnotations from "../../../src/util/stateManager/summarizeAnnotations";
+import * as Dataframe from "../../../src/util/dataframe";
+
+function float32Conversion(f) {
+  return new Float32Array([39.3])[0];
+}
 
 describe("summarizeAnnotations", () => {
   const schema = {
@@ -20,7 +25,8 @@ describe("summarizeAnnotations", () => {
   };
 
   test("empty test", () => {
-    const summary = summarizeAnnotations(schema, [], []);
+    const df = Dataframe.Dataframe.empty();
+    const summary = summarizeAnnotations(schema, df, df.clone());
     expect(summary).toEqual(
       expect.objectContaining({
         obs: {
@@ -69,18 +75,27 @@ describe("summarizeAnnotations", () => {
   });
 
   test("simple test", () => {
-    const obsAnnotations = [
+    const obsAnnotations = new Dataframe.Dataframe(
-      {
+      [1, 6],
-        __index__: 0,
+      [
-        name: "n1",
+        ["n1"],
-        nameString: "hi",
+        ["hi"],
-        nameBoolean: true,
+        [true],
-        nameFloat32: 39.3,
+        new Float32Array([39.3]),
-        nameInt32: 99,
+        new Int32Array([99]),
-        nameCategorical: 1
+        [1]
-      }
+      ],
-    ];
+      null,
-    const varAnnotations = [];
+      new Dataframe.KeyIndex([
+        "name",
+        "nameString",
+        "nameBoolean",
+        "nameFloat32",
+        "nameInt32",
+        "nameCategorical"
+      ])
+    );
+    const varAnnotations = Dataframe.Dataframe.empty();
 
     const summary = summarizeAnnotations(
       schema,
@@ -105,7 +120,13 @@ describe("summarizeAnnotations", () => {
           },
           nameFloat32: {
             categorical: false,
-            range: { min: 39.3, max: 39.3, nan: 0, ninf: 0, pinf: 0 }
+            range: {
+              min: float32Conversion(39.3),
+              max: float32Conversion(39.3),
+              nan: 0,
+              ninf: 0,
+              pinf: 0
+            }
           },
           nameInt32: {
             categorical: false,
@@ -124,36 +145,27 @@ describe("summarizeAnnotations", () => {
   });
 
   test("multi test", () => {
-    const obsAnnotations = [
+    const obsAnnotations = new Dataframe.Dataframe(
-      {
+      [3, 6],
-        __index__: 0,
+      [
-        name: "n0",
+        ["n0", "n1", "n2"],
-        nameString: "hi",
+        ["hi", "hi", "bye"],
-        nameBoolean: false,
+        [false, true, true],
-        nameFloat32: 39.3,
+        new Float32Array([39.3, 39.3, 0]),
-        nameInt32: 99,
+        new Int32Array([99, 99, 99]),
-        nameCategorical: 1
+        [1, false, "0"]
-      },
+      ],
-      {
+      null,
-        __index__: 1,
+      new Dataframe.KeyIndex([
-        name: "n1",
+        "name",
-        nameString: "hi",
+        "nameString",
-        nameBoolean: true,
+        "nameBoolean",
-        nameFloat32: 39.3,
+        "nameFloat32",
-        nameInt32: 99,
+        "nameInt32",
-        nameCategorical: false
+        "nameCategorical"
-      },
+      ])
-      {
+    );
-        __index__: 2,
+    const varAnnotations = Dataframe.Dataframe.empty();
-        name: "n2",
-        nameString: "bye",
-        nameBoolean: true,
-        nameFloat32: 0,
-        nameInt32: 99,
-        nameCategorical: "0"
-      }
-    ];
-    const varAnnotations = [];
 
     const summary = summarizeAnnotations(
       schema,
@@ -178,7 +190,13 @@ describe("summarizeAnnotations", () => {
           },
           nameFloat32: {
             categorical: false,
-            range: { min: 0, max: 39.3, nan: 0, ninf: 0, pinf: 0 }
+            range: {
+              min: 0,
+              max: float32Conversion(39.3),
+              nan: 0,
+              ninf: 0,
+              pinf: 0
+            }
           },
           nameInt32: {
             categorical: false,
@@ -197,45 +215,32 @@ describe("summarizeAnnotations", () => {
   });
 
   test("non-finite numbers", () => {
-    const obsAnnotations = [
+    const obsAnnotations = new Dataframe.Dataframe(
-      {
+      [4, 6],
-        __index__: 0,
+      [
-        name: "n0",
+        ["n0", "n1", "n2", "n2"],
-        nameString: "hi",
+        ["hi", "hi", "bye", "bye"],
-        nameBoolean: false,
+        [false, true, true, true],
-        nameFloat32: 39.3,
+        new Float32Array([
-        nameInt32: 99,
+          39.3,
-        nameCategorical: 1
+          Number.NEGATIVE_INFINITY,
-      },
+          Number.NaN,
-      {
+          Number.POSITIVE_INFINITY
-        __index__: 1,
+        ]),
-        name: "n1",
+        new Int32Array([99, 99, 99, 99]),
-        nameString: "hi",
+        [1, false, "0", "0"]
-        nameBoolean: true,
+      ],
-        nameFloat32: Number.NEGATIVE_INFINITY,
+      null,
-        nameInt32: 99,
+      new Dataframe.KeyIndex([
-        nameCategorical: false
+        "name",
-      },
+        "nameString",
-      {
+        "nameBoolean",
-        __index__: 2,
+        "nameFloat32",
-        name: "n2",
+        "nameInt32",
-        nameString: "bye",
+        "nameCategorical"
-        nameBoolean: true,
+      ])
-        nameFloat32: Number.NaN,
+    );
-        nameInt32: 99,
+    const varAnnotations = Dataframe.Dataframe.empty();
-        nameCategorical: "0"
-      },
-      {
-        __index__: 3,
-        name: "n2",
-        nameString: "bye",
-        nameBoolean: true,
-        nameFloat32: Number.POSITIVE_INFINITY,
-        nameInt32: 99,
-        nameCategorical: "0"
-      }
-    ];
-    const varAnnotations = [];
 
     const summary = summarizeAnnotations(
       schema,
@@ -260,7 +265,13 @@ describe("summarizeAnnotations", () => {
           },
           nameFloat32: {
             categorical: false,
-            range: { min: 39.3, max: 39.3, nan: 1, ninf: 1, pinf: 1 }
+            range: {
+              min: float32Conversion(39.3),
+              max: float32Conversion(39.3),
+              nan: 1,
+              ninf: 1,
+              pinf: 1
+            }
           },
           nameInt32: {
             categorical: false,

client/__tests__/util/stateManager/universe.test.js

@@ -1,13 +1,13 @@
-import _ from "lodash";
 import * as Universe from "../../../src/util/stateManager/universe";
+import * as Dataframe from "../../../src/util/dataframe";
 import * as REST from "./sampleResponses";
 
-describe("createUniverseFromRestV02Response", () => {
+describe("createUniverseFromResponse", () => {
   /*
-  test createUniverseFromRestV02Response - this function converts
+  test createUniverseFromResponse - this function converts
   a set of REST 0.2 responses into a "new" Universe.
 
-  createUniverseFromRestV02Response(
+  createUniverseFromResponse(
     configResponse,
     schemaResponse,
     annotationsObsResponse,
@@ -30,7 +30,7 @@ describe("createUniverseFromRestV02Response", () => {
     create a universe from sample data nad validate its shape & contents
     */
     const { nObs, nVar } = REST.schema.schema.dataframe;
-    const universe = Universe.createUniverseFromRestV02Response(
+    const universe = Universe.createUniverseFromResponse(
       REST.config,
       REST.schema,
       REST.annotationsObs,
@@ -45,23 +45,23 @@ describe("createUniverseFromRestV02Response", () => {
         nObs,
         nVar,
         schema: REST.schema.schema,
-        obsAnnotations: expect.any(Array),
+        obsAnnotations: expect.any(Dataframe.Dataframe),
-        varAnnotations: expect.any(Array),
+        varAnnotations: expect.any(Dataframe.Dataframe),
-        obsNameToIndexMap: expect.any(Object),
+        obsLayout: expect.any(Dataframe.Dataframe),
-        varNameToIndexMap: expect.any(Object),
+        summary: expect.any(Object),
-        obsLayout: expect.objectContaining({
-          X: expect.any(Float32Array),
-          Y: expect.any(Float32Array)
-        }),
         varDataCache: expect.any(Object)
       })
     );
 
-    expect(universe.obsAnnotations).toHaveLength(nObs);
+    expect(universe.obsAnnotations.dims).toEqual([
-    expect(_.keys(universe.obsNameToIndexMap)).toHaveLength(nObs);
+      nObs,
-    expect(universe.obsLayout.X).toHaveLength(nObs);
+      REST.schema.schema.annotations.obs.length
-    expect(universe.obsLayout.Y).toHaveLength(nObs);
+    ]);
-    expect(universe.varAnnotations).toHaveLength(nVar);
+    expect(universe.obsLayout.dims).toEqual([nObs, 2]);
-    expect(_.keys(universe.varNameToIndexMap)).toHaveLength(nVar);
+    expect(universe.obsLayout.colIndex.keys()).toEqual(["X", "Y"]);
+    expect(universe.varAnnotations.dims).toEqual([
+      nVar,
+      REST.schema.schema.annotations.var.length
+    ]);
   });
 });

client/__tests__/util/stateManager/world.test.js

@@ -1,6 +1,7 @@
 import _ from "lodash";
 import * as Universe from "../../../src/util/stateManager/universe";
 import * as World from "../../../src/util/stateManager/world";
+import * as Dataframe from "../../../src/util/dataframe";
 import Crossfilter from "../../../src/util/typedCrossfilter";
 import * as REST from "./sampleResponses";
 import {
@@ -16,7 +17,7 @@ the default REST test response.
 const defaultBigBang = () => {
   /* create unverse, world, crossfilter and dimensionMap */
   /* create universe */
-  const universe = Universe.createUniverseFromRestV02Response(
+  const universe = Universe.createUniverseFromResponse(
     REST.config,
     REST.schema,
     REST.annotationsObs,
@@ -40,7 +41,7 @@ const defaultBigBang = () => {
 
 describe("createWorldFromEntireUniverse", () => {
   test("create from REST sample", () => {
-    const universe = Universe.createUniverseFromRestV02Response(
+    const universe = Universe.createUniverseFromResponse(
       REST.config,
       REST.schema,
       REST.annotationsObs,
@@ -75,10 +76,7 @@ describe("createWorldFromEntireUniverse", () => {
             .value()
         }),
 
-        varDataCache: expect.any(Object),
+        varDataCache: expect.any(Object)
-
-        obsIndex: null, // null indicating full universe
-        obsBackIndex: null
       })
     );
   });
@@ -111,51 +109,43 @@ describe("createWorldFromCurrentSelection", () => {
     */
 
     /* matchFilter must match the dimension filters above */
-    const matchFilter = val => val.field1 >= 0 && val.field1 < 5 && !val.field3;
+    const matchFilter = (df, row) => {
-    const universeIndices = _()
+      const field1 = df.at(row, "field1");
-      .range(universe.nObs)
+      const field3 = df.at(row, "field3");
-      .filter(idx => matchFilter(universe.obsAnnotations[idx]))
+      return field1 >= 0 && field1 < 5 && !field3;
-      .value();
-
-    const expected = {
-      nObs: universeIndices.length,
-      obsAnnotations: _.map(universeIndices, i => universe.obsAnnotations[i]),
-      obsLayout: {
-        X: new Float32Array(
-          _.map(universeIndices, i => universe.obsLayout.X[i])
-        ),
-        Y: new Float32Array(
-          _.map(universeIndices, i => universe.obsLayout.Y[i])
-        )
-      },
-      obsBackIndex: _.transform(
-        universeIndices,
-        (result, univIdx, worldIdx) => {
-          result[univIdx] = worldIdx;
-        },
-        new Uint32Array(universe.nObs).fill(-1)
-      ),
-      obsIndex: new Uint32Array(universeIndices)
     };
+    const matchingIndices = _()
+      .range(universe.nObs)
+      .filter(idx => matchFilter(universe.obsAnnotations, idx))
+      .value();
 
     expect(world).toMatchObject(
       expect.objectContaining({
         api: "0.2",
-        nObs: expected.nObs,
+        nObs: matchingIndices.length,
         nVar: universe.nVar,
         schema: universe.schema,
-        obsAnnotations: expected.obsAnnotations,
+        obsAnnotations: expect.any(Dataframe.Dataframe),
         varAnnotations: universe.varAnnotations,
-        obsLayout: expected.obsLayout,
+        obsLayout: expect.any(Dataframe.Dataframe),
         summary: {
           obs: expect.any(Object) /* we could do better! */,
           var: expect.any(Object) /* we could do better! */
         },
-        varDataCache: expect.any(Object),
+        varDataCache: expect.any(Object)
-        obsIndex: expected.obsIndex,
-        obsBackIndex: expected.obsBackIndex
       })
     );
+
+    expect(world.obsAnnotations.rowIndex.keys()).toEqual(
+      new Int32Array(matchingIndices)
+    );
+    expect(world.obsAnnotations.colIndex.keys()).toEqual(
+      universe.obsAnnotations.colIndex.keys()
+    );
+    expect(world.obsLayout.rowIndex.keys()).toEqual(
+      new Int32Array(matchingIndices)
+    );
+    expect(world.obsLayout.colIndex.keys()).toEqual(["X", "Y"]);
   });
 });
 
@@ -219,7 +209,9 @@ describe("subsetVarData", () => {
       world,
       crossfilter
     );
-    expect(newWorld.obsIndex).toMatchObject(new Uint32Array([0, 2]));
+    expect(newWorld.obsAnnotations.rowIndex.keys()).toEqual(
+      new Int32Array([0, 2])
+    );
 
     /* expect a subset */
     const result = World.subsetVarData(newWorld, universe, sourceVarData);

client/__tests__/util/stateManager/worldUtil.test.js

@@ -2,16 +2,27 @@ import {
   countCategoryValues2D,
   clearCaches
 } from "../../../src/util/stateManager/worldUtil";
+import * as Dataframe from "../../../src/util/dataframe";
 
 describe("WorldUtil cache management", () => {
   test("empty", () => {
-    const count = countCategoryValues2D("a", "b", []);
+    const count = countCategoryValues2D(
+      "a",
+      "b",
+      new Dataframe.Dataframe([0, 0], [])
+    );
     expect(count).toMatchObject(new Map());
+    expect(count.size).toBe(0);
   });
 
   test("simple couts", () => {
-    const rows = [{ a: 0, b: false }, { a: 0, b: true }, { a: 1, b: false }];
+    const df = new Dataframe.Dataframe(
-    const count = countCategoryValues2D("a", "b", rows);
+      [3, 2],
+      [[0, 0, 1], [false, true, false]],
+      null,
+      new Dataframe.KeyIndex(["a", "b"])
+    );
+    const count = countCategoryValues2D("a", "b", df);
     expect(count).toMatchObject(
       new Map([
         [0, new Map([[true, 1], [false, 1]])],
@@ -22,16 +33,22 @@ describe("WorldUtil cache management", () => {
 
   test("memo cache clear", () => {
     clearCaches();
-    const row1 = [];
+    const df1 = new Dataframe.Dataframe([0, 0], []);
-    const row2 = [{ a: 0, b: false }, { a: 0, b: true }, { a: 1, b: false }];
+    const df2 = new Dataframe.Dataframe(
-    const count1 = countCategoryValues2D("a", "b", row1);
+      [3, 2],
-    const count2 = countCategoryValues2D("a", "b", row1);
+      [[0, 0, 1], [false, true, false]],
-    const count3 = countCategoryValues2D("a", "b", []);
+      null,
-    const count4 = countCategoryValues2D("a", "b", row2);
+      new Dataframe.KeyIndex(["a", "b"])
+    );
+
+    const count1 = countCategoryValues2D("a", "b", df1);
+    const count2 = countCategoryValues2D("a", "b", df1);
+    const count3 = countCategoryValues2D("a", "b", df1.clone());
+    const count4 = countCategoryValues2D("a", "b", df2);
 
     clearCaches();
-    const count10 = countCategoryValues2D("a", "b", row1);
+    const count10 = countCategoryValues2D("a", "b", df1);
-    const count11 = countCategoryValues2D("a", "b", row2);
+    const count11 = countCategoryValues2D("a", "b", df2);
 
     expect(count1).toEqual(count2);
     expect(count1).toEqual(count3);

client/__tests__/util/typedCrossfilter/bitArray.test.js

@@ -118,16 +118,16 @@ describe("selectionCount", () => {
     const dim2 = ba.allocDimension();
     expect(dim2).toBeDefined();
 
-    expect(ba.selectionCount).toEqual(0);
+    expect(ba.selectionCount()).toEqual(0);
     ba.selectAll(dim1);
-    expect(ba.selectionCount).toEqual(0);
+    expect(ba.selectionCount()).toEqual(0);
     ba.selectAll(dim2);
-    expect(ba.selectionCount).toEqual(defaultTestLength);
+    expect(ba.selectionCount()).toEqual(defaultTestLength);
 
     for (let i = 0; i < defaultTestLength; i += 1) {
       ba.deselectOne(dim1, i);
-      expect(ba.selectionCount).toEqual(defaultTestLength - i - 1);
+      expect(ba.selectionCount()).toEqual(defaultTestLength - i - 1);
-      expect(ba.selectionCount).toEqual(ba.countAllOnes());
+      expect(ba.selectionCount()).toEqual(ba.countAllOnes());
     }
 
     ba.freeDimension(dim1);

client/__tests__/util/typedCrossfilter/typedCrossfilter.test.js

@@ -119,14 +119,22 @@ function groupReduce(data, valueMap, valueReduce, valueInit) {
 }
 
 function groupCount(data, map) {
-  return groupReduce(data, map, (p, v) => p + 1, () => 0);
+  return groupReduce(data, map, p => p + 1, () => 0);
 }
 
 function groupSum(data, map) {
-  return groupReduce(data, map, (p, v) => (p += map(v)), () => 0);
+  return groupReduce(
+    data,
+    map,
+    (p, v) => {
+      p += map(v);
+      return p;
+    },
+    () => 0
+  );
 }
 
-var payments = null;
+let payments = null;
 beforeEach(() => {
   payments = crossfilter(someData);
 });
@@ -139,7 +147,7 @@ describe("typedCrossfilter", () => {
 
     const quantity = payments.dimension(
       crossfilter.ScalarDimension,
-      r => r.quantity,
+      (i, data) => data[i].quantity,
       Int32Array
     );
     expect(quantity).toBeDefined();
@@ -154,20 +162,23 @@ describe("typedCrossfilter", () => {
     expect(payments).toBeDefined();
     const quantity = payments.dimension(
       crossfilter.ScalarDimension,
-      r => r.quantity,
+      (i, data) => data[i].quantity,
       Int32Array
     );
     const tip = payments.dimension(
       crossfilter.ScalarDimension,
-      r => r.tip,
+      (i, data) => data[i].tip,
       Float32Array
     );
     const total = payments.dimension(
       crossfilter.ScalarDimension,
-      r => r.total,
+      (i, data) => data[i].total,
       Float32Array
     );
-    const type = payments.dimension(crossfilter.EnumDimension, r => r.type);
+    const type = payments.dimension(
+      crossfilter.EnumDimension,
+      (i, data) => data[i].type
+    );
 
     expect(quantity).toBeDefined();
     expect(tip).toBeDefined();
@@ -214,20 +225,18 @@ describe("typedCrossfilter", () => {
     expect(payments).toBeDefined();
     const quantity = payments.dimension(
       crossfilter.ScalarDimension,
-      r => r.quantity,
+      (i, data) => data[i].quantity,
       Int32Array
     );
     const tip = payments.dimension(
       crossfilter.ScalarDimension,
-      r => r.tip,
+      (i, data) => data[i].tip,
       Float32Array
     );
-    const total = payments.dimension(
+    const type = payments.dimension(
-      crossfilter.ScalarDimension,
+      crossfilter.EnumDimension,
-      r => r.total,
+      (i, data) => data[i].type
-      Float32Array
     );
-    const type = payments.dimension(crossfilter.EnumDimension, r => r.type);
 
     quantity.filterExact(1);
     expect(payments.countFiltered()).toEqual(
@@ -250,20 +259,23 @@ describe("typedCrossfilter", () => {
     expect(payments).toBeDefined();
     const quantity = payments.dimension(
       crossfilter.ScalarDimension,
-      r => r.quantity,
+      (i, data) => data[i].quantity,
       Int32Array
     );
     const tip = payments.dimension(
       crossfilter.ScalarDimension,
-      r => r.tip,
+      (i, data) => data[i].tip,
       Float32Array
     );
     const total = payments.dimension(
       crossfilter.ScalarDimension,
-      r => r.total,
+      (i, data) => data[i].total,
       Float32Array
     );
-    const type = payments.dimension(crossfilter.EnumDimension, r => r.type);
+    const type = payments.dimension(
+      crossfilter.EnumDimension,
+      (i, data) => data[i].type
+    );
 
     tip.filterRange([0, 91]);
     expect(payments.allFiltered()).toEqual(
@@ -291,20 +303,23 @@ describe("typedCrossfilter", () => {
     expect(payments).toBeDefined();
     const quantity = payments.dimension(
       crossfilter.ScalarDimension,
-      r => r.quantity,
+      (i, data) => data[i].quantity,
       Int32Array
     );
     const tip = payments.dimension(
       crossfilter.ScalarDimension,
-      r => r.tip,
+      (i, data) => data[i].tip,
       Float32Array
     );
     const total = payments.dimension(
       crossfilter.ScalarDimension,
-      r => r.total,
+      (i, data) => data[i].total,
       Float32Array
     );
-    const type = payments.dimension(crossfilter.EnumDimension, r => r.type);
+    const type = payments.dimension(
+      crossfilter.EnumDimension,
+      (i, data) => data[i].type
+    );
 
     type.filterEnum(["tab", "cash"]);
     expect(payments.allFiltered()).toEqual(
@@ -326,27 +341,30 @@ describe("typedCrossfilter", () => {
     expect(payments).toBeDefined();
     const quantity = payments.dimension(
       crossfilter.ScalarDimension,
-      r => r.quantity,
+      (i, data) => data[i].quantity,
       Int32Array
     );
     const tip = payments.dimension(
       crossfilter.ScalarDimension,
-      r => r.tip,
+      (i, data) => data[i].tip,
       Float32Array
     );
     const total = payments.dimension(
       crossfilter.ScalarDimension,
-      r => r.total,
+      (i, data) => data[i].total,
       Float32Array
     );
-    const type = payments.dimension(crossfilter.EnumDimension, r => r.type);
+    const type = payments.dimension(
+      crossfilter.EnumDimension,
+      (i, data) => data[i].type
+    );
 
     // Create a bunch of fake dimensions to ensure we can handle > 32
     let dimMap = {};
     for (let i = 0; i < 65; i++) {
       dimMap[i] = payments.dimension(
         crossfilter.ScalarDimension,
-        r => Math.random(),
+        () => Math.random(),
         Float32Array
       );
       expect(dimMap[i]).toBeDefined();
@@ -372,18 +390,21 @@ describe("typedCrossfilter", () => {
 
     const quantity = payments.dimension(
       crossfilter.ScalarDimension,
-      r => r.quantity,
+      (i, data) => data[i].quantity,
       Int32Array
     );
     const tip = payments.dimension(
       crossfilter.ScalarDimension,
-      r => r.tip,
+      (i, data) => data[i].tip,
       Int32Array
     );
-    const type = payments.dimension(crossfilter.EnumDimension, r => r.type);
+    const type = payments.dimension(
+      crossfilter.EnumDimension,
+      (i, data) => data[i].type
+    );
     const total = payments.dimension(
       crossfilter.ScalarDimension,
-      r => r.total,
+      (i, data) => data[i].total,
       Int32Array
     );
 
@@ -411,15 +432,18 @@ describe("typedCrossfilter", () => {
 
     const tip = payments.dimension(
       crossfilter.ScalarDimension,
-      r => r.tip,
+      (i, data) => data[i].tip,
       Int32Array
     );
     const totalX10 = payments.dimension(
       crossfilter.ScalarDimension,
-      r => r.total * 10,
+      (i, data) => data[i].total * 10,
       Int32Array
     );
-    const type = payments.dimension(crossfilter.EnumDimension, r => r.type);
+    const type = payments.dimension(
+      crossfilter.EnumDimension,
+      (i, data) => data[i].type
+    );
 
     const paymentsByTip_A = tip.group();
     const paymentsByTip_B = tip.group(r => 10 * r);
@@ -458,10 +482,13 @@ describe("typedCrossfilter", () => {
 
     const total = payments.dimension(
       crossfilter.ScalarDimension,
-      r => r.total,
+      (i, data) => data[i].total,
       Float32Array
     );
-    const type = payments.dimension(crossfilter.EnumDimension, r => r.type);
+    const type = payments.dimension(
+      crossfilter.EnumDimension,
+      (i, data) => data[i].type
+    );
 
     const paymentsByTotal = total.group();
     const paymentsByType = type.group();
@@ -499,15 +526,18 @@ describe("typedCrossfilter", () => {
 
     const tip = payments.dimension(
       crossfilter.ScalarDimension,
-      r => r.tip,
+      (i, data) => data[i].tip,
       Int32Array
     );
     const total = payments.dimension(
       crossfilter.ScalarDimension,
-      r => r.total,
+      (i, data) => data[i].total,
       Int32Array
     );
-    const type = payments.dimension(crossfilter.EnumDimension, r => r.type);
+    const type = payments.dimension(
+      crossfilter.EnumDimension,
+      (i, data) => data[i].type
+    );
 
     const paymentsByTip = tip.group();
     const paymentsByTotal = total.group();

client/src/actions/index.js

@@ -40,7 +40,7 @@ const doInitialDataLoad = () =>
       /* set config defaults */
       const config = { ...globals.configDefaults, ...results[0].config };
       const [, schema, obsAnno, varAnno, obsLayout] = [...results];
-      const universe = Universe.createUniverseFromRestV02Response(
+      const universe = Universe.createUniverseFromResponse(
         config,
         schema,
         obsAnno,
@@ -242,12 +242,14 @@ const requestDifferentialExpression = (set1, set2, num_genes = 10) => async (
     */
     const state = getState();
     const { universe } = state.controls;
-    const set1ByIndex = rangeEncodeIndices(
+
-      _.map(set1, s => universe.obsNameToIndexMap[s])
+    // These lines ensure that we convert any TypedArray to an Array.
-    );
+    // This is necessary because JSON.stringify() does some very strange
-    const set2ByIndex = rangeEncodeIndices(
+    // things with TypedArrays (they are marshalled to JSON objects, rather
-      _.map(set2, s => universe.obsNameToIndexMap[s])
+    // than being marshalled as a JSON array).
-    );
+    const aset1 = Array.isArray(set1) ? set1 : Array.from(set1);
+    const aset2 = Array.isArray(set2) ? set2 : Array.from(set2);
+
     const res = await fetch(
       `${globals.API.prefix}${globals.API.version}diffexp/obs`,
       {
@@ -259,8 +261,8 @@ const requestDifferentialExpression = (set1, set2, num_genes = 10) => async (
         body: JSON.stringify({
           mode: "topN",
           count: num_genes,
-          set1: { filter: { obs: { index: set1ByIndex } } },
+          set1: { filter: { obs: { index: aset1 } } },
-          set2: { filter: { obs: { index: set2ByIndex } } }
+          set2: { filter: { obs: { index: aset2 } } }
         })
       }
     );
@@ -271,7 +273,9 @@ const requestDifferentialExpression = (set1, set2, num_genes = 10) => async (
 
     const data = await res.json();
     // result is [ [varIdx, ...], ... ]
-    const topNGenes = _.map(data, r => universe.varAnnotations[r[0]].name);
+    const topNGenes = _.map(data, r =>
+      universe.varAnnotations.at(r[0], "name")
+    );
 
     /*
     Kick off secondary action to fetch all of the expression data for the

client/src/components/brushableHistogram/index.js

@@ -35,9 +35,11 @@ class HistogramBrush extends React.Component {
       .scaleLinear()
       .range([this.height - this.marginBottom, 0]);
 
-    if (obsAnnotations[0][field] !== undefined) {
+    if (obsAnnotations.col(field)) {
       // recalculate expensive stuff
-      const allValuesForContinuousFieldAsArray = _.map(obsAnnotations, field);
+      const allValuesForContinuousFieldAsArray = obsAnnotations
+        .col(field)
+        .asArray();
 
       histogramCache.x = d3
         .scaleLinear()
@@ -149,7 +151,7 @@ class HistogramBrush extends React.Component {
       initializeRanges
     } = this.props;
 
-    if (obsAnnotations[0][field]) {
+    if (obsAnnotations.col(field)) {
       dispatch({
         type: "color by continuous metadata",
         colorAccessor: field,

client/src/components/categorical/value.js

@@ -65,7 +65,11 @@ class CategoryValue extends React.Component {
       })[0].categories;
     }
 
-    if (colorAccessor && !isColorBy) {
+    if (
+      colorAccessor &&
+      !isColorBy &&
+      categoricalSelectionState[colorAccessor]
+    ) {
       occupancy = countCategoryValues2D(
         metadataField,
         colorAccessor,

client/src/components/continuous/continuous.js

@@ -10,7 +10,6 @@ import HistogramBrush from "../brushableHistogram";
 
 @connect(state => ({
   ranges: _.get(state.controls.world, "summary.obs", null),
-  metadata: _.get(state.controls.world, "obsAnnotations", null),
   colorAccessor: state.controls.colorAccessor,
   colorScale: state.controls.colorScale,
   selectionUpdate: _.get(state.controls, "crossfilter.updateTime", null),
@@ -39,7 +38,7 @@ class Continuous extends React.Component {
   }
 
   render() {
-    const { ranges, obsAnnotations, schema } = this.props;
+    const { ranges, schema } = this.props;
     if (schema && !this.continuousChecked) {
       this.hasContinuous = _.some(
         schema.annotations.obs,
@@ -73,7 +72,6 @@ class Continuous extends React.Component {
                 field={key}
                 isObs
                 zebra={zebra % 2 === 0}
-                fieldValues={obsAnnotations}
                 ranges={value.range}
                 handleColorAction={this.handleColorAction(key).bind(this)}
               />

client/src/components/geneExpression/cellSetButtons.js

@@ -1,8 +1,8 @@
 // jshint esversion: 6
 import React from "react";
-import _ from "lodash";
 import { AnchorButton, Tooltip } from "@blueprintjs/core";
 import { connect } from "react-redux";
+import { World } from "../../util/stateManager";
 
 @connect()
 class CellSetButton extends React.Component {
@@ -14,7 +14,7 @@ class CellSetButton extends React.Component {
       eitherCellSetOneOrTwo
     } = this.props;
 
-    const set = _.map(crossfilter.allFiltered(), "name");
+    const set = World.getSelectedByIndex(crossfilter);
 
     if (!differential.diffExp) {
       /* diffexp needs to be cleared before we store a new set */

client/src/components/geneExpression/index.js

@@ -29,8 +29,7 @@ const renderGene = (fuzzySortResult, { handleClick, modifiers, query }) => {
     return null;
   }
   /* the fuzzysort wraps the object with other properties, like a score */
-  const gene = fuzzySortResult.obj;
+  const geneName = fuzzySortResult.target;
-  const text = gene.name;
 
   return (
     <MenuItem
@@ -39,39 +38,34 @@ const renderGene = (fuzzySortResult, { handleClick, modifiers, query }) => {
       // Use of annotations in this way is incorrect and dataset specific.
       // See https://github.com/chanzuckerberg/cellxgene/issues/483
       // label={gene.n_counts}
-      key={gene.name}
+      key={geneName}
-      onClick={g => {
+      onClick={g =>
         /* this fires when user clicks a menu item */
-        handleClick(g);
+        handleClick(g)
-      }}
+      }
-      text={text}
+      text={geneName}
     />
   );
 };
 
-const filterGenes = (query, genes) => {
+const filterGenes = (query, genes) =>
   /* fires on load, once, and then for each character typed into the input */
-  return fuzzysort.go(query, genes, {
+  fuzzysort.go(query, genes, {
-    key: "name",
     limit: 5,
     threshold: -10000 // don't return bad results
   });
-};
 
 @connect(state => {
-  const metadata = _.get(state.controls.world, "obsAnnotations", null);
   const ranges = _.get(state.controls.world, "summary.obs", null);
   const initializeRanges = _.get(state.controls.world, "summary.obs");
 
   return {
     ranges,
-    metadata,
     initializeRanges,
     userDefinedGenes: state.controls.userDefinedGenes,
     userDefinedGenesLoading: state.controls.userDefinedGenesLoading,
     world: state.controls.world,
     colorAccessor: state.controls.colorAccessor,
-    allGeneNames: state.controls.allGeneNames,
     differential: state.differential
   };
 })
@@ -84,6 +78,37 @@ class GeneExpression extends React.Component {
     };
   }
 
+  placeholderGeneNames() {
+    /*
+    return a string containing gene name suggestions for use as a user hint.
+    Eg.,    Apod, Cd74, ...
+    Will return a max of 3 genes, totalling 15 characters in length.
+    Randomly selects gene names.
+
+    NOTE: the random selection means it will re-render constantly.
+    */
+    const { world } = this.props;
+    const { varAnnotations } = world;
+    const geneNames = varAnnotations.col("name").asArray();
+    if (geneNames.length > 0) {
+      const placeholder = [];
+      let len = geneNames.length;
+      const maxGeneNameCount = 3;
+      const maxStrLength = 15;
+      len = len < maxGeneNameCount ? len : maxGeneNameCount;
+      for (let i = 0, strLen = 0; i < len && strLen < maxStrLength; i += 1) {
+        const deal = Math.floor(Math.random() * geneNames.length);
+        const geneName = geneNames[deal];
+        placeholder.push(geneName);
+        strLen += geneName.length + 2; // '2' is the length of a comma and space
+      }
+      placeholder.push("...");
+      return placeholder.join(", ");
+    }
+    // default - should never happen.
+    return "Apod, Cd74, ...";
+  }
+
   handleClick(g) {
     const { world, dispatch, userDefinedGenes } = this.props;
     const gene = g.target;
@@ -93,7 +118,7 @@ class GeneExpression extends React.Component {
       postUserErrorToast(
         "That's too many genes, you can have at most 15 user defined genes"
       );
-    } else if (!_.find(world.varAnnotations, { name: gene })) {
+    } else if (world.varAnnotations.col("name").indexOf(gene) === undefined) {
       postUserErrorToast("That doesn't appear to be a valid gene name.");
     } else {
       dispatch(actions.requestUserDefinedGene(gene));
@@ -116,9 +141,13 @@ class GeneExpression extends React.Component {
       const genes = _.pull(_.uniq(bulkAdd.split(/[ ,]+/)), "");
 
       genes.forEach(gene => {
-        if (userDefinedGenes.indexOf(gene) !== -1) {
+        if (gene.length === 0) {
+          keepAroundErrorToast("Must enter a gene name.");
+        } else if (userDefinedGenes.indexOf(gene) !== -1) {
           keepAroundErrorToast("That gene already exists");
-        } else if (!_.find(world.varAnnotations, { name: gene })) {
+        } else if (
+          world.varAnnotations.col("name").indexOf(gene) === undefined
+        ) {
           keepAroundErrorToast(
             `${gene} doesn't appear to be a valid gene name.`
           );
@@ -214,8 +243,8 @@ class GeneExpression extends React.Component {
                 itemRenderer={renderGene.bind(this)}
                 items={
                   world && world.varAnnotations
-                    ? world.varAnnotations
+                    ? world.varAnnotations.col("name").asArray()
-                    : [{ name: "No genes" }]
+                    : ["No genes"]
                 }
                 popoverProps={{ minimal: true }}
               />
@@ -245,7 +274,7 @@ class GeneExpression extends React.Component {
                         this.setState({ bulkAdd: e.target.value });
                       }}
                       id="text-input-bulk-add"
-                      placeholder="Apod, Cd74, ..."
+                      placeholder={this.placeholderGeneNames()}
                       value={bulkAdd}
                     />
                     <Button
@@ -290,8 +319,7 @@ class GeneExpression extends React.Component {
           <ExpressionButtons />
           {differential.diffExp
             ? _.map(differential.diffExp, (value, index) => {
-                const annotations = world.varAnnotations[value[0]];
+                const name = world.varAnnotations.at(value[0], "name");
-                const { name } = annotations;
                 const values = world.varDataCache[name];
                 if (!values) {
                   return null;

client/src/components/graph/graph.js

@@ -35,7 +35,8 @@ class Graph extends React.Component {
     this.graphPaddingRight = globals.leftSidebarWidth;
     this.renderCache = {
       positions: null,
-      colors: null
+      colors: null,
+      sizes: null
     };
     this.state = {
       svg: null,
@@ -83,12 +84,13 @@ class Graph extends React.Component {
   }
 
   componentDidUpdate(prevProps) {
+    const { renderCache } = this;
     const {
       world,
       crossfilter,
-      selectionUpdate,
       colorRGB,
-      responsive
+      responsive,
+      selectionUpdate
     } = this.props;
     const {
       reglRender,
@@ -109,35 +111,27 @@ class Graph extends React.Component {
 
     if (regl && world) {
       /* update the regl state */
-      const { obsLayout } = world;
+      const { obsLayout, nObs } = world;
-      const cellCount = crossfilter.size();
+      const X = obsLayout.col("X").asArray();
+      const Y = obsLayout.col("Y").asArray();
 
       // X/Y positions for each point - a cached value that only
       // changes if we have loaded entirely new cell data
       //
-      if (
+      if (!renderCache.positions || world !== prevProps.world) {
-        !this.renderCache.positions ||
+        renderCache.positions = new Float32Array(2 * nObs);
-        selectionUpdate !== prevProps.selectionUpdate
-      ) {
-        if (!this.renderCache.positions) {
-          this.renderCache.positions = new Float32Array(2 * cellCount);
-        }
 
         const glScaleX = scaleLinear([0, 1], [-1, 1]);
         const glScaleY = scaleLinear([0, 1], [1, -1]);
 
-        const offset = [d3.mean(obsLayout.X) - 0.5, d3.mean(obsLayout.Y) - 0.5];
+        const offset = [d3.mean(X) - 0.5, d3.mean(Y) - 0.5];
 
-        for (
+        for (let i = 0, { positions } = renderCache; i < nObs; i += 1) {
-          let i = 0, { positions } = this.renderCache;
+          positions[2 * i] = glScaleX(X[i] - offset[0]);
-          i < cellCount;
+          positions[2 * i + 1] = glScaleY(Y[i] - offset[1]);
-          i += 1
-        ) {
-          positions[2 * i] = glScaleX(obsLayout.X[i] - offset[0]);
-          positions[2 * i + 1] = glScaleY(obsLayout.Y[i] - offset[1]);
         }
         pointBuffer({
-          data: this.renderCache.positions,
+          data: renderCache.positions,
           dimension: 2
         });
 
@@ -152,30 +146,28 @@ class Graph extends React.Component {
       // could have changed for some other reason, but for now color is
       // the only metadata that changes client-side.  If this is problematic,
       // we could add some sort of color-specific indicator to the app state.
-      if (!this.renderCache.colors || colorRGB !== prevProps.colorRGB) {
+      if (!renderCache.colors || colorRGB !== prevProps.colorRGB) {
         const rgb = colorRGB;
-        if (!this.renderCache.colors) {
+        if (!renderCache.colors) {
-          this.renderCache.colors = new Float32Array(3 * rgb.length);
+          renderCache.colors = new Float32Array(3 * rgb.length);
         }
-        for (let i = 0, { colors } = this.renderCache; i < rgb.length; i += 1) {
+        for (let i = 0, { colors } = renderCache; i < rgb.length; i += 1) {
           colors.set(rgb[i], 3 * i);
         }
-        colorBuffer({ data: this.renderCache.colors, dimension: 3 });
+        colorBuffer({ data: renderCache.colors, dimension: 3 });
       }
 
-      // Sizes for each point - this is presumed to change each time the
+      // Sizes for each point - updates are triggered only when selected
-      // component receives new props.   Almost always a true assumption, as
+      // obs change
-      // most property upates are due to changes driving a crossfilter
+      if (!renderCache.sizes || selectionUpdate !== prevProps.selectionUpdate) {
-      // selection set change.
+        if (!renderCache.sizes) {
-      //
+          renderCache.sizes = new Float32Array(nObs);
-      if (!this.renderCache.sizes) {
+        }
-        this.renderCache.sizes = new Float32Array(cellCount);
+        crossfilter.fillByIsFiltered(renderCache.sizes, 4, 0.2);
+        sizeBuffer({ data: renderCache.sizes, dimension: 1 });
       }
 
-      crossfilter.fillByIsFiltered(this.renderCache.sizes, 4, 0.2);
+      this.count = nObs;
-      sizeBuffer({ data: this.renderCache.sizes, dimension: 1 });
-
-      this.count = cellCount;
 
       regl._refresh();
       this.reglDraw(

client/src/components/scatterplot/scatterplot.js

@@ -65,12 +65,17 @@ class Scatterplot extends React.Component {
     super(props);
     this.count = 0;
     this.axes = false;
-    this.state = {
+    this.renderCache = {
-      svg: null,
+      positions: null,
-      minimized: null,
+      colors: null,
+      sizes: null,
       xScale: null,
       yScale: null
     };
+    this.state = {
+      svg: null,
+      minimized: null
+    };
   }
 
   componentDidMount() {
@@ -81,6 +86,7 @@ class Scatterplot extends React.Component {
     if (svg && expressionX && expressionY) {
       scales = Scatterplot.setupScales(expressionX, expressionY);
       this.drawAxesSVG(scales.xScale, scales.yScale, svg);
+      this.renderCache = { ...this.renderCache, ...scales };
     }
 
     const camera = _camera(this.reglCanvas, { scale: true, rotate: false });
@@ -113,8 +119,6 @@ class Scatterplot extends React.Component {
       pointBuffer,
       colorBuffer,
       svg,
-      xScale: scales ? scales.xScale : null,
-      yScale: scales ? scales.yScale : null,
       reglRender,
       camera,
       drawPoints
@@ -129,12 +133,11 @@ class Scatterplot extends React.Component {
       scatterplotYYaccessor,
       expressionX,
       expressionY,
-      colorRGB
+      colorRGB,
+      selectionUpdate
     } = this.props;
     const {
       reglRender,
-      xScale,
-      yScale,
       regl,
       pointBuffer,
       colorBuffer,
@@ -145,17 +148,12 @@ class Scatterplot extends React.Component {
     } = this.state;
 
     if (
-      world &&
+      scatterplotXXaccessor !== prevProps.scatterplotXXaccessor || // was CLU now FTH1 etc
-      svg &&
+      scatterplotYYaccessor !== prevProps.scatterplotYYaccessor // was CLU now FTH1 etc
-      xScale &&
-      yScale &&
-      scatterplotXXaccessor &&
-      scatterplotYYaccessor &&
-      (scatterplotXXaccessor !== prevProps.scatterplotXXaccessor || // was CLU now FTH1 etc
-      scatterplotYYaccessor !== prevProps.scatterplotYYaccessor || // was CLU now FTH1 etc
-        !this.axes) // clicked off the tab and back again, rerender
     ) {
-      this.drawAxesSVG(xScale, yScale, svg);
+      const scales = Scatterplot.setupScales(expressionX, expressionY);
+      this.drawAxesSVG(scales.xScale, scales.yScale, svg);
+      this.renderCache = { ...this.renderCache, ...scales };
     }
 
     if (reglRender && this.reglRenderState === "rendering") {
@@ -172,35 +170,51 @@ class Scatterplot extends React.Component {
       expressionX &&
       expressionY &&
       scatterplotXXaccessor &&
-      scatterplotYYaccessor &&
+      scatterplotYYaccessor
-      xScale &&
-      yScale
     ) {
+      const { renderCache } = this;
+      const { xScale, yScale } = this.renderCache;
       const cellCount = expressionX.length;
-      const positionsBuf = new Float32Array(2 * cellCount);
-      const colorsBuf = new Float32Array(3 * cellCount);
-      const sizesBuf = new Float32Array(cellCount);
 
-      const glScaleX = scaleLinear([0, width], [-0.95, 0.95]);
+      // Points change when expressionX or expressionY change.
-      const glScaleY = scaleLinear([0, height], [-1, 1]);
+      if (
-
+        !renderCache.positions ||
-      /*
+        expressionX !== prevProps.expressionX ||
-        Construct Vectors
+        expressionY !== prevProps.expressionY
-      */
+      ) {
-      for (let i = 0; i < cellCount; i += 1) {
+        if (!renderCache.positions) {
-        positionsBuf[2 * i] = glScaleX(xScale(expressionX[i]));
+          renderCache.positions = new Float32Array(2 * cellCount);
-        positionsBuf[2 * i + 1] = glScaleY(yScale(expressionY[i]));
+        }
+        const glScaleX = scaleLinear([0, width], [-0.95, 0.95]);
+        const glScaleY = scaleLinear([0, height], [-1, 1]);
+        for (let i = 0, { positions } = renderCache; i < cellCount; i += 1) {
+          positions[2 * i] = glScaleX(xScale(expressionX[i]));
+          positions[2 * i + 1] = glScaleY(yScale(expressionY[i]));
+        }
+        pointBuffer({ data: renderCache.positions, dimension: 2 });
       }
 
-      for (let i = 0; i < cellCount; i += 1) {
+      // Colors for each point - change only when props.colorsRGB change.
-        colorsBuf.set(colorRGB[i], 3 * i);
+      if (!renderCache.colors || colorRGB !== prevProps.colorRGB) {
+        if (!renderCache.colors) {
+          renderCache.colors = new Float32Array(3 * cellCount);
+        }
+        for (let i = 0, { colors } = renderCache; i < cellCount; i += 1) {
+          colors.set(colorRGB[i], 3 * i);
+        }
+        colorBuffer({ data: renderCache.colors, dimension: 3 });
       }
 
-      crossfilter.fillByIsFiltered(sizesBuf, 4, 0.2);
+      // Sizes for each point - updates are triggered only when selected
+      // obs change
+      if (!renderCache.sizes || selectionUpdate !== prevProps.selctionUpdate) {
+        if (!renderCache.sizes) {
+          renderCache.sizes = new Float32Array(cellCount);
+        }
+        crossfilter.fillByIsFiltered(renderCache.sizes, 4, 0.2);
+        sizeBuffer({ data: renderCache.sizes, dimension: 1 });
+      }
 
-      pointBuffer({ data: positionsBuf, dimension: 2 });
-      colorBuffer({ data: colorsBuf, dimension: 3 });
-      sizeBuffer({ data: sizesBuf, dimension: 1 });
       this.count = cellCount;
 
       regl._refresh();
@@ -213,16 +227,6 @@ class Scatterplot extends React.Component {
         camera
       );
     }
-
-    if (
-      expressionX &&
-      expressionY &&
-      (scatterplotXXaccessor !== prevProps.scatterplotXXaccessor || // was CLU now FTH1 etc
-        scatterplotYYaccessor !== prevProps.scatterplotYYaccessor)
-    ) {
-      const scales = Scatterplot.setupScales(expressionX, expressionY);
-      this.setState(scales);
-    }
   }
 
   static setupScales(expressionX, expressionY) {

client/src/middleware/updateCellColors.js

@@ -41,13 +41,13 @@ const updateCellColorsMiddleware = store => next => action => {
     action.type === "color by continuous metadata" ||
     action.type === "color by categorical metadata";
 
-  if (!filterJustChanged || !s.controls.world.obsAnnotations) {
+  const obsAnnotations = _.get(s.controls, "world.obsAnnotations", null);
+  if (!filterJustChanged || !obsAnnotations) {
     return next(
       action
     ); /* if the cells haven't loaded or the action wasn't a color change, bail */
   }
 
-  const { obsAnnotations } = s.controls.world;
   let colorScale;
   const colorsByRGB = new Array(obsAnnotations.length);
 
@@ -73,9 +73,9 @@ const updateCellColorsMiddleware = store => next => action => {
     });
 
     const key = action.colorAccessor;
+    const col = obsAnnotations.col(key).asArray();
     for (let i = 0, len = obsAnnotations.length; i < len; i += 1) {
-      const obs = obsAnnotations[i];
+      const cat = col[i];
-      const cat = obs[key];
       colorsByRGB[i] = colors[cat];
     }
   }
@@ -96,8 +96,9 @@ const updateCellColorsMiddleware = store => next => action => {
 
     const key = action.colorAccessor;
     const nonFiniteColor = parseRGB(globals.nonFiniteCellColor);
+    const col = obsAnnotations.col(key).asArray();
     for (let i = 0, len = obsAnnotations.length; i < len; i += 1) {
-      const val = obsAnnotations[i][key];
+      const val = col[i];
       if (Number.isFinite(val)) {
         const c = colorScale(val);
         colorsByRGB[i] = colors[c];

client/src/reducers/controls.js

@@ -100,6 +100,28 @@ function selectedValuesForCategory(categorySelectionState) {
   return selectedValues;
 }
 
+/*
+build a crossfilter dimension map for all gene expression related dimensions.
+*/
+function createGenesDimMap(userDefinedGenes, diffexpGenes, world, crossfilter) {
+  function _createGenesDimMap(genes, nameF) {
+    return genes.reduce((acc, gene) => {
+      acc[nameF(gene)] = World.createVarDimension(
+        world,
+        world.varDataCache,
+        crossfilter,
+        gene
+      );
+      return acc;
+    }, {});
+  }
+
+  return {
+    ..._createGenesDimMap(userDefinedGenes, userDefinedDimensionName),
+    ..._createGenesDimMap(diffexpGenes, diffexpDimensionName)
+  };
+}
+
 const Controls = (
   state = {
     // data loading flag
@@ -111,6 +133,7 @@ const Controls = (
 
     // the whole big bang
     universe: null,
+    fullUniverseCache: null,
 
     // all of the data + selection state
     world: null,
@@ -164,9 +187,7 @@ const Controls = (
     case "initial data load start": {
       return { ...state, loading: true };
     }
-    case "initial data load complete (universe exists)":
+    case "initial data load complete (universe exists)": {
-    case "reset World to eq Universe": {
-      const { userDefinedGenes, diffexpGenes } = state;
       /* first light - create world & other data-driven defaults */
       const { universe } = action;
       const world = World.createWorldFromEntireUniverse(universe);
@@ -181,51 +202,53 @@ const Controls = (
       const dimensionMap = World.createObsDimensionMap(crossfilter, world);
       WorldUtil.clearCaches();
 
-      const worldVarDataCache = world.varDataCache;
-
-      // dimensionMap = {
-      //     layout_X: dim-for-X,
-      //     obsAnno_name: dim for an annotation,
-      //     varData_userDefined_genename: dim for user defined expression,
-      //     varData_diffexp_genename: dim for diff-exp added gene expression
-      // }
-      /* var dimensions */
-      if (userDefinedGenes.length > 0) {
-        /*
-          verbose & slightly confusing that we also access this as an object
-          in controls rather than an array, should be abstracted into
-          util ie., createDimensionsFromBothListsOfGenes(userGenes, diffExp)
-        */
-        _.forEach(userDefinedGenes, gene => {
-          dimensionMap[
-            userDefinedDimensionName(gene)
-          ] = World.createVarDimension(
-            /* "__var__" + */
-            world,
-            worldVarDataCache,
-            crossfilter,
-            gene
-          );
-        });
-      }
-
-      if (diffexpGenes.length > 0) {
-        _.forEach(diffexpGenes, gene => {
-          dimensionMap[diffexpDimensionName(gene)] = World.createVarDimension(
-            /* "__var__" + */
-            world,
-            worldVarDataCache,
-            crossfilter,
-            gene
-          );
-        });
-      }
-
       return {
         ...state,
         loading: false,
         error: null,
         universe,
+        fullUniverseCache: { world, crossfilter, dimensionMap },
+        world,
+        colorRGB,
+        categoricalSelectionState,
+        crossfilter,
+        dimensionMap,
+        colorAccessor: null,
+        resettingInterface: false
+      };
+    }
+    case "reset World to eq Universe": {
+      const {
+        userDefinedGenes,
+        diffexpGenes,
+        universe,
+        fullUniverseCache
+      } = state;
+      const { world, crossfilter } = fullUniverseCache;
+      // reset all crossfilter dimensions
+      _.forEach(fullUniverseCache.dimensionMap, dim => dim.filterAll());
+      const colorRGB = new Array(universe.nObs).fill(
+        parseRGB(globals.defaultCellColor)
+      );
+      const categoricalSelectionState = createCategoricalSelectionState(
+        state,
+        world
+      );
+
+      /* free dimensions not in cache (otherwise they leak) */
+      _.forEach(state.dimensionMap, (dim, dimName) => {
+        if (!fullUniverseCache.dimensionMap[dimName]) {
+          dim.dispose();
+        }
+      });
+      const dimensionMap = {
+        ...fullUniverseCache.dimensionMap,
+        ...createGenesDimMap(userDefinedGenes, diffexpGenes, world, crossfilter)
+      };
+      WorldUtil.clearCaches();
+
+      return {
+        ...state,
         world,
         colorRGB,
         categoricalSelectionState,
@@ -252,43 +275,12 @@ const Controls = (
         world
       );
       const crossfilter = Crossfilter(world.obsAnnotations);
-      const dimensionMap = World.createObsDimensionMap(crossfilter, world);
+      const dimensionMap = {
+        ...World.createObsDimensionMap(crossfilter, world),
+        ...createGenesDimMap(userDefinedGenes, diffexpGenes, world, crossfilter)
+      };
       WorldUtil.clearCaches();
 
-      const worldVarDataCache = world.varDataCache;
-      /* var dimensions */
-
-      if (userDefinedGenes.length > 0) {
-        /*
-          verbose & slightly confusing that we also access this as an object
-          in controls rather than an array, should be abstracted into
-          util ie., createDimensionsFromBothListsOfGenes(userGenes, diffExp)
-        */
-        _.forEach(userDefinedGenes, gene => {
-          dimensionMap[
-            userDefinedDimensionName(gene)
-          ] = World.createVarDimension(
-            /* "__var__" + */
-            world,
-            worldVarDataCache,
-            crossfilter,
-            gene
-          );
-        });
-      }
-
-      if (diffexpGenes.length > 0) {
-        _.forEach(diffexpGenes, gene => {
-          dimensionMap[diffexpDimensionName(gene)] = World.createVarDimension(
-            /* "__var__" + */
-            world,
-            worldVarDataCache,
-            crossfilter,
-            gene
-          );
-        });
-      }
-
       return {
         ...state,
         loading: false,
@@ -367,7 +359,7 @@ const Controls = (
       const _diffexpGenes = [];
 
       action.data.forEach(d => {
-        _diffexpGenes.push(world.varAnnotations[d[0]].name);
+        _diffexpGenes.push(world.varAnnotations.at(d[0], "name"));
       });
 
       _.forEach(_diffexpGenes, gene => {
@@ -393,7 +385,7 @@ const Controls = (
       const worldVarDataCache = world.varDataCache;
 
       _.forEach(action.diffExp, values => {
-        const { name } = world.varAnnotations[values[0]];
+        const name = world.varAnnotations.at(values[0], "name");
         // clean up crossfilter dimensions
         const dimension = dimensionMap[diffexpDimensionName(name)];
         dimension.dispose();

client/src/util/dataframe/dataframe.js

@@ -0,0 +1,465 @@
+import { IdentityInt32Index } from "./labelIndex";
+// weird cross-dependency that we should clean up someday...
+import { sort } from "../typedCrossfilter/sort";
+import { isTypedArray, isArrayOrTypedArray, callOnceLazy } from "./util";
+import { summarizeContinuous, summarizeCategorical } from "./summarize";
+
+/*
+Dataframe is an immutable 2D matrix similiar to Python Pandas Dataframe,
+but (currently) without all of the surrounding support functions.
+Data is stored in column-major layout, and each column is monomorphic.
+
+It supports:
+* Relatively efficient creation, cloning and subsetting ("cut")
+* Very efficient columnar access (eg, sum down a column), and access
+  to the underlying column arrays.
+* Data access by row/col offset or label.  Labels are reasonably well
+  optimized for both numeric lables and arbitrary (eg, sting) labels.
+
+It does not currently support:
+* Views on matrix subset - for currently known access patterns,
+  it is more effiicent to copy on subsetting, optimizing for access
+  speed over memory use.
+* JS iterators - they are too slow.  Use explicit iteration over
+  offest or labels.
+
+Important assumptions embedded in the API:
+* Columns are implicitly categorical if they are a JS Array and numeric
+  (aka continuous) if they are a TypedArray.
+
+There are three index types for row/col indexing:
+* IdentityInt32Index - noop index, where the index label is the offset.
+* KeyIndex - index arbitrary JS objects.
+* DenseInt32Index - integer indexing.  Optimization over KeyIndex as it uses
+  Int32Array as a back-map to offsets.  This means that the index array
+  must be sized to [minLabel, maxLabel), so this is only useful when the label
+  range is relatively close the underlying offset range [minOffset, maxOffset).
+
+All private functions/methods/fields are prefixed by '__', eg, __compile().
+Don't use them outside of this file.
+
+Simple example:
+
+  // default indexing is integer offset.
+  const df = Dataframe.create([2,2], [['a', 'b'], [0, 1]])
+  console.log(df.at(0,0));  // outputs: a
+  console.log(df.col(1).asArray());   // outputs: [0, 1]
+
+  // KeyIndex
+  const df = new Dataframe([1,2], [['a'], ['b']], null, new KeyIndex(['A', 'B']))
+  console.log(df.at(0, 'A')); // outputs: a
+  console.log(df.col('A').asArray(); // outputs: ['a']
+
+Performance tuning is primarily focused on columnar access patterns, which is the
+dominant pattern in cellxgene.
+*/
+
+/**
+Dataframe
+**/
+
+class Dataframe {
+  /**
+  Constructors & factories
+  **/
+
+  constructor(dims, columnarData, rowIndex = null, colIndex = null) {
+    /*
+    The base constructor is relatively hard to use - as an alternative,
+    see factory methods and clone/slice, below.
+
+    Parameters:
+      * dims - 2D array describing intendend dimensionality: [nRows,nCols].
+      * columnarData - JS array, nCols in length, containing array
+        or TypedArray of length nRows.
+      * rowIndex/colIndex - null (create default index using offsets as key),
+        or a caller-provided index.
+    All columns and indices must have appropriate dimensionality.
+    */
+    Dataframe.__errorChecks(dims, columnarData, rowIndex, colIndex);
+    const [nRows, nCols] = dims;
+    if (!rowIndex) {
+      rowIndex = new IdentityInt32Index(nRows);
+    }
+    if (!colIndex) {
+      colIndex = new IdentityInt32Index(nCols);
+    }
+
+    this.__columns = Array.from(columnarData);
+    this.dims = dims;
+    this.length = nRows; // convenience accessor for row dimension
+    this.rowIndex = rowIndex;
+    this.colIndex = colIndex;
+
+    this.__compile();
+  }
+
+  static __errorChecks(dims, columnarData) {
+    const [nRows, nCols] = dims;
+    if (nRows < 0 || nCols < 0) {
+      throw new RangeError("Dataframe dimensions must be positive");
+    }
+    if (!Array.isArray(columnarData)) {
+      throw new TypeError("Dataframe constructor requires array of columns");
+    }
+    if (!columnarData.every(c => isArrayOrTypedArray(c))) {
+      throw new TypeError("Dataframe columns must all be Array or TypedArray");
+    }
+    if (
+      nCols !== columnarData.length ||
+      !columnarData.every(c => c.length === nRows)
+    ) {
+      throw new RangeError(
+        "Dataframe dimension does not match column data shape"
+      );
+    }
+  }
+
+  __compile() {
+    /*
+    Compile data accessors for each column.
+
+    Each column accessor is a function which will lookup data by
+    index (ie, is equivalent to dataframe.get(row, col), where 'col'
+    is fixed.
+
+    In addition, each column accessor has several functions:
+
+    asArray() -- return the entire column as a native Array or TypedArray.
+      Crucially, this native array only supports label indexing.
+      Example:
+        const arr = df.col('a').asArray();
+
+    has(rlabel) -- return boolean indicating of the row label
+      is contained within the column.  Example:
+        const isInColumn = df.col('a').includes(99)
+      For the default offset indexing, this is identical to:
+        const isInColumn = (99 > 0) && (99 < df.nRows);
+
+    ihas(roffset) -- same as has(), but accepts a row offset
+      instead of a row label.
+
+    indexOf(value) -- return the label (not offset) of the first instance of
+      'value' in the column.  If you want the offset, just use the builtin JS
+      indexOf() function, available on both Array and TypedArray.
+
+    iget(offset) -- return the value at 'offset'
+
+    */
+    const { getOffset, getLabel } = this.rowIndex;
+    this.__columnsAccessor = this.__columns.map(column => {
+      const { length } = column;
+
+      /* get value by row label */
+      const get = function get(rlabel) {
+        return column[getOffset(rlabel)];
+      };
+
+      /* get value by row offset */
+      const iget = function iget(roffset) {
+        return column[roffset];
+      };
+
+      /* full column array access */
+      const asArray = function asArray() {
+        return column;
+      };
+
+      /* test for row label inclusion in column */
+      const has = function has(rlabel) {
+        const offset = getOffset(rlabel);
+        return offset >= 0 && offset < length;
+      };
+
+      const ihas = function ihas(offset) {
+        return offset >= 0 && offset < length;
+      };
+
+      /*
+      return first label (index) at which the value is found in this column,
+      or undefined if not found.
+
+      NOTE: not found return is DIFFERENT than the default Array.indexOf as
+      -1 is a plausible Dataframe row/col label.
+      */
+      const indexOf = function indexOf(value) {
+        const offset = column.indexOf(value);
+        if (offset === -1) {
+          return undefined;
+        }
+        return getLabel(offset);
+      };
+
+      /*
+      Summarize the column data. Lazy eval;
+      */
+      const summarize = callOnceLazy(() =>
+        isTypedArray(column)
+          ? summarizeContinuous(column)
+          : summarizeCategorical(column)
+      );
+
+      get.summarize = summarize;
+      get.asArray = asArray;
+      get.has = has;
+      get.ihas = ihas;
+      get.indexOf = indexOf;
+      get.iget = iget;
+      return get;
+    });
+  }
+
+  clone() {
+    return new this.constructor(
+      this.dims,
+      [...this.__columns],
+      this.rowIndex,
+      this.colIndex
+    );
+  }
+
+  static empty() {
+    return new Dataframe([0, 0], []);
+  }
+
+  static create(dims, columnarData) {
+    /*
+    Create a dataframe from raw columnar data.  All column arrays
+    must have the same length.   Identity indexing will be used.
+
+    Example:
+    const df = Dataframe.create([2,2], [new Uint32Array(2), new Float32Array(2)]);
+    */
+    return new Dataframe(dims, columnarData, null, null);
+  }
+
+  __cut(rowOffsets, colOffsets) {
+    const dims = [...this.dims];
+
+    const getSortedLabelAndOffsets = (offsets, index) => {
+      /*
+      Given offsets, return both offsets and associated lables,
+      sorted by offset.
+      */
+      if (!offsets) {
+        return [null, null];
+      }
+      const sortedOffsets = sort(offsets);
+      const sortedLabels = new Array(sortedOffsets.length);
+      for (let i = 0, l = sortedOffsets.length; i < l; i += 1) {
+        sortedLabels[i] = index.getLabel(sortedOffsets[i]);
+      }
+      return [sortedLabels, sortedOffsets];
+    };
+
+    let { colIndex } = this;
+    if (colOffsets) {
+      let colLabels;
+      [colLabels, colOffsets] = getSortedLabelAndOffsets(
+        colOffsets,
+        this.colIndex
+      );
+      dims[1] = colOffsets.length;
+      colIndex = this.colIndex.cut(colLabels);
+    }
+
+    let { rowIndex } = this;
+    if (rowOffsets) {
+      let rowLabels;
+      [rowLabels, rowOffsets] = getSortedLabelAndOffsets(
+        rowOffsets,
+        this.rowIndex
+      );
+      dims[0] = rowLabels.length;
+      rowIndex = this.rowIndex.cut(rowLabels);
+    }
+
+    /* cut columns */
+    let columns = this.__columns;
+    if (colOffsets) {
+      columns = new Array(colOffsets.length);
+      for (let i = 0, l = colOffsets.length; i < l; i += 1) {
+        columns[i] = this.__columns[colOffsets[i]];
+      }
+    }
+
+    /* cut rows */
+    if (rowOffsets) {
+      columns = columns.map(col => {
+        const newCol = new col.constructor(rowOffsets.length);
+        for (let i = 0, l = rowOffsets.length; i < l; i += 1) {
+          newCol[i] = col[rowOffsets[i]];
+        }
+        return newCol;
+      });
+    }
+    return new Dataframe(dims, columns, rowIndex, colIndex);
+  }
+
+  cutByList(rowLabels, colLabels = null) {
+    const toOffsets = (labels, index) => {
+      if (!labels) {
+        return null;
+      }
+      return labels.map(label => {
+        const off = index.getOffset(label);
+        if (off === undefined) {
+          throw new RangeError(`unknown label: ${label}`);
+        }
+        return off;
+      });
+    };
+
+    const rowOffsets = toOffsets(rowLabels, this.rowIndex);
+    const colOffsets = toOffsets(colLabels, this.colIndex);
+    return this.__cut(rowOffsets, colOffsets);
+  }
+
+  icutByList(rowOffsets, colOffsets = null) {
+    return this.__cut(rowOffsets, colOffsets);
+  }
+
+  icutByMask(rowMask, colMask = null) {
+    /*
+    Cut on row/column based upon a truthy/falsey array.
+    */
+    const [nRows, nCols] = this.dims;
+    if (
+      (rowMask && rowMask.length !== nRows) ||
+      (colMask && colMask.length !== nCols)
+    ) {
+      throw new RangeError("boolean arrays must match row/col dimensions");
+    }
+
+    /* convert masks to lists - method wastes space, but is fast */
+    const toList = (mask, maxSize) => {
+      if (!mask) {
+        return null;
+      }
+      const list = new Int32Array(maxSize);
+      let elems = 0;
+      for (let i = 0, l = mask.length; i < l; i += 1) {
+        if (mask[i]) {
+          list[elems] = i;
+          elems += 1;
+        }
+      }
+      return new Int32Array(list.buffer, 0, elems);
+    };
+    const rowOffsets = toList(rowMask, nRows);
+    const colOffsets = toList(colMask, nCols);
+    return this.__cut(rowOffsets, colOffsets);
+  }
+
+  /**
+  Data access with row/col.
+  **/
+
+  col(columnLabel) {
+    /*
+    Return accessor bound to a column.  Allows random row access
+    based upon the row indexing.  Returns undefined if the
+    columnLabel is not present in the dataframe.
+
+    Example for a dataframe with string labeled columns, and
+    default (offset) indices for rows (eg, [0, 'foo'])
+
+      const getValue = df.col('foo');
+      for (let r = 0; r < df.nRows; r += 1) {
+        console.log(r, getValue(r));
+      }
+
+    See __compile() for the functions available in a column accessor.
+    */
+    const coff = this.colIndex.getOffset(columnLabel);
+    return this.__columnsAccessor[coff];
+  }
+
+  icol(columnOffset) {
+    /*
+    Return column accessor by offset.
+    */
+    return this.__columnsAccessor[columnOffset];
+  }
+
+  at(r, c) {
+    /*
+    Access a single value, for a row/col label pair.
+
+    For performance reasons, there are no bounds or existance
+    checks on labels, and no defined behavior when these are supplied.
+    May return undefined, throw an Error, or do something else for
+    non-existant labels.  If you want predictable out-of-bounds
+    behavior, use has(), eg,
+
+    const myVal = df.has(r,l) ? df.at(r,l) : undefined;
+    */
+    const coff = this.colIndex.getOffset(c);
+    const roff = this.rowIndex.getOffset(r);
+    return this.__columns[coff][roff];
+  }
+
+  iat(r, c) {
+    /*
+    Access a single value, for a row/col offset (integer) position.
+
+    For performance reasons, there are no bounds checks on row/col offsets
+    or other well-defined behavior for out-of-bounds values.  If you want
+    well-defined bounds checking, use ihas(), eg,
+
+    const myVal = df.ihas(r, c) ? df.iat(r, c) : undefined;
+    */
+    return this.__columns[c][r];
+  }
+
+  has(r, c) {
+    /*
+    Test if row/col labels exist in the dataframe - returns true/false
+    */
+    const [nRows, nCols] = this.dims;
+    const coff = this.colIndex.getOffset(c);
+    const roff = this.rowIndex.getOffset(r);
+    return coff >= 0 && coff < nCols && roff >= 0 && roff < nRows;
+  }
+
+  ihas(r, c) {
+    /*
+    Test if row/col offset (integer) position exists in the
+    dataframe - returns true/false
+    */
+    const [nRows, nCols] = this.dims;
+    return c >= 0 && c < nCols && r >= 0 && r < nRows;
+  }
+
+  /****
+  Functional (map/reduce/etc) data access
+
+  XXX: not yet implemented, as there is no clear use case.   Can easily
+  add these as useful.
+  ****/
+
+  /*
+  Map & reduce of column or row
+
+  XXX TODO remainder of map/reduce functions:  mapCol, mapRow, reduceRow, ...
+  */
+  /* comment out until we have a use for this
+
+  reduceCol(clabel, callback, initialValue) {
+    const coff = this.colIndex.getOffset(clabel);
+    const column = this.__columns[coff];
+    let start = 0;
+    let acc = initialValue;
+    if (initialValue === undefined) {
+      acc = column[0];
+      start = 1;
+    }
+    for (let i = start, l = column.length; i < l; i += 1) {
+      acc = callback(acc, column[i]);
+    }
+    return acc;
+  }
+  */
+}
+
+export default Dataframe;

client/src/util/dataframe/index.js

@@ -0,0 +1,2 @@
+export { default as Dataframe } from "./dataframe";
+export { DenseInt32Index, IdentityInt32Index, KeyIndex } from "./labelIndex";

client/src/util/dataframe/labelIndex.js

@@ -0,0 +1,188 @@
+/**
+Label indexing - map a label to & from an integer offset.  See Dataframe
+for how this is used.
+**/
+
+/*
+Private utility functions
+*/
+function extent(tarr) {
+  let min = 0x7fffffff;
+  let max = ~min; // eslint-disable-line no-bitwise
+  for (let i = 0, l = tarr.length; i < l; i += 1) {
+    const v = tarr[i];
+    if (v < min) {
+      min = v;
+    }
+    if (v > max) {
+      max = v;
+    }
+  }
+  return [min, max];
+}
+
+function fillRange(arr, start = 0) {
+  const larr = arr;
+  for (let i = 0, l = larr.length; i < l; i += 1) {
+    larr[i] = i + start;
+  }
+  return larr;
+}
+
+/* eslint-disable class-methods-use-this */
+class IdentityInt32Index {
+  /*
+  identity/noop index, with small assumptions that labels are int32
+  */
+  constructor(maxOffset) {
+    this.maxOffset = maxOffset;
+  }
+
+  keys() {
+    // memoize
+    const k = fillRange(new Int32Array(this.maxOffset));
+    this.keys = function keys() {
+      return k;
+    };
+    return k;
+  }
+
+  getOffset(i) {
+    // label to offset
+    return i;
+  }
+
+  getLabel(i) {
+    // offset to label
+    return i;
+  }
+
+  getMaxOffset() {
+    return this.maxOffset;
+  }
+
+  cut(labelArray) {
+    /*
+    if density of resulting integer
+    */
+    const [minLabel, maxLabel] = extent(labelArray);
+    const labelSpaceSize = maxLabel - minLabel + 1;
+    const density = labelSpaceSize / this.maxOffset;
+    /* 0.1 is a magic number, that needs testing to optimize */
+    if (density < 0.1) {
+      return new KeyIndex(labelArray);
+    }
+    return new DenseInt32Index(labelArray, [minLabel, maxLabel]);
+  }
+}
+/* eslint-enable class-methods-use-this */
+
+/* eslint-disable class-methods-use-this */
+class DenseInt32Index {
+  /*
+  DenseInt32Index indexes integer labels, and uses Int32Array typed arrays
+  for both forward and reverse indexing.   This means that the min/max range
+  of the forward index labels must be known a priori (so that the index
+  array can be pre-allocated).
+  */
+  constructor(labels, labelRange = null) {
+    if (labels.constructor !== Int32Array) {
+      labels = new Int32Array(labels);
+    }
+
+    if (!labelRange) {
+      labelRange = extent(labels);
+    }
+    const [minLabel, maxLabel] = labelRange;
+    const labelSpaceSize = maxLabel - minLabel + 1;
+    const index = new Int32Array(labelSpaceSize).fill(-1);
+    for (let i = 0, l = labels.length; i < l; i += 1) {
+      const label = labels[i];
+      index[label - minLabel] = i;
+    }
+
+    this.minLabel = minLabel;
+    this.rindex = labels;
+    this.index = index;
+    this.__compile();
+  }
+
+  __compile() {
+    const { minLabel, index, rindex } = this;
+    this.getOffset = function getOffset(l) {
+      return index[l - minLabel];
+    };
+    this.getLabel = function getLabel(i) {
+      return rindex[i];
+    };
+  }
+
+  keys() {
+    return this.rindex;
+  }
+
+  getMaxOffset() {
+    return this.rindex.length;
+  }
+
+  cut(labelArray) {
+    /*
+    time/space decision - if we are going to use less than 10% of the
+    dense index space, switch to a KeyIndex (which is slower, but uses
+    less memory for sparse label spaces).
+    */
+    const [minLabel, maxLabel] = extent(labelArray);
+    const labelSpaceSize = maxLabel - minLabel + 1;
+    const density = labelSpaceSize / this.rindex.length;
+    /* 0.1 is a magic number, that needs testing to optimize */
+    if (density < 0.1) {
+      return new KeyIndex(labelArray);
+    }
+    return new DenseInt32Index(labelArray, [minLabel, maxLabel]);
+  }
+}
+/* eslint-enable class-methods-use-this */
+
+/* eslint-disable class-methods-use-this */
+class KeyIndex {
+  /*
+  KeyIndex indexes arbitrary JS primitive types, and uses a Map()
+  as its core data structure.
+  */
+  constructor(labels) {
+    const index = new Map();
+    const rindex = labels;
+    labels.forEach((v, i) => {
+      index.set(v, i);
+    });
+
+    this.index = index;
+    this.rindex = rindex;
+    this.__compile();
+  }
+
+  __compile() {
+    const { index, rindex } = this;
+    this.getOffset = function getOffset(k) {
+      return index.get(k);
+    };
+    this.getLabel = function getLabel(i) {
+      return rindex[i];
+    };
+  }
+
+  keys() {
+    return this.rindex;
+  }
+
+  getMaxOffset() {
+    return this.rindex.length;
+  }
+
+  cut(labelArray) {
+    return new KeyIndex(labelArray);
+  }
+}
+/* eslint-enable class-methods-use-this */
+
+export { DenseInt32Index, IdentityInt32Index, KeyIndex };

client/src/util/dataframe/summarize.js

@@ -0,0 +1,57 @@
+/*
+Private dataframe support functions
+*/
+
+export function summarizeContinuous(col) {
+  let min;
+  let max;
+  let nan = 0;
+  let pinf = 0;
+  let ninf = 0;
+  if (col) {
+    for (let r = 0, l = col.length; r < l; r += 1) {
+      const val = Number(col[r]);
+      if (Number.isFinite(val)) {
+        if (min === undefined) {
+          min = val;
+          max = val;
+        } else {
+          min = val < min ? val : min;
+          max = val > max ? val : max;
+        }
+      } else if (Number.isNaN(val)) {
+        nan += 1;
+      } else if (val > 0) {
+        pinf += 1;
+      } else {
+        ninf += 1;
+      }
+    }
+  }
+  return {
+    categorical: false,
+    min,
+    max,
+    nan,
+    pinf,
+    ninf
+  };
+}
+
+export function summarizeCategorical(col) {
+  const categoryCounts = new Map();
+  if (col) {
+    for (let r = 0, l = col.length; r < l; r += 1) {
+      const val = col[r];
+      let curCount = categoryCounts.get(val);
+      if (curCount === undefined) curCount = 0;
+      categoryCounts.set(val, curCount + 1);
+    }
+  }
+  return {
+    categorical: true,
+    categories: [...categoryCounts.keys()],
+    categoryCounts,
+    numCategories: categoryCounts.size
+  };
+}

client/src/util/dataframe/util.js

@@ -0,0 +1,28 @@
+/*
+Private utility code for dataframe
+*/
+
+export function isTypedArray(x) {
+  return (
+    ArrayBuffer.isView(x) &&
+    Object.prototype.toString.call(x) !== "[object DataView]"
+  );
+}
+
+export function isArrayOrTypedArray(x) {
+  return Array.isArray(x) || isTypedArray(x);
+}
+
+export function callOnceLazy(f) {
+  let value;
+  let calledOnce = false;
+  const result = function result(...args) {
+    if (!calledOnce) {
+      value = f(...args);
+      calledOnce = true;
+    }
+    return value;
+  };
+
+  return result;
+}

client/src/util/stateManager/summarizeAnnotations.js

@@ -53,13 +53,15 @@ Example:
 NOTE: will not summarize the required 'name' annotation, as that is
 specified as unique per element.
 */
-function _summarizeAnnotations(_schema, annotations) {
+function _summarizeAnnotations(_schema, df) {
   const summary = _(_schema) // lodash wrapping: https://lodash.com/docs/4.17.11#lodash
-    .filter(v => v.name !== "name")
+    .filter(v => v.name !== "name") // don't summarize name
     .keyBy("name")
     .mapValues(anno => {
       const { name, type } = anno;
       const continuous = type === "int32" || type === "float32";
+      const numRows = df.length;
+      const col = df.col(name) ? df.col(name).asArray() : null;
 
       if (continuous) {
         let min;
@@ -67,22 +69,24 @@ function _summarizeAnnotations(_schema, annotations) {
         let nan = 0;
         let pinf = 0;
         let ninf = 0;
-        for (let r = 0; r < annotations.length; r += 1) {
+        if (col) {
-          const val = Number(annotations[r][name]);
+          for (let r = 0; r < numRows; r += 1) {
-          if (Number.isFinite(val)) {
+            const val = Number(col[r]);
-            if (min === undefined) {
+            if (Number.isFinite(val)) {
-              min = val;
+              if (min === undefined) {
-              max = val;
+                min = val;
+                max = val;
+              } else {
+                min = val < min ? val : min;
+                max = val > max ? val : max;
+              }
+            } else if (Number.isNaN(val)) {
+              nan += 1;
+            } else if (val > 0) {
+              pinf += 1;
             } else {
-              min = val < min ? val : min;
+              ninf += 1;
-              max = val > max ? val : max;
             }
-          } else if (Number.isNaN(val)) {
-            nan += 1;
-          } else if (val > 0) {
-            pinf += 1;
-          } else {
-            ninf += 1;
           }
         }
         return {
@@ -93,11 +97,13 @@ function _summarizeAnnotations(_schema, annotations) {
 
       /* else categorical */
       const categoryCounts = new Map();
-      for (let r = 0; r < annotations.length; r += 1) {
+      if (col) {
-        const val = annotations[r][name];
+        for (let r = 0; r < numRows; r += 1) {
-        let curCount = categoryCounts.get(val);
+          const val = col[r];
-        if (curCount === undefined) curCount = 0;
+          let curCount = categoryCounts.get(val);
-        categoryCounts.set(val, curCount + 1);
+          if (curCount === undefined) curCount = 0;
+          categoryCounts.set(val, curCount + 1);
+        }
       }
       return {
         categorical: true,

client/src/util/stateManager/universe.js

@@ -5,6 +5,7 @@ import _ from "lodash";
 import * as kvCache from "./keyvalcache";
 import summarizeAnnotations from "./summarizeAnnotations";
 import decodeMatrixFBS from "./matrix";
+import * as Dataframe from "../dataframe";
 
 /*
 Private helper function - create and return a template Universe
@@ -27,13 +28,10 @@ function templateUniverse() {
     /*
     Annotations
     */
-    obsAnnotations: [] /* all obs annotations, by obs index */,
+    obsAnnotations: null,
-    varAnnotations: [] /* all var annotations, by var index */,
+    varAnnotations: null,
-    obsNameToIndexMap: {} /* reverse map 'name' to index */,
+    obsLayout: null,
-    varNameToIndexMap: {} /* reverse map 'name' to index */,
+    summary: null /* derived data summaries. XXX: consider exploding in place */,
-    summary: null /* derived data summaries XXX: consider exploding in place */,
-
-    obsLayout: { X: [], Y: [] } /* xy layout */,
 
     /*
     Cache of var data (expression), by var annotation name.   Data can be
@@ -61,9 +59,8 @@ function finalize(universe) {
   /* A bit of sanity checking! */
   const { nObs, nVar } = universe;
   if (
+    nObs !== universe.obsLayout.length ||
     nObs !== universe.obsAnnotations.length ||
-    nObs !== universe.obsLayout.X.length ||
-    nObs !== universe.obsLayout.Y.length ||
     nVar !== universe.varAnnotations.length
   ) {
     throw new Error("Universe dimensionality mismatch - failed to load");
@@ -73,61 +70,33 @@ function finalize(universe) {
   //  - layout has supported number of dimensions
   //  - ...
 
-  /*
-  Create all derived (convenience) data structures.
-  */
-  universe.obsNameToIndexMap = _.transform(
-    universe.obsAnnotations,
-    (acc, value, idx) => {
-      acc[value.name] = idx;
-    },
-    {}
-  );
-  universe.varNameToIndexMap = _.transform(
-    universe.varAnnotations,
-    (acc, value, idx) => {
-      acc[value.name] = idx;
-    },
-    {}
-  );
   universe.finalized = true;
   return universe;
 }
 
-function RESTv02AnotationsFBSResponseToInternal(arrayBuffer) {
+function AnnotationsFBSToDataframe(arrayBuffer) {
   /*
-  Convert a Matrix FBS to our internal format -- row-major array of
+  Convert a Matrix FBS to a Dataframe.
-  observations/cells, stored as an object.  Each obs has a key for each
-  annotation, plus __index__ containing its obsIndex.
-
-  Example:
-  [
-    { __index__: 0, tissue_type: "lung", sex: "F", ... },
-    ...
-  ]
-
-  XXX TODO: we could make use of the columns in building crossfilter
-  dimensions (they have to be recreated).  Future optimization.
   */
   const fbs = decodeMatrixFBS(arrayBuffer);
-  const keys = fbs.colIdx;
+  const df = new Dataframe.Dataframe(
-  const result = Array(fbs.nRows);
+    [fbs.nRows, fbs.nCols],
-  for (let row = 0; row < fbs.nRows; row += 1) {
+    fbs.columns,
-    const rec = { __index__: row };
+    null,
-    for (let col = 0; col < fbs.nCols; col += 1) {
+    new Dataframe.KeyIndex(fbs.colIdx)
-      rec[keys[col]] = fbs.columns[col][row];
+  );
-    }
+  return df;
-    result[row] = rec;
-  }
-  return result;
 }
 
-function RESTv02LayoutFBSResponseToInternal(arrayBuffer) {
+function LayoutFBSToDataframe(arrayBuffer) {
   const fbs = decodeMatrixFBS(arrayBuffer, true);
-  return {
+  const df = new Dataframe.Dataframe(
-    X: fbs.columns[0],
+    [fbs.nRows, fbs.nCols],
-    Y: fbs.columns[1]
+    fbs.columns,
-  };
+    null,
+    new Dataframe.KeyIndex(["X", "Y"])
+  );
+  return df;
 }
 
 function reconcileSchemaCategoriesWithSummary(universe) {
@@ -156,7 +125,7 @@ function reconcileSchemaCategoriesWithSummary(universe) {
   });
 }
 
-export function createUniverseFromRestV02Response(
+export function createUniverseFromResponse(
   configResponse,
   schemaResponse,
   annotationsObsResponse,
@@ -178,15 +147,10 @@ export function createUniverseFromRestV02Response(
   universe.nVar = schema.dataframe.nVar;
 
   /* annotations */
-  universe.obsAnnotations = RESTv02AnotationsFBSResponseToInternal(
+  universe.obsAnnotations = AnnotationsFBSToDataframe(annotationsObsResponse);
-    annotationsObsResponse
+  universe.varAnnotations = AnnotationsFBSToDataframe(annotationsVarResponse);
-  );
-  universe.varAnnotations = RESTv02AnotationsFBSResponseToInternal(
-    annotationsVarResponse
-  );
-
   /* layout */
-  universe.obsLayout = RESTv02LayoutFBSResponseToInternal(layoutFBSResponse);
+  universe.obsLayout = LayoutFBSToDataframe(layoutFBSResponse);
 
   universe.summary = summarizeAnnotations(
     universe.schema,
@@ -214,8 +178,8 @@ export function convertDataFBStoObject(universe, arrayBuffer) {
   const result = {};
 
   for (let c = 0; c < colIdx.length; c += 1) {
-    const gene = universe.varAnnotations[colIdx[c]].name;
+    const varName = universe.varAnnotations.at(colIdx[c], "name");
-    result[gene] = columns[c];
+    result[varName] = columns[c];
   }
   return result;
 }

client/src/util/stateManager/world.js

@@ -8,6 +8,7 @@ import Crossfilter from "../typedCrossfilter";
 import { sliceByIndex } from "../typedCrossfilter/util";
 
 /*
+
 World is a subset of universe.   Most code should use world, and should
 (generally) not use Universe.   World contains any per-obs or per-var data
 that must be consistent acorss the app when we view/manipulate subsets
@@ -16,37 +17,32 @@ of Universe.
 Private API indicated by leading underscore in key name (eg, _foo).  Anything else
 is public.
 
-World contains several public keys, obsAnnotations, and obsLayout, which are
+Notable keys in the world object:
-arrays contianing information about an OBS in the same order/offset.  In
+
-other words, world.obsAnnotations[0] and world.obsLayout.X[0] refer to the same
+* nObs, nVar: dimensions
-obs/cell.
+
+* schema: data schema from the server
 
 * obsAnnotations:
 
-  obsAnnotations will return an array of objects.  Each object contains all annotation
+  Dataframe containing obs annotations.  Columns are indexed by annotation
-  values for a given observation/cell, keyed by annotation name, PLUS a key
+  name (eg, 'tissue type'), and rows are indexed by the REST API obsIndex
-  '__cellId__', containing a REST API ID for this obs/cell (referred to as the
+  (ie, the offset into the underlying server-side dataframe).
-  obsIndex in the REST 0.2 spec or cellIndex in the 0.1 spec.
 
-  Example:  [ { __cellId__: 99, cluster: 'blue', numReads: 93933 } ]
+  This indexing means that you can access data by _either_ the server's
-
+  obxIndex, or the offset into the client-side column array .  Be careful
-  NOTE: world.obsAnnotation should be identical to the old state.cells value,
+  to know which you want and are using.
-  EXCEPT that
-    * __cellIndex__ renamed to __index__
-    * __x__ and __y__ are now in world.obsLayout
-    * __color__ and __colorRBG__ should be moved to controls reducer
 
 * obsLayout:
 
-  obsLayout will return an object containing two arrays, containing X and Y
+  A dataframe containing the X/Y layout for all obs.  Columns are named
-  coordinates respectively.
+  'X' and 'Y', and rows are indexed in the same way as obsAnnotation.
 
-  Example: { X: [ 0.33, 0.23, ... ], Y: [ 0.8, 0.777, ... ]}
+* summary: summary of each obsAnnotation column (eg, numeric extent for
+  continuous data, category counts for categorical metadata)
 
-* crossfilter - a crossfilter object across world.obsAnnotations
+* varDataCache: expression columns, in a kvCache.   TODO: maybe move to a
-
+  Dataframe in the future.
-* dimensionMap - an object mapping annotation names to dimensions on
-  the crossfilter
 
 */
 
@@ -56,11 +52,6 @@ const VarDataCacheTTLMs = 1000; // min cache time in MS
 
 function templateWorld() {
   return {
-    // map from universe obsIndex to world offset.
-    // Undefined / null indicates identity mapping.
-    obsIndex: null,
-    obsBackIndex: null,
-
     /* schema/version related */
     api: null,
     schema: null,
@@ -71,7 +62,7 @@ function templateWorld() {
     obsAnnotations: null,
     varAnnotations: null,
 
-    /* layout of graph */
+    /* layout of graph. Dataframe. */
     obsLayout: null,
 
     /* derived data summaries XXX: consider exploding in place */
@@ -91,15 +82,6 @@ export function createWorldFromEntireUniverse(universe) {
 
   const world = templateWorld();
 
-  // map from the universe obsIndex to our world offset.
-  // undefined/null indicates identity map.
-  // In other words   obsBackIndex[universeIdx] -> worldIdx
-  world.obsBackIndex = null;
-  // Map to the universe index for each element in world.
-  // Null indicates identity map (aka world === universe)
-  // In other wrods  obsIndex[worldIdx]  ->  universeIdx
-  world.obsIndex = null;
-
   /*
   public interface follows
   */
@@ -143,35 +125,11 @@ export function createWorldFromCurrentSelection(universe, world, crossfilter) {
   newWorld.schema = universe.schema;
   newWorld.varAnnotations = universe.varAnnotations;
 
-  /* build index maps and back maps based upon current selection state */
+  /* now subset/cut obs */
-  const obsBackIndex = new Uint32Array(universe.nObs);
+  const mask = crossfilter.allFilteredMask();
-  obsBackIndex.fill(-1); // default - aka unused
+  newWorld.obsAnnotations = world.obsAnnotations.icutByMask(mask);
-  const notSelected = obsBackIndex[0];
+  newWorld.obsLayout = world.obsLayout.icutByMask(mask);
-  let nObs = 0;
+  newWorld.nObs = newWorld.obsAnnotations.dims[0];
-  for (let i = 0; i < universe.nObs; i += 1) {
-    if (crossfilter.isElementFiltered(i)) {
-      obsBackIndex[i] = nObs;
-      nObs += 1;
-    }
-  }
-  const obsIndex = new Uint32Array(nObs);
-  for (let i = 0; i < universe.nObs; i += 1) {
-    const worldIdx = obsBackIndex[i];
-    if (worldIdx !== notSelected) {
-      obsIndex[worldIdx] = i;
-    }
-  }
-
-  newWorld.nObs = nObs;
-  newWorld.obsIndex = obsIndex;
-  newWorld.obsBackIndex = obsBackIndex;
-
-  /* now slice */
-  newWorld.obsAnnotations = sliceByIndex(universe.obsAnnotations, obsIndex);
-  newWorld.obsLayout = {
-    X: sliceByIndex(universe.obsLayout.X, obsIndex),
-    Y: sliceByIndex(universe.obsLayout.Y, obsIndex)
-  };
 
   /* derived data & summaries */
   newWorld.summary = summarizeAnnotations(
@@ -245,22 +203,23 @@ export function createObsDimensionMap(crossfilter, world) {
   create and return a crossfilter dimension for every obs annotation
   for which we have a supported type.
   */
-  const { schema, obsLayout } = world;
+  const { schema, obsLayout, obsAnnotations } = world;
 
   // Create a crossfilter dimension for all obs annotations *except* 'name'
   const dimensionMap = _(schema.annotations.obs)
     .filter(anno => anno.name !== "name")
     .transform((result, anno) => {
       const dimType = deduceDimensionType(anno, anno.name);
+      const colData = obsAnnotations.col(anno.name).asArray();
       if (dimType === "enum") {
         result[obsAnnoDimensionName(anno.name)] = crossfilter.dimension(
           Crossfilter.EnumDimension,
-          r => r[anno.name]
+          colData
         );
-      } else {
+      } else if (dimType) {
         result[obsAnnoDimensionName(anno.name)] = crossfilter.dimension(
           Crossfilter.ScalarDimension,
-          r => r[anno.name],
+          colData,
           dimType
         );
       } // else ignore the annotation
@@ -272,8 +231,8 @@ export function createObsDimensionMap(crossfilter, world) {
   */
   dimensionMap[layoutDimensionName("XY")] = crossfilter.dimension(
     Crossfilter.SpatialDimension,
-    obsLayout.X,
+    obsLayout.col("X").asArray(),
-    obsLayout.Y
+    obsLayout.col("Y").asArray()
   );
 
   return dimensionMap;
@@ -288,5 +247,23 @@ export function subsetVarData(world, universe, varData) {
   if (worldEqUniverse(world, universe)) {
     return varData;
   }
-  return sliceByIndex(varData, world.obsIndex);
+  return sliceByIndex(varData, world.obsAnnotations.rowIndex.keys());
+}
+
+export function getSelectedByIndex(crossfilter) {
+  /*
+  return array of obsIndex, containing all selected obs/cells.
+  */
+  const selected = crossfilter.allFilteredMask(); // array of bool-ish
+  const keys = crossfilter.data.rowIndex.keys(); // row keys, aka universe rowIndex
+
+  const set = new Int32Array(selected.length);
+  let numElems = 0;
+  for (let i = 0, l = selected.length; i < l; i += 1) {
+    if (selected[i]) {
+      set[numElems] = keys[i];
+      numElems += 1;
+    }
+  }
+  return new Int32Array(set.buffer, 0, numElems);
 }

client/src/util/stateManager/worldUtil.js

@@ -18,13 +18,23 @@ Map {
    ...
 }
 
+Parameters are:
+- dim1: dimension 1 name/label
+- dim2: dimension 2 name/label
+- df: dataframe containing dim1 and dim2 on the column axis
+
 */
-function _countCategoryValues2D(dim1, dim2, rows) {
+function _countCategoryValues2D(dim1, dim2, df) {
   const dimMap = new Map();
-  for (let r = 0; r < rows.length; r += 1) {
+  const col1 = df.col(dim1) ? df.col(dim1).asArray() : null;
-    const row = rows[r];
+  const col2 = df.col(dim2) ? df.col(dim2).asArray() : null;
-    const val1 = row[dim1];
+  if (!col1 || !col2) {
-    const val2 = row[dim2];
+    return dimMap;
+  }
+
+  for (let r = 0, l = df.length; r < l; r += 1) {
+    const val1 = col1[r];
+    const val2 = col2[r];
     let d2Map = dimMap.get(val1);
     if (d2Map === undefined) {
       d2Map = new Map();

client/src/util/typedCrossfilter/bitArray.js

@@ -40,7 +40,7 @@ class BitArray {
   // Return the number of records that are selected, ie, have a one bit in
   // all allocated dimensions.
   //
-  get selectionCount() {
+  selectionCount() {
     return this.countAllOnes();
   }
 
@@ -48,16 +48,27 @@ class BitArray {
   //
   countAllOnes() {
     let count = 0;
-    const { bitarray, bitmask, length, width } = this;
+    const { bitarray, length, width } = this;
-    for (let l = 0; l < length; l += 1) {
+    if (width === 1) {
-      let dimensionsSet = 0;
+      // special case, width === 1, for performance
-      for (let w = 0; w < width; w += 1) {
+      const bitmask = this.bitmask[0];
-        if (bitarray[w * length + l] === bitmask[w]) {
+      for (let l = 0; l < length; l += 1) {
-          dimensionsSet += 1;
+        if (bitarray[l] === bitmask) {
+          count += 1;
         }
       }
-      if (dimensionsSet === width) {
+    } else {
-        count += 1;
+      const { bitmask } = this;
+      for (let l = 0; l < length; l += 1) {
+        let dimensionsSet = 0;
+        for (let w = 0; w < width; w += 1) {
+          if (bitarray[w * length + l] === bitmask[w]) {
+            dimensionsSet += 1;
+          }
+        }
+        if (dimensionsSet === width) {
+          count += 1;
+        }
       }
     }
     return count;
@@ -233,12 +244,14 @@ class BitArray {
   fillBySelection(result, selectedValue, deselectedValue) {
     // special case (width === 1) for performance
     if (this.width === 1) {
-      const bitmask = this.bitmask[0];
+      const { bitmask, bitarray } = this;
-      for (let i = 0, len = this.length; i < len; i += 1) {
+      const mask = bitmask[0];
-        result[i] =
+      if (!mask) {
-          bitmask && this.bitarray[i] === bitmask
+        result.fill(deselectedValue);
-            ? selectedValue
+      } else {
-            : deselectedValue;
+        for (let i = 0, len = this.length; i < len; i += 1) {
+          result[i] = bitarray[i] === mask ? selectedValue : deselectedValue;
+        }
       }
     } else {
       for (let i = 0, len = this.length; i < len; i += 1) {

client/src/util/typedCrossfilter/index.js

@@ -39,6 +39,14 @@ import {
   upperBoundIndirect
 } from "./util";
 
+function isArrayOrTypedArray(x) {
+  return (
+    Array.isArray(x) ||
+    (ArrayBuffer.isView(x) &&
+      Object.prototype.toString.call(x) !== "[object DataView]")
+  );
+}
+
 class NotImplementedError extends Error {
   constructor(...params) {
     super(...params);
@@ -52,6 +60,11 @@ class NotImplementedError extends Error {
 
 class TypedCrossfilter {
   constructor(data) {
+    /*
+    Typically, data is one of:
+      - Array of objects/records
+      - Dataframe (util/dataframe)
+    */
     this.data = data;
 
     // filters: array of { id, dimension }
@@ -97,18 +110,32 @@ class TypedCrossfilter {
   // return array of all records that are selected/filtered
   // by all dimensions.
   allFiltered() {
-    const { selection } = this;
+    const { data, selection } = this;
-    const res = [];
+    if (Array.isArray(data)) {
-    for (let i = 0, len = this.data.length; i < len; i += 1) {
+      const res = [];
-      if (selection.isSelected(i)) {
+      for (let i = 0, len = data.length; i < len; i += 1) {
-        res.push(this.data[i]);
+        if (selection.isSelected(i)) {
+          res.push(data[i]);
+        }
       }
+      return res;
     }
-    return res;
+    /* else, Dataframe-like */
+    return data.icutByMask(this.allFilteredMask());
+  }
+
+  // return Uint8array containing selection state (truthy/falsey) for each record.
+  //
+  allFilteredMask() {
+    return this.selection.fillBySelection(
+      new Uint8Array(this.data.length),
+      1,
+      0
+    );
   }
 
   countFiltered() {
-    return this.selection.selectionCount;
+    return this.selection.selectionCount();
   }
 
   isElementFiltered(i) {
@@ -161,6 +188,7 @@ class ScalarDimension extends _Dimension {
     // or a map function which will create it.
     let array;
     if (value instanceof ValueArrayType) {
+      // user has provided the final typed array - just use it
       if (value.length !== this.crossfilter.data.length) {
         throw new RangeError(
           "ScalarDimension values length must equal crossfilter data record count"
@@ -168,11 +196,18 @@ class ScalarDimension extends _Dimension {
       }
       array = value;
     } else if (value instanceof Function) {
-      // Create value array
+      // Create value array from user-provided map function.
       array = this._createValueArray(
         value,
         new ValueArrayType(this.crossfilter.data.length)
       );
+    } else if (isArrayOrTypedArray(value)) {
+      // Create value array from user-provided array.  Typically used
+      // only by enumerated dimensions
+      array = this._createValueArray(
+        i => value[i],
+        new ValueArrayType(this.crossfilter.data.length)
+      );
     } else {
       throw new NotImplementedError(
         "dimension value must be function or value array type"
@@ -190,7 +225,7 @@ class ScalarDimension extends _Dimension {
     const len = data.length;
     const larray = array;
     for (let i = 0; i < len; i += 1) {
-      larray[i] = value(data[i]);
+      larray[i] = value(i, data);
     }
     return larray;
   }
@@ -387,7 +422,7 @@ class EnumDimension extends ScalarDimension {
     // and the enum.
     const s = new Set();
     for (let i = 0; i < len; i += 1) {
-      s.add(value(data[i]));
+      s.add(value(i, data));
     }
     this.enumIndex = Array.from(s);
     this.enumIndex.sort();
@@ -395,7 +430,7 @@ class EnumDimension extends ScalarDimension {
     // create dimension value array
     const enumLen = this.enumIndex.length;
     for (let i = 0; i < len; i += 1) {
-      const v = value(data[i]);
+      const v = value(i, data);
       const e = lowerBound(this.enumIndex, v, 0, enumLen);
       larray[i] = e;
     }