initial bug fixes and test improvements for the matrix refactor (#1503)

* initial bug fixes and test improvements for the matrix refactor

* lint

client/__tests__/util/centroid.test.js

@@ -4,6 +4,7 @@ import calcCentroid from "../../src/util/centroid";
 
 import quantile from "../../src/util/quantile";
 import * as Universe from "../../src/util/stateManager/universe";
+import { matrixFBSToDataframe } from "../../src/util/stateManager/matrix";
 import * as World from "../../src/util/stateManager/world";
 import * as REST from "./stateManager/sampleResponses";
 import { ControlsHelpers as CH } from "../../src/util/stateManager";
@@ -22,16 +23,13 @@ describe("centroid", () => {
       ...universe,
       ...Universe.addObsAnnotations(
         universe,
-        Universe.matrixFBSToDataframe(REST.annotationsObs)
+        matrixFBSToDataframe(REST.annotationsObs)
       ),
       ...Universe.addVarAnnotations(
         universe,
-        Universe.matrixFBSToDataframe(REST.annotationsVar)
+        matrixFBSToDataframe(REST.annotationsVar)
-      ),
-      ...Universe.addObsLayout(
-        universe,
-        Universe.matrixFBSToDataframe(REST.layoutObs)
       ),
+      ...Universe.addObsLayout(universe, matrixFBSToDataframe(REST.layoutObs)),
     };
     world = World.createWorldFromEntireUniverse(universe);
 

client/__tests__/util/dataframe/dataframe.test.js

@@ -38,8 +38,8 @@ describe("dataframe constructor", () => {
 
     expect(df.rowIndex).toBeInstanceOf(Dataframe.DenseInt32Index);
     expect(df.colIndex).toBeInstanceOf(Dataframe.KeyIndex);
-    expect(df.rowIndex.keys()).toEqual(new Int32Array([2, 1, 0]));
+    expect(df.rowIndex.labels()).toEqual(new Int32Array([2, 1, 0]));
-    expect(df.colIndex.keys()).toEqual(["A", "B"]);
+    expect(df.colIndex.labels()).toEqual(["A", "B"]);
 
     expect(df.at(0, "A")).toEqual(2);
     expect(df.at(2, "B")).toEqual(3);
@@ -138,7 +138,7 @@ describe("dataframe subsetting", () => {
         new Float32Array([4.4, 5.5, 6.6]),
         ["red", "green", "blue"],
       ],
-      null,
+      null, // identity index
       new Dataframe.KeyIndex(["int32", "string", "float32", "colors"])
     );
 
@@ -153,12 +153,12 @@ describe("dataframe subsetting", () => {
       expect(dfA.col("colors").asArray()).toEqual(
         sourceDf.col("colors").asArray()
       );
-      expect(dfA.rowIndex.keys()).toEqual(sourceDf.rowIndex.keys());
+      expect(dfA.rowIndex.labels()).toEqual(sourceDf.rowIndex.labels());
-      expect(dfA.colIndex.keys()).toEqual(["colors"]);
+      expect(dfA.colIndex.labels()).toEqual(["colors"]);
     });
 
     test("all rows, two columns", () => {
-      const dfB = sourceDf.subset(null, ["colors", "float32"]);
+      const dfB = sourceDf.subset(null, ["float32", "colors"]);
       expect(dfB).toBeDefined();
       expect(dfB.dims).toEqual([3, 2]);
       expect(dfB.iat(0, 0)).toBeCloseTo(4.4);
@@ -177,8 +177,8 @@ describe("dataframe subsetting", () => {
       expect(dfB.col("float32").asArray()).toEqual(
         sourceDf.col("float32").asArray()
       );
-      expect(dfB.rowIndex.keys()).toEqual(sourceDf.rowIndex.keys());
+      expect(dfB.rowIndex.labels()).toEqual(sourceDf.rowIndex.labels());
-      expect(dfB.colIndex.keys()).toEqual(["float32", "colors"]);
+      expect(dfB.colIndex.labels()).toEqual(["float32", "colors"]);
     });
 
     test("one row, all columns", () => {
@@ -189,8 +189,8 @@ describe("dataframe subsetting", () => {
       expect(dfC.iat(0, 1)).toEqual("B");
       expect(dfC.iat(0, 2)).toBeCloseTo(5.5);
       expect(dfC.iat(0, 3)).toEqual("green");
-      expect(dfC.rowIndex.keys()).toEqual(new Int32Array([1]));
+      expect(dfC.rowIndex.labels()).toEqual(new Int32Array([1]));
-      expect(dfC.colIndex.keys()).toEqual(sourceDf.colIndex.keys());
+      expect(dfC.colIndex.labels()).toEqual(sourceDf.colIndex.labels());
     });
 
     test("two rows, all columns", () => {
@@ -201,8 +201,17 @@ describe("dataframe subsetting", () => {
       expect(dfD.icol(1).asArray()).toEqual(["A", "C"]);
       expect(dfD.icol(2).asArray()).toEqual(new Float32Array([4.4, 6.6]));
       expect(dfD.icol(3).asArray()).toEqual(["red", "blue"]);
-      expect(dfD.rowIndex.keys()).toEqual(new Int32Array([0, 2]));
+      expect(dfD.rowIndex.labels()).toEqual(new Int32Array([0, 2]));
-      expect(dfD.colIndex.keys()).toEqual(sourceDf.colIndex.keys());
+      expect(dfD.colIndex.labels()).toEqual(sourceDf.colIndex.labels());
+
+      // reverse the row order
+      const dfDr = sourceDf.subset([2, 0], null);
+      expect(dfDr.icol(0).asArray()).toEqual(new Int32Array([2, 0]));
+      expect(dfDr.icol(1).asArray()).toEqual(["C", "A"]);
+      expect(dfDr.icol(2).asArray()).toEqual(new Float32Array([6.6, 4.4]));
+      expect(dfDr.icol(3).asArray()).toEqual(["blue", "red"]);
+      expect(dfDr.rowIndex.labels()).toEqual(new Int32Array([2, 0]));
+      expect(dfDr.colIndex.labels()).toEqual(sourceDf.colIndex.labels());
     });
 
     test("all rows, all columns", () => {
@@ -213,8 +222,8 @@ describe("dataframe subsetting", () => {
       expect(dfE.icol(1).asArray()).toEqual(sourceDf.icol(1).asArray());
       expect(dfE.icol(2).asArray()).toEqual(sourceDf.icol(2).asArray());
       expect(dfE.icol(3).asArray()).toEqual(sourceDf.icol(3).asArray());
-      expect(dfE.rowIndex.keys()).toEqual(sourceDf.rowIndex.keys());
+      expect(dfE.rowIndex.labels()).toEqual(sourceDf.rowIndex.labels());
-      expect(dfE.colIndex.keys()).toEqual(sourceDf.colIndex.keys());
+      expect(dfE.colIndex.labels()).toEqual(sourceDf.colIndex.labels());
     });
 
     test("two rows, two colums", () => {
@@ -223,8 +232,17 @@ describe("dataframe subsetting", () => {
       expect(dfF.dims).toEqual([2, 2]);
       expect(dfF.icol(0).asArray()).toEqual(new Int32Array([0, 2]));
       expect(dfF.icol(1).asArray()).toEqual(new Float32Array([4.4, 6.6]));
-      expect(dfF.rowIndex.keys()).toEqual(new Int32Array([0, 2]));
+      expect(dfF.rowIndex.labels()).toEqual(new Int32Array([0, 2]));
-      expect(dfF.colIndex.keys()).toEqual(["int32", "float32"]);
+      expect(dfF.colIndex.labels()).toEqual(["int32", "float32"]);
+
+      // reverse the row and column order
+      const dfFr = sourceDf.subset([2, 0], ["float32", "int32"]);
+      expect(dfFr).toBeDefined();
+      expect(dfFr.dims).toEqual([2, 2]);
+      expect(dfFr.icol(0).asArray()).toEqual(new Float32Array([6.6, 4.4]));
+      expect(dfFr.icol(1).asArray()).toEqual(new Int32Array([2, 0]));
+      expect(dfFr.rowIndex.labels()).toEqual(new Int32Array([2, 0]));
+      expect(dfFr.colIndex.labels()).toEqual(["float32", "int32"]);
     });
 
     test("withRowIndex", () => {
@@ -271,8 +289,49 @@ describe("dataframe subsetting", () => {
     expect(dfA.dims).toEqual([2, 2]);
     expect(dfA.icol(0).asArray()).toEqual(new Int32Array([1, 2]));
     expect(dfA.icol(1).asArray()).toEqual(["green", "blue"]);
-    expect(dfA.rowIndex.keys()).toEqual(new Int32Array([4, 6]));
+    expect(dfA.rowIndex.labels()).toEqual(new Int32Array([4, 6]));
-    expect(dfA.colIndex.keys()).toEqual(["int32", "colors"]);
+    expect(dfA.colIndex.labels()).toEqual(["int32", "colors"]);
+  });
+
+  describe("isubset", () => {
+    const sourceDf = new Dataframe.Dataframe(
+      [3, 4],
+      [
+        new Int32Array([0, 1, 2]),
+        ["A", "B", "C"],
+        new Float32Array([4.4, 5.5, 6.6]),
+        ["red", "green", "blue"],
+      ],
+      null, // identity index
+      new Dataframe.KeyIndex(["int32", "string", "float32", "colors"])
+    );
+
+    test("one row, all cols", () => {
+      const dfA = sourceDf.isubset([1], null);
+      expect(dfA.dims).toEqual([1, 4]);
+      expect(dfA.icol(0).asArray()).toEqual(new Int32Array([1]));
+      expect(dfA.icol(1).asArray()).toEqual(["B"]);
+      expect(dfA.icol(2).asArray()).toEqual(new Float32Array([5.5]));
+      expect(dfA.icol(3).asArray()).toEqual(["green"]);
+    });
+
+    test("all rows, two cols", () => {
+      const dfA = sourceDf.isubset(null, [1, 2]);
+      expect(dfA.dims).toEqual([3, 2]);
+      expect(dfA.icol(0).asArray()).toEqual(["A", "B", "C"]);
+      expect(dfA.icol(1).asArray()).toEqual(new Float32Array([4.4, 5.5, 6.6]));
+      expect(dfA.col("string")).toBe(dfA.icol(0));
+      expect(dfA.col("float32")).toBe(dfA.icol(1));
+    });
+
+    test("out of order rows", () => {
+      const dfA = sourceDf.isubset([2, 0], null);
+      expect(dfA.dims).toEqual([2, 4]);
+      expect(dfA.icol(0).asArray()).toEqual(new Int32Array([2, 0]));
+      expect(dfA.icol(1).asArray()).toEqual(["C", "A"]);
+      expect(dfA.icol(2).asArray()).toEqual(new Float32Array([6.6, 4.4]));
+      expect(dfA.icol(3).asArray()).toEqual(["blue", "red"]);
+    });
   });
 });
 
@@ -310,8 +369,8 @@ describe("dataframe factories", () => {
     expect(dfB).not.toBe(dfA);
     expect(dfB.dims).toEqual(dfA.dims);
     expect(dfB).toHaveLength(dfA.length);
-    expect(dfB.rowIndex.keys()).toEqual(dfA.rowIndex.keys());
+    expect(dfB.rowIndex.labels()).toEqual(dfA.rowIndex.labels());
-    expect(dfB.colIndex.keys()).toEqual(dfA.colIndex.keys());
+    expect(dfB.colIndex.labels()).toEqual(dfA.colIndex.labels());
     for (let i = 0, l = dfB.dims[1]; i < l; i += 1) {
       expect(dfB.icol(i).asArray()).toEqual(dfA.icol(i).asArray());
     }
@@ -336,9 +395,9 @@ describe("dataframe factories", () => {
       expect(dfA.icol(1).asArray()).toEqual([true, false]);
       expect(dfA.icol(2).asArray()).toEqual([1, 0]);
       expect(dfA.col("numbers").asArray()).toEqual([1, 0]);
-      expect(dfA.colIndex.keys()).toEqual(["colors", "bools", "numbers"]);
+      expect(dfA.colIndex.labels()).toEqual(["colors", "bools", "numbers"]);
-      expect(df.colIndex.keys()).toEqual(["colors", "bools"]);
+      expect(df.colIndex.labels()).toEqual(["colors", "bools"]);
-      expect(df.rowIndex.keys()).toEqual(dfA.rowIndex.keys());
+      expect(df.rowIndex.labels()).toEqual(dfA.rowIndex.labels());
     });
 
     test("DenseInt32Index", () => {
@@ -361,9 +420,9 @@ describe("dataframe factories", () => {
       expect(dfA.col(74).asArray()).toEqual(["red", "blue"]);
       expect(dfA.col(75).asArray()).toEqual([true, false]);
       expect(dfA.col(72).asArray()).toEqual([1, 0]);
-      expect(dfA.colIndex.keys()).toEqual(new Int32Array([74, 75, 72]));
+      expect(dfA.colIndex.labels()).toEqual(new Int32Array([74, 75, 72]));
-      expect(df.colIndex.keys()).toEqual(new Int32Array([74, 75]));
+      expect(df.colIndex.labels()).toEqual(new Int32Array([74, 75]));
-      expect(df.rowIndex.keys()).toEqual(dfA.rowIndex.keys());
+      expect(df.rowIndex.labels()).toEqual(dfA.rowIndex.labels());
     });
 
     test("DenseInt32Index promote", () => {
@@ -386,9 +445,9 @@ describe("dataframe factories", () => {
       expect(dfA.col(74).asArray()).toEqual(["red", "blue"]);
       expect(dfA.col(75).asArray()).toEqual([true, false]);
       expect(dfA.col(999).asArray()).toEqual([1, 0]);
-      expect(dfA.colIndex.keys()).toEqual(new Int32Array([74, 75, 999]));
+      expect(dfA.colIndex.labels()).toEqual(new Int32Array([74, 75, 999]));
-      expect(df.colIndex.keys()).toEqual(new Int32Array([74, 75]));
+      expect(df.colIndex.labels()).toEqual(new Int32Array([74, 75]));
-      expect(df.rowIndex.keys()).toEqual(dfA.rowIndex.keys());
+      expect(df.rowIndex.labels()).toEqual(dfA.rowIndex.labels());
     });
 
     test("IdentityInt32Index with last", () => {
@@ -411,9 +470,9 @@ describe("dataframe factories", () => {
       expect(dfA.col(0).asArray()).toEqual(["red", "blue"]);
       expect(dfA.col(1).asArray()).toEqual([true, false]);
       expect(dfA.col(2).asArray()).toEqual([1, 0]);
-      expect(dfA.colIndex.keys()).toEqual(new Int32Array([0, 1, 2]));
+      expect(dfA.colIndex.labels()).toEqual(new Int32Array([0, 1, 2]));
-      expect(df.colIndex.keys()).toEqual(new Int32Array([0, 1]));
+      expect(df.colIndex.labels()).toEqual(new Int32Array([0, 1]));
-      expect(df.rowIndex.keys()).toEqual(dfA.rowIndex.keys());
+      expect(df.rowIndex.labels()).toEqual(dfA.rowIndex.labels());
     });
 
     test("IdentityInt32Index promote", () => {
@@ -436,9 +495,9 @@ describe("dataframe factories", () => {
       expect(dfA.col(0).asArray()).toEqual(["red", "blue"]);
       expect(dfA.col(1).asArray()).toEqual([true, false]);
       expect(dfA.col(99).asArray()).toEqual([1, 0]);
-      expect(dfA.colIndex.keys()).toEqual(new Int32Array([0, 1, 99]));
+      expect(dfA.colIndex.labels()).toEqual(new Int32Array([0, 1, 99]));
-      expect(df.colIndex.keys()).toEqual(new Int32Array([0, 1]));
+      expect(df.colIndex.labels()).toEqual(new Int32Array([0, 1]));
-      expect(df.rowIndex.keys()).toEqual(dfA.rowIndex.keys());
+      expect(df.rowIndex.labels()).toEqual(dfA.rowIndex.labels());
     });
 
     describe("handle column dimensions correctly", () => {
@@ -517,25 +576,25 @@ describe("dataframe factories", () => {
       const dfLikeA = dfEmpty.withColsFrom(dfA);
       expect(dfLikeA).toBeDefined();
       expect(dfLikeA.dims).toEqual(dfA.dims);
-      expect(dfLikeA.colIndex.keys()).toEqual(dfA.colIndex.keys());
+      expect(dfLikeA.colIndex.labels()).toEqual(dfA.colIndex.labels());
       expect(dfLikeA.rowIndex).toEqual(dfA.rowIndex);
-      expect(dfLikeA.rowIndex.keys()).toEqual(dfA.rowIndex.keys());
+      expect(dfLikeA.rowIndex.labels()).toEqual(dfA.rowIndex.labels());
       expect(dfLikeA.icol(0).asArray()).toEqual(dfA.icol(0).asArray());
 
       const dfAlsoLikeA = dfA.withColsFrom(dfEmpty);
       expect(dfAlsoLikeA).toBeDefined();
       expect(dfAlsoLikeA.dims).toEqual(dfA.dims);
-      expect(dfAlsoLikeA.colIndex.keys()).toEqual(dfA.colIndex.keys());
+      expect(dfAlsoLikeA.colIndex.labels()).toEqual(dfA.colIndex.labels());
       expect(dfAlsoLikeA.rowIndex).toEqual(dfA.rowIndex);
-      expect(dfAlsoLikeA.rowIndex.keys()).toEqual(dfA.rowIndex.keys());
+      expect(dfAlsoLikeA.rowIndex.labels()).toEqual(dfA.rowIndex.labels());
       expect(dfAlsoLikeA.icol(0).asArray()).toEqual(dfA.icol(0).asArray());
 
       const dfC = dfA.withColsFrom(dfB);
       expect(dfC).toBeDefined();
       expect(dfC.dims).toEqual([2, 2]);
-      expect(dfC.colIndex.keys()).toEqual(["colors", "bools"]);
+      expect(dfC.colIndex.labels()).toEqual(["colors", "bools"]);
       expect(dfC.rowIndex).toEqual(dfA.rowIndex);
-      expect(dfC.rowIndex.keys()).toEqual(dfA.rowIndex.keys());
+      expect(dfC.rowIndex.labels()).toEqual(dfA.rowIndex.labels());
       expect(dfC.col("colors").asArray()).toEqual(["red", "blue"]);
       expect(dfC.col("bools").asArray()).toEqual([true, false]);
     });
@@ -562,21 +621,21 @@ describe("dataframe factories", () => {
       const dfX = dfEmpty.withColsFrom(dfB, ["colors", "bools"]);
       expect(dfX).toBeDefined();
       expect(dfX.dims).toEqual([2, 2]);
-      expect(dfX.colIndex.keys()).toEqual(["colors", "bools"]);
+      expect(dfX.colIndex.labels()).toEqual(["colors", "bools"]);
       expect(dfX.rowIndex).toEqual(dfB.rowIndex);
       expect(dfX.icol(0).asArray()).toEqual(dfB.icol(0).asArray());
 
       const dfY = dfA.withColsFrom(dfB, ["numbers"]);
       expect(dfY).toBeDefined();
       expect(dfY.dims).toEqual([2, 2]);
-      expect(dfY.colIndex.keys()).toEqual(["colors", "numbers"]);
+      expect(dfY.colIndex.labels()).toEqual(["colors", "numbers"]);
       expect(dfY.rowIndex).toEqual(dfA.rowIndex);
       expect(dfY.icol(0).asArray()).toEqual(dfA.icol(0).asArray());
 
       const dfZ = dfA.withColsFrom(dfEmpty, []);
       expect(dfZ).toBeDefined();
       expect(dfZ.dims).toEqual(dfA.dims);
-      expect(dfZ.colIndex.keys()).toEqual(dfA.colIndex.keys());
+      expect(dfZ.colIndex.labels()).toEqual(dfA.colIndex.labels());
       expect(dfZ.rowIndex).toEqual(dfA.rowIndex);
       expect(dfZ.icol(0).asArray()).toEqual(dfA.icol(0).asArray());
 
@@ -604,7 +663,7 @@ describe("dataframe factories", () => {
       const dfX = dfA.withColsFrom(dfB, { colors: "_colors", bools: "_bools" });
       expect(dfX).toBeDefined();
       expect(dfX.dims).toEqual([2, 3]);
-      expect(dfX.colIndex.keys()).toEqual(["colors", "_colors", "_bools"]);
+      expect(dfX.colIndex.labels()).toEqual(["colors", "_colors", "_bools"]);
       expect(dfX.rowIndex).toEqual(dfA.rowIndex);
       expect(dfX.icol(0).asArray()).toEqual(dfA.icol(0).asArray());
       expect(dfX.col("_colors").asArray()).toBe(dfB.col("colors").asArray());
@@ -630,9 +689,9 @@ describe("dataframe factories", () => {
       expect(dfA.icol(0).asArray()).toEqual([true, false]);
       expect(dfA.icol(1).asArray()).toEqual([1, 0]);
       expect(dfA.col("numbers").asArray()).toEqual([1, 0]);
-      expect(dfA.colIndex.keys()).toEqual(["bools", "numbers"]);
+      expect(dfA.colIndex.labels()).toEqual(["bools", "numbers"]);
-      expect(df.colIndex.keys()).toEqual(["colors", "bools", "numbers"]);
+      expect(df.colIndex.labels()).toEqual(["colors", "bools", "numbers"]);
-      expect(df.rowIndex.keys()).toEqual(dfA.rowIndex.keys());
+      expect(df.rowIndex.labels()).toEqual(dfA.rowIndex.labels());
     });
 
     test("IdentityInt32Index drop first", () => {
@@ -654,9 +713,9 @@ describe("dataframe factories", () => {
       expect(dfA.icol(1).asArray()).toEqual([1, 0]);
       expect(df.col(1).asArray()).toEqual(dfA.col(1).asArray());
       expect(df.col(2).asArray()).toEqual(dfA.col(2).asArray());
-      expect(dfA.colIndex.keys()).toEqual(new Int32Array([1, 2]));
+      expect(dfA.colIndex.labels()).toEqual(new Int32Array([1, 2]));
-      expect(df.colIndex.keys()).toEqual(new Int32Array([0, 1, 2]));
+      expect(df.colIndex.labels()).toEqual(new Int32Array([0, 1, 2]));
-      expect(df.rowIndex.keys()).toEqual(dfA.rowIndex.keys());
+      expect(df.rowIndex.labels()).toEqual(dfA.rowIndex.labels());
     });
 
     test("IdentityInt32Index drop last", () => {
@@ -678,9 +737,9 @@ describe("dataframe factories", () => {
       expect(dfA.icol(1).asArray()).toEqual([true, false]);
       expect(df.col(0).asArray()).toEqual(dfA.col(0).asArray());
       expect(df.col(1).asArray()).toEqual(dfA.col(1).asArray());
-      expect(dfA.colIndex.keys()).toEqual(new Int32Array([0, 1]));
+      expect(dfA.colIndex.labels()).toEqual(new Int32Array([0, 1]));
-      expect(df.colIndex.keys()).toEqual(new Int32Array([0, 1, 2]));
+      expect(df.colIndex.labels()).toEqual(new Int32Array([0, 1, 2]));
-      expect(df.rowIndex.keys()).toEqual(dfA.rowIndex.keys());
+      expect(df.rowIndex.labels()).toEqual(dfA.rowIndex.labels());
     });
 
     test("DenseInt32Index", () => {
@@ -702,9 +761,9 @@ describe("dataframe factories", () => {
       expect(dfA.icol(1).asArray()).toEqual([1, 0]);
       expect(dfA.col(100).asArray()).toEqual([1, 0]);
       expect(dfA.col(102).asArray()).toEqual(["red", "blue"]);
-      expect(dfA.colIndex.keys()).toEqual(new Int32Array([102, 100]));
+      expect(dfA.colIndex.labels()).toEqual(new Int32Array([102, 100]));
-      expect(df.colIndex.keys()).toEqual(new Int32Array([102, 101, 100]));
+      expect(df.colIndex.labels()).toEqual(new Int32Array([102, 101, 100]));
-      expect(df.rowIndex.keys()).toEqual(dfA.rowIndex.keys());
+      expect(df.rowIndex.labels()).toEqual(dfA.rowIndex.labels());
     });
   });
 
@@ -766,8 +825,8 @@ describe("dataframe factories", () => {
         new Dataframe.KeyIndex(["A", "B"])
       );
       const dfB = dfA.renameCol("B", "C");
-      expect(dfA.colIndex.keys()).toEqual(["A", "B"]);
+      expect(dfA.colIndex.labels()).toEqual(["A", "B"]);
-      expect(dfB.colIndex.keys()).toEqual(["A", "C"]);
+      expect(dfB.colIndex.labels()).toEqual(["A", "C"]);
       expect(dfA.dims).toMatchObject(dfB.dims);
       expect(dfA.columns()).toMatchObject(dfB.columns());
     });
@@ -857,3 +916,84 @@ describe("dataframe col", () => {
     expect(df.col("B").indexOf(true)).toBeUndefined();
   });
 });
+
+describe("label indexing", () => {
+
+  test("IdentityInt32Index", () => {
+    const idx = new Dataframe.IdentityInt32Index(12); // [0, 12)
+
+    expect(Dataframe.isLabelIndex(idx)).toBeTruthy();
+
+    expect(idx.labels()).toEqual(new Int32Array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]));
+    expect(idx.getLabel(1)).toEqual(1);
+    expect(idx.getOffset(1)).toEqual(1);
+    expect(idx.getOffsets([1,3])).toEqual([1,3])
+    expect(idx.getLabels([1, 3])).toEqual([1,3])
+    expect(idx.size()).toEqual(12);
+
+    expect(idx.subset([2]).labels()).toEqual([2]);
+    expect(idx.subset([2, 3, 4]).labels()).toEqual(new Int32Array([2, 3, 4]));
+    expect(idx.subset([0, 1, 2, 3]).labels()).toEqual(new Int32Array([0, 1, 2, 3]));
+
+    expect(idx.isubset([2]).labels()).toEqual([2]);
+    expect(idx.isubset([2, 3, 4]).labels()).toEqual(new Int32Array([2, 3, 4]));
+    expect(idx.isubset([0, 1, 2, 3]).labels()).toEqual(new Int32Array([0, 1, 2, 3]));
+
+    expect(idx.subset([0, 1, 2, 3, 4])).toBeInstanceOf(Dataframe.IdentityInt32Index);
+    expect(idx.subset([2, 1, 0])).toBeInstanceOf(Dataframe.IdentityInt32Index);
+    expect(idx.subset([1, 2, 3, 4])).toBeInstanceOf(Dataframe.DenseInt32Index);
+    expect(idx.subset([0, 1, 3, 4])).toBeInstanceOf(Dataframe.DenseInt32Index);
+    expect(idx.subset([0, 1, 2, 3, 10])).toBeInstanceOf(Dataframe.DenseInt32Index);
+    expect(idx.subset([4, 3, 2, 1])).toBeInstanceOf(Dataframe.DenseInt32Index);
+    expect(idx.subset([4])).toBeInstanceOf(Dataframe.KeyIndex);
+
+    expect(idx.withLabel(99).labels()).toEqual(new Int32Array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 99]));
+    expect(idx.dropLabel(0).labels()).toEqual(new Int32Array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]));
+    expect(idx.dropLabel(11).labels()).toEqual(new Int32Array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10]));
+    expect(idx.dropLabel(5).labels()).toEqual(new Int32Array([0, 1, 2, 3, 4, 6, 7, 8, 9, 10, 11]));
+  });
+
+  test("DenseInt32Index", () => {
+
+    const idx = new Dataframe.DenseInt32Index([99, 1002, 48, 0, 22]);
+
+    expect(Dataframe.isLabelIndex(idx)).toBeTruthy();
+
+    expect(idx.labels()).toEqual(new Int32Array([99, 1002, 48, 0, 22]));
+    expect(idx.size()).toEqual(5);
+    expect(idx.getOffset(1002)).toEqual(1);
+    expect(idx.getOffset(0)).toEqual(3);
+    expect(idx.getLabel(0)).toEqual(99);
+    expect(idx.getLabels(new Int32Array([2, 4]))).toEqual(new Int32Array([48, 22]));
+    expect(idx.getLabels([2, 4])).toEqual([48, 22]);
+    expect(idx.getOffsets([0, 48])).toEqual([3, 2]);
+
+    expect(idx.subset([1002, 0, 99]).labels()).toEqual(new Int32Array([1002, 0, 99]))
+    expect(idx.getOffsets(idx.subset([1002, 0, 99]).labels())).toEqual(new Int32Array([1, 3, 0]));
+    expect(idx.isubset([4, 1, 2]).labels()).toEqual(new Int32Array([22, 1002, 48]));
+
+    expect(idx.withLabel(88).labels()).toEqual(new Int32Array([99, 1002, 48, 0, 22, 88]));
+    expect(idx.withLabel(88).getOffset(88)).toEqual(5);
+    expect(idx.dropLabel(48).labels()).toEqual(new Int32Array([99, 1002, 0, 22]));
+  });
+
+  test("KeyIndex", () => {
+    const idx = new Dataframe.KeyIndex(["red", "green", "blue"]);
+
+    expect(Dataframe.isLabelIndex(idx)).toBeTruthy();
+
+    expect(idx.labels()).toEqual(["red", "green", "blue"]);
+    expect(idx.size()).toEqual(3);
+    expect(idx.getOffset("blue")).toEqual(2);
+    expect(idx.getLabel(1)).toEqual("green");
+
+    expect(idx.subset(["green"]).labels()).toEqual(["green"]);
+    expect(idx.subset(["green", "red"]).labels()).toEqual(["green", "red"]);
+    expect(idx.isubset([2, 1, 0]).labels()).toEqual(["blue", "green", "red"]);
+
+    expect(idx.withLabel("yo").labels()).toEqual(["red", "green", "blue", "yo"]);
+    expect(idx.withLabel("yo").getOffset("yo")).toEqual(3);
+    expect(idx.dropLabel("blue").labels()).toEqual(["red", "green"]);
+});
+
+})

client/__tests__/util/stateManager/fbs.test.js

@@ -27,7 +27,7 @@ describe("encode/decode", () => {
     const dfWithColIdx = new Dataframe([3, 4], columns, null, colIndex);
     const dfB = decodeMatrixFBS(encodeMatrixFBS(dfWithColIdx));
     expect([dfB.nRows, dfB.nCols]).toEqual(dfWithColIdx.dims);
-    expect(dfB.colIdx).toEqual(colIndex.keys());
+    expect(dfB.colIdx).toEqual(colIndex.labels());
     expect(dfB.rowIdx).toBeNull();
     expect(dfB.columns).toEqual(columns);
   });

client/__tests__/util/stateManager/universe.test.js

@@ -1,4 +1,5 @@
 import * as Universe from "../../../src/util/stateManager/universe";
+import { matrixFBSToDataframe } from "../../../src/util/stateManager/matrix";
 import * as Dataframe from "../../../src/util/dataframe";
 import * as REST from "./sampleResponses";
 
@@ -51,16 +52,13 @@ describe("createUniverseFromResponse", () => {
       ...universe,
       ...Universe.addObsAnnotations(
         universe,
-        Universe.matrixFBSToDataframe(REST.annotationsObs)
+        matrixFBSToDataframe(REST.annotationsObs)
       ),
       ...Universe.addVarAnnotations(
         universe,
-        Universe.matrixFBSToDataframe(REST.annotationsVar)
+        matrixFBSToDataframe(REST.annotationsVar)
-      ),
-      ...Universe.addObsLayout(
-        universe,
-        Universe.matrixFBSToDataframe(REST.layoutObs)
       ),
+      ...Universe.addObsLayout(universe, matrixFBSToDataframe(REST.layoutObs)),
     };
 
     expect(universe).toMatchObject(
@@ -80,7 +78,7 @@ describe("createUniverseFromResponse", () => {
       REST.schema.schema.annotations.obs.columns.length,
     ]);
     expect(universe.obsLayout.dims).toEqual([nObs, 2]);
-    expect(universe.obsLayout.colIndex.keys()).toEqual(
+    expect(universe.obsLayout.colIndex.labels()).toEqual(
       universe.schema.layout.obs[0].dims
     );
     expect(universe.varAnnotations.dims).toEqual([

client/__tests__/util/stateManager/world.test.js

@@ -1,5 +1,6 @@
 import _ from "lodash";
 import * as Universe from "../../../src/util/stateManager/universe";
+import { matrixFBSToDataframe } from "../../../src/util/stateManager/matrix";
 import * as World from "../../../src/util/stateManager/world";
 import * as Dataframe from "../../../src/util/dataframe";
 import Crossfilter from "../../../src/util/typedCrossfilter";
@@ -26,16 +27,13 @@ const defaultBigBang = () => {
     ...universe,
     ...Universe.addObsAnnotations(
       universe,
-      Universe.matrixFBSToDataframe(REST.annotationsObs)
+      matrixFBSToDataframe(REST.annotationsObs)
     ),
     ...Universe.addVarAnnotations(
       universe,
-      Universe.matrixFBSToDataframe(REST.annotationsVar)
+      matrixFBSToDataframe(REST.annotationsVar)
-    ),
-    ...Universe.addObsLayout(
-      universe,
-      Universe.matrixFBSToDataframe(REST.layoutObs)
     ),
+    ...Universe.addObsLayout(universe, matrixFBSToDataframe(REST.layoutObs)),
   };
 
   /* create world */
@@ -59,9 +57,9 @@ describe("createWorldFromEntireUniverse", () => {
     const universe = Universe.createUniverseFromResponse(
       _.cloneDeep(REST.config),
       _.cloneDeep(REST.schema),
-      Universe.matrixFBSToDataframe(_.cloneDeep(REST.annotationsObs)),
+      matrixFBSToDataframe(_.cloneDeep(REST.annotationsObs)),
-      Universe.matrixFBSToDataframe(_.cloneDeep(REST.annotationsVar)),
+      matrixFBSToDataframe(_.cloneDeep(REST.annotationsVar)),
-      Universe.matrixFBSToDataframe(_.cloneDeep(REST.layoutObs))
+      matrixFBSToDataframe(_.cloneDeep(REST.layoutObs))
     );
     expect(universe).toBeDefined();
 
@@ -144,16 +142,16 @@ describe("createWorldFromCurrentSelection", () => {
       })
     );
 
-    expect(world.obsAnnotations.rowIndex.keys()).toEqual(
+    expect(world.obsAnnotations.rowIndex.labels()).toEqual(
       new Int32Array(matchingIndices)
     );
-    expect(world.obsAnnotations.colIndex.keys()).toEqual(
+    expect(world.obsAnnotations.colIndex.labels()).toEqual(
-      universe.obsAnnotations.colIndex.keys()
+      universe.obsAnnotations.colIndex.labels()
     );
-    expect(world.obsLayout.rowIndex.keys()).toEqual(
+    expect(world.obsLayout.rowIndex.labels()).toEqual(
       new Int32Array(matchingIndices)
     );
-    expect(world.obsLayout.colIndex.keys()).toEqual(
+    expect(world.obsLayout.colIndex.labels()).toEqual(
       world.schema.layout.obs[0].dims
     );
   });

client/src/actions/index.js

@@ -29,7 +29,7 @@ async function obsAnnotationFetchAndLoad(dispatch, schema) {
         fetchBinary(
           `annotations/obs?annotation-name=${encodeURIComponent(col.name)}`
         )
-          .then((buffer) => Universe.matrixFBSToDataframe(buffer))
+          .then((buffer) => MatrixFBS.matrixFBSToDataframe(buffer))
           .then((df) =>
             dispatch({
               type: "universe: column load success",
@@ -52,7 +52,7 @@ async function varAnnotationFetchAndLoad(dispatch, schema) {
   return Promise.all(
     names.map((name) =>
       fetchBinary(`annotations/var?annotation-name=${encodeURIComponent(name)}`)
-        .then((buffer) => Universe.matrixFBSToDataframe(buffer))
+        .then((buffer) => MatrixFBS.matrixFBSToDataframe(buffer))
         .then((df) =>
           dispatch({
             type: "universe: column load success",
@@ -77,7 +77,7 @@ function layoutFetchAndLoad(dispatch, schema) {
       plimit.add(() =>
         fetchBinary(
           `layout/obs?layout-name=${encodeURIComponent(e)}`
-        ).then((buffer) => Universe.matrixFBSToDataframe(buffer))
+        ).then((buffer) => MatrixFBS.matrixFBSToDataframe(buffer))
       )
     )
   ).then((dfs) =>

client/src/actions/reembed.js

@@ -1,5 +1,5 @@
 import { API } from "../globals";
-import { Universe } from "../util/stateManager";
+import { MatrixFBS } from "../util/stateManager";
 import {
   postNetworkErrorToast,
   postAsyncSuccessToast,
@@ -24,7 +24,7 @@ function abortableFetch(request, opts, timeout = 0) {
 
 async function doReembedFetch(dispatch, getState) {
   const state = getState();
-  let cells = state.world.obsAnnotations.rowIndex.keys();
+  let cells = state.world.obsAnnotations.rowIndex.labels();
 
   // These lines ensure that we convert any TypedArray to an Array.
   // This is necessary because JSON.stringify() does some very strange
@@ -80,7 +80,7 @@ export function requestReembed() {
       const res = await doReembedFetch(dispatch, getState);
       const schema = JSON.parse(res.headers.get("CxG-Schema"));
       const buffer = await res.arrayBuffer();
-      const df = Universe.matrixFBSToDataframe(buffer);
+      const df = MatrixFBS.matrixFBSToDataframe(buffer);
       dispatch({
         type: "reembed: request completed",
       });

client/src/reducers/categoricalSelection.js

@@ -31,7 +31,7 @@ const CategoricalSelection = (
       const names = CH.selectableCategoryNames(
         world.schema,
         CH.maxCategoryItems(prevSharedState.config),
-        dataframe.colIndex.keys()
+        dataframe.colIndex.labels()
       );
       if (names.length === 0) return state;
       return {

client/src/reducers/world.js

@@ -93,7 +93,7 @@ const WorldReducer = (
           let worldValSlice = val;
           if (!World.worldEqUniverse(state, universe)) {
             worldValSlice = universeVarData
-              .subset(state.obsAnnotations.rowIndex.keys(), [key], null)
+              .subset(state.obsAnnotations.rowIndex.labels(), [key], null)
               .icol(0)
               .asArray();
           }
@@ -129,10 +129,10 @@ const WorldReducer = (
       //
       let clippedVarData = state.varData;
       const keysToDrop = clippedVarData.colIndex
-        .keys()
+        .labels()
         .filter((k) => !unclippedVarData.hasCol(k));
       const keysToAdd = unclippedVarData.colIndex
-        .keys()
+        .labels()
         .filter((k) => !clippedVarData.hasCol(k));
       keysToDrop.forEach((k) => {
         clippedVarData = clippedVarData.dropCol(k);
@@ -171,7 +171,7 @@ const WorldReducer = (
       let newAnnotation = null;
       if (!World.worldEqUniverse(state, universe)) {
         newAnnotation = universe.obsAnnotations
-          .subset(state.obsAnnotations.rowIndex.keys(), [name], null)
+          .subset(state.obsAnnotations.rowIndex.labels(), [name], null)
           .icol(0)
           .asArray();
       } else {
@@ -303,7 +303,7 @@ const WorldReducer = (
       let schema = origSchema;
 
       // alias the names the server sent us, in case they were not the same as the schema
-      const embedingLabels = embedding.colIndex.keys();
+      const embedingLabels = embedding.colIndex.labels();
       const labels = {
         [embedingLabels[0]]: dims[0],
         [embedingLabels[1]]: dims[1],

client/src/util/dataframe/dataframe.js

@@ -1,6 +1,5 @@
 import { IdentityInt32Index, isLabelIndex } from "./labelIndex";
 // weird cross-dependency that we should clean up someday...
-import { sortArray } from "../typedCrossfilter/sort";
 import {
   isTypedArray,
   isArrayOrTypedArray,
@@ -389,7 +388,7 @@ class Dataframe {
     let dstLabels;
     if (!labels) {
       // combine all columns
-      dstLabels = dataframe.colIndex.keys();
+      dstLabels = dataframe.colIndex.labels();
       srcLabels = dstLabels;
     } else if (Array.isArray(labels)) {
       // combine subset of keys with no aliasing
@@ -537,7 +536,12 @@ class Dataframe {
   }
 
   static empty(rowIndex = null, colIndex = null) {
-    return new Dataframe([0, 0], [], rowIndex, colIndex);
+    const dims = [
+      rowIndex ? rowIndex.size() : 0,
+      colIndex ? colIndex.size() : 0,
+    ];
+    if (dims[0] && dims[1]) throw new Error("not an empty dataframe");
+    return new Dataframe(dims, new Array(dims[1]), rowIndex, colIndex);
   }
 
   static create(dims, columnarData) {
@@ -551,97 +555,59 @@ class Dataframe {
     return new Dataframe(dims, columnarData, null, null);
   }
 
-  __subset(rowOffsets, colOffsets, withRowIndex) {
+  __subset(newRowIndex, newColIndex) {
     const dims = [...this.dims];
 
-    const getSortedLabelAndOffsets = (offsets, index) => {
+    /* subset columns */
-      /*
+    let { __columns, colIndex } = this;
-      Given offsets, return both offsets and associated lables,
+    if (newColIndex) {
-      sorted by offset.
+      const colOffsets = this.colIndex.getOffsets(newColIndex.labels());
-      */
+      __columns = new Array(colOffsets.length);
-      if (!offsets) {
+      for (let i = 0, l = colOffsets.length; i < l; i += 1) {
-        return [null, null];
+        __columns[i] = this.__columns[colOffsets[i]];
       }
-      const sortedOffsets = sortArray(offsets);
+      colIndex = newColIndex;
-      const sortedLabels = new Array(sortedOffsets.length);
-      for (let i = 0, l = sortedOffsets.length; i < l; i += 1) {
-        sortedLabels[i] = index.getLabel(sortedOffsets[i]);
-      }
-      return [sortedLabels, sortedOffsets];
-    };
-
-    let { colIndex } = this;
-    if (colOffsets) {
-      let colLabels;
-      [colLabels, colOffsets] = getSortedLabelAndOffsets(
-        colOffsets,
-        this.colIndex
-      );
       dims[1] = colOffsets.length;
-      colIndex = this.colIndex.subsetLabels(colLabels);
     }
 
     let { rowIndex } = this;
-    if (withRowIndex) rowIndex = withRowIndex;
+    if (newRowIndex) {
-    if (rowOffsets) {
+      const rowOffsets = this.rowIndex.getOffsets(newRowIndex.labels());
-      let rowLabels;
+      __columns = __columns.map((col) => {
-      [rowLabels, rowOffsets] = getSortedLabelAndOffsets(
-        rowOffsets,
-        this.rowIndex
-      );
-      dims[0] = rowLabels.length;
-      if (!withRowIndex) rowIndex = this.rowIndex.subsetLabels(rowLabels);
-    }
-
-    /* subset columns */
-    let columns = this.__columns;
-    if (colOffsets) {
-      columns = new Array(colOffsets.length);
-      for (let i = 0, l = colOffsets.length; i < l; i += 1) {
-        columns[i] = this.__columns[colOffsets[i]];
-      }
-    }
-
-    /* subset rows */
-    if (rowOffsets) {
-      columns = columns.map((col) => {
         const newCol = new col.constructor(rowOffsets.length);
         for (let i = 0, l = rowOffsets.length; i < l; i += 1) {
           newCol[i] = col[rowOffsets[i]];
         }
         return newCol;
       });
+      rowIndex = newRowIndex;
+      dims[0] = rowOffsets.length;
     }
 
     if (dims[0] === 0 || dims[1] === 0) return Dataframe.empty();
-    return new Dataframe(dims, columns, rowIndex, colIndex);
+    return new Dataframe(dims, __columns, rowIndex, colIndex);
   }
 
   subset(rowLabels, colLabels = null, withRowIndex = null) {
     /*
     Subset by row/col labels.
 
-    withRowIndex allows assignment of new row index during subset operation.
+    withRowIndex allows subset with an index, rather than rowLabels.
-    If withRowIndex === null, it will reset the index to identity (offset)
+    If withRowIndex is specified, rowLabels is ignored.
-    indexing.  if withRowIndex is a label index object, it will be used
-    for the new dataframe.
     */
-    const toOffsets = (labels, index) => {
+    let rowIndex = null;
-      if (!labels) {
+    if (withRowIndex) {
-        return null;
+      rowIndex = withRowIndex;
-      }
+    } else if (rowLabels) {
-      return labels.map((label) => {
+      rowIndex = this.rowIndex.subset(rowLabels);
-        const off = index.getOffset(label);
+    }
-        if (off === undefined) {
-          throw new RangeError(`unknown label: ${label}`);
-        }
-        return off;
-      });
-    };
 
-    const rowOffsets = toOffsets(rowLabels, this.rowIndex);
+    let colIndex = null;
-    const colOffsets = toOffsets(colLabels, this.colIndex);
+    if (colLabels) {
-    return this.__subset(rowOffsets, colOffsets, withRowIndex);
+      colIndex = this.colIndex.subset(colLabels);
+    }
+
+    return this.__subset(rowIndex, colIndex);
   }
 
   isubset(rowOffsets, colOffsets = null, withRowIndex = null) {
@@ -653,7 +619,19 @@ class Dataframe {
     indexing. If withRowIndex is a label index object, it will be used
     for the new dataframe.
     */
-    return this.__subset(rowOffsets, colOffsets, withRowIndex);
+    let rowIndex = null;
+    if (withRowIndex) {
+      rowIndex = withRowIndex;
+    } else if (rowOffsets) {
+      rowIndex = this.rowIndex.isubset(rowOffsets);
+    }
+
+    let colIndex = null;
+    if (colOffsets) {
+      colIndex = this.colIndex.isubset(colOffsets);
+    }
+
+    return this.__subset(rowIndex, colIndex);
   }
 
   isubsetMask(rowMask, colMask = null, withRowIndex = null) {
@@ -690,7 +668,7 @@ class Dataframe {
     };
     const rowOffsets = toList(rowMask, nRows);
     const colOffsets = toList(colMask, nCols);
-    return this.__subset(rowOffsets, colOffsets, withRowIndex);
+    return this.isubset(rowOffsets, colOffsets, withRowIndex);
   }
 
   /**
@@ -790,7 +768,7 @@ class Dataframe {
     Return true if this is an empty dataframe, ie, has dimensions [0,0]
     */
     const [rows, cols] = this.dims;
-    return rows === 0 && cols === 0;
+    return rows === 0 || cols === 0;
   }
 
   /****

client/src/util/dataframe/index.js

@@ -1,2 +1,7 @@
 export { default as Dataframe } from "./dataframe";
-export { DenseInt32Index, IdentityInt32Index, KeyIndex } from "./labelIndex";
+export {
+  DenseInt32Index,
+  IdentityInt32Index,
+  KeyIndex,
+  isLabelIndex,
+} from "./labelIndex";

client/src/util/dataframe/labelIndex.js

@@ -32,10 +32,10 @@ class IdentityInt32Index {
     this.maxOffset = maxOffset;
   }
 
-  keys() {
+  labels() {
     // memoize
     const k = fillRange(new Int32Array(this.maxOffset));
-    this.keys = function keys() {
+    this.labels = function labels() {
       return k;
     };
     return k;
@@ -47,12 +47,24 @@ class IdentityInt32Index {
     return i;
   }
 
+  // eslint-disable-next-line class-methods-use-this
+  getOffsets(arr) {
+    // labels to offsets
+    return arr;
+  }
+
   // eslint-disable-next-line class-methods-use-this
   getLabel(i) {
     // offset to label
     return i;
   }
 
+  // eslint-disable-next-line class-methods-use-this
+  getLabels(arr) {
+    // offsets to labels
+    return arr;
+  }
+
   size() {
     return this.maxOffset;
   }
@@ -62,6 +74,9 @@ class IdentityInt32Index {
     time/space decision - based on the resulting density
     */
     const [minLabel, maxLabel] = extent(labelArray);
+    if (minLabel === 0 && maxLabel === labelArray.length - 1)
+      return new IdentityInt32Index(labelArray.length);
+
     const labelSpaceSize = maxLabel - minLabel + 1;
     const density = labelSpaceSize / this.maxOffset;
     /* 0.1 is a magic number, that needs testing to optimize */
@@ -71,30 +86,43 @@ class IdentityInt32Index {
     return new DenseInt32Index(labelArray, [minLabel, maxLabel]);
   }
 
-  subsetLabels(labelArray) {
+  subset(labels) {
-    return this.__promote(labelArray);
+    /* validate subset */
+    const { maxOffset } = this;
+    for (let i = 0, l = labels.length; i < l; i += 1) {
+      const label = labels[i];
+      if (label < 0 || label >= maxOffset)
+        throw new RangeError(`offset or label: ${label}`);
+    }
+    return this.__promote(labels);
+  }
+
+  /* identity index - labels are offsets */
+  isubset(offsets) {
+    return this.subset(offsets);
   }
 
   withLabel(label) {
     if (label === this.maxOffset) {
       return new IdentityInt32Index(label + 1);
     }
-    return this.__promote([...this.keys(), label]);
+    return this.__promote([...this.labels(), label]);
   }
 
   withLabels(labels) {
-    return this.__promote([...this.keys(), ...labels]);
+    return this.__promote([...this.labels(), ...labels]);
   }
 
   dropLabel(label) {
     if (label === this.maxOffset - 1) {
       return new IdentityInt32Index(label);
     }
-    const labelArray = [...this.keys()];
+    const labelArray = [...this.labels()];
     labelArray.splice(labelArray.indexOf(label), 1);
     return this.__promote(labelArray);
   }
 }
+
 class DenseInt32Index {
   /*
   DenseInt32Index indexes integer labels, and uses Int32Array typed arrays
@@ -129,12 +157,29 @@ class DenseInt32Index {
     this.getOffset = function getOffset(l) {
       return index[l - minLabel];
     };
+
+    this.getOffsets = function getOffsets(arr) {
+      const res = new arr.constructor(arr.length);
+      for (let i = 0, len = arr.length; i < len; i += 1) {
+        res[i] = index[arr[i] - minLabel];
+      }
+      return res;
+    };
+
     this.getLabel = function getLabel(i) {
       return rindex[i];
     };
+
+    this.getLabels = function getLabels(arr) {
+      const res = new arr.constructor(arr.length);
+      for (let i = 0, len = arr.length; i < len; i += 1) {
+        res[i] = rindex[arr[i]];
+      }
+      return res;
+    };
   }
 
-  keys() {
+  labels() {
     return this.rindex;
   }
 
@@ -158,20 +203,44 @@ class DenseInt32Index {
     return new DenseInt32Index(labelArray, [minLabel, maxLabel]);
   }
 
-  subsetLabels(labelArray) {
+  subset(labels) {
-    return this.__promote(labelArray);
+    /* validate subset */
+    for (let i = 0, l = labels.length; i < l; i += 1) {
+      const label = labels[i];
+      const offset = this.getOffset(label);
+      if (offset === undefined || offset === -1)
+        throw new RangeError(`unknown label: ${label}`);
+    }
+
+    return this.__promote(labels);
+  }
+
+  // eslint-disable-next-line class-methods-use-this
+  isubset(offsets) {
+    /* validate subset */
+    const { rindex } = this;
+    const maxOffset = rindex.length;
+    const labels = new Int32Array(offsets.length);
+    for (let i = 0, l = offsets.length; i < l; i += 1) {
+      const offset = offsets[i];
+      if (offset < 0 || offset >= maxOffset)
+        throw new RangeError(`out of bounds offset: ${offset}`);
+      labels[i] = rindex[offset];
+    }
+
+    return this.__promote(labels);
   }
 
   withLabel(label) {
-    return this.__promote([...this.keys(), label]);
+    return this.__promote([...this.labels(), label]);
   }
 
   withLabels(labels) {
-    return this.__promote([...this.keys(), ...labels]);
+    return this.__promote([...this.labels(), ...labels]);
   }
 
   dropLabel(label) {
-    const labelArray = [...this.keys()];
+    const labelArray = [...this.labels()];
     labelArray.splice(labelArray.indexOf(label), 1);
     return this.__promote(labelArray);
   }
@@ -207,12 +276,29 @@ class KeyIndex {
     this.getOffset = function getOffset(k) {
       return index.get(k);
     };
+
+    this.getOffsets = function getOffsets(arr) {
+      const res = new arr.constructor(arr.length);
+      for (let i = 0, len = arr.length; i < len; i += 1) {
+        res[i] = index.get(arr[i]);
+      }
+      return res;
+    };
+
     this.getLabel = function getLabel(i) {
       return rindex[i];
     };
+
+    this.getLabels = function getLabels(arr) {
+      const res = new arr.constructor(arr.length);
+      for (let i = 0, len = arr.length; i < len; i += 1) {
+        res[i] = rindex[arr[i]];
+      }
+      return res;
+    };
   }
 
-  keys() {
+  labels() {
     return this.rindex;
   }
 
@@ -220,9 +306,30 @@ class KeyIndex {
     return this.rindex.length;
   }
 
+  subset(labels) {
+    /* validate subset */
+    for (let i = 0, l = labels.length; i < l; i += 1) {
+      const label = labels[i];
+      const offset = this.getOffset(label);
+      if (offset === undefined) throw new RangeError(`unknown label: ${label}`);
+    }
+
+    return new KeyIndex(labels);
+  }
+
   // eslint-disable-next-line class-methods-use-this
-  subsetLabels(labelArray) {
+  isubset(offsets) {
-    return new KeyIndex(labelArray);
+    const { rindex } = this;
+    const maxOffset = rindex.length;
+    const labels = new Array(offsets.length);
+    for (let i = 0, l = offsets.length; i < l; i += 1) {
+      const offset = offsets[i];
+      if (offset < 0 || offset >= maxOffset)
+        throw new RangeError(`out of bounds offset: ${offset}`);
+      labels[i] = rindex[offset];
+    }
+
+    return new KeyIndex(labels);
   }
 
   withLabel(label) {

client/src/util/stateManager/annotationsHelpers.js

@@ -100,7 +100,7 @@ export function setLabelByValue(df, colName, fromLabel, toLabel) {
   /* 
 	in the dataframe column `colName`, set any value of `fromLabel` to `toLabel`
 	*/
-  const keys = df.colIndex.keys();
+  const keys = df.colIndex.labels();
   const ndf = df.mapColumns((col, colIdx) => {
     if (colName !== keys[colIdx]) return col;
 
@@ -118,7 +118,7 @@ export function setLabelByMask(df, colName, mask, label) {
   /* 
 	in the dataframe column `colName`, set the masked rows to 'label'
 	*/
-  const keys = df.colIndex.keys();
+  const keys = df.colIndex.labels();
   const ndf = df.mapColumns((col, colIdx) => {
     if (colName !== keys[colIdx]) return col;
 

client/src/util/stateManager/controlsHelpers.js

@@ -187,7 +187,7 @@ export function pruneVarDataCache(varData, needed) {
   if (numOverWatermark <= 0) return varData;
 
   const { colIndex } = varData;
-  const all = colIndex.keys();
+  const all = colIndex.labels();
   const unused = _.difference(all, needed);
   if (unused.length > 0) {
     // sort by offset in the dataframe - ie, psuedo-LRU

client/src/util/stateManager/matrix.js

@@ -1,7 +1,12 @@
 import { flatbuffers } from "flatbuffers";
 import { NetEncoding } from "./matrix_generated";
-import { isTypedArray } from "../typeHelpers";
+import { isTypedArray, isFpTypedArray } from "../typeHelpers";
-import { IdentityInt32Index, DenseInt32Index, KeyIndex } from "../dataframe";
+import {
+  Dataframe,
+  IdentityInt32Index,
+  DenseInt32Index,
+  KeyIndex,
+} from "../dataframe";
 
 const utf8Decoder = new TextDecoder("utf-8");
 
@@ -133,14 +138,14 @@ export function encodeMatrixFBS(df) {
         encColIndex = encodeTypedArray(
           builder,
           encColIndexUType,
-          df.colIndex.keys()
+          df.colIndex.labels()
         );
       } else if (colIndexType === KeyIndex) {
         encColIndexUType = NetEncoding.TypedArray.JSONEncodedArray;
         encColIndex = encodeTypedArray(
           builder,
           encColIndexUType,
-          utf8Encoder.encode(JSON.stringify(df.colIndex.keys()))
+          utf8Encoder.encode(JSON.stringify(df.colIndex.labels()))
         );
       } else {
         throw new Error("Index type FBS encoding unsupported");
@@ -162,3 +167,79 @@ export function encodeMatrixFBS(df) {
   builder.finish(root);
   return builder.asUint8Array();
 }
+
+function promoteTypedArray(o) {
+  /*
+  Decide what internal data type to use for the data returned from 
+  the server.
+
+  TODO - future optimization: not all int32/uint32 data series require
+  promotion to float64.  We COULD simply look at the data to decide. 
+  */
+  if (isFpTypedArray(o) || Array.isArray(o)) return o;
+
+  let TyepdArrayCtor;
+  switch (o.constructor) {
+    case Int8Array:
+    case Uint8Array:
+    case Uint8ClampedArray:
+    case Int16Array:
+    case Uint16Array:
+      TyepdArrayCtor = Float32Array;
+      break;
+
+    case Int32Array:
+    case Uint32Array:
+      TyepdArrayCtor = Float64Array;
+      break;
+
+    default:
+      throw new Error("Unexpected data type returned from server.");
+  }
+  if (o.constructor === TyepdArrayCtor) return o;
+  return new TyepdArrayCtor(o);
+}
+
+export function matrixFBSToDataframe(arrayBuffers) {
+  /*
+  Convert array of Matrix FBS to a Dataframe.
+
+  The application has strong assumptions that all scalar data will be
+  stored as a float32 or float64 (regardless of underlying data types).
+  For example, clipping of value ranges (eg, user-selected percentiles)
+  depends on the ability to use NaN in any numeric type.
+
+  All float data from the server is left as is.  All non-float is promoted
+  to an appropriate float.
+  */
+  if (!Array.isArray(arrayBuffers)) {
+    arrayBuffers = [arrayBuffers];
+  }
+  if (arrayBuffers.length === 0) {
+    return Dataframe.Dataframe.empty();
+  }
+
+  const fbs = arrayBuffers.map((ab) => decodeMatrixFBS(ab, true)); // leave in place
+  /* check that all FBS have same row dimensionality */
+  const { nRows } = fbs[0];
+  fbs.forEach((b) => {
+    if (b.nRows !== nRows)
+      throw new Error("FBS with inconsistent dimensionality");
+  });
+  const columns = fbs
+    .map((fb) =>
+      fb.columns.map((c) => {
+        if (isFpTypedArray(c) || Array.isArray(c)) return c;
+        return promoteTypedArray(c);
+      })
+    )
+    .flat();
+  // colIdx may be TypedArray or Array
+  const colIdx = fbs
+    .map((b) => (Array.isArray(b.colIdx) ? b.colIdx : Array.from(b.colIdx)))
+    .flat();
+  const nCols = columns.length;
+
+  const df = new Dataframe([nRows, nCols], columns, null, new KeyIndex(colIdx));
+  return df;
+}

client/src/util/stateManager/universe.js

@@ -40,84 +40,6 @@ These functions are used exclusively by the actions and reducers to
 build an internal POJO for use by the rendering components.
 */
 
-function promoteTypedArray(o) {
-  /*
-  Decide what internal data type to use for the data returned from 
-  the server.
-
-  TODO - future optimization: not all int32/uint32 data series require
-  promotion to float64.  We COULD simply look at the data to decide. 
-  */
-  if (isFpTypedArray(o) || Array.isArray(o)) return o;
-
-  let TyepdArrayCtor;
-  switch (o.constructor) {
-    case Int8Array:
-    case Uint8Array:
-    case Uint8ClampedArray:
-    case Int16Array:
-    case Uint16Array:
-      TyepdArrayCtor = Float32Array;
-      break;
-
-    case Int32Array:
-    case Uint32Array:
-      TyepdArrayCtor = Float64Array;
-      break;
-
-    default:
-      throw new Error("Unexpected data type returned from server.");
-  }
-  if (o.constructor === TyepdArrayCtor) return o;
-  return new TyepdArrayCtor(o);
-}
-
-export function matrixFBSToDataframe(arrayBuffers) {
-  /*
-  Convert array of Matrix FBS to a Dataframe.
-
-  The application has strong assumptions that all scalar data will be
-  stored as a float32 or float64 (regardless of underlying data types).
-  For example, clipping of value ranges (eg, user-selected percentiles)
-  depends on the ability to use NaN in any numeric type.
-
-  All float data from the server is left as is.  All non-float is promoted
-  to an appropriate float.
-  */
-  if (!Array.isArray(arrayBuffers)) {
-    arrayBuffers = [arrayBuffers];
-  }
-  if (arrayBuffers.length === 0) {
-    return Dataframe.Dataframe.empty();
-  }
-
-  const fbs = arrayBuffers.map((ab) => decodeMatrixFBS(ab, true)); // leave in place
-  /* check that all FBS have same row dimensionality */
-  const { nRows } = fbs[0];
-  fbs.forEach((b) => {
-    if (b.nRows !== nRows)
-      throw new Error("FBS with inconsistent dimensionality");
-  });
-  const columns = fbs
-    .map((fb) =>
-      fb.columns.map((c) => {
-        if (isFpTypedArray(c) || Array.isArray(c)) return c;
-        return promoteTypedArray(c);
-      })
-    )
-    .flat();
-  const colIdx = fbs.map((b) => b.colIdx).flat();
-  const nCols = columns.length;
-
-  const df = new Dataframe.Dataframe(
-    [nRows, nCols],
-    columns,
-    null,
-    new Dataframe.KeyIndex(colIdx)
-  );
-  return df;
-}
-
 export function createUniverseFromResponse(configResponse, schemaResponse) {
   /*
   build & return universe from a REST 0.2 /config, /schema and /annotations/obs response
@@ -178,7 +100,7 @@ export function addObsAnnotations(universe, df) {
 
   // for all of the new data, reconcile with schema and sort categories.
   const dfs = Array.isArray(df) ? df : [df];
-  const keys = dfs.map((d) => d.colIndex.keys()).flat();
+  const keys = dfs.map((d) => d.colIndex.labels()).flat();
   const { schema } = universe;
   keys.forEach((k) => {
     const colSchema = schema.annotations.obsByName[k];

client/src/util/stateManager/world.js

@@ -99,7 +99,7 @@ function clipDataframe(
   if (upperQuantile > 1) upperQuantile = 1;
   if (lowerQuantile === 0 && upperQuantile === 1) return df;
 
-  const keys = df.colIndex.keys();
+  const keys = df.colIndex.labels();
   return df.mapColumns((col, colIdx) => {
     const colLabel = keys[colIdx];
     if (!clipPredicate(df, colIdx, colLabel)) return col;
@@ -277,7 +277,7 @@ export function addObsDimensions(crossfilter, world) {
   but not yet in the crossfilter
   */
   const schema = world.schema.annotations.obsByName;
-  const dimsWeNeed = world.obsAnnotations.colIndex.keys();
+  const dimsWeNeed = world.obsAnnotations.colIndex.labels();
   crossfilter = dimsWeNeed.reduce((xfltr, name) => {
     const dimName = obsAnnoDimensionName(name);
     if (xfltr.hasDimension(dimName)) return xfltr;
@@ -321,7 +321,7 @@ export function getSelectedByIndex(crossfilter) {
   return array of obsIndex, containing all selected obs/cells.
   */
   const selected = crossfilter.allSelectedMask(); // array of bool-ish
-  const keys = crossfilter.data.rowIndex.keys(); // row keys, aka universe rowIndex
+  const keys = crossfilter.data.rowIndex.labels(); // row keys, aka universe rowIndex
 
   const set = new Int32Array(selected.length);
   let numElems = 0;

client/src/util/typedCrossfilter/crossfilter.js

@@ -60,9 +60,7 @@ export default class ImmutableTypedCrossfilter {
   }
 
   setData(data) {
-    const { selectionCache } = this;
+    return new ImmutableTypedCrossfilter(data, this.dimensions);
-    this.selectionCache = {};
-    return new ImmutableTypedCrossfilter(data, this.dimensions, selectionCache);
   }
 
   dimensionNames() {