diff --git a/client/__tests__/util/centroid.test.js b/client/__tests__/util/centroid.test.js index 28467cde..9ed75c3a 100644 --- a/client/__tests__/util/centroid.test.js +++ b/client/__tests__/util/centroid.test.js @@ -4,6 +4,7 @@ import calcCentroid from "../../src/util/centroid"; import quantile from "../../src/util/quantile"; import * as Universe from "../../src/util/stateManager/universe"; +import { matrixFBSToDataframe } from "../../src/util/stateManager/matrix"; import * as World from "../../src/util/stateManager/world"; import * as REST from "./stateManager/sampleResponses"; import { ControlsHelpers as CH } from "../../src/util/stateManager"; @@ -22,16 +23,13 @@ describe("centroid", () => { ...universe, ...Universe.addObsAnnotations( universe, - Universe.matrixFBSToDataframe(REST.annotationsObs) + matrixFBSToDataframe(REST.annotationsObs) ), ...Universe.addVarAnnotations( universe, - Universe.matrixFBSToDataframe(REST.annotationsVar) - ), - ...Universe.addObsLayout( - universe, - Universe.matrixFBSToDataframe(REST.layoutObs) + matrixFBSToDataframe(REST.annotationsVar) ), + ...Universe.addObsLayout(universe, matrixFBSToDataframe(REST.layoutObs)), }; world = World.createWorldFromEntireUniverse(universe); diff --git a/client/__tests__/util/dataframe/dataframe.test.js b/client/__tests__/util/dataframe/dataframe.test.js index 3ad843a9..b15df2e1 100644 --- a/client/__tests__/util/dataframe/dataframe.test.js +++ b/client/__tests__/util/dataframe/dataframe.test.js @@ -38,8 +38,8 @@ describe("dataframe constructor", () => { expect(df.rowIndex).toBeInstanceOf(Dataframe.DenseInt32Index); expect(df.colIndex).toBeInstanceOf(Dataframe.KeyIndex); - expect(df.rowIndex.keys()).toEqual(new Int32Array([2, 1, 0])); - expect(df.colIndex.keys()).toEqual(["A", "B"]); + expect(df.rowIndex.labels()).toEqual(new Int32Array([2, 1, 0])); + expect(df.colIndex.labels()).toEqual(["A", "B"]); expect(df.at(0, "A")).toEqual(2); expect(df.at(2, "B")).toEqual(3); @@ -138,7 +138,7 @@ describe("dataframe subsetting", () => { new Float32Array([4.4, 5.5, 6.6]), ["red", "green", "blue"], ], - null, + null, // identity index new Dataframe.KeyIndex(["int32", "string", "float32", "colors"]) ); @@ -153,12 +153,12 @@ describe("dataframe subsetting", () => { expect(dfA.col("colors").asArray()).toEqual( sourceDf.col("colors").asArray() ); - expect(dfA.rowIndex.keys()).toEqual(sourceDf.rowIndex.keys()); - expect(dfA.colIndex.keys()).toEqual(["colors"]); + expect(dfA.rowIndex.labels()).toEqual(sourceDf.rowIndex.labels()); + expect(dfA.colIndex.labels()).toEqual(["colors"]); }); test("all rows, two columns", () => { - const dfB = sourceDf.subset(null, ["colors", "float32"]); + const dfB = sourceDf.subset(null, ["float32", "colors"]); expect(dfB).toBeDefined(); expect(dfB.dims).toEqual([3, 2]); expect(dfB.iat(0, 0)).toBeCloseTo(4.4); @@ -177,8 +177,8 @@ describe("dataframe subsetting", () => { expect(dfB.col("float32").asArray()).toEqual( sourceDf.col("float32").asArray() ); - expect(dfB.rowIndex.keys()).toEqual(sourceDf.rowIndex.keys()); - expect(dfB.colIndex.keys()).toEqual(["float32", "colors"]); + expect(dfB.rowIndex.labels()).toEqual(sourceDf.rowIndex.labels()); + expect(dfB.colIndex.labels()).toEqual(["float32", "colors"]); }); test("one row, all columns", () => { @@ -189,8 +189,8 @@ describe("dataframe subsetting", () => { expect(dfC.iat(0, 1)).toEqual("B"); expect(dfC.iat(0, 2)).toBeCloseTo(5.5); expect(dfC.iat(0, 3)).toEqual("green"); - expect(dfC.rowIndex.keys()).toEqual(new Int32Array([1])); - expect(dfC.colIndex.keys()).toEqual(sourceDf.colIndex.keys()); + expect(dfC.rowIndex.labels()).toEqual(new Int32Array([1])); + expect(dfC.colIndex.labels()).toEqual(sourceDf.colIndex.labels()); }); test("two rows, all columns", () => { @@ -201,8 +201,17 @@ describe("dataframe subsetting", () => { expect(dfD.icol(1).asArray()).toEqual(["A", "C"]); expect(dfD.icol(2).asArray()).toEqual(new Float32Array([4.4, 6.6])); expect(dfD.icol(3).asArray()).toEqual(["red", "blue"]); - expect(dfD.rowIndex.keys()).toEqual(new Int32Array([0, 2])); - expect(dfD.colIndex.keys()).toEqual(sourceDf.colIndex.keys()); + expect(dfD.rowIndex.labels()).toEqual(new Int32Array([0, 2])); + expect(dfD.colIndex.labels()).toEqual(sourceDf.colIndex.labels()); + + // reverse the row order + const dfDr = sourceDf.subset([2, 0], null); + expect(dfDr.icol(0).asArray()).toEqual(new Int32Array([2, 0])); + expect(dfDr.icol(1).asArray()).toEqual(["C", "A"]); + expect(dfDr.icol(2).asArray()).toEqual(new Float32Array([6.6, 4.4])); + expect(dfDr.icol(3).asArray()).toEqual(["blue", "red"]); + expect(dfDr.rowIndex.labels()).toEqual(new Int32Array([2, 0])); + expect(dfDr.colIndex.labels()).toEqual(sourceDf.colIndex.labels()); }); test("all rows, all columns", () => { @@ -213,8 +222,8 @@ describe("dataframe subsetting", () => { expect(dfE.icol(1).asArray()).toEqual(sourceDf.icol(1).asArray()); expect(dfE.icol(2).asArray()).toEqual(sourceDf.icol(2).asArray()); expect(dfE.icol(3).asArray()).toEqual(sourceDf.icol(3).asArray()); - expect(dfE.rowIndex.keys()).toEqual(sourceDf.rowIndex.keys()); - expect(dfE.colIndex.keys()).toEqual(sourceDf.colIndex.keys()); + expect(dfE.rowIndex.labels()).toEqual(sourceDf.rowIndex.labels()); + expect(dfE.colIndex.labels()).toEqual(sourceDf.colIndex.labels()); }); test("two rows, two colums", () => { @@ -223,8 +232,17 @@ describe("dataframe subsetting", () => { expect(dfF.dims).toEqual([2, 2]); expect(dfF.icol(0).asArray()).toEqual(new Int32Array([0, 2])); expect(dfF.icol(1).asArray()).toEqual(new Float32Array([4.4, 6.6])); - expect(dfF.rowIndex.keys()).toEqual(new Int32Array([0, 2])); - expect(dfF.colIndex.keys()).toEqual(["int32", "float32"]); + expect(dfF.rowIndex.labels()).toEqual(new Int32Array([0, 2])); + expect(dfF.colIndex.labels()).toEqual(["int32", "float32"]); + + // reverse the row and column order + const dfFr = sourceDf.subset([2, 0], ["float32", "int32"]); + expect(dfFr).toBeDefined(); + expect(dfFr.dims).toEqual([2, 2]); + expect(dfFr.icol(0).asArray()).toEqual(new Float32Array([6.6, 4.4])); + expect(dfFr.icol(1).asArray()).toEqual(new Int32Array([2, 0])); + expect(dfFr.rowIndex.labels()).toEqual(new Int32Array([2, 0])); + expect(dfFr.colIndex.labels()).toEqual(["float32", "int32"]); }); test("withRowIndex", () => { @@ -271,8 +289,49 @@ describe("dataframe subsetting", () => { expect(dfA.dims).toEqual([2, 2]); expect(dfA.icol(0).asArray()).toEqual(new Int32Array([1, 2])); expect(dfA.icol(1).asArray()).toEqual(["green", "blue"]); - expect(dfA.rowIndex.keys()).toEqual(new Int32Array([4, 6])); - expect(dfA.colIndex.keys()).toEqual(["int32", "colors"]); + expect(dfA.rowIndex.labels()).toEqual(new Int32Array([4, 6])); + expect(dfA.colIndex.labels()).toEqual(["int32", "colors"]); + }); + + describe("isubset", () => { + const sourceDf = new Dataframe.Dataframe( + [3, 4], + [ + new Int32Array([0, 1, 2]), + ["A", "B", "C"], + new Float32Array([4.4, 5.5, 6.6]), + ["red", "green", "blue"], + ], + null, // identity index + new Dataframe.KeyIndex(["int32", "string", "float32", "colors"]) + ); + + test("one row, all cols", () => { + const dfA = sourceDf.isubset([1], null); + expect(dfA.dims).toEqual([1, 4]); + expect(dfA.icol(0).asArray()).toEqual(new Int32Array([1])); + expect(dfA.icol(1).asArray()).toEqual(["B"]); + expect(dfA.icol(2).asArray()).toEqual(new Float32Array([5.5])); + expect(dfA.icol(3).asArray()).toEqual(["green"]); + }); + + test("all rows, two cols", () => { + const dfA = sourceDf.isubset(null, [1, 2]); + expect(dfA.dims).toEqual([3, 2]); + expect(dfA.icol(0).asArray()).toEqual(["A", "B", "C"]); + expect(dfA.icol(1).asArray()).toEqual(new Float32Array([4.4, 5.5, 6.6])); + expect(dfA.col("string")).toBe(dfA.icol(0)); + expect(dfA.col("float32")).toBe(dfA.icol(1)); + }); + + test("out of order rows", () => { + const dfA = sourceDf.isubset([2, 0], null); + expect(dfA.dims).toEqual([2, 4]); + expect(dfA.icol(0).asArray()).toEqual(new Int32Array([2, 0])); + expect(dfA.icol(1).asArray()).toEqual(["C", "A"]); + expect(dfA.icol(2).asArray()).toEqual(new Float32Array([6.6, 4.4])); + expect(dfA.icol(3).asArray()).toEqual(["blue", "red"]); + }); }); }); @@ -310,8 +369,8 @@ describe("dataframe factories", () => { expect(dfB).not.toBe(dfA); expect(dfB.dims).toEqual(dfA.dims); expect(dfB).toHaveLength(dfA.length); - expect(dfB.rowIndex.keys()).toEqual(dfA.rowIndex.keys()); - expect(dfB.colIndex.keys()).toEqual(dfA.colIndex.keys()); + expect(dfB.rowIndex.labels()).toEqual(dfA.rowIndex.labels()); + expect(dfB.colIndex.labels()).toEqual(dfA.colIndex.labels()); for (let i = 0, l = dfB.dims[1]; i < l; i += 1) { expect(dfB.icol(i).asArray()).toEqual(dfA.icol(i).asArray()); } @@ -336,9 +395,9 @@ describe("dataframe factories", () => { expect(dfA.icol(1).asArray()).toEqual([true, false]); expect(dfA.icol(2).asArray()).toEqual([1, 0]); expect(dfA.col("numbers").asArray()).toEqual([1, 0]); - expect(dfA.colIndex.keys()).toEqual(["colors", "bools", "numbers"]); - expect(df.colIndex.keys()).toEqual(["colors", "bools"]); - expect(df.rowIndex.keys()).toEqual(dfA.rowIndex.keys()); + expect(dfA.colIndex.labels()).toEqual(["colors", "bools", "numbers"]); + expect(df.colIndex.labels()).toEqual(["colors", "bools"]); + expect(df.rowIndex.labels()).toEqual(dfA.rowIndex.labels()); }); test("DenseInt32Index", () => { @@ -361,9 +420,9 @@ describe("dataframe factories", () => { expect(dfA.col(74).asArray()).toEqual(["red", "blue"]); expect(dfA.col(75).asArray()).toEqual([true, false]); expect(dfA.col(72).asArray()).toEqual([1, 0]); - expect(dfA.colIndex.keys()).toEqual(new Int32Array([74, 75, 72])); - expect(df.colIndex.keys()).toEqual(new Int32Array([74, 75])); - expect(df.rowIndex.keys()).toEqual(dfA.rowIndex.keys()); + expect(dfA.colIndex.labels()).toEqual(new Int32Array([74, 75, 72])); + expect(df.colIndex.labels()).toEqual(new Int32Array([74, 75])); + expect(df.rowIndex.labels()).toEqual(dfA.rowIndex.labels()); }); test("DenseInt32Index promote", () => { @@ -386,9 +445,9 @@ describe("dataframe factories", () => { expect(dfA.col(74).asArray()).toEqual(["red", "blue"]); expect(dfA.col(75).asArray()).toEqual([true, false]); expect(dfA.col(999).asArray()).toEqual([1, 0]); - expect(dfA.colIndex.keys()).toEqual(new Int32Array([74, 75, 999])); - expect(df.colIndex.keys()).toEqual(new Int32Array([74, 75])); - expect(df.rowIndex.keys()).toEqual(dfA.rowIndex.keys()); + expect(dfA.colIndex.labels()).toEqual(new Int32Array([74, 75, 999])); + expect(df.colIndex.labels()).toEqual(new Int32Array([74, 75])); + expect(df.rowIndex.labels()).toEqual(dfA.rowIndex.labels()); }); test("IdentityInt32Index with last", () => { @@ -411,9 +470,9 @@ describe("dataframe factories", () => { expect(dfA.col(0).asArray()).toEqual(["red", "blue"]); expect(dfA.col(1).asArray()).toEqual([true, false]); expect(dfA.col(2).asArray()).toEqual([1, 0]); - expect(dfA.colIndex.keys()).toEqual(new Int32Array([0, 1, 2])); - expect(df.colIndex.keys()).toEqual(new Int32Array([0, 1])); - expect(df.rowIndex.keys()).toEqual(dfA.rowIndex.keys()); + expect(dfA.colIndex.labels()).toEqual(new Int32Array([0, 1, 2])); + expect(df.colIndex.labels()).toEqual(new Int32Array([0, 1])); + expect(df.rowIndex.labels()).toEqual(dfA.rowIndex.labels()); }); test("IdentityInt32Index promote", () => { @@ -436,9 +495,9 @@ describe("dataframe factories", () => { expect(dfA.col(0).asArray()).toEqual(["red", "blue"]); expect(dfA.col(1).asArray()).toEqual([true, false]); expect(dfA.col(99).asArray()).toEqual([1, 0]); - expect(dfA.colIndex.keys()).toEqual(new Int32Array([0, 1, 99])); - expect(df.colIndex.keys()).toEqual(new Int32Array([0, 1])); - expect(df.rowIndex.keys()).toEqual(dfA.rowIndex.keys()); + expect(dfA.colIndex.labels()).toEqual(new Int32Array([0, 1, 99])); + expect(df.colIndex.labels()).toEqual(new Int32Array([0, 1])); + expect(df.rowIndex.labels()).toEqual(dfA.rowIndex.labels()); }); describe("handle column dimensions correctly", () => { @@ -517,25 +576,25 @@ describe("dataframe factories", () => { const dfLikeA = dfEmpty.withColsFrom(dfA); expect(dfLikeA).toBeDefined(); expect(dfLikeA.dims).toEqual(dfA.dims); - expect(dfLikeA.colIndex.keys()).toEqual(dfA.colIndex.keys()); + expect(dfLikeA.colIndex.labels()).toEqual(dfA.colIndex.labels()); expect(dfLikeA.rowIndex).toEqual(dfA.rowIndex); - expect(dfLikeA.rowIndex.keys()).toEqual(dfA.rowIndex.keys()); + expect(dfLikeA.rowIndex.labels()).toEqual(dfA.rowIndex.labels()); expect(dfLikeA.icol(0).asArray()).toEqual(dfA.icol(0).asArray()); const dfAlsoLikeA = dfA.withColsFrom(dfEmpty); expect(dfAlsoLikeA).toBeDefined(); expect(dfAlsoLikeA.dims).toEqual(dfA.dims); - expect(dfAlsoLikeA.colIndex.keys()).toEqual(dfA.colIndex.keys()); + expect(dfAlsoLikeA.colIndex.labels()).toEqual(dfA.colIndex.labels()); expect(dfAlsoLikeA.rowIndex).toEqual(dfA.rowIndex); - expect(dfAlsoLikeA.rowIndex.keys()).toEqual(dfA.rowIndex.keys()); + expect(dfAlsoLikeA.rowIndex.labels()).toEqual(dfA.rowIndex.labels()); expect(dfAlsoLikeA.icol(0).asArray()).toEqual(dfA.icol(0).asArray()); const dfC = dfA.withColsFrom(dfB); expect(dfC).toBeDefined(); expect(dfC.dims).toEqual([2, 2]); - expect(dfC.colIndex.keys()).toEqual(["colors", "bools"]); + expect(dfC.colIndex.labels()).toEqual(["colors", "bools"]); expect(dfC.rowIndex).toEqual(dfA.rowIndex); - expect(dfC.rowIndex.keys()).toEqual(dfA.rowIndex.keys()); + expect(dfC.rowIndex.labels()).toEqual(dfA.rowIndex.labels()); expect(dfC.col("colors").asArray()).toEqual(["red", "blue"]); expect(dfC.col("bools").asArray()).toEqual([true, false]); }); @@ -562,21 +621,21 @@ describe("dataframe factories", () => { const dfX = dfEmpty.withColsFrom(dfB, ["colors", "bools"]); expect(dfX).toBeDefined(); expect(dfX.dims).toEqual([2, 2]); - expect(dfX.colIndex.keys()).toEqual(["colors", "bools"]); + expect(dfX.colIndex.labels()).toEqual(["colors", "bools"]); expect(dfX.rowIndex).toEqual(dfB.rowIndex); expect(dfX.icol(0).asArray()).toEqual(dfB.icol(0).asArray()); const dfY = dfA.withColsFrom(dfB, ["numbers"]); expect(dfY).toBeDefined(); expect(dfY.dims).toEqual([2, 2]); - expect(dfY.colIndex.keys()).toEqual(["colors", "numbers"]); + expect(dfY.colIndex.labels()).toEqual(["colors", "numbers"]); expect(dfY.rowIndex).toEqual(dfA.rowIndex); expect(dfY.icol(0).asArray()).toEqual(dfA.icol(0).asArray()); const dfZ = dfA.withColsFrom(dfEmpty, []); expect(dfZ).toBeDefined(); expect(dfZ.dims).toEqual(dfA.dims); - expect(dfZ.colIndex.keys()).toEqual(dfA.colIndex.keys()); + expect(dfZ.colIndex.labels()).toEqual(dfA.colIndex.labels()); expect(dfZ.rowIndex).toEqual(dfA.rowIndex); expect(dfZ.icol(0).asArray()).toEqual(dfA.icol(0).asArray()); @@ -604,7 +663,7 @@ describe("dataframe factories", () => { const dfX = dfA.withColsFrom(dfB, { colors: "_colors", bools: "_bools" }); expect(dfX).toBeDefined(); expect(dfX.dims).toEqual([2, 3]); - expect(dfX.colIndex.keys()).toEqual(["colors", "_colors", "_bools"]); + expect(dfX.colIndex.labels()).toEqual(["colors", "_colors", "_bools"]); expect(dfX.rowIndex).toEqual(dfA.rowIndex); expect(dfX.icol(0).asArray()).toEqual(dfA.icol(0).asArray()); expect(dfX.col("_colors").asArray()).toBe(dfB.col("colors").asArray()); @@ -630,9 +689,9 @@ describe("dataframe factories", () => { expect(dfA.icol(0).asArray()).toEqual([true, false]); expect(dfA.icol(1).asArray()).toEqual([1, 0]); expect(dfA.col("numbers").asArray()).toEqual([1, 0]); - expect(dfA.colIndex.keys()).toEqual(["bools", "numbers"]); - expect(df.colIndex.keys()).toEqual(["colors", "bools", "numbers"]); - expect(df.rowIndex.keys()).toEqual(dfA.rowIndex.keys()); + expect(dfA.colIndex.labels()).toEqual(["bools", "numbers"]); + expect(df.colIndex.labels()).toEqual(["colors", "bools", "numbers"]); + expect(df.rowIndex.labels()).toEqual(dfA.rowIndex.labels()); }); test("IdentityInt32Index drop first", () => { @@ -654,9 +713,9 @@ describe("dataframe factories", () => { expect(dfA.icol(1).asArray()).toEqual([1, 0]); expect(df.col(1).asArray()).toEqual(dfA.col(1).asArray()); expect(df.col(2).asArray()).toEqual(dfA.col(2).asArray()); - expect(dfA.colIndex.keys()).toEqual(new Int32Array([1, 2])); - expect(df.colIndex.keys()).toEqual(new Int32Array([0, 1, 2])); - expect(df.rowIndex.keys()).toEqual(dfA.rowIndex.keys()); + expect(dfA.colIndex.labels()).toEqual(new Int32Array([1, 2])); + expect(df.colIndex.labels()).toEqual(new Int32Array([0, 1, 2])); + expect(df.rowIndex.labels()).toEqual(dfA.rowIndex.labels()); }); test("IdentityInt32Index drop last", () => { @@ -678,9 +737,9 @@ describe("dataframe factories", () => { expect(dfA.icol(1).asArray()).toEqual([true, false]); expect(df.col(0).asArray()).toEqual(dfA.col(0).asArray()); expect(df.col(1).asArray()).toEqual(dfA.col(1).asArray()); - expect(dfA.colIndex.keys()).toEqual(new Int32Array([0, 1])); - expect(df.colIndex.keys()).toEqual(new Int32Array([0, 1, 2])); - expect(df.rowIndex.keys()).toEqual(dfA.rowIndex.keys()); + expect(dfA.colIndex.labels()).toEqual(new Int32Array([0, 1])); + expect(df.colIndex.labels()).toEqual(new Int32Array([0, 1, 2])); + expect(df.rowIndex.labels()).toEqual(dfA.rowIndex.labels()); }); test("DenseInt32Index", () => { @@ -702,9 +761,9 @@ describe("dataframe factories", () => { expect(dfA.icol(1).asArray()).toEqual([1, 0]); expect(dfA.col(100).asArray()).toEqual([1, 0]); expect(dfA.col(102).asArray()).toEqual(["red", "blue"]); - expect(dfA.colIndex.keys()).toEqual(new Int32Array([102, 100])); - expect(df.colIndex.keys()).toEqual(new Int32Array([102, 101, 100])); - expect(df.rowIndex.keys()).toEqual(dfA.rowIndex.keys()); + expect(dfA.colIndex.labels()).toEqual(new Int32Array([102, 100])); + expect(df.colIndex.labels()).toEqual(new Int32Array([102, 101, 100])); + expect(df.rowIndex.labels()).toEqual(dfA.rowIndex.labels()); }); }); @@ -766,8 +825,8 @@ describe("dataframe factories", () => { new Dataframe.KeyIndex(["A", "B"]) ); const dfB = dfA.renameCol("B", "C"); - expect(dfA.colIndex.keys()).toEqual(["A", "B"]); - expect(dfB.colIndex.keys()).toEqual(["A", "C"]); + expect(dfA.colIndex.labels()).toEqual(["A", "B"]); + expect(dfB.colIndex.labels()).toEqual(["A", "C"]); expect(dfA.dims).toMatchObject(dfB.dims); expect(dfA.columns()).toMatchObject(dfB.columns()); }); @@ -857,3 +916,84 @@ describe("dataframe col", () => { expect(df.col("B").indexOf(true)).toBeUndefined(); }); }); + +describe("label indexing", () => { + + test("IdentityInt32Index", () => { + const idx = new Dataframe.IdentityInt32Index(12); // [0, 12) + + expect(Dataframe.isLabelIndex(idx)).toBeTruthy(); + + expect(idx.labels()).toEqual(new Int32Array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11])); + expect(idx.getLabel(1)).toEqual(1); + expect(idx.getOffset(1)).toEqual(1); + expect(idx.getOffsets([1,3])).toEqual([1,3]) + expect(idx.getLabels([1, 3])).toEqual([1,3]) + expect(idx.size()).toEqual(12); + + expect(idx.subset([2]).labels()).toEqual([2]); + expect(idx.subset([2, 3, 4]).labels()).toEqual(new Int32Array([2, 3, 4])); + expect(idx.subset([0, 1, 2, 3]).labels()).toEqual(new Int32Array([0, 1, 2, 3])); + + expect(idx.isubset([2]).labels()).toEqual([2]); + expect(idx.isubset([2, 3, 4]).labels()).toEqual(new Int32Array([2, 3, 4])); + expect(idx.isubset([0, 1, 2, 3]).labels()).toEqual(new Int32Array([0, 1, 2, 3])); + + expect(idx.subset([0, 1, 2, 3, 4])).toBeInstanceOf(Dataframe.IdentityInt32Index); + expect(idx.subset([2, 1, 0])).toBeInstanceOf(Dataframe.IdentityInt32Index); + expect(idx.subset([1, 2, 3, 4])).toBeInstanceOf(Dataframe.DenseInt32Index); + expect(idx.subset([0, 1, 3, 4])).toBeInstanceOf(Dataframe.DenseInt32Index); + expect(idx.subset([0, 1, 2, 3, 10])).toBeInstanceOf(Dataframe.DenseInt32Index); + expect(idx.subset([4, 3, 2, 1])).toBeInstanceOf(Dataframe.DenseInt32Index); + expect(idx.subset([4])).toBeInstanceOf(Dataframe.KeyIndex); + + expect(idx.withLabel(99).labels()).toEqual(new Int32Array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 99])); + expect(idx.dropLabel(0).labels()).toEqual(new Int32Array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11])); + expect(idx.dropLabel(11).labels()).toEqual(new Int32Array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10])); + expect(idx.dropLabel(5).labels()).toEqual(new Int32Array([0, 1, 2, 3, 4, 6, 7, 8, 9, 10, 11])); + }); + + test("DenseInt32Index", () => { + + const idx = new Dataframe.DenseInt32Index([99, 1002, 48, 0, 22]); + + expect(Dataframe.isLabelIndex(idx)).toBeTruthy(); + + expect(idx.labels()).toEqual(new Int32Array([99, 1002, 48, 0, 22])); + expect(idx.size()).toEqual(5); + expect(idx.getOffset(1002)).toEqual(1); + expect(idx.getOffset(0)).toEqual(3); + expect(idx.getLabel(0)).toEqual(99); + expect(idx.getLabels(new Int32Array([2, 4]))).toEqual(new Int32Array([48, 22])); + expect(idx.getLabels([2, 4])).toEqual([48, 22]); + expect(idx.getOffsets([0, 48])).toEqual([3, 2]); + + expect(idx.subset([1002, 0, 99]).labels()).toEqual(new Int32Array([1002, 0, 99])) + expect(idx.getOffsets(idx.subset([1002, 0, 99]).labels())).toEqual(new Int32Array([1, 3, 0])); + expect(idx.isubset([4, 1, 2]).labels()).toEqual(new Int32Array([22, 1002, 48])); + + expect(idx.withLabel(88).labels()).toEqual(new Int32Array([99, 1002, 48, 0, 22, 88])); + expect(idx.withLabel(88).getOffset(88)).toEqual(5); + expect(idx.dropLabel(48).labels()).toEqual(new Int32Array([99, 1002, 0, 22])); + }); + + test("KeyIndex", () => { + const idx = new Dataframe.KeyIndex(["red", "green", "blue"]); + + expect(Dataframe.isLabelIndex(idx)).toBeTruthy(); + + expect(idx.labels()).toEqual(["red", "green", "blue"]); + expect(idx.size()).toEqual(3); + expect(idx.getOffset("blue")).toEqual(2); + expect(idx.getLabel(1)).toEqual("green"); + + expect(idx.subset(["green"]).labels()).toEqual(["green"]); + expect(idx.subset(["green", "red"]).labels()).toEqual(["green", "red"]); + expect(idx.isubset([2, 1, 0]).labels()).toEqual(["blue", "green", "red"]); + + expect(idx.withLabel("yo").labels()).toEqual(["red", "green", "blue", "yo"]); + expect(idx.withLabel("yo").getOffset("yo")).toEqual(3); + expect(idx.dropLabel("blue").labels()).toEqual(["red", "green"]); +}); + +}) \ No newline at end of file diff --git a/client/__tests__/util/stateManager/fbs.test.js b/client/__tests__/util/stateManager/fbs.test.js index c7c639e7..b8bd3742 100644 --- a/client/__tests__/util/stateManager/fbs.test.js +++ b/client/__tests__/util/stateManager/fbs.test.js @@ -27,7 +27,7 @@ describe("encode/decode", () => { const dfWithColIdx = new Dataframe([3, 4], columns, null, colIndex); const dfB = decodeMatrixFBS(encodeMatrixFBS(dfWithColIdx)); expect([dfB.nRows, dfB.nCols]).toEqual(dfWithColIdx.dims); - expect(dfB.colIdx).toEqual(colIndex.keys()); + expect(dfB.colIdx).toEqual(colIndex.labels()); expect(dfB.rowIdx).toBeNull(); expect(dfB.columns).toEqual(columns); }); diff --git a/client/__tests__/util/stateManager/universe.test.js b/client/__tests__/util/stateManager/universe.test.js index 76964723..bf89058c 100644 --- a/client/__tests__/util/stateManager/universe.test.js +++ b/client/__tests__/util/stateManager/universe.test.js @@ -1,4 +1,5 @@ import * as Universe from "../../../src/util/stateManager/universe"; +import { matrixFBSToDataframe } from "../../../src/util/stateManager/matrix"; import * as Dataframe from "../../../src/util/dataframe"; import * as REST from "./sampleResponses"; @@ -51,16 +52,13 @@ describe("createUniverseFromResponse", () => { ...universe, ...Universe.addObsAnnotations( universe, - Universe.matrixFBSToDataframe(REST.annotationsObs) + matrixFBSToDataframe(REST.annotationsObs) ), ...Universe.addVarAnnotations( universe, - Universe.matrixFBSToDataframe(REST.annotationsVar) - ), - ...Universe.addObsLayout( - universe, - Universe.matrixFBSToDataframe(REST.layoutObs) + matrixFBSToDataframe(REST.annotationsVar) ), + ...Universe.addObsLayout(universe, matrixFBSToDataframe(REST.layoutObs)), }; expect(universe).toMatchObject( @@ -80,7 +78,7 @@ describe("createUniverseFromResponse", () => { REST.schema.schema.annotations.obs.columns.length, ]); expect(universe.obsLayout.dims).toEqual([nObs, 2]); - expect(universe.obsLayout.colIndex.keys()).toEqual( + expect(universe.obsLayout.colIndex.labels()).toEqual( universe.schema.layout.obs[0].dims ); expect(universe.varAnnotations.dims).toEqual([ diff --git a/client/__tests__/util/stateManager/world.test.js b/client/__tests__/util/stateManager/world.test.js index 6b6a5bfb..ec6adc76 100644 --- a/client/__tests__/util/stateManager/world.test.js +++ b/client/__tests__/util/stateManager/world.test.js @@ -1,5 +1,6 @@ import _ from "lodash"; import * as Universe from "../../../src/util/stateManager/universe"; +import { matrixFBSToDataframe } from "../../../src/util/stateManager/matrix"; import * as World from "../../../src/util/stateManager/world"; import * as Dataframe from "../../../src/util/dataframe"; import Crossfilter from "../../../src/util/typedCrossfilter"; @@ -26,16 +27,13 @@ const defaultBigBang = () => { ...universe, ...Universe.addObsAnnotations( universe, - Universe.matrixFBSToDataframe(REST.annotationsObs) + matrixFBSToDataframe(REST.annotationsObs) ), ...Universe.addVarAnnotations( universe, - Universe.matrixFBSToDataframe(REST.annotationsVar) - ), - ...Universe.addObsLayout( - universe, - Universe.matrixFBSToDataframe(REST.layoutObs) + matrixFBSToDataframe(REST.annotationsVar) ), + ...Universe.addObsLayout(universe, matrixFBSToDataframe(REST.layoutObs)), }; /* create world */ @@ -59,9 +57,9 @@ describe("createWorldFromEntireUniverse", () => { const universe = Universe.createUniverseFromResponse( _.cloneDeep(REST.config), _.cloneDeep(REST.schema), - Universe.matrixFBSToDataframe(_.cloneDeep(REST.annotationsObs)), - Universe.matrixFBSToDataframe(_.cloneDeep(REST.annotationsVar)), - Universe.matrixFBSToDataframe(_.cloneDeep(REST.layoutObs)) + matrixFBSToDataframe(_.cloneDeep(REST.annotationsObs)), + matrixFBSToDataframe(_.cloneDeep(REST.annotationsVar)), + matrixFBSToDataframe(_.cloneDeep(REST.layoutObs)) ); expect(universe).toBeDefined(); @@ -144,16 +142,16 @@ describe("createWorldFromCurrentSelection", () => { }) ); - expect(world.obsAnnotations.rowIndex.keys()).toEqual( + expect(world.obsAnnotations.rowIndex.labels()).toEqual( new Int32Array(matchingIndices) ); - expect(world.obsAnnotations.colIndex.keys()).toEqual( - universe.obsAnnotations.colIndex.keys() + expect(world.obsAnnotations.colIndex.labels()).toEqual( + universe.obsAnnotations.colIndex.labels() ); - expect(world.obsLayout.rowIndex.keys()).toEqual( + expect(world.obsLayout.rowIndex.labels()).toEqual( new Int32Array(matchingIndices) ); - expect(world.obsLayout.colIndex.keys()).toEqual( + expect(world.obsLayout.colIndex.labels()).toEqual( world.schema.layout.obs[0].dims ); }); diff --git a/client/src/actions/index.js b/client/src/actions/index.js index 0bfffd51..ada975d3 100644 --- a/client/src/actions/index.js +++ b/client/src/actions/index.js @@ -29,7 +29,7 @@ async function obsAnnotationFetchAndLoad(dispatch, schema) { fetchBinary( `annotations/obs?annotation-name=${encodeURIComponent(col.name)}` ) - .then((buffer) => Universe.matrixFBSToDataframe(buffer)) + .then((buffer) => MatrixFBS.matrixFBSToDataframe(buffer)) .then((df) => dispatch({ type: "universe: column load success", @@ -52,7 +52,7 @@ async function varAnnotationFetchAndLoad(dispatch, schema) { return Promise.all( names.map((name) => fetchBinary(`annotations/var?annotation-name=${encodeURIComponent(name)}`) - .then((buffer) => Universe.matrixFBSToDataframe(buffer)) + .then((buffer) => MatrixFBS.matrixFBSToDataframe(buffer)) .then((df) => dispatch({ type: "universe: column load success", @@ -77,7 +77,7 @@ function layoutFetchAndLoad(dispatch, schema) { plimit.add(() => fetchBinary( `layout/obs?layout-name=${encodeURIComponent(e)}` - ).then((buffer) => Universe.matrixFBSToDataframe(buffer)) + ).then((buffer) => MatrixFBS.matrixFBSToDataframe(buffer)) ) ) ).then((dfs) => diff --git a/client/src/actions/reembed.js b/client/src/actions/reembed.js index f337d70d..7d73337e 100644 --- a/client/src/actions/reembed.js +++ b/client/src/actions/reembed.js @@ -1,5 +1,5 @@ import { API } from "../globals"; -import { Universe } from "../util/stateManager"; +import { MatrixFBS } from "../util/stateManager"; import { postNetworkErrorToast, postAsyncSuccessToast, @@ -24,7 +24,7 @@ function abortableFetch(request, opts, timeout = 0) { async function doReembedFetch(dispatch, getState) { const state = getState(); - let cells = state.world.obsAnnotations.rowIndex.keys(); + let cells = state.world.obsAnnotations.rowIndex.labels(); // These lines ensure that we convert any TypedArray to an Array. // This is necessary because JSON.stringify() does some very strange @@ -80,7 +80,7 @@ export function requestReembed() { const res = await doReembedFetch(dispatch, getState); const schema = JSON.parse(res.headers.get("CxG-Schema")); const buffer = await res.arrayBuffer(); - const df = Universe.matrixFBSToDataframe(buffer); + const df = MatrixFBS.matrixFBSToDataframe(buffer); dispatch({ type: "reembed: request completed", }); diff --git a/client/src/reducers/categoricalSelection.js b/client/src/reducers/categoricalSelection.js index 39008666..3f3589f2 100644 --- a/client/src/reducers/categoricalSelection.js +++ b/client/src/reducers/categoricalSelection.js @@ -31,7 +31,7 @@ const CategoricalSelection = ( const names = CH.selectableCategoryNames( world.schema, CH.maxCategoryItems(prevSharedState.config), - dataframe.colIndex.keys() + dataframe.colIndex.labels() ); if (names.length === 0) return state; return { diff --git a/client/src/reducers/world.js b/client/src/reducers/world.js index 6a62df26..b1f2e410 100644 --- a/client/src/reducers/world.js +++ b/client/src/reducers/world.js @@ -93,7 +93,7 @@ const WorldReducer = ( let worldValSlice = val; if (!World.worldEqUniverse(state, universe)) { worldValSlice = universeVarData - .subset(state.obsAnnotations.rowIndex.keys(), [key], null) + .subset(state.obsAnnotations.rowIndex.labels(), [key], null) .icol(0) .asArray(); } @@ -129,10 +129,10 @@ const WorldReducer = ( // let clippedVarData = state.varData; const keysToDrop = clippedVarData.colIndex - .keys() + .labels() .filter((k) => !unclippedVarData.hasCol(k)); const keysToAdd = unclippedVarData.colIndex - .keys() + .labels() .filter((k) => !clippedVarData.hasCol(k)); keysToDrop.forEach((k) => { clippedVarData = clippedVarData.dropCol(k); @@ -171,7 +171,7 @@ const WorldReducer = ( let newAnnotation = null; if (!World.worldEqUniverse(state, universe)) { newAnnotation = universe.obsAnnotations - .subset(state.obsAnnotations.rowIndex.keys(), [name], null) + .subset(state.obsAnnotations.rowIndex.labels(), [name], null) .icol(0) .asArray(); } else { @@ -303,7 +303,7 @@ const WorldReducer = ( let schema = origSchema; // alias the names the server sent us, in case they were not the same as the schema - const embedingLabels = embedding.colIndex.keys(); + const embedingLabels = embedding.colIndex.labels(); const labels = { [embedingLabels[0]]: dims[0], [embedingLabels[1]]: dims[1], diff --git a/client/src/util/dataframe/dataframe.js b/client/src/util/dataframe/dataframe.js index 162cfa6e..85a01e6d 100644 --- a/client/src/util/dataframe/dataframe.js +++ b/client/src/util/dataframe/dataframe.js @@ -1,6 +1,5 @@ import { IdentityInt32Index, isLabelIndex } from "./labelIndex"; // weird cross-dependency that we should clean up someday... -import { sortArray } from "../typedCrossfilter/sort"; import { isTypedArray, isArrayOrTypedArray, @@ -389,7 +388,7 @@ class Dataframe { let dstLabels; if (!labels) { // combine all columns - dstLabels = dataframe.colIndex.keys(); + dstLabels = dataframe.colIndex.labels(); srcLabels = dstLabels; } else if (Array.isArray(labels)) { // combine subset of keys with no aliasing @@ -537,7 +536,12 @@ class Dataframe { } static empty(rowIndex = null, colIndex = null) { - return new Dataframe([0, 0], [], rowIndex, colIndex); + const dims = [ + rowIndex ? rowIndex.size() : 0, + colIndex ? colIndex.size() : 0, + ]; + if (dims[0] && dims[1]) throw new Error("not an empty dataframe"); + return new Dataframe(dims, new Array(dims[1]), rowIndex, colIndex); } static create(dims, columnarData) { @@ -551,97 +555,59 @@ class Dataframe { return new Dataframe(dims, columnarData, null, null); } - __subset(rowOffsets, colOffsets, withRowIndex) { + __subset(newRowIndex, newColIndex) { const dims = [...this.dims]; - const getSortedLabelAndOffsets = (offsets, index) => { - /* - Given offsets, return both offsets and associated lables, - sorted by offset. - */ - if (!offsets) { - return [null, null]; + /* subset columns */ + let { __columns, colIndex } = this; + if (newColIndex) { + const colOffsets = this.colIndex.getOffsets(newColIndex.labels()); + __columns = new Array(colOffsets.length); + for (let i = 0, l = colOffsets.length; i < l; i += 1) { + __columns[i] = this.__columns[colOffsets[i]]; } - const sortedOffsets = sortArray(offsets); - const sortedLabels = new Array(sortedOffsets.length); - for (let i = 0, l = sortedOffsets.length; i < l; i += 1) { - sortedLabels[i] = index.getLabel(sortedOffsets[i]); - } - return [sortedLabels, sortedOffsets]; - }; - - let { colIndex } = this; - if (colOffsets) { - let colLabels; - [colLabels, colOffsets] = getSortedLabelAndOffsets( - colOffsets, - this.colIndex - ); + colIndex = newColIndex; dims[1] = colOffsets.length; - colIndex = this.colIndex.subsetLabels(colLabels); } let { rowIndex } = this; - if (withRowIndex) rowIndex = withRowIndex; - if (rowOffsets) { - let rowLabels; - [rowLabels, rowOffsets] = getSortedLabelAndOffsets( - rowOffsets, - this.rowIndex - ); - dims[0] = rowLabels.length; - if (!withRowIndex) rowIndex = this.rowIndex.subsetLabels(rowLabels); - } - - /* subset columns */ - let columns = this.__columns; - if (colOffsets) { - columns = new Array(colOffsets.length); - for (let i = 0, l = colOffsets.length; i < l; i += 1) { - columns[i] = this.__columns[colOffsets[i]]; - } - } - - /* subset rows */ - if (rowOffsets) { - columns = columns.map((col) => { + if (newRowIndex) { + const rowOffsets = this.rowIndex.getOffsets(newRowIndex.labels()); + __columns = __columns.map((col) => { const newCol = new col.constructor(rowOffsets.length); for (let i = 0, l = rowOffsets.length; i < l; i += 1) { newCol[i] = col[rowOffsets[i]]; } return newCol; }); + rowIndex = newRowIndex; + dims[0] = rowOffsets.length; } if (dims[0] === 0 || dims[1] === 0) return Dataframe.empty(); - return new Dataframe(dims, columns, rowIndex, colIndex); + return new Dataframe(dims, __columns, rowIndex, colIndex); } subset(rowLabels, colLabels = null, withRowIndex = null) { /* Subset by row/col labels. - withRowIndex allows assignment of new row index during subset operation. - If withRowIndex === null, it will reset the index to identity (offset) - indexing. if withRowIndex is a label index object, it will be used - for the new dataframe. + withRowIndex allows subset with an index, rather than rowLabels. + If withRowIndex is specified, rowLabels is ignored. */ - const toOffsets = (labels, index) => { - if (!labels) { - return null; - } - return labels.map((label) => { - const off = index.getOffset(label); - if (off === undefined) { - throw new RangeError(`unknown label: ${label}`); - } - return off; - }); - }; + let rowIndex = null; + if (withRowIndex) { + rowIndex = withRowIndex; + } else if (rowLabels) { + rowIndex = this.rowIndex.subset(rowLabels); + } - const rowOffsets = toOffsets(rowLabels, this.rowIndex); - const colOffsets = toOffsets(colLabels, this.colIndex); - return this.__subset(rowOffsets, colOffsets, withRowIndex); + let colIndex = null; + if (colLabels) { + colIndex = this.colIndex.subset(colLabels); + } + + return this.__subset(rowIndex, colIndex); } isubset(rowOffsets, colOffsets = null, withRowIndex = null) { @@ -653,7 +619,19 @@ class Dataframe { indexing. If withRowIndex is a label index object, it will be used for the new dataframe. */ - return this.__subset(rowOffsets, colOffsets, withRowIndex); + let rowIndex = null; + if (withRowIndex) { + rowIndex = withRowIndex; + } else if (rowOffsets) { + rowIndex = this.rowIndex.isubset(rowOffsets); + } + + let colIndex = null; + if (colOffsets) { + colIndex = this.colIndex.isubset(colOffsets); + } + + return this.__subset(rowIndex, colIndex); } isubsetMask(rowMask, colMask = null, withRowIndex = null) { @@ -690,7 +668,7 @@ class Dataframe { }; const rowOffsets = toList(rowMask, nRows); const colOffsets = toList(colMask, nCols); - return this.__subset(rowOffsets, colOffsets, withRowIndex); + return this.isubset(rowOffsets, colOffsets, withRowIndex); } /** @@ -790,7 +768,7 @@ class Dataframe { Return true if this is an empty dataframe, ie, has dimensions [0,0] */ const [rows, cols] = this.dims; - return rows === 0 && cols === 0; + return rows === 0 || cols === 0; } /**** diff --git a/client/src/util/dataframe/index.js b/client/src/util/dataframe/index.js index 68ccba29..15d95c1b 100644 --- a/client/src/util/dataframe/index.js +++ b/client/src/util/dataframe/index.js @@ -1,2 +1,7 @@ export { default as Dataframe } from "./dataframe"; -export { DenseInt32Index, IdentityInt32Index, KeyIndex } from "./labelIndex"; +export { + DenseInt32Index, + IdentityInt32Index, + KeyIndex, + isLabelIndex, +} from "./labelIndex"; diff --git a/client/src/util/dataframe/labelIndex.js b/client/src/util/dataframe/labelIndex.js index 79f69802..c3eae9ed 100644 --- a/client/src/util/dataframe/labelIndex.js +++ b/client/src/util/dataframe/labelIndex.js @@ -32,10 +32,10 @@ class IdentityInt32Index { this.maxOffset = maxOffset; } - keys() { + labels() { // memoize const k = fillRange(new Int32Array(this.maxOffset)); - this.keys = function keys() { + this.labels = function labels() { return k; }; return k; @@ -47,12 +47,24 @@ class IdentityInt32Index { return i; } + // eslint-disable-next-line class-methods-use-this + getOffsets(arr) { + // labels to offsets + return arr; + } + // eslint-disable-next-line class-methods-use-this getLabel(i) { // offset to label return i; } + // eslint-disable-next-line class-methods-use-this + getLabels(arr) { + // offsets to labels + return arr; + } + size() { return this.maxOffset; } @@ -62,6 +74,9 @@ class IdentityInt32Index { time/space decision - based on the resulting density */ const [minLabel, maxLabel] = extent(labelArray); + if (minLabel === 0 && maxLabel === labelArray.length - 1) + return new IdentityInt32Index(labelArray.length); + const labelSpaceSize = maxLabel - minLabel + 1; const density = labelSpaceSize / this.maxOffset; /* 0.1 is a magic number, that needs testing to optimize */ @@ -71,30 +86,43 @@ class IdentityInt32Index { return new DenseInt32Index(labelArray, [minLabel, maxLabel]); } - subsetLabels(labelArray) { - return this.__promote(labelArray); + subset(labels) { + /* validate subset */ + const { maxOffset } = this; + for (let i = 0, l = labels.length; i < l; i += 1) { + const label = labels[i]; + if (label < 0 || label >= maxOffset) + throw new RangeError(`offset or label: ${label}`); + } + return this.__promote(labels); + } + + /* identity index - labels are offsets */ + isubset(offsets) { + return this.subset(offsets); } withLabel(label) { if (label === this.maxOffset) { return new IdentityInt32Index(label + 1); } - return this.__promote([...this.keys(), label]); + return this.__promote([...this.labels(), label]); } withLabels(labels) { - return this.__promote([...this.keys(), ...labels]); + return this.__promote([...this.labels(), ...labels]); } dropLabel(label) { if (label === this.maxOffset - 1) { return new IdentityInt32Index(label); } - const labelArray = [...this.keys()]; + const labelArray = [...this.labels()]; labelArray.splice(labelArray.indexOf(label), 1); return this.__promote(labelArray); } } + class DenseInt32Index { /* DenseInt32Index indexes integer labels, and uses Int32Array typed arrays @@ -129,12 +157,29 @@ class DenseInt32Index { this.getOffset = function getOffset(l) { return index[l - minLabel]; }; + + this.getOffsets = function getOffsets(arr) { + const res = new arr.constructor(arr.length); + for (let i = 0, len = arr.length; i < len; i += 1) { + res[i] = index[arr[i] - minLabel]; + } + return res; + }; + this.getLabel = function getLabel(i) { return rindex[i]; }; + + this.getLabels = function getLabels(arr) { + const res = new arr.constructor(arr.length); + for (let i = 0, len = arr.length; i < len; i += 1) { + res[i] = rindex[arr[i]]; + } + return res; + }; } - keys() { + labels() { return this.rindex; } @@ -158,20 +203,44 @@ class DenseInt32Index { return new DenseInt32Index(labelArray, [minLabel, maxLabel]); } - subsetLabels(labelArray) { - return this.__promote(labelArray); + subset(labels) { + /* validate subset */ + for (let i = 0, l = labels.length; i < l; i += 1) { + const label = labels[i]; + const offset = this.getOffset(label); + if (offset === undefined || offset === -1) + throw new RangeError(`unknown label: ${label}`); + } + + return this.__promote(labels); + } + + // eslint-disable-next-line class-methods-use-this + isubset(offsets) { + /* validate subset */ + const { rindex } = this; + const maxOffset = rindex.length; + const labels = new Int32Array(offsets.length); + for (let i = 0, l = offsets.length; i < l; i += 1) { + const offset = offsets[i]; + if (offset < 0 || offset >= maxOffset) + throw new RangeError(`out of bounds offset: ${offset}`); + labels[i] = rindex[offset]; + } + + return this.__promote(labels); } withLabel(label) { - return this.__promote([...this.keys(), label]); + return this.__promote([...this.labels(), label]); } withLabels(labels) { - return this.__promote([...this.keys(), ...labels]); + return this.__promote([...this.labels(), ...labels]); } dropLabel(label) { - const labelArray = [...this.keys()]; + const labelArray = [...this.labels()]; labelArray.splice(labelArray.indexOf(label), 1); return this.__promote(labelArray); } @@ -207,12 +276,29 @@ class KeyIndex { this.getOffset = function getOffset(k) { return index.get(k); }; + + this.getOffsets = function getOffsets(arr) { + const res = new arr.constructor(arr.length); + for (let i = 0, len = arr.length; i < len; i += 1) { + res[i] = index.get(arr[i]); + } + return res; + }; + this.getLabel = function getLabel(i) { return rindex[i]; }; + + this.getLabels = function getLabels(arr) { + const res = new arr.constructor(arr.length); + for (let i = 0, len = arr.length; i < len; i += 1) { + res[i] = rindex[arr[i]]; + } + return res; + }; } - keys() { + labels() { return this.rindex; } @@ -220,9 +306,30 @@ class KeyIndex { return this.rindex.length; } + subset(labels) { + /* validate subset */ + for (let i = 0, l = labels.length; i < l; i += 1) { + const label = labels[i]; + const offset = this.getOffset(label); + if (offset === undefined) throw new RangeError(`unknown label: ${label}`); + } + + return new KeyIndex(labels); + } + // eslint-disable-next-line class-methods-use-this - subsetLabels(labelArray) { - return new KeyIndex(labelArray); + isubset(offsets) { + const { rindex } = this; + const maxOffset = rindex.length; + const labels = new Array(offsets.length); + for (let i = 0, l = offsets.length; i < l; i += 1) { + const offset = offsets[i]; + if (offset < 0 || offset >= maxOffset) + throw new RangeError(`out of bounds offset: ${offset}`); + labels[i] = rindex[offset]; + } + + return new KeyIndex(labels); } withLabel(label) { diff --git a/client/src/util/stateManager/annotationsHelpers.js b/client/src/util/stateManager/annotationsHelpers.js index 08249449..525e5b3b 100644 --- a/client/src/util/stateManager/annotationsHelpers.js +++ b/client/src/util/stateManager/annotationsHelpers.js @@ -100,7 +100,7 @@ export function setLabelByValue(df, colName, fromLabel, toLabel) { /* in the dataframe column `colName`, set any value of `fromLabel` to `toLabel` */ - const keys = df.colIndex.keys(); + const keys = df.colIndex.labels(); const ndf = df.mapColumns((col, colIdx) => { if (colName !== keys[colIdx]) return col; @@ -118,7 +118,7 @@ export function setLabelByMask(df, colName, mask, label) { /* in the dataframe column `colName`, set the masked rows to 'label' */ - const keys = df.colIndex.keys(); + const keys = df.colIndex.labels(); const ndf = df.mapColumns((col, colIdx) => { if (colName !== keys[colIdx]) return col; diff --git a/client/src/util/stateManager/controlsHelpers.js b/client/src/util/stateManager/controlsHelpers.js index 176edeb5..3206b63f 100644 --- a/client/src/util/stateManager/controlsHelpers.js +++ b/client/src/util/stateManager/controlsHelpers.js @@ -187,7 +187,7 @@ export function pruneVarDataCache(varData, needed) { if (numOverWatermark <= 0) return varData; const { colIndex } = varData; - const all = colIndex.keys(); + const all = colIndex.labels(); const unused = _.difference(all, needed); if (unused.length > 0) { // sort by offset in the dataframe - ie, psuedo-LRU diff --git a/client/src/util/stateManager/matrix.js b/client/src/util/stateManager/matrix.js index 9f467680..05d4e61d 100644 --- a/client/src/util/stateManager/matrix.js +++ b/client/src/util/stateManager/matrix.js @@ -1,7 +1,12 @@ import { flatbuffers } from "flatbuffers"; import { NetEncoding } from "./matrix_generated"; -import { isTypedArray } from "../typeHelpers"; -import { IdentityInt32Index, DenseInt32Index, KeyIndex } from "../dataframe"; +import { isTypedArray, isFpTypedArray } from "../typeHelpers"; +import { + Dataframe, + IdentityInt32Index, + DenseInt32Index, + KeyIndex, +} from "../dataframe"; const utf8Decoder = new TextDecoder("utf-8"); @@ -133,14 +138,14 @@ export function encodeMatrixFBS(df) { encColIndex = encodeTypedArray( builder, encColIndexUType, - df.colIndex.keys() + df.colIndex.labels() ); } else if (colIndexType === KeyIndex) { encColIndexUType = NetEncoding.TypedArray.JSONEncodedArray; encColIndex = encodeTypedArray( builder, encColIndexUType, - utf8Encoder.encode(JSON.stringify(df.colIndex.keys())) + utf8Encoder.encode(JSON.stringify(df.colIndex.labels())) ); } else { throw new Error("Index type FBS encoding unsupported"); @@ -162,3 +167,79 @@ export function encodeMatrixFBS(df) { builder.finish(root); return builder.asUint8Array(); } + +function promoteTypedArray(o) { + /* + Decide what internal data type to use for the data returned from + the server. + + TODO - future optimization: not all int32/uint32 data series require + promotion to float64. We COULD simply look at the data to decide. + */ + if (isFpTypedArray(o) || Array.isArray(o)) return o; + + let TyepdArrayCtor; + switch (o.constructor) { + case Int8Array: + case Uint8Array: + case Uint8ClampedArray: + case Int16Array: + case Uint16Array: + TyepdArrayCtor = Float32Array; + break; + + case Int32Array: + case Uint32Array: + TyepdArrayCtor = Float64Array; + break; + + default: + throw new Error("Unexpected data type returned from server."); + } + if (o.constructor === TyepdArrayCtor) return o; + return new TyepdArrayCtor(o); +} + +export function matrixFBSToDataframe(arrayBuffers) { + /* + Convert array of Matrix FBS to a Dataframe. + + The application has strong assumptions that all scalar data will be + stored as a float32 or float64 (regardless of underlying data types). + For example, clipping of value ranges (eg, user-selected percentiles) + depends on the ability to use NaN in any numeric type. + + All float data from the server is left as is. All non-float is promoted + to an appropriate float. + */ + if (!Array.isArray(arrayBuffers)) { + arrayBuffers = [arrayBuffers]; + } + if (arrayBuffers.length === 0) { + return Dataframe.Dataframe.empty(); + } + + const fbs = arrayBuffers.map((ab) => decodeMatrixFBS(ab, true)); // leave in place + /* check that all FBS have same row dimensionality */ + const { nRows } = fbs[0]; + fbs.forEach((b) => { + if (b.nRows !== nRows) + throw new Error("FBS with inconsistent dimensionality"); + }); + const columns = fbs + .map((fb) => + fb.columns.map((c) => { + if (isFpTypedArray(c) || Array.isArray(c)) return c; + return promoteTypedArray(c); + }) + ) + .flat(); + // colIdx may be TypedArray or Array + const colIdx = fbs + .map((b) => (Array.isArray(b.colIdx) ? b.colIdx : Array.from(b.colIdx))) + .flat(); + const nCols = columns.length; + + const df = new Dataframe([nRows, nCols], columns, null, new KeyIndex(colIdx)); + return df; +} diff --git a/client/src/util/stateManager/universe.js b/client/src/util/stateManager/universe.js index c9f5d06e..1b23b85d 100644 --- a/client/src/util/stateManager/universe.js +++ b/client/src/util/stateManager/universe.js @@ -40,84 +40,6 @@ These functions are used exclusively by the actions and reducers to build an internal POJO for use by the rendering components. */ -function promoteTypedArray(o) { - /* - Decide what internal data type to use for the data returned from - the server. - - TODO - future optimization: not all int32/uint32 data series require - promotion to float64. We COULD simply look at the data to decide. - */ - if (isFpTypedArray(o) || Array.isArray(o)) return o; - - let TyepdArrayCtor; - switch (o.constructor) { - case Int8Array: - case Uint8Array: - case Uint8ClampedArray: - case Int16Array: - case Uint16Array: - TyepdArrayCtor = Float32Array; - break; - - case Int32Array: - case Uint32Array: - TyepdArrayCtor = Float64Array; - break; - - default: - throw new Error("Unexpected data type returned from server."); - } - if (o.constructor === TyepdArrayCtor) return o; - return new TyepdArrayCtor(o); -} - -export function matrixFBSToDataframe(arrayBuffers) { - /* - Convert array of Matrix FBS to a Dataframe. - - The application has strong assumptions that all scalar data will be - stored as a float32 or float64 (regardless of underlying data types). - For example, clipping of value ranges (eg, user-selected percentiles) - depends on the ability to use NaN in any numeric type. - - All float data from the server is left as is. All non-float is promoted - to an appropriate float. - */ - if (!Array.isArray(arrayBuffers)) { - arrayBuffers = [arrayBuffers]; - } - if (arrayBuffers.length === 0) { - return Dataframe.Dataframe.empty(); - } - - const fbs = arrayBuffers.map((ab) => decodeMatrixFBS(ab, true)); // leave in place - /* check that all FBS have same row dimensionality */ - const { nRows } = fbs[0]; - fbs.forEach((b) => { - if (b.nRows !== nRows) - throw new Error("FBS with inconsistent dimensionality"); - }); - const columns = fbs - .map((fb) => - fb.columns.map((c) => { - if (isFpTypedArray(c) || Array.isArray(c)) return c; - return promoteTypedArray(c); - }) - ) - .flat(); - const colIdx = fbs.map((b) => b.colIdx).flat(); - const nCols = columns.length; - - const df = new Dataframe.Dataframe( - [nRows, nCols], - columns, - null, - new Dataframe.KeyIndex(colIdx) - ); - return df; -} - export function createUniverseFromResponse(configResponse, schemaResponse) { /* build & return universe from a REST 0.2 /config, /schema and /annotations/obs response @@ -178,7 +100,7 @@ export function addObsAnnotations(universe, df) { // for all of the new data, reconcile with schema and sort categories. const dfs = Array.isArray(df) ? df : [df]; - const keys = dfs.map((d) => d.colIndex.keys()).flat(); + const keys = dfs.map((d) => d.colIndex.labels()).flat(); const { schema } = universe; keys.forEach((k) => { const colSchema = schema.annotations.obsByName[k]; diff --git a/client/src/util/stateManager/world.js b/client/src/util/stateManager/world.js index 8c69c836..da219e0f 100644 --- a/client/src/util/stateManager/world.js +++ b/client/src/util/stateManager/world.js @@ -99,7 +99,7 @@ function clipDataframe( if (upperQuantile > 1) upperQuantile = 1; if (lowerQuantile === 0 && upperQuantile === 1) return df; - const keys = df.colIndex.keys(); + const keys = df.colIndex.labels(); return df.mapColumns((col, colIdx) => { const colLabel = keys[colIdx]; if (!clipPredicate(df, colIdx, colLabel)) return col; @@ -277,7 +277,7 @@ export function addObsDimensions(crossfilter, world) { but not yet in the crossfilter */ const schema = world.schema.annotations.obsByName; - const dimsWeNeed = world.obsAnnotations.colIndex.keys(); + const dimsWeNeed = world.obsAnnotations.colIndex.labels(); crossfilter = dimsWeNeed.reduce((xfltr, name) => { const dimName = obsAnnoDimensionName(name); if (xfltr.hasDimension(dimName)) return xfltr; @@ -321,7 +321,7 @@ export function getSelectedByIndex(crossfilter) { return array of obsIndex, containing all selected obs/cells. */ const selected = crossfilter.allSelectedMask(); // array of bool-ish - const keys = crossfilter.data.rowIndex.keys(); // row keys, aka universe rowIndex + const keys = crossfilter.data.rowIndex.labels(); // row keys, aka universe rowIndex const set = new Int32Array(selected.length); let numElems = 0; diff --git a/client/src/util/typedCrossfilter/crossfilter.js b/client/src/util/typedCrossfilter/crossfilter.js index 3ee99386..c1ccaa41 100644 --- a/client/src/util/typedCrossfilter/crossfilter.js +++ b/client/src/util/typedCrossfilter/crossfilter.js @@ -60,9 +60,7 @@ export default class ImmutableTypedCrossfilter { } setData(data) { - const { selectionCache } = this; - this.selectionCache = {}; - return new ImmutableTypedCrossfilter(data, this.dimensions, selectionCache); + return new ImmutableTypedCrossfilter(data, this.dimensions); } dimensionNames() {