From ffd627341990a1e85384141a4378df5cce4221bb Mon Sep 17 00:00:00 2001 From: Bruce Martin Date: Thu, 28 Feb 2019 08:34:22 -0800 Subject: [PATCH] Dataframe, part deux - add varData and summarize() (#608) * initial dataframe commit * initial dataframe port of core app * rename variables for clarity * remove unused import * comment out unused code * fix array handling bug in crossfilter dimension creation * allow creation of empty dataframes * handle non-existent columns * handle non-existent columns * revise tests for new dataframe * comments for clarity * comments for clarity * generate bulk add placeholder with real gene names * fix bug in gene name adding * more dataframe unit tests * fix bug - subset from current world, not universe * put cut and pasted code into a single function * improve caching of crossfilter * remove cascading update bug from graph * more performance work * improve state handling for scatterplot * performance optimization of critical path * add column summarization * dataframe utils * add callOnceLazy * fix tests * minor updates found during review * fix misspelling * remove RESTv02 from function names * comment cleanup * cut/icut col parameter defaults to null * break up large test * improve tests and comments on dataframe at/has functions * add Dataframe withCol/dropCol * expression varData now stored in a dataframe * dead code cleanup * use dataframe.summarize() * test cases for Dataframe.col.summarize * update test cases for new dataframe summarize * improve naming * use new hasCol API * add comments * add more Dataframe.withCol tests * add ability to specify row index in cut operation * retire subsetVarData function * correctly handle expression subsetting * lint and improve comments * rename cut to subset * changes based on PR review --- .../util/dataframe/dataframe.test.js | 257 +++++++++++++++- .../util/dataframe/summarize.test.js | 253 +++++++++++++++ .../util/stateManager/keyvalcache.test.js | 249 --------------- .../stateManager/summarizeAnnotations.test.js | 291 ------------------ .../util/stateManager/universe.test.js | 5 +- .../__tests__/util/stateManager/world.test.js | 77 +---- client/src/actions/index.js | 23 +- .../components/brushableHistogram/index.js | 23 +- .../src/components/continuous/continuous.js | 47 +-- .../geneExpression/cellSetButtons.js | 6 +- client/src/components/geneExpression/index.js | 16 +- .../src/components/scatterplot/scatterplot.js | 13 +- client/src/reducers/controls.js | 138 ++++++--- client/src/util/dataframe/dataframe.js | 157 ++++++++-- client/src/util/dataframe/labelIndex.js | 72 ++++- client/src/util/stateManager/index.js | 1 - client/src/util/stateManager/keyvalcache.js | 122 -------- .../util/stateManager/summarizeAnnotations.js | 128 -------- client/src/util/stateManager/universe.js | 67 +--- client/src/util/stateManager/world.js | 103 ++----- client/src/util/typedCrossfilter/index.js | 2 +- 21 files changed, 910 insertions(+), 1140 deletions(-) create mode 100644 client/__tests__/util/dataframe/summarize.test.js delete mode 100644 client/__tests__/util/stateManager/keyvalcache.test.js delete mode 100644 client/__tests__/util/stateManager/summarizeAnnotations.test.js delete mode 100644 client/src/util/stateManager/keyvalcache.js delete mode 100644 client/src/util/stateManager/summarizeAnnotations.js diff --git a/client/__tests__/util/dataframe/dataframe.test.js b/client/__tests__/util/dataframe/dataframe.test.js index b1139984..a91ca893 100644 --- a/client/__tests__/util/dataframe/dataframe.test.js +++ b/client/__tests__/util/dataframe/dataframe.test.js @@ -129,7 +129,7 @@ describe("simple data access", () => { }); describe("dataframe subsetting", () => { - describe("cutByList", () => { + describe("subset", () => { const sourceDf = new Dataframe.Dataframe( [3, 4], [ @@ -143,7 +143,7 @@ describe("dataframe subsetting", () => { ); test("all rows, one column", () => { - const dfA = sourceDf.cutByList(null, ["colors"]); + const dfA = sourceDf.subset(null, ["colors"]); expect(dfA).toBeDefined(); expect(dfA.dims).toEqual([3, 1]); expect(dfA.iat(0, 0)).toEqual("red"); @@ -158,7 +158,7 @@ describe("dataframe subsetting", () => { }); test("all rows, two columns", () => { - const dfB = sourceDf.cutByList(null, ["colors", "float32"]); + const dfB = sourceDf.subset(null, ["colors", "float32"]); expect(dfB).toBeDefined(); expect(dfB.dims).toEqual([3, 2]); expect(dfB.iat(0, 0)).toBeCloseTo(4.4); @@ -182,7 +182,7 @@ describe("dataframe subsetting", () => { }); test("one row, all columns", () => { - const dfC = sourceDf.cutByList([1], null); + const dfC = sourceDf.subset([1], null); expect(dfC).toBeDefined(); expect(dfC.dims).toEqual([1, 4]); expect(dfC.iat(0, 0)).toEqual(1); @@ -194,7 +194,7 @@ describe("dataframe subsetting", () => { }); test("two rows, all columns", () => { - const dfD = sourceDf.cutByList([0, 2], null); + const dfD = sourceDf.subset([0, 2], null); expect(dfD).toBeDefined(); expect(dfD.dims).toEqual([2, 4]); expect(dfD.icol(0).asArray()).toEqual(new Int32Array([0, 2])); @@ -206,7 +206,7 @@ describe("dataframe subsetting", () => { }); test("all rows, all columns", () => { - const dfE = sourceDf.cutByList(null, null); + const dfE = sourceDf.subset(null, null); expect(dfE).toBeDefined(); expect(dfE.dims).toEqual([3, 4]); expect(dfE.icol(0).asArray()).toEqual(sourceDf.icol(0).asArray()); @@ -218,7 +218,7 @@ describe("dataframe subsetting", () => { }); test("two rows, two colums", () => { - const dfF = sourceDf.cutByList([0, 2], ["int32", "float32"]); + const dfF = sourceDf.subset([0, 2], ["int32", "float32"]); expect(dfF).toBeDefined(); expect(dfF.dims).toEqual([2, 2]); expect(dfF.icol(0).asArray()).toEqual(new Int32Array([0, 2])); @@ -226,9 +226,32 @@ describe("dataframe subsetting", () => { expect(dfF.rowIndex.keys()).toEqual(new Int32Array([0, 2])); expect(dfF.colIndex.keys()).toEqual(["int32", "float32"]); }); + + test("withRowIndex", () => { + const df = sourceDf.subset( + null, + ["int32", "float32"], + new Dataframe.DenseInt32Index([3, 2, 1]) + ); + expect(df.colIndex).toBeInstanceOf(Dataframe.KeyIndex); + expect(df.rowIndex).toBeInstanceOf(Dataframe.DenseInt32Index); + expect(df.at(3, "int32")).toEqual(df.iat(0, 0)); + }); + + test("withRowIndex error checks", () => { + expect(() => + sourceDf.subset(null, ["red"], new Dataframe.IdentityInt32Index(1)) + ).toThrow(RangeError); + expect(() => + sourceDf.subset(null, ["red"], new Dataframe.DenseInt32Index([0, 1])) + ).toThrow(RangeError); + expect(() => + sourceDf.subset(null, ["red"], new Dataframe.KeyIndex([0, 1, 2, 3])) + ).toThrow(RangeError); + }); }); - test("icutByMask", () => { + test("isubsetMask", () => { const sourceDf = new Dataframe.Dataframe( [3, 4], [ @@ -241,7 +264,7 @@ describe("dataframe subsetting", () => { new Dataframe.KeyIndex(["int32", "string", "float32", "colors"]) ); - const dfA = sourceDf.icutByMask( + const dfA = sourceDf.isubsetMask( new Uint8Array([0, 1, 1]), new Uint8Array([1, 0, 0, 1]) ); @@ -293,6 +316,222 @@ describe("dataframe factories", () => { expect(dfB.icol(i).asArray()).toEqual(dfA.icol(i).asArray()); } }); + + describe("withCol", () => { + test("KeyIndex", () => { + const df = new Dataframe.Dataframe( + [2, 2], + [["red", "blue"], [true, false]], + null, + new Dataframe.KeyIndex(["colors", "bools"]) + ); + const dfA = df.withCol("numbers", [1, 0]); + + expect(dfA).toBeDefined(); + expect(dfA.dims).toEqual([2, 3]); + expect(dfA.icol(0).asArray()).toEqual(["red", "blue"]); + expect(dfA.icol(1).asArray()).toEqual([true, false]); + expect(dfA.icol(2).asArray()).toEqual([1, 0]); + expect(dfA.col("numbers").asArray()).toEqual([1, 0]); + expect(dfA.colIndex.keys()).toEqual(["colors", "bools", "numbers"]); + expect(df.colIndex.keys()).toEqual(["colors", "bools"]); + expect(df.rowIndex.keys()).toEqual(dfA.rowIndex.keys()); + }); + + test("DenseInt32Index", () => { + const df = new Dataframe.Dataframe( + [2, 2], + [["red", "blue"], [true, false]], + null, + new Dataframe.DenseInt32Index([74, 75]) + ); + const dfA = df.withCol(72, [1, 0]); + + expect(dfA).toBeDefined(); + expect(dfA.dims).toEqual([2, 3]); + expect(dfA.icol(0).asArray()).toEqual(["red", "blue"]); + expect(dfA.icol(1).asArray()).toEqual([true, false]); + expect(dfA.icol(2).asArray()).toEqual([1, 0]); + expect(dfA.col(74).asArray()).toEqual(["red", "blue"]); + expect(dfA.col(75).asArray()).toEqual([true, false]); + expect(dfA.col(72).asArray()).toEqual([1, 0]); + expect(dfA.colIndex.keys()).toEqual(new Int32Array([74, 75, 72])); + expect(df.colIndex.keys()).toEqual(new Int32Array([74, 75])); + expect(df.rowIndex.keys()).toEqual(dfA.rowIndex.keys()); + }); + + test("DenseInt32Index promote", () => { + const df = new Dataframe.Dataframe( + [2, 2], + [["red", "blue"], [true, false]], + null, + new Dataframe.DenseInt32Index([74, 75]) + ); + const dfA = df.withCol(999, [1, 0]); + + expect(dfA).toBeDefined(); + expect(dfA.dims).toEqual([2, 3]); + expect(dfA.icol(0).asArray()).toEqual(["red", "blue"]); + expect(dfA.icol(1).asArray()).toEqual([true, false]); + expect(dfA.icol(2).asArray()).toEqual([1, 0]); + expect(dfA.col(74).asArray()).toEqual(["red", "blue"]); + expect(dfA.col(75).asArray()).toEqual([true, false]); + expect(dfA.col(999).asArray()).toEqual([1, 0]); + expect(dfA.colIndex.keys()).toEqual(new Int32Array([74, 75, 999])); + expect(df.colIndex.keys()).toEqual(new Int32Array([74, 75])); + expect(df.rowIndex.keys()).toEqual(dfA.rowIndex.keys()); + }); + + test("IdentityInt32Index with last", () => { + const df = new Dataframe.Dataframe( + [2, 2], + [["red", "blue"], [true, false]], + null, + null + ); + const dfA = df.withCol(2, [1, 0]); + + expect(dfA).toBeDefined(); + expect(dfA.dims).toEqual([2, 3]); + expect(dfA.icol(0).asArray()).toEqual(["red", "blue"]); + expect(dfA.icol(1).asArray()).toEqual([true, false]); + expect(dfA.icol(2).asArray()).toEqual([1, 0]); + expect(dfA.col(0).asArray()).toEqual(["red", "blue"]); + expect(dfA.col(1).asArray()).toEqual([true, false]); + expect(dfA.col(2).asArray()).toEqual([1, 0]); + expect(dfA.colIndex.keys()).toEqual(new Int32Array([0, 1, 2])); + expect(df.colIndex.keys()).toEqual(new Int32Array([0, 1])); + expect(df.rowIndex.keys()).toEqual(dfA.rowIndex.keys()); + }); + + test("IdentityInt32Index promote", () => { + const df = new Dataframe.Dataframe( + [2, 2], + [["red", "blue"], [true, false]], + null, + null + ); + const dfA = df.withCol(99, [1, 0]); + + expect(dfA).toBeDefined(); + expect(dfA.dims).toEqual([2, 3]); + expect(dfA.icol(0).asArray()).toEqual(["red", "blue"]); + expect(dfA.icol(1).asArray()).toEqual([true, false]); + expect(dfA.icol(2).asArray()).toEqual([1, 0]); + expect(dfA.col(0).asArray()).toEqual(["red", "blue"]); + expect(dfA.col(1).asArray()).toEqual([true, false]); + expect(dfA.col(99).asArray()).toEqual([1, 0]); + expect(dfA.colIndex.keys()).toEqual(new Int32Array([0, 1, 99])); + expect(df.colIndex.keys()).toEqual(new Int32Array([0, 1])); + expect(df.rowIndex.keys()).toEqual(dfA.rowIndex.keys()); + }); + + describe("handle column dimensions correctly", () => { + /* + there are two conditions: + - empty dataframe - will accept an add of any dimensionality + - non-empty dataframe - added column must match row-count dimension + */ + test("empty.withCol", () => { + const edf = Dataframe.Dataframe.empty(); + const df = edf.withCol("foo", [1, 2, 3]); + + expect(edf).toBeDefined(); + expect(df).toBeDefined(); + expect(edf).not.toEqual(df); + expect(df.dims).toEqual([3, 1]); + expect(df.icol(0).asArray()).toEqual([1, 2, 3]); + }); + + test("withCol dimension check", () => { + const dfA = new Dataframe.Dataframe([1, 1], [["a"]]); + expect(() => { + dfA.withCol(1, []); + }).toThrow(RangeError); + }); + }); + }); + + describe("dropCol", () => { + test("KeyIndex", () => { + const df = new Dataframe.Dataframe( + [2, 3], + [["red", "blue"], [true, false], [1, 0]], + null, + new Dataframe.KeyIndex(["colors", "bools", "numbers"]) + ); + const dfA = df.dropCol("colors"); + + expect(dfA).toBeDefined(); + expect(dfA.dims).toEqual([2, 2]); + expect(dfA.icol(0).asArray()).toEqual([true, false]); + expect(dfA.icol(1).asArray()).toEqual([1, 0]); + expect(dfA.col("numbers").asArray()).toEqual([1, 0]); + expect(dfA.colIndex.keys()).toEqual(["bools", "numbers"]); + expect(df.colIndex.keys()).toEqual(["colors", "bools", "numbers"]); + expect(df.rowIndex.keys()).toEqual(dfA.rowIndex.keys()); + }); + + test("IdentityInt32Index drop first", () => { + const df = new Dataframe.Dataframe( + [2, 3], + [["red", "blue"], [true, false], [1, 0]], + null, + null + ); + const dfA = df.dropCol(0); + + expect(dfA).toBeDefined(); + expect(dfA.dims).toEqual([2, 2]); + expect(dfA.icol(0).asArray()).toEqual([true, false]); + expect(dfA.icol(1).asArray()).toEqual([1, 0]); + expect(df.col(1).asArray()).toEqual(dfA.col(1).asArray()); + expect(df.col(2).asArray()).toEqual(dfA.col(2).asArray()); + expect(dfA.colIndex.keys()).toEqual(new Int32Array([1, 2])); + expect(df.colIndex.keys()).toEqual(new Int32Array([0, 1, 2])); + expect(df.rowIndex.keys()).toEqual(dfA.rowIndex.keys()); + }); + + test("IdentityInt32Index drop last", () => { + const df = new Dataframe.Dataframe( + [2, 3], + [["red", "blue"], [true, false], [1, 0]], + null, + null + ); + const dfA = df.dropCol(2); + + expect(dfA).toBeDefined(); + expect(dfA.dims).toEqual([2, 2]); + expect(dfA.icol(0).asArray()).toEqual(["red", "blue"]); + expect(dfA.icol(1).asArray()).toEqual([true, false]); + expect(df.col(0).asArray()).toEqual(dfA.col(0).asArray()); + expect(df.col(1).asArray()).toEqual(dfA.col(1).asArray()); + expect(dfA.colIndex.keys()).toEqual(new Int32Array([0, 1])); + expect(df.colIndex.keys()).toEqual(new Int32Array([0, 1, 2])); + expect(df.rowIndex.keys()).toEqual(dfA.rowIndex.keys()); + }); + + test("DenseInt32Index", () => { + const df = new Dataframe.Dataframe( + [2, 3], + [["red", "blue"], [true, false], [1, 0]], + null, + new Dataframe.DenseInt32Index([102, 101, 100]) + ); + const dfA = df.dropCol(101); + + expect(dfA).toBeDefined(); + expect(dfA.dims).toEqual([2, 2]); + expect(dfA.icol(0).asArray()).toEqual(["red", "blue"]); + expect(dfA.icol(1).asArray()).toEqual([1, 0]); + expect(dfA.col(100).asArray()).toEqual([1, 0]); + expect(dfA.col(102).asArray()).toEqual(["red", "blue"]); + expect(dfA.colIndex.keys()).toEqual(new Int32Array([102, 100])); + expect(df.colIndex.keys()).toEqual(new Int32Array([102, 101, 100])); + expect(df.rowIndex.keys()).toEqual(dfA.rowIndex.keys()); + }); + }); }); describe("dataframe col", () => { diff --git a/client/__tests__/util/dataframe/summarize.test.js b/client/__tests__/util/dataframe/summarize.test.js new file mode 100644 index 00000000..702d0fae --- /dev/null +++ b/client/__tests__/util/dataframe/summarize.test.js @@ -0,0 +1,253 @@ +import * as Dataframe from "../../../src/util/dataframe"; + +function float32Conversion(f) { + return new Float32Array([f])[0]; +} + +describe("Dataframe column summary", () => { + test("empty column test", () => { + const df = Dataframe.Dataframe.create([0, 1], [[]]); + const summary = df.icol(0).summarize(); + expect(summary).toEqual( + expect.objectContaining({ + categorical: true, + categories: [], + categoryCounts: new Map(), + numCategories: 0 + }) + ); + }); + + test("simple test", () => { + const df = new Dataframe.Dataframe( + [1, 6], + [ + ["n1"], + ["hi"], + [true], + new Float32Array([39.3]), + new Int32Array([99]), + [1] + ], + null, + new Dataframe.KeyIndex([ + "name", + "nameString", + "nameBoolean", + "nameFloat32", + "nameInt32", + "nameCategorical" + ]) + ); + + expect(df.icol(0).summarize()).toEqual( + expect.objectContaining({ + categorical: true, + categories: ["n1"], + categoryCounts: new Map([["n1", 1]]), + numCategories: 1 + }) + ); + expect(df.icol(1).summarize()).toEqual( + expect.objectContaining({ + categorical: true, + categories: ["hi"], + categoryCounts: new Map([["hi", 1]]), + numCategories: 1 + }) + ); + expect(df.icol(2).summarize()).toEqual( + expect.objectContaining({ + categorical: true, + categories: [true], + categoryCounts: new Map([[true, 1]]), + numCategories: 1 + }) + ); + expect(df.icol(3).summarize()).toEqual( + expect.objectContaining({ + categorical: false, + min: float32Conversion(39.3), + max: float32Conversion(39.3), + nan: 0, + ninf: 0, + pinf: 0 + }) + ); + expect(df.icol(4).summarize()).toEqual( + expect.objectContaining({ + categorical: false, + min: 99, + max: 99, + nan: 0, + ninf: 0, + pinf: 0 + }) + ); + expect(df.icol(5).summarize()).toEqual( + expect.objectContaining({ + categorical: true, + categories: [1], + categoryCounts: new Map([[1, 1]]), + numCategories: 1 + }) + ); + }); + + test("multi test", () => { + const df = new Dataframe.Dataframe( + [3, 6], + [ + ["n0", "n1", "n2"], + ["hi", "hi", "bye"], + [false, true, true], + new Float32Array([39.3, 39.3, 0]), + new Int32Array([99, 99, 99]), + [1, false, "0"] + ], + null, + new Dataframe.KeyIndex([ + "name", + "nameString", + "nameBoolean", + "nameFloat32", + "nameInt32", + "nameCategorical" + ]) + ); + + expect(df.icol(0).summarize()).toEqual( + expect.objectContaining({ + categorical: true, + categories: expect.arrayContaining(["n0", "n1", "n2"]), + categoryCounts: new Map([["n0", 1], ["n1", 1], ["n2", 1]]), + numCategories: 3 + }) + ); + expect(df.icol(1).summarize()).toEqual( + expect.objectContaining({ + categorical: true, + categories: expect.arrayContaining(["hi", "bye"]), + categoryCounts: new Map([["hi", 2], ["bye", 1]]), + numCategories: 2 + }) + ); + expect(df.icol(2).summarize()).toEqual( + expect.objectContaining({ + categorical: true, + categories: expect.arrayContaining([true, false]), + categoryCounts: new Map([[true, 2], [false, 1]]), + numCategories: 2 + }) + ); + expect(df.icol(3).summarize()).toEqual( + expect.objectContaining({ + categorical: false, + min: 0, + max: float32Conversion(39.3), + nan: 0, + ninf: 0, + pinf: 0 + }) + ); + expect(df.icol(4).summarize()).toEqual( + expect.objectContaining({ + categorical: false, + min: 99, + max: 99, + nan: 0, + ninf: 0, + pinf: 0 + }) + ); + expect(df.icol(5).summarize()).toEqual( + expect.objectContaining({ + categorical: true, + categories: expect.arrayContaining([1, false, "0"]), + categoryCounts: new Map([[1, 1], [false, 1], ["0", 1]]), + numCategories: 3 + }) + ); + }); + + test("non-finite numbers", () => { + const df = new Dataframe.Dataframe( + [4, 6], + [ + ["n0", "n1", "n2", "n2"], + ["hi", "hi", "bye", "bye"], + [false, true, true, true], + new Float32Array([ + 39.3, + Number.NEGATIVE_INFINITY, + Number.NaN, + Number.POSITIVE_INFINITY + ]), + new Int32Array([99, 99, 99, 99]), + [1, false, "0", "0"] + ], + null, + new Dataframe.KeyIndex([ + "name", + "nameString", + "nameBoolean", + "nameFloat32", + "nameInt32", + "nameCategorical" + ]) + ); + + expect(df.icol(0).summarize()).toEqual( + expect.objectContaining({ + categorical: true, + categories: expect.arrayContaining(["n0", "n1", "n2"]), + categoryCounts: new Map([["n0", 1], ["n1", 1], ["n2", 2]]), + numCategories: 3 + }) + ); + expect(df.icol(1).summarize()).toEqual( + expect.objectContaining({ + categorical: true, + categories: expect.arrayContaining(["hi", "bye"]), + categoryCounts: new Map([["hi", 2], ["bye", 1]]), + numCategories: 2 + }) + ); + expect(df.icol(2).summarize()).toEqual( + expect.objectContaining({ + categorical: true, + categories: expect.arrayContaining([true, false]), + categoryCounts: new Map([[true, 2], [false, 1]]), + numCategories: 2 + }) + ); + expect(df.icol(3).summarize()).toEqual( + expect.objectContaining({ + categorical: false, + min: float32Conversion(39.3), + max: float32Conversion(39.3), + nan: 1, + ninf: 1, + pinf: 1 + }) + ); + expect(df.icol(4).summarize()).toEqual( + expect.objectContaining({ + categorical: false, + min: 99, + max: 99, + nan: 0, + ninf: 0, + pinf: 0 + }) + ); + expect(df.icol(5).summarize()).toEqual( + expect.objectContaining({ + categorical: true, + categories: expect.arrayContaining([1, false, "0"]), + categoryCounts: new Map([[1, 1], [false, 1], ["0", 1]]), + numCategories: 3 + }) + ); + }); +}); diff --git a/client/__tests__/util/stateManager/keyvalcache.test.js b/client/__tests__/util/stateManager/keyvalcache.test.js deleted file mode 100644 index aa21bc96..00000000 --- a/client/__tests__/util/stateManager/keyvalcache.test.js +++ /dev/null @@ -1,249 +0,0 @@ -import _ from "lodash"; -import * as kvCache from "../../../src/util/stateManager/keyvalcache"; - -/* -This is PRIVATE to keyvalcache and must be kept in sync with -any changs ot that module. Need to Know - to enable error handling test -*/ -const cachePrivateKey = "__kvcachekey__"; - -/* -helper function - promisify setTimeout() -*/ -function timeout(ms) { - return new Promise(resolve => setTimeout(resolve, ms)); -} - -describe("kvcache API", () => { - /* - test the happy path create/set/get API - */ - - test("simple create", () => { - /* with defaults */ - const kvc = kvCache.create(); - expect(kvc).toBeDefined(); - expect(kvc).toEqual(expect.objectContaining({})); - expect(kvCache.get(kvc, "test")).toBeUndefined(); - - /* with params */ - const kvc1 = kvCache.create(/* lowWatermark */ 99, /* minTTL */ 0); - expect(kvc1).toBeDefined(); - expect(kvc1).toEqual(expect.objectContaining({})); - }); - - test("set/get", () => { - /* - - check basic get/set functionality - - check set does not mutate source cache - */ - const keyName = "foo"; - const kvc1 = kvCache.create(); - expect(kvc1).toBeDefined(); - expect(kvCache.get(kvc1, keyName)).toBeUndefined(); - - const val2 = [2]; - const kvc2 = kvCache.set(kvc1, keyName, val2); - expect(kvc2).toBeDefined(); - expect(kvc2).not.toBe(kvc1); - expect(kvCache.get(kvc1, keyName)).toBeUndefined(); - expect(kvCache.get(kvc2, keyName)).toBe(val2); - - const val3 = [3]; - const kvc3 = kvCache.set(kvc2, keyName, val3); - expect(kvc3).toBeDefined(); - expect(kvc3).not.toBe(kvc1); - expect(kvc3).not.toBe(kvc2); - expect(kvCache.get(kvc1, keyName)).toBeUndefined(); - expect(kvCache.get(kvc2, keyName)).toBe(val2); - expect(kvCache.get(kvc3, keyName)).toBe(val3); - }); -}); - -describe("common error handling", () => { - /* - Test common error handlers - */ - - test("set() protection from namespace pollution", () => { - /* - Test that set() will not allow use of the private cache key - */ - const kvc = kvCache.create(); - expect(() => { - kvCache.set(kvc, cachePrivateKey, {}); - }).toThrow(); - }); - - test("create() does not accept bogus config", () => { - expect(() => { - kvCache.create([], {}); - }).toThrow(); - expect(() => { - kvCache.create(-99, 0); - }).toThrow(); - expect(() => { - kvCache.create(100, -1); - }).toThrow(); - expect(() => { - kvCache.create(1000, "foobar"); - }).toThrow(); - expect(() => { - kvCache.create(null, 8); - }).toThrow(); - }); -}); - -describe("map", () => { - /* - Test kvCache.map() - create new cache that is a transformation of an - existing cache - */ - test("map of empty cache", () => { - const kvc = kvCache.create(); - const callback = jest.fn(); - const kvcMapped = kvCache.map(kvc, callback); - expect(callback).not.toHaveBeenCalled(); - expect(kvcMapped).toBeDefined(); - expect(kvcMapped).not.toBe(kvc); // immutable operation - expect(kvcMapped).toEqual(kvc); - }); - - test("map of non-empty cache", () => { - const key = "aKey"; - const val = [0, 1, 2]; - let kvc = kvCache.create(); - kvc = kvCache.set(kvc, key, val); - const mockCB = jest.fn().mockImplementation(v => [...v]); - const kvcMapped = kvCache.map(kvc, mockCB); - - expect(kvcMapped).toBeDefined(); - expect(kvcMapped).not.toBe(kvc); // immutable operation - expect(_.isEqual(kvc, kvcMapped)).toBe(true); - - expect(mockCB).toHaveBeenCalledTimes(1); - expect(mockCB).toHaveBeenLastCalledWith(val, key); - }); -}); - -describe("flush", () => { - /* - test various cache flush behavior - */ - test("flush - lowWatermark, disable minTTL", () => { - /* - verify lowWatermark functions correctly - */ - - // set lowWatermark to 2, set three times - only the final two - // should remain. - let kvc = kvCache.create(2, 0); - ["a", "b", "c"].forEach(k => { - kvc = kvCache.set(kvc, k, []); - }); - - expect(kvc).toEqual( - expect.objectContaining({ - b: expect.arrayContaining([]), - c: expect.arrayContaining([]) - }) - ); - expect(kvc).toEqual( - expect.not.objectContaining({ - a: expect.arrayContaining([]) - }) - ); - }); - - test("flush - minTTL, disable lowWatermark", async () => { - /* - verify minTTL functions correctly - */ - - // set minTTL to 1 ms - let kvc = kvCache.create(0, 10); - kvc = kvCache.set(kvc, "a", []); - await timeout(20); - ["b", "c"].forEach(k => { - kvc = kvCache.set(kvc, k, []); - }); - - expect(kvc).toEqual( - expect.objectContaining({ - b: expect.arrayContaining([]), - c: expect.arrayContaining([]) - }) - ); - expect(kvc).toEqual( - expect.not.objectContaining({ - a: expect.arrayContaining([]) - }) - ); - }); - - test("flush - minTTL and lowWatermark", async () => { - /* - verify minTTL functions correctly - */ - - // set lowwatermark to 3, minTTL to 1 ms - let kvc = kvCache.create(3, 10); - kvc = kvCache.set(kvc, "a", []); - // delay - await timeout(20); - ["b", "c"].forEach(k => { - kvc = kvCache.set(kvc, k, []); - }); - - expect(kvc).toEqual( - expect.objectContaining({ - a: expect.arrayContaining([]), - b: expect.arrayContaining([]), - c: expect.arrayContaining([]) - }) - ); - - kvc = kvCache.set(kvc, "d", []); - expect(kvc).toEqual( - expect.objectContaining({ - b: expect.arrayContaining([]), - c: expect.arrayContaining([]), - d: expect.arrayContaining([]) - }) - ); - expect(kvc).toEqual( - expect.not.objectContaining({ - a: expect.arrayContaining([]) - }) - ); - }); - - test("manual flush", async () => { - let kvc = kvCache.create(1, 10); - ["a", "b", "c", "d"].forEach(k => { - kvc = kvCache.set(kvc, k, []); - }); - - // Before TTL has expired, should have all values in cache. - expect(kvc).toEqual( - expect.objectContaining({ - a: expect.arrayContaining([]), - b: expect.arrayContaining([]), - c: expect.arrayContaining([]) - }) - ); - - // let TTL expire - await timeout(10); - - // manually flush - const postFlushKvc = kvCache.flush(kvc); - expect(postFlushKvc).toBeDefined(); - expect(postFlushKvc).not.toBe(kvc); - expect(postFlushKvc).toEqual( - expect.objectContaining({ - d: expect.arrayContaining([]) - }) - ); - }); -}); diff --git a/client/__tests__/util/stateManager/summarizeAnnotations.test.js b/client/__tests__/util/stateManager/summarizeAnnotations.test.js deleted file mode 100644 index 2ebc2e4a..00000000 --- a/client/__tests__/util/stateManager/summarizeAnnotations.test.js +++ /dev/null @@ -1,291 +0,0 @@ -import summarizeAnnotations from "../../../src/util/stateManager/summarizeAnnotations"; -import * as Dataframe from "../../../src/util/dataframe"; - -function float32Conversion(f) { - return new Float32Array([39.3])[0]; -} - -describe("summarizeAnnotations", () => { - const schema = { - annotations: { - obs: [ - { name: "name", type: "string" }, - { name: "nameString", type: "string" }, - { name: "nameBoolean", type: "boolean" }, - { name: "nameFloat32", type: "float32" }, - { name: "nameInt32", type: "int32" }, - { - name: "nameCategorical", - type: "categorical", - categories: [true, false, 1, 0, 0.00001, 4383.4833, "test", "", "0"] - } - ], - var: [{ name: "name", type: "string" }] - } - }; - - test("empty test", () => { - const df = Dataframe.Dataframe.empty(); - const summary = summarizeAnnotations(schema, df, df.clone()); - expect(summary).toEqual( - expect.objectContaining({ - obs: { - nameString: { - categorical: true, - categories: [], - categoryCounts: new Map(), - numCategories: 0 - }, - nameBoolean: { - categorical: true, - categories: [], - categoryCounts: new Map(), - numCategories: 0 - }, - nameFloat32: { - categorical: false, - range: { - max: undefined, - min: undefined, - nan: 0, - ninf: 0, - pinf: 0 - } - }, - nameInt32: { - categorical: false, - range: { - max: undefined, - min: undefined, - nan: 0, - ninf: 0, - pinf: 0 - } - }, - nameCategorical: { - categorical: true, - categories: [], - categoryCounts: new Map(), - numCategories: 0 - } - }, - var: {} - }) - ); - }); - - test("simple test", () => { - const obsAnnotations = new Dataframe.Dataframe( - [1, 6], - [ - ["n1"], - ["hi"], - [true], - new Float32Array([39.3]), - new Int32Array([99]), - [1] - ], - null, - new Dataframe.KeyIndex([ - "name", - "nameString", - "nameBoolean", - "nameFloat32", - "nameInt32", - "nameCategorical" - ]) - ); - const varAnnotations = Dataframe.Dataframe.empty(); - - const summary = summarizeAnnotations( - schema, - obsAnnotations, - varAnnotations - ); - - expect(summary).toEqual( - expect.objectContaining({ - obs: { - nameString: { - categorical: true, - categories: ["hi"], - categoryCounts: new Map([["hi", 1]]), - numCategories: 1 - }, - nameBoolean: { - categorical: true, - categories: [true], - categoryCounts: new Map([[true, 1]]), - numCategories: 1 - }, - nameFloat32: { - categorical: false, - range: { - min: float32Conversion(39.3), - max: float32Conversion(39.3), - nan: 0, - ninf: 0, - pinf: 0 - } - }, - nameInt32: { - categorical: false, - range: { min: 99, max: 99, nan: 0, ninf: 0, pinf: 0 } - }, - nameCategorical: { - categorical: true, - categories: [1], - categoryCounts: new Map([[1, 1]]), - numCategories: 1 - } - }, - var: {} - }) - ); - }); - - test("multi test", () => { - const obsAnnotations = new Dataframe.Dataframe( - [3, 6], - [ - ["n0", "n1", "n2"], - ["hi", "hi", "bye"], - [false, true, true], - new Float32Array([39.3, 39.3, 0]), - new Int32Array([99, 99, 99]), - [1, false, "0"] - ], - null, - new Dataframe.KeyIndex([ - "name", - "nameString", - "nameBoolean", - "nameFloat32", - "nameInt32", - "nameCategorical" - ]) - ); - const varAnnotations = Dataframe.Dataframe.empty(); - - const summary = summarizeAnnotations( - schema, - obsAnnotations, - varAnnotations - ); - - expect(summary).toMatchObject( - expect.objectContaining({ - obs: { - nameString: { - categorical: true, - categories: expect.arrayContaining(["hi", "bye"]), - categoryCounts: new Map([["hi", 2], ["bye", 1]]), - numCategories: 2 - }, - nameBoolean: { - categorical: true, - categories: expect.arrayContaining([true, false]), - categoryCounts: new Map([[true, 2], [false, 1]]), - numCategories: 2 - }, - nameFloat32: { - categorical: false, - range: { - min: 0, - max: float32Conversion(39.3), - nan: 0, - ninf: 0, - pinf: 0 - } - }, - nameInt32: { - categorical: false, - range: { min: 99, max: 99, nan: 0, ninf: 0, pinf: 0 } - }, - nameCategorical: { - categorical: true, - categories: expect.arrayContaining([1, false, "0"]), - categoryCounts: new Map([[1, 1], [false, 1], ["0", 1]]), - numCategories: 3 - } - }, - var: {} - }) - ); - }); - - test("non-finite numbers", () => { - const obsAnnotations = new Dataframe.Dataframe( - [4, 6], - [ - ["n0", "n1", "n2", "n2"], - ["hi", "hi", "bye", "bye"], - [false, true, true, true], - new Float32Array([ - 39.3, - Number.NEGATIVE_INFINITY, - Number.NaN, - Number.POSITIVE_INFINITY - ]), - new Int32Array([99, 99, 99, 99]), - [1, false, "0", "0"] - ], - null, - new Dataframe.KeyIndex([ - "name", - "nameString", - "nameBoolean", - "nameFloat32", - "nameInt32", - "nameCategorical" - ]) - ); - const varAnnotations = Dataframe.Dataframe.empty(); - - const summary = summarizeAnnotations( - schema, - obsAnnotations, - varAnnotations - ); - - expect(summary).toMatchObject( - expect.objectContaining({ - obs: { - nameString: { - categorical: true, - categories: expect.arrayContaining(["hi", "bye"]), - categoryCounts: new Map([["hi", 2], ["bye", 1]]), - numCategories: 2 - }, - nameBoolean: { - categorical: true, - categories: expect.arrayContaining([true, false]), - categoryCounts: new Map([[true, 2], [false, 1]]), - numCategories: 2 - }, - nameFloat32: { - categorical: false, - range: { - min: float32Conversion(39.3), - max: float32Conversion(39.3), - nan: 1, - ninf: 1, - pinf: 1 - } - }, - nameInt32: { - categorical: false, - range: { min: 99, max: 99, nan: 0, ninf: 0, pinf: 0 } - }, - nameCategorical: { - categorical: true, - categories: expect.arrayContaining([1, false, "0"]), - categoryCounts: new Map([[1, 1], [false, 1], ["0", 1]]), - numCategories: 3 - } - }, - var: {} - }) - ); - }); -}); diff --git a/client/__tests__/util/stateManager/universe.test.js b/client/__tests__/util/stateManager/universe.test.js index f45fd7c5..b58e176b 100644 --- a/client/__tests__/util/stateManager/universe.test.js +++ b/client/__tests__/util/stateManager/universe.test.js @@ -41,15 +41,13 @@ describe("createUniverseFromResponse", () => { expect(universe).toBeDefined(); expect(universe).toMatchObject( expect.objectContaining({ - api: "0.2", nObs, nVar, schema: REST.schema.schema, obsAnnotations: expect.any(Dataframe.Dataframe), varAnnotations: expect.any(Dataframe.Dataframe), obsLayout: expect.any(Dataframe.Dataframe), - summary: expect.any(Object), - varDataCache: expect.any(Object) + varData: expect.any(Dataframe.Dataframe) }) ); @@ -63,5 +61,6 @@ describe("createUniverseFromResponse", () => { nVar, REST.schema.schema.annotations.var.length ]); + expect(universe.varData.isEmpty()).toBeTruthy(); }); }); diff --git a/client/__tests__/util/stateManager/world.test.js b/client/__tests__/util/stateManager/world.test.js index c0aafb28..94c1b9c8 100644 --- a/client/__tests__/util/stateManager/world.test.js +++ b/client/__tests__/util/stateManager/world.test.js @@ -8,7 +8,6 @@ import { obsAnnoDimensionName, layoutDimensionName } from "../../../src/util/nameCreators"; -import * as kvCache from "../../../src/util/stateManager/keyvalcache"; /* Helper - creates universe, world, corssfilter and dimensionMap from @@ -55,28 +54,13 @@ describe("createWorldFromEntireUniverse", () => { expect(world).toMatchObject( expect.objectContaining({ - api: "0.2", nObs: universe.nObs, nVar: universe.nVar, schema: universe.schema, obsAnnotations: universe.obsAnnotations, varAnnotations: universe.varAnnotations, obsLayout: universe.obsLayout, - - summary: expect.objectContaining({ - obs: _(REST.schema.schema.annotations.obs) - .filter(v => v.name !== "name") - .keyBy("name") - .mapValues(() => expect.any(Object)) - .value(), - var: _(REST.schema.schema.annotations.var) - .filter(v => v.name !== "name") - .keyBy("name") - .mapValues(() => expect.any(Object)) - .value() - }), - - varDataCache: expect.any(Object) + varData: expect.any(Dataframe.Dataframe) }) ); }); @@ -121,18 +105,13 @@ describe("createWorldFromCurrentSelection", () => { expect(world).toMatchObject( expect.objectContaining({ - api: "0.2", nObs: matchingIndices.length, nVar: universe.nVar, schema: universe.schema, obsAnnotations: expect.any(Dataframe.Dataframe), varAnnotations: universe.varAnnotations, obsLayout: expect.any(Dataframe.Dataframe), - summary: { - obs: expect.any(Object) /* we could do better! */, - var: expect.any(Object) /* we could do better! */ - }, - varDataCache: expect.any(Object) + varData: expect.any(Dataframe.Dataframe) }) ); @@ -183,60 +162,14 @@ describe("createObsDimensionMap", () => { }); }); -describe("subsetVarData", () => { - test("when world eq universe", () => { - const { universe, world } = defaultBigBang(); - /* create a mock varData array for subsetting */ - const sourceVarData = new Float32Array(universe.nObs); - - /* expect literally the same object back */ - const result = World.subsetVarData(world, universe, sourceVarData); - expect(result).toBe(sourceVarData); - }); - - test("when world neq universe", () => { - const { universe, world, crossfilter, dimensionMap } = defaultBigBang(); - /* create a mock varData array for subsetting */ - const sourceVarData = Float32Array.from(_.range(universe.nObs)); - - /* mock a selection */ - dimensionMap[obsAnnoDimensionName("field1")].filterRange([0, 5]); - dimensionMap[obsAnnoDimensionName("field3")].filterExact(false); - - /* create the world from the selection */ - const newWorld = World.createWorldFromCurrentSelection( - universe, - world, - crossfilter - ); - expect(newWorld.obsAnnotations.rowIndex.keys()).toEqual( - new Int32Array([0, 2]) - ); - - /* expect a subset */ - const result = World.subsetVarData(newWorld, universe, sourceVarData); - expect(result).not.toBe(sourceVarData); - expect(result).toHaveLength(newWorld.nObs); - /* check that we have expected source var content */ - expect(result).toMatchObject(new Float32Array([0, 2])); - }); -}); - -describe("createVarDimension", () => { +describe("createVarDataDimension", () => { /* create default universe */ const { world, crossfilter } = defaultBigBang(); - /* create a mock var data cache */ - const varDataCache = kvCache.set( - kvCache.create(), + world.varData = world.varData.withCol( "GENE", Float32Array.from(_.range(world.nObs)) ); - const result = World.createVarDimension( - world, - varDataCache, - crossfilter, - "GENE" - ); + const result = World.createVarDataDimension(world, crossfilter, "GENE"); expect(result).toBeInstanceOf(Crossfilter.ScalarDimension); }); diff --git a/client/src/actions/index.js b/client/src/actions/index.js index 7964fadf..7a62f3b4 100644 --- a/client/src/actions/index.js +++ b/client/src/actions/index.js @@ -1,12 +1,11 @@ // jshint esversion: 6 import _ from "lodash"; import * as globals from "../globals"; -import { Universe, kvCache } from "../util/stateManager"; +import { Universe } from "../util/stateManager"; import { catchErrorsWrap, doJsonRequest, doBinaryRequest, - rangeEncodeIndices, dispatchNetworkErrorMessageToUser } from "../util/actionHelpers"; @@ -130,9 +129,9 @@ async function _doRequestExpressionData(dispatch, getState, genes) { let expressionData = _.transform( genes, (expData, g) => { - const data = kvCache.get(universe.varDataCache, g); + const data = universe.varData.col(g); if (data) { - expData[g] = data; + expData[g] = data.asArray(); } }, {} @@ -170,7 +169,7 @@ function requestSingleGeneExpressionCountsForColoringPOST(gene) { type: "color by expression", gene, data: { - [gene]: kvCache.get(world.varDataCache, gene) + [gene]: world.varData.col(gene).asArray() } }); } catch (error) { @@ -193,7 +192,7 @@ const requestUserDefinedGene = gene => async (dispatch, getState) => { type: "request user defined gene success", data: { genes: [gene], - expression: kvCache.get(world.varDataCache, gene) + expression: world.varData.col(gene).asArray() } }); } catch (error) { @@ -243,12 +242,16 @@ const requestDifferentialExpression = (set1, set2, num_genes = 10) => async ( const state = getState(); const { universe } = state.controls; + // Legal values are null, Array or TypedArray. Null is initial state. + if (!set1) set1 = []; + if (!set2) set2 = []; + // These lines ensure that we convert any TypedArray to an Array. // This is necessary because JSON.stringify() does some very strange // things with TypedArrays (they are marshalled to JSON objects, rather // than being marshalled as a JSON array). - const aset1 = Array.isArray(set1) ? set1 : Array.from(set1); - const aset2 = Array.isArray(set2) ? set2 : Array.from(set2); + set1 = Array.isArray(set1) ? set1 : Array.from(set1); + set2 = Array.isArray(set2) ? set2 : Array.from(set2); const res = await fetch( `${globals.API.prefix}${globals.API.version}diffexp/obs`, @@ -261,8 +264,8 @@ const requestDifferentialExpression = (set1, set2, num_genes = 10) => async ( body: JSON.stringify({ mode: "topN", count: num_genes, - set1: { filter: { obs: { index: aset1 } } }, - set2: { filter: { obs: { index: aset2 } } } + set1: { filter: { obs: { index: set1 } } }, + set2: { filter: { obs: { index: set2 } } } }) } ); diff --git a/client/src/components/brushableHistogram/index.js b/client/src/components/brushableHistogram/index.js index 19bf0f73..657703bf 100644 --- a/client/src/components/brushableHistogram/index.js +++ b/client/src/components/brushableHistogram/index.js @@ -10,7 +10,6 @@ import { Button, ButtonGroup, Tooltip } from "@blueprintjs/core"; import { connect } from "react-redux"; import * as d3 from "d3"; import memoize from "memoize-one"; -import { kvCache } from "../../util/stateManager"; import * as globals from "../../globals"; import actions from "../../actions"; import finiteExtent from "../../util/finiteExtent"; @@ -21,7 +20,6 @@ import finiteExtent from "../../util/finiteExtent"; scatterplotYYaccessor: state.controls.scatterplotYYaccessor, crossfilter: state.controls.crossfilter, differential: state.differential, - initializeRanges: _.get(state.controls.world, "summary.obs"), colorAccessor: state.controls.colorAccessor, colorScale: state.controls.colorScale, obsAnnotations: _.get(state.controls.world, "obsAnnotations", null) @@ -35,7 +33,7 @@ class HistogramBrush extends React.Component { .scaleLinear() .range([this.height - this.marginBottom, 0]); - if (obsAnnotations.col(field)) { + if (obsAnnotations.hasCol(field)) { // recalculate expensive stuff const allValuesForContinuousFieldAsArray = obsAnnotations .col(field) @@ -52,9 +50,8 @@ class HistogramBrush extends React.Component { .thresholds(40)(allValuesForContinuousFieldAsArray); histogramCache.numValues = allValuesForContinuousFieldAsArray.length; - } else if (kvCache.get(world.varDataCache, field)) { - /* it's not in observations, so it's a gene, but let's check to make sure */ - const varValues = kvCache.get(world.varDataCache, field); + } else if (world.varData.hasCol(field)) { + const varValues = world.varData.col(field).asArray(); histogramCache.x = d3 .scaleLinear() @@ -143,21 +140,15 @@ class HistogramBrush extends React.Component { } handleColorAction() { - const { - obsAnnotations, - dispatch, - field, - world, - initializeRanges - } = this.props; + const { obsAnnotations, dispatch, field, world, ranges } = this.props; - if (obsAnnotations.col(field)) { + if (obsAnnotations.hasCol(field)) { dispatch({ type: "color by continuous metadata", colorAccessor: field, - rangeMaxForColorAccessor: initializeRanges[field].range.max + rangeMaxForColorAccessor: ranges.max }); - } else if (kvCache.get(world.varDataCache, field)) { + } else if (world.varData.hasCol(field)) { dispatch(actions.requestSingleGeneExpressionCountsForColoringPOST(field)); } } diff --git a/client/src/components/continuous/continuous.js b/client/src/components/continuous/continuous.js index 27b2d509..0ea5f022 100644 --- a/client/src/components/continuous/continuous.js +++ b/client/src/components/continuous/continuous.js @@ -9,7 +9,7 @@ import * as globals from "../../globals"; import HistogramBrush from "../brushableHistogram"; @connect(state => ({ - ranges: _.get(state.controls.world, "summary.obs", null), + obsAnnotations: _.get(state.controls.world, "obsAnnotations", null), colorAccessor: state.controls.colorAccessor, colorScale: state.controls.colorScale, selectionUpdate: _.get(state.controls, "crossfilter.updateTime", null), @@ -28,17 +28,18 @@ class Continuous extends React.Component { handleColorAction(key) { return () => { - const { dispatch, ranges } = this.props; + const { dispatch, obsAnnotations } = this.props; + const summary = obsAnnotations.col(key).summarize(); dispatch({ type: "color by continuous metadata", colorAccessor: key, - rangeMaxForColorAccessor: ranges[key].range.max + rangeMaxForColorAccessor: summary.max }); }; } render() { - const { ranges, schema } = this.props; + const { obsAnnotations, schema } = this.props; if (schema && !this.continuousChecked) { this.hasContinuous = _.some( schema.annotations.obs, @@ -62,23 +63,27 @@ class Continuous extends React.Component { Continuous metadata

) : null} - {_.map(ranges, (value, key) => { - const isColorField = key.includes("color") || key.includes("Color"); - zebra += 1; - if (value.range && key !== "name" && !isColorField) { - return ( - - ); - } - return null; - })} + {obsAnnotations + ? _.map(obsAnnotations.colIndex.keys(), key => { + const summary = obsAnnotations.col(key).summarize(); + const isColorField = + key.includes("color") || key.includes("Color"); + zebra += 1; + if (!summary.categorical && key !== "name" && !isColorField) { + return ( + + ); + } + return null; + }) + : null} ); } diff --git a/client/src/components/geneExpression/cellSetButtons.js b/client/src/components/geneExpression/cellSetButtons.js index 2deda7fb..de3b7620 100644 --- a/client/src/components/geneExpression/cellSetButtons.js +++ b/client/src/components/geneExpression/cellSetButtons.js @@ -14,7 +14,11 @@ class CellSetButton extends React.Component { eitherCellSetOneOrTwo } = this.props; - const set = World.getSelectedByIndex(crossfilter); + // Reducer and components assume that value will be null if + // no selection made. World..getSelectedByIndex() returns a + // zero length TypedArray when nothing is selected. + let set = World.getSelectedByIndex(crossfilter); + if (set.length === 0) set = null; if (!differential.diffExp) { /* diffexp needs to be cleared before we store a new set */ diff --git a/client/src/components/geneExpression/index.js b/client/src/components/geneExpression/index.js index bfa35d10..42df8583 100644 --- a/client/src/components/geneExpression/index.js +++ b/client/src/components/geneExpression/index.js @@ -56,12 +56,8 @@ const filterGenes = (query, genes) => }); @connect(state => { - const ranges = _.get(state.controls.world, "summary.obs", null); - const initializeRanges = _.get(state.controls.world, "summary.obs"); - return { - ranges, - initializeRanges, + obsAnnotations: _.get(state.controls.world, "obsAnnotations", null), userDefinedGenes: state.controls.userDefinedGenes, userDefinedGenesLoading: state.controls.userDefinedGenesLoading, world: state.controls.world, @@ -293,16 +289,17 @@ class GeneExpression extends React.Component { ) : null} {world && userDefinedGenes.length > 0 ? _.map(userDefinedGenes, (geneName, index) => { - const values = world.varDataCache[geneName]; + const values = world.varData.col(geneName); if (!values) { return null; } + const summary = values.summarize(); return ( ); @@ -322,16 +319,17 @@ class GeneExpression extends React.Component { {differential.diffExp ? _.map(differential.diffExp, (value, index) => { const name = world.varAnnotations.at(value[0], "name"); - const values = world.varDataCache[name]; + const values = world.varData.col(name); if (!values) { return null; } + const summary = values.summarize(); return ( { @@ -30,12 +29,16 @@ import finiteExtent from "../../util/finiteExtent"; scatterplotYYaccessor } = state.controls; const expressionX = - world && scatterplotXXaccessor - ? kvCache.get(world.varDataCache, scatterplotXXaccessor) + world && + scatterplotXXaccessor && + world.varData.hasCol(scatterplotXXaccessor) + ? world.varData.col(scatterplotXXaccessor).asArray() : null; const expressionY = - world && scatterplotYYaccessor - ? kvCache.get(world.varDataCache, scatterplotYYaccessor) + world && + scatterplotYYaccessor && + world.varData.hasCol(scatterplotYYaccessor) + ? world.varData.col(scatterplotYYaccessor).asArray() : null; return { diff --git a/client/src/reducers/controls.js b/client/src/reducers/controls.js index c29c6e0f..736cdd80 100644 --- a/client/src/reducers/controls.js +++ b/client/src/reducers/controls.js @@ -1,9 +1,8 @@ // jshint esversion: 6 import _ from "lodash"; -import { polygonContains } from "d3"; -import { World, kvCache, WorldUtil } from "../util/stateManager"; +import { World, WorldUtil } from "../util/stateManager"; import parseRGB from "../util/parseRGB"; import Crossfilter from "../util/typedCrossfilter"; import * as globals from "../globals"; @@ -61,19 +60,20 @@ function topNCategories(summary) { function createCategoricalSelectionState(state, world) { const res = {}; - _.forEach(world.summary.obs, (value, key) => { - if (value.categories) { + _.forEach(world.obsAnnotations.colIndex.keys(), key => { + const summary = world.obsAnnotations.col(key).summarize(); + if (summary.categories) { const isColorField = key.includes("color") || key.includes("Color"); const isSelectableCategory = !isColorField && key !== "name" && - value.categories.length < state.maxCategoryItems; + summary.categories.length < state.maxCategoryItems; if (isSelectableCategory) { - const [categoryValues, categoryCounts] = topNCategories(value); + const [categoryValues, categoryCounts] = topNCategories(summary); const categoryIndices = new Map(categoryValues.map((v, i) => [v, i])); const numCategories = categoryIndices.size; const categorySelected = new Array(numCategories).fill(true); - const isTruncated = categoryValues.length < value.numCategories; + const isTruncated = categoryValues.length < summary.numCategories; res[key] = { categoryValues, // array: of natively typed category values categoryIndices, // map: category value (native type) -> category index @@ -104,11 +104,10 @@ function selectedValuesForCategory(categorySelectionState) { build a crossfilter dimension map for all gene expression related dimensions. */ function createGenesDimMap(userDefinedGenes, diffexpGenes, world, crossfilter) { - function _createGenesDimMap(genes, nameF) { + function _createGenesDimMap(genes, nameCreator) { return genes.reduce((acc, gene) => { - acc[nameF(gene)] = World.createVarDimension( + acc[nameCreator(gene)] = World.createVarDataDimension( world, - world.varDataCache, crossfilter, gene ); @@ -122,6 +121,46 @@ function createGenesDimMap(userDefinedGenes, diffexpGenes, world, crossfilter) { }; } +function pruneVarDataCache(varData, needed) { + /* + Remove any unneeded columns from the varData dataframe. Will only + prune / remove if the total column count exceeds VarDataCacheLowWatermark + + Note: this code leverages the fact that dataframe offsets indicate + the order in which the columns were added. This crudely provides + LRU semantics, so we can delete "older" columns first. + */ + + /* + VarDataCacheLowWatermark - this cofig value sets the minimum cache size, + in columns, below which we don't throw away data. + + The value should be high enough so we are caching the maximum which will + "typically" be used in the UI (currently: 10 for diffexp, and N for user- + specified genes), and low enough to account for memory use (any single + column size is 4 bytes * numObs, so a column can be multi-megabyte in common + use cases). + */ + const VarDataCacheLowWatermark = 32; + + const numOverWatermark = varData.dims[1] - VarDataCacheLowWatermark; + if (numOverWatermark <= 0) return varData; + + const { colIndex } = varData; + const all = colIndex.keys(); + const unused = _.difference(all, needed); + if (unused.length > 0) { + // sort by offset in the dataframe - ie, psuedo-LRU + unused.sort((a, b) => colIndex.getOffset(a) - colIndex.getOffset(b)); + const numToDrop = + unused.length < numOverWatermark ? unused.length : numOverWatermark; + for (let i = 0; i < numToDrop; i += 1) { + varData = varData.dropCol(unused[i]); + } + } + return varData; +} + const Controls = ( state = { // data loading flag @@ -295,28 +334,60 @@ const Controls = ( } case "expression load success": { const { world, universe } = state; - let universeVarDataCache = universe.varDataCache; - let worldVarDataCache = world.varDataCache; + let universeVarData = universe.varData; + let worldVarData = world.varData; + + // Load new expression data into the varData dataframes, if + // not already present. _.forEach(action.expressionData, (val, key) => { - universeVarDataCache = kvCache.set(universeVarDataCache, key, val); - if (kvCache.get(worldVarDataCache, key) === undefined) { - worldVarDataCache = kvCache.set( - worldVarDataCache, + // If not already in universe.varData, save entire expression column + if (!universeVarData.hasCol(key)) { + universeVarData = universeVarData.withCol(key, val); + } + + // If not already in world.varData, save sliced expression column + if (!worldVarData.hasCol(key)) { + // Slice if world !== universe, else just use whole column. + // Use the obsAnnotation index as the cut key, as we keep + // all world dataframes in sync. + let worldValSlice = val; + if (!World.worldEqUniverse(world, universe)) { + worldValSlice = universeVarData + .subset(world.obsAnnotations.rowIndex.keys(), [key], null) + .icol(0) + .asArray(); + } + + // Now build world's varData dataframe + worldVarData = worldVarData.withCol( key, - World.subsetVarData(world, universe, val) + worldValSlice, + world.obsAnnotations.rowIndex ); } }); + // Prune size of varData "cache" if getting out of hand.... + const { userDefinedGenes, diffexpGenes } = state; + const allTheGenesWeNeed = _.uniq( + [].concat( + userDefinedGenes, + diffexpGenes, + Object.keys(action.expressionData) + ) + ); + universeVarData = pruneVarDataCache(universeVarData, allTheGenesWeNeed); + worldVarData = pruneVarDataCache(worldVarData, allTheGenesWeNeed); + return { ...state, universe: { ...universe, - varDataCache: universeVarDataCache + varData: universeVarData }, world: { ...world, - varDataCache: worldVarDataCache + varData: worldVarData } }; } @@ -334,17 +405,12 @@ const Controls = ( } case "request user defined gene success": { const { world, crossfilter, dimensionMap, userDefinedGenes } = state; - const worldVarDataCache = world.varDataCache; const _userDefinedGenes = userDefinedGenes.slice(); const gene = action.data.genes[0]; - dimensionMap[userDefinedDimensionName(gene)] = World.createVarDimension( - /* "__var__" + */ - world, - worldVarDataCache, - crossfilter, - gene - ); + dimensionMap[ + userDefinedDimensionName(gene) + ] = World.createVarDataDimension(world, crossfilter, gene); return { ...state, @@ -355,7 +421,6 @@ const Controls = ( } case "request differential expression success": { const { world, crossfilter, dimensionMap } = state; - const worldVarDataCache = world.varDataCache; const _diffexpGenes = []; action.data.forEach(d => { @@ -363,10 +428,8 @@ const Controls = ( }); _.forEach(_diffexpGenes, gene => { - dimensionMap[diffexpDimensionName(gene)] = World.createVarDimension( - /* "__var__" + */ + dimensionMap[diffexpDimensionName(gene)] = World.createVarDataDimension( world, - worldVarDataCache, crossfilter, gene ); @@ -381,9 +444,6 @@ const Controls = ( case "clear differential expression": { const { world, universe, dimensionMap } = state; const _dimensionMap = dimensionMap; - const universeVarDataCache = universe.varDataCache; - const worldVarDataCache = world.varDataCache; - _.forEach(action.diffExp, values => { const name = world.varAnnotations.at(values[0], "name"); // clean up crossfilter dimensions @@ -394,15 +454,7 @@ const Controls = ( return { ...state, dimensionMap: _dimensionMap, - diffexpGenes: [], - universe: { - ...universe, - varDataCache: universeVarDataCache - }, - world: { - ...world, - varDataCache: worldVarDataCache - } + diffexpGenes: [] }; } case "user defined gene": { diff --git a/client/src/util/dataframe/dataframe.js b/client/src/util/dataframe/dataframe.js index 23b9abfd..5e6d624f 100644 --- a/client/src/util/dataframe/dataframe.js +++ b/client/src/util/dataframe/dataframe.js @@ -1,4 +1,4 @@ -import { IdentityInt32Index } from "./labelIndex"; +import { IdentityInt32Index, isLabelIndex } from "./labelIndex"; // weird cross-dependency that we should clean up someday... import { sort } from "../typedCrossfilter/sort"; import { isTypedArray, isArrayOrTypedArray, callOnceLazy } from "./util"; @@ -10,7 +10,7 @@ but (currently) without all of the surrounding support functions. Data is stored in column-major layout, and each column is monomorphic. It supports: -* Relatively efficient creation, cloning and subsetting ("cut") +* Relatively efficient creation, cloning and subsetting * Very efficient columnar access (eg, sum down a column), and access to the underlying column arrays. * Data access by row/col offset or label. Labels are reasonably well @@ -76,14 +76,17 @@ class Dataframe { or a caller-provided index. All columns and indices must have appropriate dimensionality. */ - Dataframe.__errorChecks(dims, columnarData, rowIndex, colIndex); const [nRows, nCols] = dims; + if (nRows < 0 || nCols < 0) { + throw new RangeError("Dataframe dimensions must be positive"); + } if (!rowIndex) { rowIndex = new IdentityInt32Index(nRows); } if (!colIndex) { colIndex = new IdentityInt32Index(nCols); } + Dataframe.__errorChecks(dims, columnarData, rowIndex, colIndex); this.__columns = Array.from(columnarData); this.dims = dims; @@ -94,23 +97,40 @@ class Dataframe { this.__compile(); } - static __errorChecks(dims, columnarData) { + static __errorChecks(dims, columnarData, rowIndex, colIndex) { const [nRows, nCols] = dims; - if (nRows < 0 || nCols < 0) { - throw new RangeError("Dataframe dimensions must be positive"); - } + + /* check for expected types */ if (!Array.isArray(columnarData)) { throw new TypeError("Dataframe constructor requires array of columns"); } if (!columnarData.every(c => isArrayOrTypedArray(c))) { throw new TypeError("Dataframe columns must all be Array or TypedArray"); } + if (!isLabelIndex(rowIndex)) { + throw new TypeError("Dataframe rowIndex is an unsupported type."); + } + if (!isLabelIndex(colIndex)) { + throw new TypeError("Dataframe colIndex is an unsupported type."); + } + + /* check for expected dimensionality / size */ if ( nCols !== columnarData.length || !columnarData.every(c => c.length === nRows) ) { throw new RangeError( - "Dataframe dimension does not match column data shape" + "Dataframe dimension does not match provided data shape" + ); + } + if (nRows !== rowIndex.size()) { + throw new RangeError( + "Dataframe rowIndex must have same size as underlying data" + ); + } + if (nCols !== colIndex.size()) { + throw new RangeError( + "Dataframe colIndex must have same size as underlying data" ); } } @@ -210,6 +230,9 @@ class Dataframe { } clone() { + /* + Clone this dataframe + */ return new this.constructor( this.dims, [...this.__columns], @@ -218,8 +241,57 @@ class Dataframe { ); } - static empty() { - return new Dataframe([0, 0], []); + withCol(label, colData, withRowIndex = null) { + /* + Create a new DF, which is `this` plus the new column. Example: + const newDf = df.withCol("foo", [1,2,3]); + + Dimensionality of new column must match existing dataframe. + + Special case: empty dataframe will accept any size column. Example: + const newDf = Dataframe.empty().withCol("foo", [1,2,3]); + + If `withRowIndex` specified, the provided index will become the + rowIndex for the newly created dataframe. If not specified, + the rowIndex from `this` will be used (ie, the rowIndex is + unchanged). + */ + let dims; + let rowIndex; + if (this.isEmpty()) { + dims = [colData.length, 1]; + rowIndex = null; + } else { + dims = [this.dims[0], this.dims[1] + 1]; + ({ rowIndex } = this); + } + + if (withRowIndex) { + rowIndex = withRowIndex; + } + + const columns = [...this.__columns]; + columns.push(colData); + const colIndex = this.colIndex.withLabel(label); + return new this.constructor(dims, columns, rowIndex, colIndex); + } + + dropCol(label) { + /* + Create a new dataframe, omitting one columns. + + const newDf = df.dropCol("colors"); + */ + const dims = [this.dims[0], this.dims[1] - 1]; + const coffset = this.colIndex.getOffset(label); + const columns = [...this.__columns]; + columns.splice(coffset, 1); + const colIndex = this.colIndex.dropLabel(label); + return new this.constructor(dims, columns, this.rowIndex, colIndex); + } + + static empty(rowIndex = null, colIndex = null) { + return new Dataframe([0, 0], [], rowIndex, colIndex); } static create(dims, columnarData) { @@ -233,7 +305,7 @@ class Dataframe { return new Dataframe(dims, columnarData, null, null); } - __cut(rowOffsets, colOffsets) { + __subset(rowOffsets, colOffsets, withRowIndex) { const dims = [...this.dims]; const getSortedLabelAndOffsets = (offsets, index) => { @@ -260,10 +332,11 @@ class Dataframe { this.colIndex ); dims[1] = colOffsets.length; - colIndex = this.colIndex.cut(colLabels); + colIndex = this.colIndex.subsetLabels(colLabels); } let { rowIndex } = this; + if (withRowIndex) rowIndex = withRowIndex; if (rowOffsets) { let rowLabels; [rowLabels, rowOffsets] = getSortedLabelAndOffsets( @@ -271,10 +344,10 @@ class Dataframe { this.rowIndex ); dims[0] = rowLabels.length; - rowIndex = this.rowIndex.cut(rowLabels); + if (!withRowIndex) rowIndex = this.rowIndex.subsetLabels(rowLabels); } - /* cut columns */ + /* subset columns */ let columns = this.__columns; if (colOffsets) { columns = new Array(colOffsets.length); @@ -283,7 +356,7 @@ class Dataframe { } } - /* cut rows */ + /* subset rows */ if (rowOffsets) { columns = columns.map(col => { const newCol = new col.constructor(rowOffsets.length); @@ -296,7 +369,15 @@ class Dataframe { return new Dataframe(dims, columns, rowIndex, colIndex); } - cutByList(rowLabels, colLabels = null) { + subset(rowLabels, colLabels = null, withRowIndex = null) { + /* + Subset by row/col labels. + + withRowIndex allows assignment of new row index during subset operation. + If withRowIndex === null, it will reset the index to identity (offset) + indexing. if withRowIndex is a label index object, it will be used + for the new dataframe. + */ const toOffsets = (labels, index) => { if (!labels) { return null; @@ -312,16 +393,29 @@ class Dataframe { const rowOffsets = toOffsets(rowLabels, this.rowIndex); const colOffsets = toOffsets(colLabels, this.colIndex); - return this.__cut(rowOffsets, colOffsets); + return this.__subset(rowOffsets, colOffsets, withRowIndex); } - icutByList(rowOffsets, colOffsets = null) { - return this.__cut(rowOffsets, colOffsets); - } - - icutByMask(rowMask, colMask = null) { + isubset(rowOffsets, colOffsets = null, withRowIndex = null) { /* - Cut on row/column based upon a truthy/falsey array. + Subset by row/col offset. + + withRowIndex allows assignment of new row index during subset operation. + If withRowIndex === null, it will reset the index to identity (offset) + indexing. if withRowIndex is a label index object, it will be used + for the new dataframe. + */ + return this.__subset(rowOffsets, colOffsets, withRowIndex); + } + + isubsetMask(rowMask, colMask = null, withRowIndex = null) { + /* + Subset on row/column based upon a truthy/falsey array (a mask). + + withRowIndex allows assignment of new row index during subset operation. + If withRowIndex === null, it will reset the index to identity (offset) + indexing. if withRowIndex is a label index object, it will be used + for the new dataframe. */ const [nRows, nCols] = this.dims; if ( @@ -348,7 +442,7 @@ class Dataframe { }; const rowOffsets = toList(rowMask, nRows); const colOffsets = toList(colMask, nCols); - return this.__cut(rowOffsets, colOffsets); + return this.__subset(rowOffsets, colOffsets, withRowIndex); } /** @@ -431,6 +525,21 @@ class Dataframe { return c >= 0 && c < nCols && r >= 0 && r < nRows; } + hasCol(c) { + /* + Test if col label exists - return true/false + */ + return !!this.col(c); + } + + isEmpty() { + /* + Return true if this is an empty dataframe, ie, has dimensions [0,0] + */ + const [rows, cols] = this.dims; + return rows === 0 && cols === 0; + } + /**** Functional (map/reduce/etc) data access diff --git a/client/src/util/dataframe/labelIndex.js b/client/src/util/dataframe/labelIndex.js index 4594879c..2b31da65 100644 --- a/client/src/util/dataframe/labelIndex.js +++ b/client/src/util/dataframe/labelIndex.js @@ -57,13 +57,13 @@ class IdentityInt32Index { return i; } - getMaxOffset() { + size() { return this.maxOffset; } - cut(labelArray) { + __promote(labelArray) { /* - if density of resulting integer + time/space decision - based on the resulting density */ const [minLabel, maxLabel] = extent(labelArray); const labelSpaceSize = maxLabel - minLabel + 1; @@ -74,6 +74,26 @@ class IdentityInt32Index { } return new DenseInt32Index(labelArray, [minLabel, maxLabel]); } + + subsetLabels(labelArray) { + return this.__promote(labelArray); + } + + withLabel(label) { + if (label === this.maxOffset) { + return new IdentityInt32Index(label + 1); + } + return this.__promote([...this.keys(), label]); + } + + dropLabel(label) { + if (label === this.maxOffset - 1) { + return new IdentityInt32Index(label); + } + const labelArray = [...this.keys()]; + labelArray.splice(labelArray.indexOf(label), 1); + return this.__promote(labelArray); + } } /* eslint-enable class-methods-use-this */ @@ -121,11 +141,11 @@ class DenseInt32Index { return this.rindex; } - getMaxOffset() { + size() { return this.rindex.length; } - cut(labelArray) { + __promote(labelArray) { /* time/space decision - if we are going to use less than 10% of the dense index space, switch to a KeyIndex (which is slower, but uses @@ -140,6 +160,20 @@ class DenseInt32Index { } return new DenseInt32Index(labelArray, [minLabel, maxLabel]); } + + subsetLabels(labelArray) { + return this.__promote(labelArray); + } + + withLabel(label) { + return this.__promote([...this.keys(), label]); + } + + dropLabel(label) { + const labelArray = [...this.keys()]; + labelArray.splice(labelArray.indexOf(label), 1); + return this.__promote(labelArray); + } } /* eslint-enable class-methods-use-this */ @@ -151,6 +185,9 @@ class KeyIndex { */ constructor(labels) { const index = new Map(); + if (labels === undefined) { + labels = []; + } const rindex = labels; labels.forEach((v, i) => { index.set(v, i); @@ -175,14 +212,33 @@ class KeyIndex { return this.rindex; } - getMaxOffset() { + size() { return this.rindex.length; } - cut(labelArray) { + subsetLabels(labelArray) { + return new KeyIndex(labelArray); + } + + withLabel(label) { + return new KeyIndex([...this.rindex, label]); + } + + dropLabel(label) { + const idx = this.rindex.indexOf(label); + const labelArray = [...this.rindex]; + labelArray.splice(idx, 1); return new KeyIndex(labelArray); } } /* eslint-enable class-methods-use-this */ -export { DenseInt32Index, IdentityInt32Index, KeyIndex }; +function isLabelIndex(i) { + return ( + i instanceof IdentityInt32Index || + i instanceof DenseInt32Index || + i instanceof KeyIndex + ); +} + +export { DenseInt32Index, IdentityInt32Index, KeyIndex, isLabelIndex }; diff --git a/client/src/util/stateManager/index.js b/client/src/util/stateManager/index.js index 9cfdc1d3..b3140a2d 100644 --- a/client/src/util/stateManager/index.js +++ b/client/src/util/stateManager/index.js @@ -16,5 +16,4 @@ exists to support those concepts. export * as Universe from "./universe"; export * as World from "./world"; -export * as kvCache from "./keyvalcache"; export * as WorldUtil from "./worldUtil"; diff --git a/client/src/util/stateManager/keyvalcache.js b/client/src/util/stateManager/keyvalcache.js deleted file mode 100644 index a2c67aa8..00000000 --- a/client/src/util/stateManager/keyvalcache.js +++ /dev/null @@ -1,122 +0,0 @@ -// jshint esversion: 6 -import _ from "lodash"; - -/* -Very simple key/value cache for use by World & Universe. Cache keys must -be a string, and values are any JS non-primitive value. - - * constructor(lowWatermark, minTTL): - - lowWatermark defines the number of cache elements below which - flushing will not occur. - - minTTL defines minimum time in milliseconds that cache entries will live. - A value of -1 disables automatic flushing (flush() can still - be called by external user). - * set() - add a key/val pair. - * get() - get a value or undefined if not present. - * flush(minAgeMs) - flush cache entries in excess of lowWatermark if those - entries are older than minAgeMs. - -*/ - -const cachePrivateKey = "__kvcachekey__"; -const defaultLowWatermark = 32; -const defaultMinTTL = 1000; - -function create(lowWatermark = defaultLowWatermark, minTTL = defaultMinTTL) { - if (typeof minTTL !== "number" || typeof lowWatermark !== "number") { - throw new TypeError( - "minTTL and lowWatermark parameters must be a primitive number" - ); - } - if (lowWatermark < 0 || minTTL < 0) { - throw new RangeError( - "minTTL and lowWatermark parameters must be number greater than zero" - ); - } - - return { - [cachePrivateKey]: { - lowWatermark, - minTTL - } - }; -} - -function get(kvcache, key) { - if (key === cachePrivateKey) { - throw new RangeError(`key parameter may not have value ${cachePrivateKey}`); - } - - const val = kvcache[key]; - if (val) { - val[cachePrivateKey] = Date.now(); - } - return val; -} - -function set(kvcache, key, val) { - if (key === cachePrivateKey) { - throw new RangeError(`key parameter may not have value ${cachePrivateKey}`); - } - - const newKvCache = { ...kvcache }; - newKvCache[key] = val; - val[cachePrivateKey] = Date.now(); - flushInPlace(newKvCache); - return newKvCache; -} - -function flush(kvcache) { - const newKvCache = { ...kvcache }; - flushInPlace(newKvCache); - return newKvCache; -} - -/* -Flush elements from cache IF cache size is greater than lowWatermark, and -those elements are older than minAgeMS -*/ -function flushInPlace(kvCache) { - const { lowWatermark, minTTL } = kvCache[cachePrivateKey]; - const eol = Date.now() - minTTL; - const allKeys = _(kvCache) - .keys() - .filter(k => k !== cachePrivateKey) - .sortBy([k => kvCache[k][cachePrivateKey]]) - .value(); - - if (allKeys.length > lowWatermark) { - const keysToDelete = _(allKeys) - .slice(0, allKeys.length - lowWatermark) - .filter(k => kvCache[k][cachePrivateKey] <= eol) - .value(); - _.forEach(keysToDelete, k => delete kvCache[k]); - } - - return kvCache; -} - -/* -use to create a cache that is a transformation of another cache. -*/ -function map(srcKvCache, cb, createOptions) { - const keysInSrcKvCache = _(srcKvCache) - .keys() - .filter(k => k !== cachePrivateKey) - .value(); - const lowWatermark = _.get( - createOptions, - "lowWatermark", - defaultLowWatermark - ); - const minTTL = _.get(createOptions, "minTTL", defaultMinTTL); - const newKvCache = create(lowWatermark, minTTL); - _.forEach(keysInSrcKvCache, key => { - const val = cb(get(srcKvCache, key), key); - newKvCache[key] = val; - val[cachePrivateKey] = Date.now(); - }); - return newKvCache; -} - -export { create, get, set, flush, map }; diff --git a/client/src/util/stateManager/summarizeAnnotations.js b/client/src/util/stateManager/summarizeAnnotations.js deleted file mode 100644 index 60116ec2..00000000 --- a/client/src/util/stateManager/summarizeAnnotations.js +++ /dev/null @@ -1,128 +0,0 @@ -import _ from "lodash"; -import finiteExtent from "../finiteExtent"; - -/* -Build and return obs/var summary using any annotation in the schema - -Summary information for each annotation, keyed by annotation name. -Value will be an object, containing summary information. - -For continuous annotations (int, float, etc): - : { - categorical: false, - range { - min: , - max: - } - } - -For categorical annotations (boolean, string, category): - : { - categorical: true, - categories: [ , , ... ] - categoryCounts: Map { - : , - ... - }, - numCategories: - } - -Summarize will be returned for BOTH obs and var annotations. - -Example: - { - "Splice_sites_Annotated": { - categorical: false, - range: { - "min": 26, - "max": 1075869 - } - }, - "Selection": { - categorical: true, - numCategories, 3, - categories: [ "Astrocytes(HEPACAM)", "Endothelial(BSC)", "Unpanned" ], - categoryCounts: Map { - "Astrocytes(HEPACAM)": 714, - "Endothelial(BSC)": 123, - "Unpanned": 665 - } - } - } - -NOTE: will not summarize the required 'name' annotation, as that is -specified as unique per element. -*/ -function _summarizeAnnotations(_schema, df) { - const summary = _(_schema) // lodash wrapping: https://lodash.com/docs/4.17.11#lodash - .filter(v => v.name !== "name") // don't summarize name - .keyBy("name") - .mapValues(anno => { - const { name, type } = anno; - const continuous = type === "int32" || type === "float32"; - const numRows = df.length; - const col = df.col(name) ? df.col(name).asArray() : null; - - if (continuous) { - let min; - let max; - let nan = 0; - let pinf = 0; - let ninf = 0; - if (col) { - for (let r = 0; r < numRows; r += 1) { - const val = Number(col[r]); - if (Number.isFinite(val)) { - if (min === undefined) { - min = val; - max = val; - } else { - min = val < min ? val : min; - max = val > max ? val : max; - } - } else if (Number.isNaN(val)) { - nan += 1; - } else if (val > 0) { - pinf += 1; - } else { - ninf += 1; - } - } - } - return { - categorical: false, - range: { min, max, nan, pinf, ninf } - }; - } - - /* else categorical */ - const categoryCounts = new Map(); - if (col) { - for (let r = 0; r < numRows; r += 1) { - const val = col[r]; - let curCount = categoryCounts.get(val); - if (curCount === undefined) curCount = 0; - categoryCounts.set(val, curCount + 1); - } - } - return { - categorical: true, - categories: [...categoryCounts.keys()], - categoryCounts, - numCategories: categoryCounts.size - }; - }) - .value(); - return summary; -} - -export default function summarizeAnnotations( - schema, - obsAnnotations, - varAnnotations -) { - return { - obs: _summarizeAnnotations(schema.annotations.obs, obsAnnotations), - var: _summarizeAnnotations(schema.annotations.var, varAnnotations) - }; -} diff --git a/client/src/util/stateManager/universe.js b/client/src/util/stateManager/universe.js index 17f8e5df..b0d92ed4 100644 --- a/client/src/util/stateManager/universe.js +++ b/client/src/util/stateManager/universe.js @@ -2,8 +2,6 @@ import _ from "lodash"; -import * as kvCache from "./keyvalcache"; -import summarizeAnnotations from "./summarizeAnnotations"; import decodeMatrixFBS from "./matrix"; import * as Dataframe from "../dataframe"; @@ -12,15 +10,7 @@ Private helper function - create and return a template Universe */ function templateUniverse() { /* default universe template */ - - /* varDataCache config - see kvCache for semantics */ - const VarDataCacheLowWatermark = 32; // cache element count - const VarDataCacheTTLMs = 1000; // min cache time in MS - return { - api: null, - finalized: false, // XXX: may not be needed - nObs: 0, nVar: 0, schema: {}, @@ -28,18 +18,14 @@ function templateUniverse() { /* Annotations */ - obsAnnotations: null, - varAnnotations: null, - obsLayout: null, - summary: null /* derived data summaries. XXX: consider exploding in place */, + obsAnnotations: Dataframe.Dataframe.empty(), + varAnnotations: Dataframe.Dataframe.empty(), + obsLayout: Dataframe.Dataframe.empty(), /* - Cache of var data (expression), by var annotation name. Data can be - accesses as a POJO, but if you want caching semantics, use the kvCache - API (eg., kvCache.get(), kvCache.set(), ...), which will maintain the - LRU semantics. + Var data columns - subset of all */ - varDataCache: kvCache.create(VarDataCacheLowWatermark, VarDataCacheTTLMs) + varData: Dataframe.Dataframe.empty(null, new Dataframe.KeyIndex()) }; } @@ -51,29 +37,6 @@ These functions are used exclusively by the actions and reducers to build an internal POJO for use by the rendering components. */ -/* -generate any client-side transformations or summarization that -is independent of REST API response formats. -*/ -function finalize(universe) { - /* A bit of sanity checking! */ - const { nObs, nVar } = universe; - if ( - nObs !== universe.obsLayout.length || - nObs !== universe.obsAnnotations.length || - nVar !== universe.varAnnotations.length - ) { - throw new Error("Universe dimensionality mismatch - failed to load"); - } - // TODO: add more sanity checks, such as: - // - all annotations in the schema - // - layout has supported number of dimensions - // - ... - - universe.finalized = true; - return universe; -} - function AnnotationsFBSToDataframe(arrayBuffer) { /* Convert a Matrix FBS to a Dataframe. @@ -118,7 +81,7 @@ function reconcileSchemaCategoriesWithSummary(universe) { ) { const categories = _.union( _.get(s, "categories", []), - _.get(universe.summary.obs[s.name], "categories", []) + _.get(universe.obsAnnotations.col(s.name).summarize(), "categories", []) ); s.categories = categories; } @@ -138,9 +101,6 @@ export function createUniverseFromResponse( const { schema } = schemaResponse; const universe = templateUniverse(); - /* constants */ - universe.api = "0.2"; - /* schema related */ universe.schema = schema; universe.nObs = schema.dataframe.nObs; @@ -152,14 +112,17 @@ export function createUniverseFromResponse( /* layout */ universe.obsLayout = LayoutFBSToDataframe(layoutFBSResponse); - universe.summary = summarizeAnnotations( - universe.schema, - universe.obsAnnotations, - universe.varAnnotations - ); + /* sanity check */ + if ( + universe.nObs !== universe.obsLayout.length || + universe.nObs !== universe.obsAnnotations.length || + universe.nVar !== universe.varAnnotations.length + ) { + throw new Error("Universe dimensionality mismatch - failed to load"); + } reconcileSchemaCategoriesWithSummary(universe); - return finalize(universe); + return universe; } export function convertDataFBStoObject(universe, arrayBuffer) { diff --git a/client/src/util/stateManager/world.js b/client/src/util/stateManager/world.js index 9a4134a5..81641c9a 100644 --- a/client/src/util/stateManager/world.js +++ b/client/src/util/stateManager/world.js @@ -1,11 +1,10 @@ // jshint esversion: 6 import _ from "lodash"; -import * as kvCache from "./keyvalcache"; -import summarizeAnnotations from "./summarizeAnnotations"; import { layoutDimensionName, obsAnnoDimensionName } from "../nameCreators"; import Crossfilter from "../typedCrossfilter"; import { sliceByIndex } from "../typedCrossfilter/util"; +import * as Dataframe from "../dataframe"; /* @@ -38,48 +37,33 @@ Notable keys in the world object: A dataframe containing the X/Y layout for all obs. Columns are named 'X' and 'Y', and rows are indexed in the same way as obsAnnotation. -* summary: summary of each obsAnnotation column (eg, numeric extent for - continuous data, category counts for categorical metadata) - -* varDataCache: expression columns, in a kvCache. TODO: maybe move to a - Dataframe in the future. +* varData: a cache of expression columns, stored in a Dataframe. Cache + managed by controls reducer. */ -/* varDataCache config - see kvCache for semantics */ -const VarDataCacheLowWatermark = 32; // cache element count -const VarDataCacheTTLMs = 1000; // min cache time in MS - function templateWorld() { return { /* schema/version related */ - api: null, schema: null, nObs: 0, nVar: 0, /* annotations */ - obsAnnotations: null, - varAnnotations: null, + obsAnnotations: Dataframe.Dataframe.empty(), + varAnnotations: Dataframe.Dataframe.empty(), /* layout of graph. Dataframe. */ - obsLayout: null, + obsLayout: Dataframe.Dataframe.empty(), - /* derived data summaries XXX: consider exploding in place */ - summary: null, - - varDataCache: kvCache.create( - VarDataCacheLowWatermark, - VarDataCacheTTLMs - ) /* cache of var data (expression) */ + /* + Var data columns - subset of all data (may be empty) + */ + varData: Dataframe.Dataframe.empty(null, new Dataframe.KeyIndex()) }; } export function createWorldFromEntireUniverse(universe) { - if (!universe.finalized) { - throw new Error("World can't be created from an partial Universe"); - } - const world = templateWorld(); /* @@ -87,31 +71,21 @@ export function createWorldFromEntireUniverse(universe) { */ /* Schema related */ - world.api = universe.api; world.schema = universe.schema; world.nObs = universe.nObs; world.nVar = universe.nVar; - /* annotations */ + /* annotation dataframes */ world.obsAnnotations = universe.obsAnnotations; world.varAnnotations = universe.varAnnotations; - /* layout and display characteristics */ + /* layout and display characteristics dataframe */ world.obsLayout = universe.obsLayout; - /* derived data & summaries */ - world.summary = summarizeAnnotations( - world.schema, - world.obsAnnotations, - world.varAnnotations - ); - - /* build the varDataCache */ - world.varDataCache = kvCache.map( - universe.varDataCache, - val => subsetVarData(world, universe, val), - { lowWatermark: VarDataCacheLowWatermark, minTTL: VarDataCacheTTLMs } - ); + /* + Var data columns - subset of all + */ + world.varData = universe.varData.clone(); return world; } @@ -120,30 +94,24 @@ export function createWorldFromCurrentSelection(universe, world, crossfilter) { const newWorld = templateWorld(); /* these don't change as only OBS are selected in our current implementation */ - newWorld.api = universe.api; newWorld.nVar = universe.nVar; newWorld.schema = universe.schema; newWorld.varAnnotations = universe.varAnnotations; /* now subset/cut obs */ const mask = crossfilter.allFilteredMask(); - newWorld.obsAnnotations = world.obsAnnotations.icutByMask(mask); - newWorld.obsLayout = world.obsLayout.icutByMask(mask); + newWorld.obsAnnotations = world.obsAnnotations.isubsetMask(mask); + newWorld.obsLayout = world.obsLayout.isubsetMask(mask); newWorld.nObs = newWorld.obsAnnotations.dims[0]; - /* derived data & summaries */ - newWorld.summary = summarizeAnnotations( - newWorld.schema, - newWorld.obsAnnotations, - newWorld.varAnnotations - ); - - /* build the varDataCache */ - newWorld.varDataCache = kvCache.map( - universe.varDataCache, - val => subsetVarData(newWorld, universe, val), - { lowWatermark: VarDataCacheLowWatermark, minTTL: VarDataCacheTTLMs } - ); + /* + Var data columns - subset of all + */ + if (world.varData.isEmpty()) { + newWorld.varData = world.varData.clone(); + } else { + newWorld.varData = world.varData.isubsetMask(mask); + } return newWorld; } @@ -183,17 +151,10 @@ function deduceDimensionType(attributes, fieldName) { when it is no longer needed (it will not be garbage collected without this call) */ - -export function createVarDimension( - world, - _worldVarDataCache, - crossfilter, - geneName -) { - // return crossfilter.dimension(_worldVarDataCache[geneName], Float32Array); +export function createVarDataDimension(world, crossfilter, name) { return crossfilter.dimension( Crossfilter.ScalarDimension, - _worldVarDataCache[geneName], + world.varData.col(name).asArray(), Float32Array ); } @@ -242,14 +203,6 @@ export function worldEqUniverse(world, universe) { return world.obsAnnotations === universe.obsAnnotations; } -export function subsetVarData(world, universe, varData) { - // If world === universe, just return the entire varData array - if (worldEqUniverse(world, universe)) { - return varData; - } - return sliceByIndex(varData, world.obsAnnotations.rowIndex.keys()); -} - export function getSelectedByIndex(crossfilter) { /* return array of obsIndex, containing all selected obs/cells. diff --git a/client/src/util/typedCrossfilter/index.js b/client/src/util/typedCrossfilter/index.js index 5730fdef..99888314 100644 --- a/client/src/util/typedCrossfilter/index.js +++ b/client/src/util/typedCrossfilter/index.js @@ -121,7 +121,7 @@ class TypedCrossfilter { return res; } /* else, Dataframe-like */ - return data.icutByMask(this.allFilteredMask()); + return data.isubsetMask(this.allFilteredMask()); } // return Uint8array containing selection state (truthy/falsey) for each record.