diff --git a/client/__tests__/util/dataframe/dataframe.test.js b/client/__tests__/util/dataframe/dataframe.test.js index a91ca893..a89b3486 100644 --- a/client/__tests__/util/dataframe/dataframe.test.js +++ b/client/__tests__/util/dataframe/dataframe.test.js @@ -532,6 +532,42 @@ describe("dataframe factories", () => { expect(df.rowIndex.keys()).toEqual(dfA.rowIndex.keys()); }); }); + + describe("mapColumns", () => { + test("identity", () => { + const dfA = Dataframe.Dataframe.create( + [3, 3], + [ + new Array(3).fill(0), + new Int16Array(3).fill(99), + new Float64Array(3).fill(1.1) + ] + ); + const dfB = dfA.mapColumns((col, idx) => { + expect(dfA.icol(idx).asArray()).toBe(col); + return col; + }); + expect(dfA).not.toBe(dfB); + expect(dfA.dims).toEqual(dfB.dims); + for (let c = 0; c < dfA.dims[1]; c += 1) { + expect(dfA.icol(c).asArray()).toBe(dfB.icol(c).asArray()); + } + }); + + test("transform", () => { + const dfA = Dataframe.Dataframe.create( + [3, 3], + [new Array(3).fill(0), new Array(3).fill(0), new Array(3).fill(0)] + ); + const dfB = dfA.mapColumns(() => { + return new Array(3).fill(1); + }); + expect(dfA).not.toBe(dfB); + expect(dfB.iat(0, 0)).toEqual(1); + expect(dfB.iat(0, 1)).toEqual(1); + expect(dfB.iat(0, 2)).toEqual(1); + }); + }); }); describe("dataframe col", () => { diff --git a/client/__tests__/util/quantile.test.js b/client/__tests__/util/quantile.test.js new file mode 100644 index 00000000..b8b49954 --- /dev/null +++ b/client/__tests__/util/quantile.test.js @@ -0,0 +1,29 @@ +import quantile from "../../src/util/quantile"; + +describe("quantile", () => { + test("single q", () => { + const arr = new Float32Array([9, 3, 5, 6, 0]); + expect(quantile([1.0], arr)).toMatchObject([9]); + expect(quantile([0.9], arr)).toMatchObject([9]); + expect(quantile([0.8], arr)).toMatchObject([9]); + expect(quantile([0.7], arr)).toMatchObject([6]); + expect(quantile([0.6], arr)).toMatchObject([6]); + expect(quantile([0.5], arr)).toMatchObject([5]); + expect(quantile([0.4], arr)).toMatchObject([5]); + expect(quantile([0.3], arr)).toMatchObject([3]); + expect(quantile([0.2], arr)).toMatchObject([3]); + expect(quantile([0.1], arr)).toMatchObject([0]); + expect(quantile([0], arr)).toMatchObject([0]); + }); + + test("multi q", () => { + const arr = new Float32Array([9, 3, 5, 6, 0]); + expect(quantile([0, 0.25, 0.5, 0.75, 1.0], arr)).toMatchObject([ + 0, + 3, + 5, + 6, + 9 + ]); + }); +}); diff --git a/client/__tests__/util/stateManager/world.test.js b/client/__tests__/util/stateManager/world.test.js index a4eeed0a..8cc5fc31 100644 --- a/client/__tests__/util/stateManager/world.test.js +++ b/client/__tests__/util/stateManager/world.test.js @@ -58,10 +58,15 @@ describe("createWorldFromEntireUniverse", () => { nObs: universe.nObs, nVar: universe.nVar, schema: universe.schema, - obsAnnotations: universe.obsAnnotations, - varAnnotations: universe.varAnnotations, - obsLayout: universe.obsLayout, - varData: expect.any(Dataframe.Dataframe) + obsAnnotations: expect.any(Dataframe.Dataframe), + varAnnotations: expect.any(Dataframe.Dataframe), + obsLayout: expect.any(Dataframe.Dataframe), + varData: expect.any(Dataframe.Dataframe), + clipQuantiles: { min: 0, max: 1 }, + unclipped: { + obsAnnotations: expect.any(Dataframe.Dataframe), + varData: expect.any(Dataframe.Dataframe) + } }) ); }); @@ -76,7 +81,7 @@ describe("createWorldFromCurrentSelection", () => { } = defaultBigBang(); /* mock a selection */ - let crossfilter = originalCrossfilter + const crossfilter = originalCrossfilter .select(obsAnnoDimensionName("field1"), { mode: "range", lo: 0, hi: 5 }) .select(obsAnnoDimensionName("field3"), { mode: "exact", @@ -84,7 +89,7 @@ describe("createWorldFromCurrentSelection", () => { }); /* create the world from the selection */ - const world = World.createWorldFromCurrentSelection( + const world = World.createWorldBySelection( universe, originalWorld, crossfilter @@ -112,10 +117,15 @@ describe("createWorldFromCurrentSelection", () => { nObs: matchingIndices.length, nVar: universe.nVar, schema: universe.schema, + clipQuantiles: { min: 0, max: 1 }, obsAnnotations: expect.any(Dataframe.Dataframe), - varAnnotations: universe.varAnnotations, + varAnnotations: expect.any(Dataframe.Dataframe), obsLayout: expect.any(Dataframe.Dataframe), - varData: expect.any(Dataframe.Dataframe) + varData: expect.any(Dataframe.Dataframe), + unclipped: { + obsAnnotations: expect.any(Dataframe.Dataframe), + varData: expect.any(Dataframe.Dataframe) + } }) ); diff --git a/client/__tests__/util/typedCrossfilter/crossfilter.test.js b/client/__tests__/util/typedCrossfilter/crossfilter.test.js index b3c9e5c1..c94fb3fd 100644 --- a/client/__tests__/util/typedCrossfilter/crossfilter.test.js +++ b/client/__tests__/util/typedCrossfilter/crossfilter.test.js @@ -11,7 +11,8 @@ const someData = [ tip: 100, type: "tab", productIDs: ["001"], - coords: [0, 0] + coords: [0, 0], + nonFinite: 0.0 }, { date: "2011-11-14T16:20:19Z", @@ -20,7 +21,8 @@ const someData = [ tip: 100, type: "tab", productIDs: ["001", "005"], - coords: [0.4, 0.4] + coords: [0.4, 0.4], + nonFinite: Number.NaN }, { date: "2011-11-14T16:28:54Z", @@ -29,7 +31,8 @@ const someData = [ tip: 200, type: "visa", productIDs: ["004", "005"], - coords: [0.3, 0.1] + coords: [0.3, 0.1], + nonFinite: Number.POSITIVE_INFINITY }, { date: "2011-11-14T16:30:43Z", @@ -38,7 +41,8 @@ const someData = [ tip: 0, type: "tab", productIDs: ["001", "002"], - coords: [0.392, 0.1] + coords: [0.392, 0.1], + nonFinite: Number.NEGATIVE_INFINITY }, { date: "2011-11-14T16:48:46Z", @@ -47,7 +51,8 @@ const someData = [ tip: 0, type: "tab", productIDs: ["005"], - coords: [0.7, 0.0482] + coords: [0.7, 0.0482], + nonFinite: 1.0 }, { date: "2011-11-14T16:53:41Z", @@ -56,7 +61,8 @@ const someData = [ tip: 0, type: "tab", productIDs: ["001", "004", "005"], - coords: [0.9999, 1.0] + coords: [0.9999, 1.0], + nonFinite: Number.NaN }, { date: "2011-11-14T16:54:06Z", @@ -65,7 +71,8 @@ const someData = [ tip: 0, type: "cash", productIDs: ["001", "002", "003", "004", "005"], - coords: [0.384, 0.6938] + coords: [0.384, 0.6938], + nonFinite: 99.0 }, { date: "2011-11-14T16:58:03Z", @@ -74,7 +81,8 @@ const someData = [ tip: 0, type: "tab", productIDs: ["001"], - coords: [0.4822, 0.482] + coords: [0.4822, 0.482], + nonFinite: Number.NaN }, { date: "2011-11-14T17:07:21Z", @@ -83,7 +91,8 @@ const someData = [ tip: 0, type: "tab", productIDs: ["004", "005"], - coords: [0.2234, 0] + coords: [0.2234, 0], + nonFinite: Number.NaN }, { date: "2011-11-14T17:22:59Z", @@ -92,7 +101,8 @@ const someData = [ tip: 0, type: "tab", productIDs: ["001", "002", "004", "005"], - coords: [0.382, 0.38485] + coords: [0.382, 0.38485], + nonFinite: -1 }, { date: "2011-11-14T17:25:45Z", @@ -101,7 +111,8 @@ const someData = [ tip: 0, type: "cash", productIDs: ["002"], - coords: [0.998, 0.8472] + coords: [0.998, 0.8472], + nonFinite: 0.0 }, { date: "2011-11-14T17:29:52Z", @@ -110,7 +121,8 @@ const someData = [ tip: 100, type: "visa", productIDs: ["004"], - coords: [0.8273, 0.3384] + coords: [0.8273, 0.3384], + nonFinite: 0.0 } ]; @@ -327,4 +339,96 @@ describe("ImmutableTypedCrossfilter", () => { ).toEqual(_.filter(someData, d => polygonContains(polygon, d.coords))); }); }); + + describe("non-finite scalars", () => { + let p; + beforeEach(() => { + p = payments + .addDimension("quantity", "scalar", (i, d) => d[i].quantity, Int32Array) + .addDimension( + "nonFinite", + "scalar", + (i, d) => d[i].nonFinite, + Float32Array + ) + .select("quantity", { mode: "all" }); + }); + + test("all or none", () => { + expect(p.select("nonFinite", { mode: "all" }).countSelected()).toEqual( + someData.length + ); + expect(p.select("nonFinite", { mode: "none" }).countSelected()).toEqual( + 0 + ); + }); + + test("exact", () => { + expect( + p.select("nonFinite", { mode: "exact", values: [0] }).countSelected() + ).toEqual(3); + expect( + p.select("nonFinite", { mode: "exact", values: [1] }).countSelected() + ).toEqual(1); + expect( + p + .select("nonFinite", { + mode: "exact", + values: [Number.POSITIVE_INFINITY] + }) + .countSelected() + ).toEqual(1); + expect( + p + .select("nonFinite", { + mode: "exact", + values: [Number.NEGATIVE_INFINITY] + }) + .countSelected() + ).toEqual(1); + expect( + p + .select("nonFinite", { mode: "exact", values: [Number.NaN] }) + .countSelected() + ).toEqual(4); + expect( + p + .select("nonFinite", { + mode: "exact", + values: [Number.POSITIVE_INFINITY, 0, 1, 99] + }) + .countSelected() + ).toEqual(6); + }); + + test("range", () => { + expect( + p + .select("nonFinite", { + mode: "range", + lo: 0, + hi: Number.POSITIVE_INFINITY + }) + .countSelected() + ).toEqual(5); + expect( + p + .select("nonFinite", { + mode: "range", + lo: 0, + hi: Number.NaN + }) + .countSelected() + ).toEqual(6); + expect( + p + .select("nonFinite", { + mode: "range", + lo: Number.NEGATIVE_INFINITY, + hi: Number.POSITIVE_INFINITY + }) + .countSelected() + ).toEqual(7); + }); + }); }); diff --git a/client/__tests__/util/typedCrossfilter/sort.test.js b/client/__tests__/util/typedCrossfilter/sort.test.js index 71ce948e..a1d1125d 100644 --- a/client/__tests__/util/typedCrossfilter/sort.test.js +++ b/client/__tests__/util/typedCrossfilter/sort.test.js @@ -1,4 +1,22 @@ -import { sort, sortIndex } from "../../../src/util/typedCrossfilter/sort"; +import { + sortArray, + sortIndex, + lowerBound, + upperBound, + lowerBoundIndirect, + upperBoundIndirect +} from "../../../src/util/typedCrossfilter/sort"; + +/* +Sort tests should keep in mind that there are separate code +paths for: + - small vs. large arrays (insertionsort only) + - float-only typed arrays vs. other array types (non-finite handling) + - indexed vs. direct sort +*/ + +const pInf = Number.POSITIVE_INFINITY; +const nInf = Number.NEGATIVE_INFINITY; function fillRange(arr, start = 0) { const larr = arr; @@ -15,42 +33,223 @@ function fillRand(arr) { return arr; } -describe("sort", () => { - [Array, Float32Array, Uint32Array, Int32Array, Float64Array].map(Type => - test(Type.name, () => { - expect(sort(Type.from([6, 5, 4, 3, 2, 1, 0]))).toMatchObject( - Type.from([0, 1, 2, 3, 4, 5, 6]) - ); - expect(sort(Type.from([6, 5, 4, 3, 2, 1]))).toMatchObject( - Type.from([1, 2, 3, 4, 5, 6]) - ); +describe("sortArray", () => { + describe("JS vals", () => { + [ + [true, false], + ["a", "b", "0", "1"], + [0, "a", true, null, undefined, 3.1415], + fillRand(new Array(1000)), + ["a", NaN, null, pInf] + ].map((val, idx) => + test(`JS vals ${idx}`, () => { + expect(sortArray(val)).toMatchObject(val.sort()); + }) + ); + }); - const source = fillRand(new Type(1000)); - expect(sort(Type.from(source))).toMatchObject(Type.from(source).sort()); - }) - ); + describe("finite numbers", () => { + [Array, Float32Array, Uint32Array, Int32Array, Float64Array].map(Type => + test(Type.name, () => { + expect(sortArray(Type.from([6, 5, 4, 3, 2, 1, 0]))).toMatchObject( + Type.from([0, 1, 2, 3, 4, 5, 6]) + ); + + expect(sortArray(Type.from([6, 5, 4, 3, 2, 1]))).toMatchObject( + Type.from([1, 2, 3, 4, 5, 6]) + ); + + const source = fillRand(new Type(1000)); + expect(sortArray(Type.from(source))).toMatchObject( + Type.from(source).sort() + ); + }) + ); + }); + + describe("non-finite numbers", () => { + test("inifinity", () => { + expect(sortArray(new Float32Array([pInf, nInf, 0, 1, 2]))).toMatchObject( + new Float32Array([nInf, 0, 1, 2, pInf]) + ); + expect( + sortArray(new Float32Array([pInf, nInf, pInf, nInf])) + ).toMatchObject(new Float32Array([nInf, nInf, pInf, pInf])); + expect( + sortArray(new Float32Array([pInf, nInf, pInf, nInf, pInf])) + ).toMatchObject(new Float32Array([nInf, nInf, pInf, pInf, pInf])); + expect( + sortArray( + new Float32Array(100).fill(Infinity, 0, 50).fill(-Infinity, 50, 100) + ) + ).toMatchObject( + new Float32Array(100).fill(-Infinity, 0, 50).fill(Infinity, 50, 100) + ); + }); + test("NaN", () => { + expect(sortArray(new Float64Array([NaN, 2, 1, 0]))).toMatchObject( + new Float64Array([0, 1, 2, NaN]) + ); + expect(sortArray(new Float32Array([NaN, 2, 1, 0]))).toMatchObject( + new Float32Array([0, 1, 2, NaN]) + ); + expect(sortArray(new Float32Array([NaN, 2, NaN, 1, 0]))).toMatchObject( + new Float32Array([0, 1, 2, NaN, NaN]) + ); + expect(sortArray(new Float32Array([NaN, 2, 1, NaN, 0]))).toMatchObject( + new Float32Array([0, 1, 2, NaN, NaN]) + ); + expect( + sortArray(fillRange(new Float32Array(100)).fill(NaN, 0, 10)) + ).toMatchObject(fillRange(new Float32Array(100), 10).fill(NaN, 90, 100)); + }); + + test("mixed numbers", () => { + expect( + sortArray(new Float32Array([NaN, pInf, nInf, NaN, NaN])) + ).toMatchObject(new Float32Array([nInf, pInf, NaN, NaN, NaN])); + expect( + sortArray(new Float32Array([NaN, pInf, nInf, NaN, 1, NaN, 2])) + ).toMatchObject(new Float32Array([nInf, 1, 2, pInf, NaN, NaN, NaN])); + expect( + sortArray(new Float32Array([NaN, pInf, nInf, 0, 1, NaN, 2])) + ).toMatchObject(new Float32Array([nInf, 0, 1, 2, pInf, NaN, NaN])); + expect( + sortArray( + fillRange(new Float32Array(100)) + .fill(NaN, 0, 10) + .fill(Infinity, 10, 20) + ) + ).toMatchObject( + fillRange(new Float32Array(100), 20) + .fill(Infinity, 80, 90) + .fill(NaN, 90, 100) + ); + }); + }); }); describe("sortIndex", () => { - [Array, Float32Array, Uint32Array, Int32Array, Float64Array].map(Type => - test(Type.name, () => { - const source1 = Type.from([6, 5, 4, 3, 2, 1, 0]); + describe("finite numbers", () => { + [Array, Float32Array, Uint32Array, Int32Array, Float64Array].map(Type => + test(Type.name, () => { + const source1 = Type.from([6, 5, 4, 3, 2, 1, 0]); + const index1 = fillRange(new Uint32Array(source1.length)); + expect(sortIndex(index1, source1)).toMatchObject( + index1.sort((a, b) => source1[a] - source1[b]) + ); + + const source2 = Type.from([6, 5, 4, 3, 2, 1]); + const index2 = fillRange(new Uint32Array(source2.length)); + expect(sortIndex(index2, source2)).toMatchObject( + index2.sort((a, b) => source1[a] - source1[b]) + ); + + const source3 = fillRand(new Type(1000)); + const index3 = fillRange(new Uint32Array(source3.length)); + expect(sortIndex(index3, source3)).toMatchObject( + index3.sort((a, b) => source1[a] - source1[b]) + ); + }) + ); + }); + + describe("non-finite numbers", () => { + test("mixed numbers", () => { + const source1 = new Float32Array([NaN, pInf, nInf, NaN, 1, NaN, 2]); const index1 = fillRange(new Uint32Array(source1.length)); expect(sortIndex(index1, source1)).toMatchObject( - index1.sort((a, b) => source1[a] - source1[b]) + new Uint32Array([2, 4, 6, 1, 0, 3, 5]) ); - const source2 = Type.from([6, 5, 4, 3, 2, 1]); + const source2 = new Float32Array([NaN, pInf, nInf, 0, 1, NaN, 2]); const index2 = fillRange(new Uint32Array(source2.length)); expect(sortIndex(index2, source2)).toMatchObject( - index2.sort((a, b) => source1[a] - source1[b]) + new Uint32Array([2, 3, 4, 6, 1, 0, 5]) ); - - const source3 = fillRand(new Type(1000)); - const index3 = fillRange(new Uint32Array(source3.length)); - expect(sortIndex(index3, source3)).toMatchObject( - index3.sort((a, b) => source1[a] - source1[b]) - ); - }) - ); + }); + }); +}); + +describe("lowerBound", () => { + test("non-float path", () => { + expect(lowerBound([], 0, 0, 0)).toEqual(0); + + expect(lowerBound([0, 1, 2, 3], -1, 0, 4)).toEqual(0); + expect(lowerBound([0, 1, 2, 3], 0, 0, 4)).toEqual(0); + expect(lowerBound([0, 1, 2, 3], 1, 0, 4)).toEqual(1); + expect(lowerBound([0, 1, 2, 3], 3, 0, 4)).toEqual(3); + expect(lowerBound([0, 1, 2, 3], 4, 0, 4)).toEqual(4); + + expect(lowerBound([0, 1, 2, 3, 4], -1, 0, 5)).toEqual(0); + expect(lowerBound([0, 1, 2, 3, 4], 0, 0, 5)).toEqual(0); + expect(lowerBound([0, 1, 2, 3, 4], 2, 0, 5)).toEqual(2); + expect(lowerBound([0, 1, 2, 3, 4], 4, 0, 5)).toEqual(4); + expect(lowerBound([0, 1, 2, 3, 4], 5, 0, 5)).toEqual(5); + + expect(lowerBound([0, 2, 4, 6, 8], 5, 0, 5)).toEqual(3); + expect(lowerBound([0, 2, 2, 2, 8], 5, 0, 5)).toEqual(4); + + expect(lowerBound([0, 1, 2, 3, 4, 5, 6, 7, 8], 3, 2, 4)).toEqual(3); + expect(lowerBound([0, 1, 2, 3, 4, 5, 6, 7, 8], 99, 2, 4)).toEqual(4); + }); + + test("float path, finites", () => { + expect(lowerBound(new Float32Array([0, 1, 2, 3]), 1, 0, 4)).toEqual(1); + + expect(lowerBound(new Float32Array([]), 0, 0, 0)).toEqual(0); + + expect(lowerBound(new Float32Array([0, 1, 2, 3]), -1, 0, 4)).toEqual(0); + expect(lowerBound(new Float32Array([0, 1, 2, 3]), 0, 0, 4)).toEqual(0); + expect(lowerBound(new Float32Array([0, 1, 2, 3]), 1, 0, 4)).toEqual(1); + expect(lowerBound(new Float32Array([0, 1, 2, 3]), 3, 0, 4)).toEqual(3); + expect(lowerBound(new Float32Array([0, 1, 2, 3]), 4, 0, 4)).toEqual(4); + + expect(lowerBound(new Float32Array([0, 1, 2, 3, 4]), -1, 0, 5)).toEqual(0); + expect(lowerBound(new Float32Array([0, 1, 2, 3, 4]), 0, 0, 5)).toEqual(0); + expect(lowerBound(new Float32Array([0, 1, 2, 3, 4]), 2, 0, 5)).toEqual(2); + expect(lowerBound(new Float32Array([0, 1, 2, 3, 4]), 4, 0, 5)).toEqual(4); + expect(lowerBound(new Float32Array([0, 1, 2, 3, 4]), 5, 0, 5)).toEqual(5); + + expect(lowerBound(new Float32Array([0, 2, 4, 6, 8]), 5, 0, 5)).toEqual(3); + expect(lowerBound(new Float32Array([0, 2, 2, 2, 8]), 5, 0, 5)).toEqual(4); + + expect( + lowerBound(new Float32Array([0, 1, 2, 3, 4, 5, 6, 7, 8]), 3, 2, 4) + ).toEqual(3); + expect( + lowerBound(new Float32Array([0, 1, 2, 3, 4, 5, 6, 7, 8]), 99, 2, 4) + ).toEqual(4); + }); + + test("float path, non-finite", () => { + expect( + lowerBound( + new Float32Array([-Infinity, 0, 1, Infinity, NaN]), + -Infinity, + 0, + 5 + ) + ).toEqual(0); + expect( + lowerBound(new Float32Array([-Infinity, 0, 1, Infinity, NaN]), 0, 0, 5) + ).toEqual(1); + expect( + lowerBound(new Float32Array([-Infinity, 0, 1, Infinity, NaN]), 1, 0, 5) + ).toEqual(2); + expect( + lowerBound(new Float32Array([-Infinity, 0, 1, Infinity, NaN]), 2, 0, 5) + ).toEqual(3); + expect( + lowerBound( + new Float32Array([-Infinity, 0, 1, Infinity, NaN]), + Infinity, + 0, + 5 + ) + ).toEqual(3); + expect( + lowerBound(new Float32Array([-Infinity, 0, 1, Infinity, NaN]), NaN, 0, 5) + ).toEqual(4); + }); }); diff --git a/client/__tests__/util/typedCrossfilter/util.test.js b/client/__tests__/util/typedCrossfilter/util.test.js index b002782a..39ae43d2 100644 --- a/client/__tests__/util/typedCrossfilter/util.test.js +++ b/client/__tests__/util/typedCrossfilter/util.test.js @@ -1,10 +1,7 @@ import { fillRange, sliceByIndex, - makeSortIndex, - lowerBound, - lowerBoundIndirect, - upperBoundIndirect + makeSortIndex } from "../../../src/util/typedCrossfilter/util"; describe("fillRange", () => { diff --git a/client/package-lock.json b/client/package-lock.json index 85ef7cc8..a423ca6d 100644 --- a/client/package-lock.json +++ b/client/package-lock.json @@ -1049,12 +1049,12 @@ } }, "@blueprintjs/core": { - "version": "3.8.0", - "resolved": "https://registry.npmjs.org/@blueprintjs/core/-/core-3.8.0.tgz", - "integrity": "sha512-maw63uME+spdyDUOJH47r+YRc8kDeux8kQz4poiAQKk4eJavc160/aU7w5k4sV29mVputp7P22qNY1oBNXjtDQ==", + "version": "3.15.0", + "resolved": "https://registry.npmjs.org/@blueprintjs/core/-/core-3.15.0.tgz", + "integrity": "sha512-znXO0UaWBJO7Nm2qEvZzURCDuogsJ9DJlIw3N2XnIi+k/c8mQ1xlvtg0TT2pS88lMG9HUEKky3b0UQaSXBw4LA==", "requires": { - "@blueprintjs/icons": "^3.3.0", - "@types/dom4": "^2.0.0", + "@blueprintjs/icons": "^3.7.0", + "@types/dom4": "^2.0.1", "classnames": "^2.2", "dom4": "^2.0.1", "normalize.css": "^8.0.0", @@ -1063,6 +1063,17 @@ "react-transition-group": "^2.2.1", "resize-observer-polyfill": "^1.5.0", "tslib": "^1.9.0" + }, + "dependencies": { + "@blueprintjs/icons": { + "version": "3.8.0", + "resolved": "https://registry.npmjs.org/@blueprintjs/icons/-/icons-3.8.0.tgz", + "integrity": "sha512-yHaRQ3vfV9Gf3foZ4ONtxddz+u5ufkHqHj8Ia5VhPbFgG4el+cPdmsGGIIM72rgKS1KQa5Ay+ggjpByUlXvrKg==", + "requires": { + "classnames": "^2.2", + "tslib": "^1.9.0" + } + } } }, "@blueprintjs/icons": { @@ -5524,7 +5535,8 @@ "ansi-regex": { "version": "2.1.1", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "aproba": { "version": "1.2.0", @@ -5545,12 +5557,14 @@ "balanced-match": { "version": "1.0.0", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "brace-expansion": { "version": "1.1.11", "bundled": true, "dev": true, + "optional": true, "requires": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" @@ -5565,17 +5579,20 @@ "code-point-at": { "version": "1.1.0", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "concat-map": { "version": "0.0.1", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "console-control-strings": { "version": "1.1.0", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "core-util-is": { "version": "1.0.2", @@ -5692,7 +5709,8 @@ "inherits": { "version": "2.0.3", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "ini": { "version": "1.3.5", @@ -5704,6 +5722,7 @@ "version": "1.0.0", "bundled": true, "dev": true, + "optional": true, "requires": { "number-is-nan": "^1.0.0" } @@ -5718,6 +5737,7 @@ "version": "3.0.4", "bundled": true, "dev": true, + "optional": true, "requires": { "brace-expansion": "^1.1.7" } @@ -5725,12 +5745,14 @@ "minimist": { "version": "0.0.8", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "minipass": { "version": "2.2.4", "bundled": true, "dev": true, + "optional": true, "requires": { "safe-buffer": "^5.1.1", "yallist": "^3.0.0" @@ -5749,6 +5771,7 @@ "version": "0.5.1", "bundled": true, "dev": true, + "optional": true, "requires": { "minimist": "0.0.8" } @@ -5829,7 +5852,8 @@ "number-is-nan": { "version": "1.0.1", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "object-assign": { "version": "4.1.1", @@ -5841,6 +5865,7 @@ "version": "1.4.0", "bundled": true, "dev": true, + "optional": true, "requires": { "wrappy": "1" } @@ -5926,7 +5951,8 @@ "safe-buffer": { "version": "5.1.1", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "safer-buffer": { "version": "2.1.2", @@ -5962,6 +5988,7 @@ "version": "1.0.2", "bundled": true, "dev": true, + "optional": true, "requires": { "code-point-at": "^1.0.0", "is-fullwidth-code-point": "^1.0.0", @@ -5981,6 +6008,7 @@ "version": "3.0.1", "bundled": true, "dev": true, + "optional": true, "requires": { "ansi-regex": "^2.0.0" } @@ -6024,12 +6052,14 @@ "wrappy": { "version": "1.0.2", "bundled": true, - "dev": true + "dev": true, + "optional": true }, "yallist": { "version": "3.0.2", "bundled": true, - "dev": true + "dev": true, + "optional": true } } }, diff --git a/client/package.json b/client/package.json index 90b8849c..ff505f45 100644 --- a/client/package.json +++ b/client/package.json @@ -31,7 +31,7 @@ "eslint-scope": "3.7.1" }, "dependencies": { - "@blueprintjs/core": "^3.8.0", + "@blueprintjs/core": "^3.15.0", "@blueprintjs/icons": "^3.3.0", "@blueprintjs/select": "^3.8.0", "canvas-fit": "^1.5.0", diff --git a/client/src/components/brushableHistogram/index.js b/client/src/components/brushableHistogram/index.js index a49ca8a2..f12063a0 100644 --- a/client/src/components/brushableHistogram/index.js +++ b/client/src/components/brushableHistogram/index.js @@ -5,14 +5,12 @@ https://bl.ocks.org/SpaceActuary/2f004899ea1b2bd78d6f1dbb2febf771 */ // jshint esversion: 6 import React from "react"; -import _ from "lodash"; import { Button, ButtonGroup, Tooltip } from "@blueprintjs/core"; import { connect } from "react-redux"; import * as d3 from "d3"; import memoize from "memoize-one"; import * as globals from "../../globals"; import actions from "../../actions"; -import finiteExtent from "../../util/finiteExtent"; import { makeContinuousDimensionName } from "../../util/nameCreators"; @connect(state => ({ @@ -21,53 +19,52 @@ import { makeContinuousDimensionName } from "../../util/nameCreators"; scatterplotYYaccessor: state.controls.scatterplotYYaccessor, continuousSelection: state.continuousSelection, differential: state.differential, - colorAccessor: state.colors.colorAccessor, - obsAnnotations: _.get(state.world, "obsAnnotations", null) + colorAccessor: state.colors.colorAccessor })) class HistogramBrush extends React.Component { - calcHistogramCache = memoize((obsAnnotations, field, rangeMin, rangeMax) => { - const { world } = this.props; - const histogramCache = {}; + static getColumn(world, field, clipped = true) { + /* + Return the underlying Dataframe column for our field. By default, + returns the clipped column. If clipped===false, will return the + unclipped column. + */ + const obsAnnotations = clipped + ? world.obsAnnotations + : world.unclipped.obsAnnotations; + const varData = clipped ? world.varData : world.unclipped.varData; + if (obsAnnotations.hasCol(field)) { + return obsAnnotations.col(field); + } + return varData.col(field); + } + calcHistogramCache = memoize((world, field) => { + /* + recalculate expensive stuff, notably bins, summaries, etc. + */ + const histogramCache = {}; + const col = HistogramBrush.getColumn(world, field); + const values = col.asArray(); + const summary = col.summarize(); + const { min: domainMin, max: domainMax } = summary; + histogramCache.x = d3 + .scaleLinear() + .domain([domainMin, domainMax]) + .range([0, this.width - this.marginRight]); + + histogramCache.bins = d3 + .histogram() + .domain(histogramCache.x.domain()) + .thresholds(40)(values); + + const yMax = histogramCache.bins + .map(b => b.length) + .reduce((a, b) => Math.max(a, b)); histogramCache.y = d3 .scaleLinear() + .domain([0, yMax]) .range([this.height - this.marginBottom, 0]); - if (obsAnnotations.hasCol(field)) { - // recalculate expensive stuff - const allValuesForContinuousFieldAsArray = obsAnnotations - .col(field) - .asArray(); - - histogramCache.x = d3 - .scaleLinear() - .domain([rangeMin, rangeMax]) - .range([0, this.width]); - - histogramCache.bins = d3 - .histogram() - .domain(histogramCache.x.domain()) - .thresholds(40)(allValuesForContinuousFieldAsArray); - - histogramCache.numValues = allValuesForContinuousFieldAsArray.length; - } else if (world.varData.hasCol(field)) { - const varValues = world.varData.col(field).asArray(); - - histogramCache.x = d3 - .scaleLinear() - .domain( - finiteExtent(varValues) - ) /* replace this if we have ranges for genes back from server like we do for annotations on cells */ - .range([0, this.width]); - - histogramCache.bins = d3 - .histogram() - .domain(histogramCache.x.domain()) - .thresholds(40)(varValues); - - histogramCache.numValues = varValues.length; - } - return histogramCache; }); @@ -76,22 +73,23 @@ class HistogramBrush extends React.Component { this.width = 340; this.height = 100; - this.marginBottom = 20; + this.marginBottom = 20; // space for X axis & labels + this.marginRight = 40; // space for Y axis & labels } componentDidMount() { const { field } = this.props; - const { x, y, bins, numValues, svgRef } = this._histogram; + const { x, y, bins, svgRef } = this._histogram; - this.renderAxesBrushBins(x, y, bins, numValues, svgRef, field); + this.renderAxesBrushBins(x, y, bins, svgRef, field); } componentDidUpdate(prevProps) { - const { field, obsAnnotations, continuousSelection } = this.props; - const { x, y, bins, numValues, svgRef } = this._histogram; + const { field, world, continuousSelection } = this.props; + const { x, y, bins, svgRef } = this._histogram; - if (obsAnnotations !== prevProps.obsAnnotations) { - this.renderAxesBrushBins(x, y, bins, numValues, svgRef, field); + if (world !== prevProps.world) { + this.renderAxesBrushBins(x, y, bins, svgRef, field); } /* @@ -172,7 +170,6 @@ class HistogramBrush extends React.Component { onBrushEnd(selection, x) { return () => { const { dispatch, field, isObs, isUserDefined, isDiffExp } = this.props; - const { brushXselection } = this.state; const minAllowedBrushSize = 10; const smallAmountToAvoidInfiniteLoop = 0.1; @@ -197,11 +194,6 @@ class HistogramBrush extends React.Component { smallAmountToAvoidInfiniteLoop; // _range = [x(d3.event.selection[0]), x(procedurallyResizedBrushWidth)]; - - d3.event.target.move(brushXselection, [ - d3.event.selection[0], - procedurallyResizedBrushWidth - ]); } dispatch({ @@ -229,23 +221,16 @@ class HistogramBrush extends React.Component { } drawHistogram(svgRef) { - const { obsAnnotations, field, ranges } = this.props; - const histogramCache = this.calcHistogramCache( - obsAnnotations, - field, - ranges.min, - ranges.max - ); - - const { x, y, bins, numValues } = histogramCache; - - this._histogram = { x, y, bins, numValues, svgRef }; + const { field, world } = this.props; + const histogramCache = this.calcHistogramCache(world, field); + const { x, y, bins } = histogramCache; + this._histogram = { x, y, bins, svgRef }; } handleColorAction() { - const { obsAnnotations, dispatch, field, world, ranges } = this.props; + const { dispatch, field, world, ranges } = this.props; - if (obsAnnotations.hasCol(field)) { + if (world.obsAnnotations.hasCol(field)) { dispatch({ type: "color by continuous metadata", colorAccessor: field, @@ -307,29 +292,29 @@ class HistogramBrush extends React.Component { }; } - renderAxesBrushBins(x, y, bins, numValues, svgRef, field) { + renderAxesBrushBins(x, y, bins, svgRef, field) { + const svg = d3.select(svgRef); + /* Remove everything */ - d3.select(svgRef) - .selectAll("*") - .remove(); + svg.selectAll("*").remove(); /* BINS */ - d3.select(svgRef) + svg .insert("g", "*") .attr("fill", "#bbb") .selectAll("rect") .data(bins) .enter() .append("rect") - .attr("class", "bar") .attr("x", d => x(d.x0) + 1) - .attr("y", d => y(d.length / numValues)) + .attr("y", d => y(d.length)) .attr("width", d => Math.abs(x(d.x1) - x(d.x0) - 1)) - .attr("height", d => y(0) - y(d.length / numValues)); + .attr("height", d => y(0) - y(d.length)); /* BRUSH */ const brushX = d3 .brushX() + .extent([[0, 0], [this.width - this.marginRight, this.height]]) /* emit start so that the Undoable history can save an undo point upon drag start, and ignore the subsequent intermediate drag events. @@ -344,24 +329,24 @@ class HistogramBrush extends React.Component { .attr("data-testid", `${svgRef.dataset.testid}-brush`) .call(brushX); - /* AXIS */ - d3.select(svgRef) + /* X AXIS */ + svg .append("g") .attr("class", "axis axis--x") .attr("transform", `translate(0,${this.height - this.marginBottom})`) .call(d3.axisBottom(x).ticks(5)); - d3.select(svgRef) - .selectAll(".axis--x text") - .style("fill", "rgb(80,80,80)"); + /* Y AXIS */ + svg + .append("g") + .attr("class", "axis axis--y") + .attr("transform", `translate(${this.width - this.marginRight},0)`) + .call(d3.axisRight(y).ticks(3)); - d3.select(svgRef) - .selectAll(".axis--x path") - .style("stroke", "rgb(230,230,230)"); - - d3.select(svgRef) - .selectAll(".axis--x line") - .style("stroke", "rgb(230,230,230)"); + /* axis style */ + svg.selectAll(".axis text").style("fill", "rgb(80,80,80)"); + svg.selectAll(".axis path").style("stroke", "rgb(230,230,230)"); + svg.selectAll(".axis line").style("stroke", "rgb(230,230,230)"); this.setState({ brushX, brushXselection }); } @@ -369,20 +354,29 @@ class HistogramBrush extends React.Component { render() { const { field, + world, colorAccessor, isUserDefined, isDiffExp, logFoldChange, - pval, pvalAdj, scatterplotXXaccessor, scatterplotYYaccessor, zebra } = this.props; - const field_for_id = field.replace(/\s/g, "_"); + const fieldForId = field.replace(/\s/g, "_"); + const { + min: unclippedRangeMin, + max: unclippedRangeMax + } = HistogramBrush.getColumn(world, field, false).summarize(); + const unclippedRangeMinColor = + world.clipQuantiles.min === 0 ? "#bbb" : globals.blue; + const unclippedRangeMaxColor = + world.clipQuantiles.max === 1 ? "#bbb" : globals.blue; + return (
-
+
{isDiffExp || isUserDefined ? ( { @@ -460,15 +460,21 @@ class HistogramBrush extends React.Component {
+ + min {unclippedRangeMin.toPrecision(4)} + {field} + + max {unclippedRangeMax.toPrecision(4)} +
{isDiffExp ? ( diff --git a/client/src/components/continuous/continuous.js b/client/src/components/continuous/continuous.js index 8e169a56..8e8438ba 100644 --- a/client/src/components/continuous/continuous.js +++ b/client/src/components/continuous/continuous.js @@ -64,18 +64,15 @@ class Continuous extends React.Component { ) : null} {obsAnnotations ? _.map(obsAnnotations.colIndex.keys(), key => { - const summary = obsAnnotations.col(key).summarize(); const isColorField = key.includes("color") || key.includes("Color"); + if (key === "name" || isColorField) return null; + + const summary = obsAnnotations.col(key).summarize(); const nonFiniteExtent = summary.min === undefined || summary.max === undefined; - zebra += 1; - if ( - !summary.categorical && - key !== "name" && - !isColorField && - !nonFiniteExtent - ) { + if (!summary.categorical && !nonFiniteExtent) { + zebra += 1; return ( { if (!modifiers.matchesPredicate) { diff --git a/client/src/components/graph/graph.js b/client/src/components/graph/graph.js index a1d42498..7fc3ad80 100644 --- a/client/src/components/graph/graph.js +++ b/client/src/components/graph/graph.js @@ -1,6 +1,5 @@ // jshint esversion: 6 import React from "react"; -import _ from "lodash"; import * as d3 from "d3"; import { connect } from "react-redux"; import mat4 from "gl-mat4"; @@ -12,7 +11,9 @@ import { Popover, Menu, MenuItem, - Position + Position, + NumericInput, + Icon } from "@blueprintjs/core"; import * as globals from "../../globals"; @@ -29,6 +30,8 @@ import { World } from "../../util/stateManager"; world: state.world, universe: state.universe, crossfilter: state.crossfilter, + clipPercentileMin: Math.round(100 * (state.world?.clipQuantiles?.min ?? 0)), + clipPercentileMax: Math.round(100 * (state.world?.clipQuantiles?.max ?? 1)), responsive: state.responsive, colorRGB: state.colors.rgb, opacityForDeselectedCells: state.controls.opacityForDeselectedCells, @@ -47,6 +50,30 @@ import { World } from "../../util/stateManager"; currentSelection: state.graphSelection.selection })) class Graph extends React.Component { + static isValidDigitKeyEvent(e) { + /* + Return true if this event is necessary to enter a percent number input. + Return false if not. + + Returns true for events with keys: backspace, control, alt, meta, [0-9], + or events that don't have a key. + */ + if (e.key === null) return true; + if (e.ctrlKey || e.altKey || e.metaKey) return true; + + // concept borrowed from blueprint's numericInputUtils: + // keys that print a single character when pressed have a `key` name of + // length 1. every other key has a longer `key` name (e.g. "Backspace", + // "ArrowUp", "Shift"). since none of those keys can print a character + // to the field--and since they may have important native behaviors + // beyond printing a character--we don't want to disable their effects. + const isSingleCharKey = e.key.length === 1; + if (!isSingleCharKey) return true; + + const key = e.key.charCodeAt(0) - 48; /* "0" */ + return key >= 0 && key <= 9; + } + constructor(props) { super(props); this.count = 0; @@ -62,7 +89,8 @@ class Graph extends React.Component { svg: null, tool: null, container: null, - mode: "select" + mode: "select", + pendingClipPercentiles: null }; } @@ -265,6 +293,7 @@ class Graph extends React.Component { * there are no userDefinedGenes or diffexpGenes displayed * scatterplot is not displayed * nothing in cellset1 or cellset2 + * clip percentiles are [0,100] */ const { crossfilter, @@ -276,7 +305,9 @@ class Graph extends React.Component { scatterplotXXaccessor, scatterplotYYaccessor, celllist1, - celllist2 + celllist2, + clipPercentileMin, + clipPercentileMax } = this.props; if (!crossfilter || !world || !universe) { @@ -294,7 +325,9 @@ class Graph extends React.Component { nothingColoredBy && noGenes && scatterNotDpl && - nothingInCellsets + nothingInCellsets && + clipPercentileMax === 100 && + clipPercentileMin === 0 ); }; @@ -306,6 +339,102 @@ class Graph extends React.Component { dispatch(actions.resetInterface()); }; + isClipDisabled = () => { + /* + return true if clip button should be disabled. + */ + const { pendingClipPercentiles } = this.state; + const clipPercentileMin = pendingClipPercentiles?.clipPercentileMin; + const clipPercentileMax = pendingClipPercentiles?.clipPercentileMax; + + const { world } = this.props; + const currentClipMin = 100 * world?.clipQuantiles?.min; + const currentClipMax = 100 * world?.clipQuantiles?.max; + + // if you change this test, be careful with logic around + // comparisons between undefined / NaN handling. + const isDisabled = + !(clipPercentileMin < clipPercentileMax) || + (clipPercentileMin === currentClipMin && + clipPercentileMax === currentClipMax); + + return isDisabled; + }; + + handleClipOnKeyPress = e => { + /* + allow only numbers, plus other critical keys which + may be required to make a number + */ + if (!Graph.isValidDigitKeyEvent(e)) { + e.preventDefault(); + } + }; + + handleClipPercentileMinValueChange = v => { + /* + Ignore anything that isn't a legit number + */ + if (!Number.isFinite(v)) return; + + const { pendingClipPercentiles } = this.state; + const clipPercentileMax = pendingClipPercentiles?.clipPercentileMax; + + /* + clamp to [0, currentClipPercentileMax] + */ + if (v <= 0) v = 0; + if (v > 100) v = 100; + const clipPercentileMin = Math.round(v); // paranoia + this.setState({ + pendingClipPercentiles: { clipPercentileMin, clipPercentileMax } + }); + }; + + handleClipPercentileMaxValueChange = v => { + /* + Ignore anything that isn't a legit number + */ + if (!Number.isFinite(v)) return; + + const { pendingClipPercentiles } = this.state; + const clipPercentileMin = pendingClipPercentiles?.clipPercentileMin; + + /* + clamp to [0, 100] + */ + if (v < 0) v = 0; + if (v > 100) v = 100; + const clipPercentileMax = Math.round(v); // paranoia + + this.setState({ + pendingClipPercentiles: { clipPercentileMin, clipPercentileMax } + }); + }; + + handleClipCommit = () => { + const { dispatch } = this.props; + const { pendingClipPercentiles } = this.state; + const { clipPercentileMin, clipPercentileMax } = pendingClipPercentiles; + const min = clipPercentileMin / 100; + const max = clipPercentileMax / 100; + dispatch({ + type: "set clip quantiles", + clipQuantiles: { min, max } + }); + }; + + handleClipOpening = () => { + const { clipPercentileMin, clipPercentileMax } = this.props; + this.setState({ + pendingClipPercentiles: { clipPercentileMin, clipPercentileMax } + }); + }; + + handleClipClosing = () => { + this.setState({ pendingClipPercentiles: null }); + }; + brushToolUpdate(tool, container, offset) { /* this is called from componentDidUpdate(), so be very careful using @@ -614,9 +743,20 @@ class Graph extends React.Component { libraryVersions, undoDisabled, redoDisabled, - selectionTool + selectionTool, + clipPercentileMin, + clipPercentileMax } = this.props; - const { mode } = this.state; + const { mode, pendingClipPercentiles } = this.state; + + const clipMin = + pendingClipPercentiles?.clipPercentileMin ?? clipPercentileMin; + const clipMax = + pendingClipPercentiles?.clipPercentileMax ?? clipPercentileMax; + const activeClipClass = + clipPercentileMin > 0 || clipPercentileMax < 100 + ? " bp3-intent-warning" + : ""; // constants used to create selection tool button let selectionTooltip; @@ -684,66 +824,175 @@ class Graph extends React.Component { reset -
-
- -
+
+ +
-
+
+ + { + dispatch({ type: "@@undoable/undo" }); + }} + style={{ + cursor: "pointer" + }} + /> + + + { + dispatch({ type: "@@undoable/redo" }); + }} + style={{ + cursor: "pointer" + }} + /> + +
+
+ + + } + onOpening={this.handleClipOpening} + onClosing={this.handleClipClosing} + content={ +
+
Clip all continuous values to percentile range
+
+ + +
+ } + /> + + {" "} + -{" "} + + + +
+ } + /> + + +
+
+ } + /> + +
+ +
@@ -786,7 +1035,7 @@ class Graph extends React.Component { >