mirror of
https://github.com/chanzuckerberg/cellxgene.git
synced 2026-09-19 10:58:10 +08:00
Dataframe (#576)
* initial dataframe commit * initial dataframe port of core app * rename variables for clarity * remove unused import * comment out unused code * fix array handling bug in crossfilter dimension creation * allow creation of empty dataframes * handle non-existent columns * handle non-existent columns * revise tests for new dataframe * comments for clarity * comments for clarity * generate bulk add placeholder with real gene names * fix bug in gene name adding * more dataframe unit tests * fix bug - subset from current world, not universe * put cut and pasted code into a single function * improve caching of crossfilter * remove cascading update bug from graph * more performance work * improve state handling for scatterplot * performance optimization of critical path * add column summarization * dataframe utils * add callOnceLazy * fix tests * minor updates found during review * fix misspelling * remove RESTv02 from function names * comment cleanup * cut/icut col parameter defaults to null * break up large test * improve tests and comments on dataframe at/has functions
This commit is contained in:
Bruce Martin
377
client/__tests__/util/dataframe/dataframe.test.js
Normal file
377
client/__tests__/util/dataframe/dataframe.test.js
Normal file
@@ -0,0 +1,377 @@
|
||||
import * as Dataframe from "../../../src/util/dataframe";
|
||||
|
||||
describe("dataframe constructor", () => {
|
||||
test("empty dataframe", () => {
|
||||
const df = new Dataframe.Dataframe([0, 0], []);
|
||||
expect(df).toBeDefined();
|
||||
expect(df.dims).toEqual([0, 0]);
|
||||
expect(df).toHaveLength(0);
|
||||
expect(df.icol(0)).not.toBeDefined();
|
||||
});
|
||||
|
||||
test("create with default indices", () => {
|
||||
const df = new Dataframe.Dataframe(
|
||||
[3, 2],
|
||||
[new Int32Array(3).fill(0), new Int32Array(3).fill(1)]
|
||||
);
|
||||
|
||||
expect(df).toBeDefined();
|
||||
expect(df.dims).toEqual([3, 2]);
|
||||
expect(df.rowIndex).toBeInstanceOf(Dataframe.IdentityInt32Index);
|
||||
expect(df.colIndex).toBeInstanceOf(Dataframe.IdentityInt32Index);
|
||||
expect(df.at(0, 0)).toEqual(0);
|
||||
expect(df.at(2, 1)).toEqual(1);
|
||||
expect(df.iat(0, 0)).toEqual(0);
|
||||
expect(df.iat(2, 1)).toEqual(1);
|
||||
});
|
||||
|
||||
test("create with labelled indices", () => {
|
||||
const df = new Dataframe.Dataframe(
|
||||
[3, 2],
|
||||
[new Int32Array([0, 1, 2]), new Int32Array([3, 4, 5])],
|
||||
new Dataframe.DenseInt32Index([2, 1, 0]),
|
||||
new Dataframe.KeyIndex(["A", "B"])
|
||||
);
|
||||
|
||||
expect(df).toBeDefined();
|
||||
expect(df.dims).toEqual([3, 2]);
|
||||
|
||||
expect(df.rowIndex).toBeInstanceOf(Dataframe.DenseInt32Index);
|
||||
expect(df.colIndex).toBeInstanceOf(Dataframe.KeyIndex);
|
||||
expect(df.rowIndex.keys()).toEqual(new Int32Array([2, 1, 0]));
|
||||
expect(df.colIndex.keys()).toEqual(["A", "B"]);
|
||||
|
||||
expect(df.at(0, "A")).toEqual(2);
|
||||
expect(df.at(2, "B")).toEqual(3);
|
||||
expect(df.iat(0, 0)).toEqual(0);
|
||||
expect(df.iat(2, 1)).toEqual(5);
|
||||
});
|
||||
});
|
||||
|
||||
describe("simple data access", () => {
|
||||
const df = new Dataframe.Dataframe(
|
||||
[4, 2],
|
||||
[
|
||||
new Float64Array([0.0, Number.NaN, Number.POSITIVE_INFINITY, 3.14159]),
|
||||
["red", "blue", "green", "nan"]
|
||||
],
|
||||
new Dataframe.DenseInt32Index([3, 2, 1, 0]),
|
||||
new Dataframe.KeyIndex(["numbers", "colors"])
|
||||
);
|
||||
|
||||
test("iat", () => {
|
||||
expect(df).toBeDefined();
|
||||
|
||||
// present
|
||||
expect(df.iat(0, 0)).toEqual(0.0);
|
||||
expect(df.iat(0, 1)).toEqual("red");
|
||||
expect(df.iat(1, 0)).toEqual(Number.NaN);
|
||||
expect(df.iat(1, 1)).toEqual("blue");
|
||||
expect(df.iat(2, 0)).toEqual(Number.POSITIVE_INFINITY);
|
||||
expect(df.iat(2, 1)).toEqual("green");
|
||||
expect(df.iat(3, 0)).toEqual(3.14159);
|
||||
expect(df.iat(3, 1)).toEqual("nan");
|
||||
|
||||
// labels out of range have no defined behavior
|
||||
});
|
||||
|
||||
test("at", () => {
|
||||
expect(df).toBeDefined();
|
||||
|
||||
// present
|
||||
expect(df.at(3, "numbers")).toEqual(0.0);
|
||||
expect(df.at(3, "colors")).toEqual("red");
|
||||
expect(df.at(2, "numbers")).toEqual(Number.NaN);
|
||||
expect(df.at(2, "colors")).toEqual("blue");
|
||||
expect(df.at(1, "numbers")).toEqual(Number.POSITIVE_INFINITY);
|
||||
expect(df.at(1, "colors")).toEqual("green");
|
||||
expect(df.at(0, "numbers")).toEqual(3.14159);
|
||||
expect(df.at(0, "colors")).toEqual("nan");
|
||||
|
||||
// labels out of range have no defined behavior
|
||||
});
|
||||
|
||||
test("ihas", () => {
|
||||
expect(df).toBeDefined();
|
||||
|
||||
// present
|
||||
expect(df.ihas(0, 0)).toBeTruthy();
|
||||
expect(df.ihas(1, 1)).toBeTruthy();
|
||||
expect(df.ihas(3, 1)).toBeTruthy();
|
||||
|
||||
// not present
|
||||
expect(df.ihas(-1, -1)).toBeFalsy();
|
||||
expect(df.ihas(0, 99)).toBeFalsy();
|
||||
expect(df.ihas(99, 0)).toBeFalsy();
|
||||
expect(df.ihas(99, 99)).toBeFalsy();
|
||||
expect(df.ihas(-1, 0)).toBeFalsy();
|
||||
expect(df.ihas(0, -1)).toBeFalsy();
|
||||
});
|
||||
|
||||
test("has", () => {
|
||||
expect(df).toBeDefined();
|
||||
|
||||
// present
|
||||
expect(df.has(3, "numbers")).toBeTruthy();
|
||||
expect(df.has(0, "numbers")).toBeTruthy();
|
||||
expect(df.has(3, "colors")).toBeTruthy();
|
||||
expect(df.has(0, "colors")).toBeTruthy();
|
||||
|
||||
// not present
|
||||
expect(df.has(3, "foo")).toBeFalsy();
|
||||
expect(df.has(-1, "numbers")).toBeFalsy();
|
||||
expect(df.has(-1, -1)).toBeFalsy();
|
||||
expect(df.has(null, null)).toBeFalsy();
|
||||
expect(df.has(0, "foo")).toBeFalsy();
|
||||
expect(df.has(99, "numbers")).toBeFalsy();
|
||||
expect(df.has(99, "foo")).toBeFalsy();
|
||||
});
|
||||
});
|
||||
|
||||
describe("dataframe subsetting", () => {
|
||||
describe("cutByList", () => {
|
||||
const sourceDf = new Dataframe.Dataframe(
|
||||
[3, 4],
|
||||
[
|
||||
new Int32Array([0, 1, 2]),
|
||||
["A", "B", "C"],
|
||||
new Float32Array([4.4, 5.5, 6.6]),
|
||||
["red", "green", "blue"]
|
||||
],
|
||||
null,
|
||||
new Dataframe.KeyIndex(["int32", "string", "float32", "colors"])
|
||||
);
|
||||
|
||||
test("all rows, one column", () => {
|
||||
const dfA = sourceDf.cutByList(null, ["colors"]);
|
||||
expect(dfA).toBeDefined();
|
||||
expect(dfA.dims).toEqual([3, 1]);
|
||||
expect(dfA.iat(0, 0)).toEqual("red");
|
||||
expect(dfA.at(2, "colors")).toEqual("blue");
|
||||
expect(dfA.col("colors").asArray()).toEqual(["red", "green", "blue"]);
|
||||
expect(dfA.icol(0).asArray()).toEqual(["red", "green", "blue"]);
|
||||
expect(dfA.col("colors").asArray()).toEqual(
|
||||
sourceDf.col("colors").asArray()
|
||||
);
|
||||
expect(dfA.rowIndex.keys()).toEqual(sourceDf.rowIndex.keys());
|
||||
expect(dfA.colIndex.keys()).toEqual(["colors"]);
|
||||
});
|
||||
|
||||
test("all rows, two columns", () => {
|
||||
const dfB = sourceDf.cutByList(null, ["colors", "float32"]);
|
||||
expect(dfB).toBeDefined();
|
||||
expect(dfB.dims).toEqual([3, 2]);
|
||||
expect(dfB.iat(0, 0)).toBeCloseTo(4.4);
|
||||
expect(dfB.iat(0, 1)).toEqual("red");
|
||||
expect(dfB.at(2, "colors")).toEqual("blue");
|
||||
expect(dfB.at(2, "float32")).toBeCloseTo(6.6);
|
||||
expect(dfB.col("colors").asArray()).toEqual(["red", "green", "blue"]);
|
||||
expect(dfB.col("float32").asArray()).toEqual(
|
||||
new Float32Array([4.4, 5.5, 6.6])
|
||||
);
|
||||
expect(dfB.icol(0).asArray()).toEqual(dfB.col("float32").asArray());
|
||||
expect(dfB.icol(1).asArray()).toEqual(dfB.col("colors").asArray());
|
||||
expect(dfB.col("colors").asArray()).toEqual(
|
||||
sourceDf.col("colors").asArray()
|
||||
);
|
||||
expect(dfB.col("float32").asArray()).toEqual(
|
||||
sourceDf.col("float32").asArray()
|
||||
);
|
||||
expect(dfB.rowIndex.keys()).toEqual(sourceDf.rowIndex.keys());
|
||||
expect(dfB.colIndex.keys()).toEqual(["float32", "colors"]);
|
||||
});
|
||||
|
||||
test("one row, all columns", () => {
|
||||
const dfC = sourceDf.cutByList([1], null);
|
||||
expect(dfC).toBeDefined();
|
||||
expect(dfC.dims).toEqual([1, 4]);
|
||||
expect(dfC.iat(0, 0)).toEqual(1);
|
||||
expect(dfC.iat(0, 1)).toEqual("B");
|
||||
expect(dfC.iat(0, 2)).toBeCloseTo(5.5);
|
||||
expect(dfC.iat(0, 3)).toEqual("green");
|
||||
expect(dfC.rowIndex.keys()).toEqual(new Int32Array([1]));
|
||||
expect(dfC.colIndex.keys()).toEqual(sourceDf.colIndex.keys());
|
||||
});
|
||||
|
||||
test("two rows, all columns", () => {
|
||||
const dfD = sourceDf.cutByList([0, 2], null);
|
||||
expect(dfD).toBeDefined();
|
||||
expect(dfD.dims).toEqual([2, 4]);
|
||||
expect(dfD.icol(0).asArray()).toEqual(new Int32Array([0, 2]));
|
||||
expect(dfD.icol(1).asArray()).toEqual(["A", "C"]);
|
||||
expect(dfD.icol(2).asArray()).toEqual(new Float32Array([4.4, 6.6]));
|
||||
expect(dfD.icol(3).asArray()).toEqual(["red", "blue"]);
|
||||
expect(dfD.rowIndex.keys()).toEqual(new Int32Array([0, 2]));
|
||||
expect(dfD.colIndex.keys()).toEqual(sourceDf.colIndex.keys());
|
||||
});
|
||||
|
||||
test("all rows, all columns", () => {
|
||||
const dfE = sourceDf.cutByList(null, null);
|
||||
expect(dfE).toBeDefined();
|
||||
expect(dfE.dims).toEqual([3, 4]);
|
||||
expect(dfE.icol(0).asArray()).toEqual(sourceDf.icol(0).asArray());
|
||||
expect(dfE.icol(1).asArray()).toEqual(sourceDf.icol(1).asArray());
|
||||
expect(dfE.icol(2).asArray()).toEqual(sourceDf.icol(2).asArray());
|
||||
expect(dfE.icol(3).asArray()).toEqual(sourceDf.icol(3).asArray());
|
||||
expect(dfE.rowIndex.keys()).toEqual(sourceDf.rowIndex.keys());
|
||||
expect(dfE.colIndex.keys()).toEqual(sourceDf.colIndex.keys());
|
||||
});
|
||||
|
||||
test("two rows, two colums", () => {
|
||||
const dfF = sourceDf.cutByList([0, 2], ["int32", "float32"]);
|
||||
expect(dfF).toBeDefined();
|
||||
expect(dfF.dims).toEqual([2, 2]);
|
||||
expect(dfF.icol(0).asArray()).toEqual(new Int32Array([0, 2]));
|
||||
expect(dfF.icol(1).asArray()).toEqual(new Float32Array([4.4, 6.6]));
|
||||
expect(dfF.rowIndex.keys()).toEqual(new Int32Array([0, 2]));
|
||||
expect(dfF.colIndex.keys()).toEqual(["int32", "float32"]);
|
||||
});
|
||||
});
|
||||
|
||||
test("icutByMask", () => {
|
||||
const sourceDf = new Dataframe.Dataframe(
|
||||
[3, 4],
|
||||
[
|
||||
new Int32Array([0, 1, 2]),
|
||||
["A", "B", "C"],
|
||||
new Float32Array([4.4, 5.5, 6.6]),
|
||||
["red", "green", "blue"]
|
||||
],
|
||||
new Dataframe.DenseInt32Index([2, 4, 6]),
|
||||
new Dataframe.KeyIndex(["int32", "string", "float32", "colors"])
|
||||
);
|
||||
|
||||
const dfA = sourceDf.icutByMask(
|
||||
new Uint8Array([0, 1, 1]),
|
||||
new Uint8Array([1, 0, 0, 1])
|
||||
);
|
||||
expect(dfA.dims).toEqual([2, 2]);
|
||||
expect(dfA.icol(0).asArray()).toEqual(new Int32Array([1, 2]));
|
||||
expect(dfA.icol(1).asArray()).toEqual(["green", "blue"]);
|
||||
expect(dfA.rowIndex.keys()).toEqual(new Int32Array([4, 6]));
|
||||
expect(dfA.colIndex.keys()).toEqual(["int32", "colors"]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("dataframe factories", () => {
|
||||
test("create", () => {
|
||||
const df = Dataframe.Dataframe.create(
|
||||
[3, 3],
|
||||
[
|
||||
new Array(3).fill(0),
|
||||
new Int16Array(3).fill(99),
|
||||
new Float64Array(3).fill(1.1)
|
||||
]
|
||||
);
|
||||
|
||||
expect(df).toBeDefined();
|
||||
expect(df.dims).toEqual([3, 3]);
|
||||
expect(df).toHaveLength(3);
|
||||
expect(df.iat(0, 0)).toEqual(0);
|
||||
expect(df.iat(1, 1)).toEqual(99);
|
||||
expect(df.iat(2, 2)).toBeCloseTo(1.1);
|
||||
expect(df.iat(0, 0)).toEqual(df.at(0, 0));
|
||||
expect(df.iat(1, 1)).toEqual(df.at(1, 1));
|
||||
expect(df.iat(2, 2)).toEqual(df.at(2, 2));
|
||||
});
|
||||
|
||||
test("clone", () => {
|
||||
const dfA = new Dataframe.Dataframe(
|
||||
[3, 2],
|
||||
[new Int32Array([0, 1, 2]), new Int32Array([3, 4, 5])],
|
||||
new Dataframe.DenseInt32Index([2, 1, 0]),
|
||||
new Dataframe.KeyIndex(["A", "B"])
|
||||
);
|
||||
|
||||
const dfB = dfA.clone();
|
||||
expect(dfB).not.toBe(dfA);
|
||||
expect(dfB.dims).toEqual(dfA.dims);
|
||||
expect(dfB).toHaveLength(dfA.length);
|
||||
expect(dfB.rowIndex.keys()).toEqual(dfA.rowIndex.keys());
|
||||
expect(dfB.colIndex.keys()).toEqual(dfA.colIndex.keys());
|
||||
for (let i = 0, l = dfB.dims[1]; i < l; i += 1) {
|
||||
expect(dfB.icol(i).asArray()).toEqual(dfA.icol(i).asArray());
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("dataframe col", () => {
|
||||
let df = null;
|
||||
beforeEach(() => {
|
||||
df = new Dataframe.Dataframe(
|
||||
[2, 2],
|
||||
[[true, false], [1, 0]],
|
||||
null,
|
||||
new Dataframe.KeyIndex(["A", "B"])
|
||||
);
|
||||
});
|
||||
|
||||
test("col", () => {
|
||||
expect(df).toBeDefined();
|
||||
expect(df.col("A")).toBe(df.icol(0));
|
||||
expect(df.col("B")).toBe(df.icol(1));
|
||||
expect(df.col("undefined")).toBeUndefined();
|
||||
expect(df.icol("undefined")).toBeUndefined();
|
||||
|
||||
const colA = df.col("A");
|
||||
expect(colA).toBeInstanceOf(Function);
|
||||
expect(colA.asArray).toBeInstanceOf(Function);
|
||||
expect(colA.has).toBeInstanceOf(Function);
|
||||
expect(colA.ihas).toBeInstanceOf(Function);
|
||||
expect(colA.indexOf).toBeInstanceOf(Function);
|
||||
expect(colA.iget).toBeInstanceOf(Function);
|
||||
});
|
||||
|
||||
test("col.asArray", () => {
|
||||
expect(df).toBeDefined();
|
||||
expect(df.col("A").asArray()).toEqual([true, false]);
|
||||
expect(df.icol(0).asArray()).toEqual([true, false]);
|
||||
expect(df.col("B").asArray()).toEqual([1, 0]);
|
||||
expect(df.icol(1).asArray()).toEqual([1, 0]);
|
||||
});
|
||||
|
||||
test("col.has", () => {
|
||||
expect(df).toBeDefined();
|
||||
expect(df.col("A").has(-1)).toBe(false);
|
||||
expect(df.col("A").has(0)).toBe(true);
|
||||
expect(df.col("A").has(1)).toBe(true);
|
||||
expect(df.col("A").has(2)).toBe(false);
|
||||
expect(df.col("B").has(-1)).toBe(false);
|
||||
expect(df.col("B").has(0)).toBe(true);
|
||||
expect(df.col("B").has(1)).toBe(true);
|
||||
expect(df.col("B").has(2)).toBe(false);
|
||||
});
|
||||
|
||||
test("col.ihas", () => {
|
||||
expect(df).toBeDefined();
|
||||
expect(df.col("A").ihas(-1)).toBe(false);
|
||||
expect(df.col("A").ihas(0)).toBe(true);
|
||||
expect(df.col("A").ihas(1)).toBe(true);
|
||||
expect(df.col("A").ihas(2)).toBe(false);
|
||||
expect(df.col("B").ihas(-1)).toBe(false);
|
||||
expect(df.col("B").ihas(0)).toBe(true);
|
||||
expect(df.col("B").ihas(1)).toBe(true);
|
||||
expect(df.col("B").ihas(2)).toBe(false);
|
||||
});
|
||||
|
||||
test("col.iget", () => {
|
||||
expect(df).toBeDefined();
|
||||
expect(df.col("A").iget(0)).toEqual(df.iat(0, 0));
|
||||
expect(df.col("B").iget(1)).toEqual(df.iat(1, 1));
|
||||
});
|
||||
|
||||
test("col.indexOf", () => {
|
||||
expect(df).toBeDefined();
|
||||
expect(df.col("A").indexOf(true)).toEqual(0);
|
||||
expect(df.col("A").indexOf(false)).toEqual(1);
|
||||
expect(df.col("A").indexOf(99)).toBeUndefined();
|
||||
expect(df.col("A").indexOf(undefined)).toBeUndefined();
|
||||
expect(df.col("A").indexOf(1)).toBeUndefined();
|
||||
|
||||
expect(df.col("B").indexOf(1)).toEqual(0);
|
||||
expect(df.col("B").indexOf(0)).toEqual(1);
|
||||
expect(df.col("B").indexOf(99)).toBeUndefined();
|
||||
expect(df.col("B").indexOf(undefined)).toBeUndefined();
|
||||
expect(df.col("B").indexOf(true)).toBeUndefined();
|
||||
});
|
||||
});
|
||||
@@ -157,16 +157,6 @@ const anAnnotationsVarFBSResponse = (() => {
|
||||
return encodeMatrix(columns, anAnnotationsVarJSONResponse.names);
|
||||
})();
|
||||
|
||||
const aLayoutJSONResponse = {
|
||||
layout: {
|
||||
ndims: 2,
|
||||
coordinates: _()
|
||||
.range(nObs)
|
||||
.map(idx => [idx, Math.random(), Math.random()])
|
||||
.value()
|
||||
}
|
||||
};
|
||||
|
||||
const aLayoutFBSResponse = (() => {
|
||||
const coords = [
|
||||
new Float32Array(nObs).fill(Math.random()),
|
||||
@@ -190,7 +180,7 @@ const aLayoutFBSResponse = (() => {
|
||||
|
||||
NetEncoding.Matrix.startMatrix(builder);
|
||||
NetEncoding.Matrix.addNRows(builder, nObs);
|
||||
NetEncoding.Matrix.addNCols(builder, nVar);
|
||||
NetEncoding.Matrix.addNCols(builder, coords.length);
|
||||
NetEncoding.Matrix.addColumns(builder, columns);
|
||||
const matrix = NetEncoding.Matrix.endMatrix(builder);
|
||||
builder.finish(matrix);
|
||||
|
||||
@@ -1,4 +1,9 @@
|
||||
import summarizeAnnotations from "../../../src/util/stateManager/summarizeAnnotations";
|
||||
import * as Dataframe from "../../../src/util/dataframe";
|
||||
|
||||
function float32Conversion(f) {
|
||||
return new Float32Array([39.3])[0];
|
||||
}
|
||||
|
||||
describe("summarizeAnnotations", () => {
|
||||
const schema = {
|
||||
@@ -20,7 +25,8 @@ describe("summarizeAnnotations", () => {
|
||||
};
|
||||
|
||||
test("empty test", () => {
|
||||
const summary = summarizeAnnotations(schema, [], []);
|
||||
const df = Dataframe.Dataframe.empty();
|
||||
const summary = summarizeAnnotations(schema, df, df.clone());
|
||||
expect(summary).toEqual(
|
||||
expect.objectContaining({
|
||||
obs: {
|
||||
@@ -69,18 +75,27 @@ describe("summarizeAnnotations", () => {
|
||||
});
|
||||
|
||||
test("simple test", () => {
|
||||
const obsAnnotations = [
|
||||
{
|
||||
__index__: 0,
|
||||
name: "n1",
|
||||
nameString: "hi",
|
||||
nameBoolean: true,
|
||||
nameFloat32: 39.3,
|
||||
nameInt32: 99,
|
||||
nameCategorical: 1
|
||||
}
|
||||
];
|
||||
const varAnnotations = [];
|
||||
const obsAnnotations = new Dataframe.Dataframe(
|
||||
[1, 6],
|
||||
[
|
||||
["n1"],
|
||||
["hi"],
|
||||
[true],
|
||||
new Float32Array([39.3]),
|
||||
new Int32Array([99]),
|
||||
[1]
|
||||
],
|
||||
null,
|
||||
new Dataframe.KeyIndex([
|
||||
"name",
|
||||
"nameString",
|
||||
"nameBoolean",
|
||||
"nameFloat32",
|
||||
"nameInt32",
|
||||
"nameCategorical"
|
||||
])
|
||||
);
|
||||
const varAnnotations = Dataframe.Dataframe.empty();
|
||||
|
||||
const summary = summarizeAnnotations(
|
||||
schema,
|
||||
@@ -105,7 +120,13 @@ describe("summarizeAnnotations", () => {
|
||||
},
|
||||
nameFloat32: {
|
||||
categorical: false,
|
||||
range: { min: 39.3, max: 39.3, nan: 0, ninf: 0, pinf: 0 }
|
||||
range: {
|
||||
min: float32Conversion(39.3),
|
||||
max: float32Conversion(39.3),
|
||||
nan: 0,
|
||||
ninf: 0,
|
||||
pinf: 0
|
||||
}
|
||||
},
|
||||
nameInt32: {
|
||||
categorical: false,
|
||||
@@ -124,36 +145,27 @@ describe("summarizeAnnotations", () => {
|
||||
});
|
||||
|
||||
test("multi test", () => {
|
||||
const obsAnnotations = [
|
||||
{
|
||||
__index__: 0,
|
||||
name: "n0",
|
||||
nameString: "hi",
|
||||
nameBoolean: false,
|
||||
nameFloat32: 39.3,
|
||||
nameInt32: 99,
|
||||
nameCategorical: 1
|
||||
},
|
||||
{
|
||||
__index__: 1,
|
||||
name: "n1",
|
||||
nameString: "hi",
|
||||
nameBoolean: true,
|
||||
nameFloat32: 39.3,
|
||||
nameInt32: 99,
|
||||
nameCategorical: false
|
||||
},
|
||||
{
|
||||
__index__: 2,
|
||||
name: "n2",
|
||||
nameString: "bye",
|
||||
nameBoolean: true,
|
||||
nameFloat32: 0,
|
||||
nameInt32: 99,
|
||||
nameCategorical: "0"
|
||||
}
|
||||
];
|
||||
const varAnnotations = [];
|
||||
const obsAnnotations = new Dataframe.Dataframe(
|
||||
[3, 6],
|
||||
[
|
||||
["n0", "n1", "n2"],
|
||||
["hi", "hi", "bye"],
|
||||
[false, true, true],
|
||||
new Float32Array([39.3, 39.3, 0]),
|
||||
new Int32Array([99, 99, 99]),
|
||||
[1, false, "0"]
|
||||
],
|
||||
null,
|
||||
new Dataframe.KeyIndex([
|
||||
"name",
|
||||
"nameString",
|
||||
"nameBoolean",
|
||||
"nameFloat32",
|
||||
"nameInt32",
|
||||
"nameCategorical"
|
||||
])
|
||||
);
|
||||
const varAnnotations = Dataframe.Dataframe.empty();
|
||||
|
||||
const summary = summarizeAnnotations(
|
||||
schema,
|
||||
@@ -178,7 +190,13 @@ describe("summarizeAnnotations", () => {
|
||||
},
|
||||
nameFloat32: {
|
||||
categorical: false,
|
||||
range: { min: 0, max: 39.3, nan: 0, ninf: 0, pinf: 0 }
|
||||
range: {
|
||||
min: 0,
|
||||
max: float32Conversion(39.3),
|
||||
nan: 0,
|
||||
ninf: 0,
|
||||
pinf: 0
|
||||
}
|
||||
},
|
||||
nameInt32: {
|
||||
categorical: false,
|
||||
@@ -197,45 +215,32 @@ describe("summarizeAnnotations", () => {
|
||||
});
|
||||
|
||||
test("non-finite numbers", () => {
|
||||
const obsAnnotations = [
|
||||
{
|
||||
__index__: 0,
|
||||
name: "n0",
|
||||
nameString: "hi",
|
||||
nameBoolean: false,
|
||||
nameFloat32: 39.3,
|
||||
nameInt32: 99,
|
||||
nameCategorical: 1
|
||||
},
|
||||
{
|
||||
__index__: 1,
|
||||
name: "n1",
|
||||
nameString: "hi",
|
||||
nameBoolean: true,
|
||||
nameFloat32: Number.NEGATIVE_INFINITY,
|
||||
nameInt32: 99,
|
||||
nameCategorical: false
|
||||
},
|
||||
{
|
||||
__index__: 2,
|
||||
name: "n2",
|
||||
nameString: "bye",
|
||||
nameBoolean: true,
|
||||
nameFloat32: Number.NaN,
|
||||
nameInt32: 99,
|
||||
nameCategorical: "0"
|
||||
},
|
||||
{
|
||||
__index__: 3,
|
||||
name: "n2",
|
||||
nameString: "bye",
|
||||
nameBoolean: true,
|
||||
nameFloat32: Number.POSITIVE_INFINITY,
|
||||
nameInt32: 99,
|
||||
nameCategorical: "0"
|
||||
}
|
||||
];
|
||||
const varAnnotations = [];
|
||||
const obsAnnotations = new Dataframe.Dataframe(
|
||||
[4, 6],
|
||||
[
|
||||
["n0", "n1", "n2", "n2"],
|
||||
["hi", "hi", "bye", "bye"],
|
||||
[false, true, true, true],
|
||||
new Float32Array([
|
||||
39.3,
|
||||
Number.NEGATIVE_INFINITY,
|
||||
Number.NaN,
|
||||
Number.POSITIVE_INFINITY
|
||||
]),
|
||||
new Int32Array([99, 99, 99, 99]),
|
||||
[1, false, "0", "0"]
|
||||
],
|
||||
null,
|
||||
new Dataframe.KeyIndex([
|
||||
"name",
|
||||
"nameString",
|
||||
"nameBoolean",
|
||||
"nameFloat32",
|
||||
"nameInt32",
|
||||
"nameCategorical"
|
||||
])
|
||||
);
|
||||
const varAnnotations = Dataframe.Dataframe.empty();
|
||||
|
||||
const summary = summarizeAnnotations(
|
||||
schema,
|
||||
@@ -260,7 +265,13 @@ describe("summarizeAnnotations", () => {
|
||||
},
|
||||
nameFloat32: {
|
||||
categorical: false,
|
||||
range: { min: 39.3, max: 39.3, nan: 1, ninf: 1, pinf: 1 }
|
||||
range: {
|
||||
min: float32Conversion(39.3),
|
||||
max: float32Conversion(39.3),
|
||||
nan: 1,
|
||||
ninf: 1,
|
||||
pinf: 1
|
||||
}
|
||||
},
|
||||
nameInt32: {
|
||||
categorical: false,
|
||||
|
||||
@@ -1,13 +1,13 @@
|
||||
import _ from "lodash";
|
||||
import * as Universe from "../../../src/util/stateManager/universe";
|
||||
import * as Dataframe from "../../../src/util/dataframe";
|
||||
import * as REST from "./sampleResponses";
|
||||
|
||||
describe("createUniverseFromRestV02Response", () => {
|
||||
describe("createUniverseFromResponse", () => {
|
||||
/*
|
||||
test createUniverseFromRestV02Response - this function converts
|
||||
test createUniverseFromResponse - this function converts
|
||||
a set of REST 0.2 responses into a "new" Universe.
|
||||
|
||||
createUniverseFromRestV02Response(
|
||||
createUniverseFromResponse(
|
||||
configResponse,
|
||||
schemaResponse,
|
||||
annotationsObsResponse,
|
||||
@@ -30,7 +30,7 @@ describe("createUniverseFromRestV02Response", () => {
|
||||
create a universe from sample data nad validate its shape & contents
|
||||
*/
|
||||
const { nObs, nVar } = REST.schema.schema.dataframe;
|
||||
const universe = Universe.createUniverseFromRestV02Response(
|
||||
const universe = Universe.createUniverseFromResponse(
|
||||
REST.config,
|
||||
REST.schema,
|
||||
REST.annotationsObs,
|
||||
@@ -45,23 +45,23 @@ describe("createUniverseFromRestV02Response", () => {
|
||||
nObs,
|
||||
nVar,
|
||||
schema: REST.schema.schema,
|
||||
obsAnnotations: expect.any(Array),
|
||||
varAnnotations: expect.any(Array),
|
||||
obsNameToIndexMap: expect.any(Object),
|
||||
varNameToIndexMap: expect.any(Object),
|
||||
obsLayout: expect.objectContaining({
|
||||
X: expect.any(Float32Array),
|
||||
Y: expect.any(Float32Array)
|
||||
}),
|
||||
obsAnnotations: expect.any(Dataframe.Dataframe),
|
||||
varAnnotations: expect.any(Dataframe.Dataframe),
|
||||
obsLayout: expect.any(Dataframe.Dataframe),
|
||||
summary: expect.any(Object),
|
||||
varDataCache: expect.any(Object)
|
||||
})
|
||||
);
|
||||
|
||||
expect(universe.obsAnnotations).toHaveLength(nObs);
|
||||
expect(_.keys(universe.obsNameToIndexMap)).toHaveLength(nObs);
|
||||
expect(universe.obsLayout.X).toHaveLength(nObs);
|
||||
expect(universe.obsLayout.Y).toHaveLength(nObs);
|
||||
expect(universe.varAnnotations).toHaveLength(nVar);
|
||||
expect(_.keys(universe.varNameToIndexMap)).toHaveLength(nVar);
|
||||
expect(universe.obsAnnotations.dims).toEqual([
|
||||
nObs,
|
||||
REST.schema.schema.annotations.obs.length
|
||||
]);
|
||||
expect(universe.obsLayout.dims).toEqual([nObs, 2]);
|
||||
expect(universe.obsLayout.colIndex.keys()).toEqual(["X", "Y"]);
|
||||
expect(universe.varAnnotations.dims).toEqual([
|
||||
nVar,
|
||||
REST.schema.schema.annotations.var.length
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import _ from "lodash";
|
||||
import * as Universe from "../../../src/util/stateManager/universe";
|
||||
import * as World from "../../../src/util/stateManager/world";
|
||||
import * as Dataframe from "../../../src/util/dataframe";
|
||||
import Crossfilter from "../../../src/util/typedCrossfilter";
|
||||
import * as REST from "./sampleResponses";
|
||||
import {
|
||||
@@ -16,7 +17,7 @@ the default REST test response.
|
||||
const defaultBigBang = () => {
|
||||
/* create unverse, world, crossfilter and dimensionMap */
|
||||
/* create universe */
|
||||
const universe = Universe.createUniverseFromRestV02Response(
|
||||
const universe = Universe.createUniverseFromResponse(
|
||||
REST.config,
|
||||
REST.schema,
|
||||
REST.annotationsObs,
|
||||
@@ -40,7 +41,7 @@ const defaultBigBang = () => {
|
||||
|
||||
describe("createWorldFromEntireUniverse", () => {
|
||||
test("create from REST sample", () => {
|
||||
const universe = Universe.createUniverseFromRestV02Response(
|
||||
const universe = Universe.createUniverseFromResponse(
|
||||
REST.config,
|
||||
REST.schema,
|
||||
REST.annotationsObs,
|
||||
@@ -75,10 +76,7 @@ describe("createWorldFromEntireUniverse", () => {
|
||||
.value()
|
||||
}),
|
||||
|
||||
varDataCache: expect.any(Object),
|
||||
|
||||
obsIndex: null, // null indicating full universe
|
||||
obsBackIndex: null
|
||||
varDataCache: expect.any(Object)
|
||||
})
|
||||
);
|
||||
});
|
||||
@@ -111,51 +109,43 @@ describe("createWorldFromCurrentSelection", () => {
|
||||
*/
|
||||
|
||||
/* matchFilter must match the dimension filters above */
|
||||
const matchFilter = val => val.field1 >= 0 && val.field1 < 5 && !val.field3;
|
||||
const universeIndices = _()
|
||||
.range(universe.nObs)
|
||||
.filter(idx => matchFilter(universe.obsAnnotations[idx]))
|
||||
.value();
|
||||
|
||||
const expected = {
|
||||
nObs: universeIndices.length,
|
||||
obsAnnotations: _.map(universeIndices, i => universe.obsAnnotations[i]),
|
||||
obsLayout: {
|
||||
X: new Float32Array(
|
||||
_.map(universeIndices, i => universe.obsLayout.X[i])
|
||||
),
|
||||
Y: new Float32Array(
|
||||
_.map(universeIndices, i => universe.obsLayout.Y[i])
|
||||
)
|
||||
},
|
||||
obsBackIndex: _.transform(
|
||||
universeIndices,
|
||||
(result, univIdx, worldIdx) => {
|
||||
result[univIdx] = worldIdx;
|
||||
},
|
||||
new Uint32Array(universe.nObs).fill(-1)
|
||||
),
|
||||
obsIndex: new Uint32Array(universeIndices)
|
||||
const matchFilter = (df, row) => {
|
||||
const field1 = df.at(row, "field1");
|
||||
const field3 = df.at(row, "field3");
|
||||
return field1 >= 0 && field1 < 5 && !field3;
|
||||
};
|
||||
const matchingIndices = _()
|
||||
.range(universe.nObs)
|
||||
.filter(idx => matchFilter(universe.obsAnnotations, idx))
|
||||
.value();
|
||||
|
||||
expect(world).toMatchObject(
|
||||
expect.objectContaining({
|
||||
api: "0.2",
|
||||
nObs: expected.nObs,
|
||||
nObs: matchingIndices.length,
|
||||
nVar: universe.nVar,
|
||||
schema: universe.schema,
|
||||
obsAnnotations: expected.obsAnnotations,
|
||||
obsAnnotations: expect.any(Dataframe.Dataframe),
|
||||
varAnnotations: universe.varAnnotations,
|
||||
obsLayout: expected.obsLayout,
|
||||
obsLayout: expect.any(Dataframe.Dataframe),
|
||||
summary: {
|
||||
obs: expect.any(Object) /* we could do better! */,
|
||||
var: expect.any(Object) /* we could do better! */
|
||||
},
|
||||
varDataCache: expect.any(Object),
|
||||
obsIndex: expected.obsIndex,
|
||||
obsBackIndex: expected.obsBackIndex
|
||||
varDataCache: expect.any(Object)
|
||||
})
|
||||
);
|
||||
|
||||
expect(world.obsAnnotations.rowIndex.keys()).toEqual(
|
||||
new Int32Array(matchingIndices)
|
||||
);
|
||||
expect(world.obsAnnotations.colIndex.keys()).toEqual(
|
||||
universe.obsAnnotations.colIndex.keys()
|
||||
);
|
||||
expect(world.obsLayout.rowIndex.keys()).toEqual(
|
||||
new Int32Array(matchingIndices)
|
||||
);
|
||||
expect(world.obsLayout.colIndex.keys()).toEqual(["X", "Y"]);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -219,7 +209,9 @@ describe("subsetVarData", () => {
|
||||
world,
|
||||
crossfilter
|
||||
);
|
||||
expect(newWorld.obsIndex).toMatchObject(new Uint32Array([0, 2]));
|
||||
expect(newWorld.obsAnnotations.rowIndex.keys()).toEqual(
|
||||
new Int32Array([0, 2])
|
||||
);
|
||||
|
||||
/* expect a subset */
|
||||
const result = World.subsetVarData(newWorld, universe, sourceVarData);
|
||||
|
||||
@@ -2,16 +2,27 @@ import {
|
||||
countCategoryValues2D,
|
||||
clearCaches
|
||||
} from "../../../src/util/stateManager/worldUtil";
|
||||
import * as Dataframe from "../../../src/util/dataframe";
|
||||
|
||||
describe("WorldUtil cache management", () => {
|
||||
test("empty", () => {
|
||||
const count = countCategoryValues2D("a", "b", []);
|
||||
const count = countCategoryValues2D(
|
||||
"a",
|
||||
"b",
|
||||
new Dataframe.Dataframe([0, 0], [])
|
||||
);
|
||||
expect(count).toMatchObject(new Map());
|
||||
expect(count.size).toBe(0);
|
||||
});
|
||||
|
||||
test("simple couts", () => {
|
||||
const rows = [{ a: 0, b: false }, { a: 0, b: true }, { a: 1, b: false }];
|
||||
const count = countCategoryValues2D("a", "b", rows);
|
||||
const df = new Dataframe.Dataframe(
|
||||
[3, 2],
|
||||
[[0, 0, 1], [false, true, false]],
|
||||
null,
|
||||
new Dataframe.KeyIndex(["a", "b"])
|
||||
);
|
||||
const count = countCategoryValues2D("a", "b", df);
|
||||
expect(count).toMatchObject(
|
||||
new Map([
|
||||
[0, new Map([[true, 1], [false, 1]])],
|
||||
@@ -22,16 +33,22 @@ describe("WorldUtil cache management", () => {
|
||||
|
||||
test("memo cache clear", () => {
|
||||
clearCaches();
|
||||
const row1 = [];
|
||||
const row2 = [{ a: 0, b: false }, { a: 0, b: true }, { a: 1, b: false }];
|
||||
const count1 = countCategoryValues2D("a", "b", row1);
|
||||
const count2 = countCategoryValues2D("a", "b", row1);
|
||||
const count3 = countCategoryValues2D("a", "b", []);
|
||||
const count4 = countCategoryValues2D("a", "b", row2);
|
||||
const df1 = new Dataframe.Dataframe([0, 0], []);
|
||||
const df2 = new Dataframe.Dataframe(
|
||||
[3, 2],
|
||||
[[0, 0, 1], [false, true, false]],
|
||||
null,
|
||||
new Dataframe.KeyIndex(["a", "b"])
|
||||
);
|
||||
|
||||
const count1 = countCategoryValues2D("a", "b", df1);
|
||||
const count2 = countCategoryValues2D("a", "b", df1);
|
||||
const count3 = countCategoryValues2D("a", "b", df1.clone());
|
||||
const count4 = countCategoryValues2D("a", "b", df2);
|
||||
|
||||
clearCaches();
|
||||
const count10 = countCategoryValues2D("a", "b", row1);
|
||||
const count11 = countCategoryValues2D("a", "b", row2);
|
||||
const count10 = countCategoryValues2D("a", "b", df1);
|
||||
const count11 = countCategoryValues2D("a", "b", df2);
|
||||
|
||||
expect(count1).toEqual(count2);
|
||||
expect(count1).toEqual(count3);
|
||||
|
||||
@@ -118,16 +118,16 @@ describe("selectionCount", () => {
|
||||
const dim2 = ba.allocDimension();
|
||||
expect(dim2).toBeDefined();
|
||||
|
||||
expect(ba.selectionCount).toEqual(0);
|
||||
expect(ba.selectionCount()).toEqual(0);
|
||||
ba.selectAll(dim1);
|
||||
expect(ba.selectionCount).toEqual(0);
|
||||
expect(ba.selectionCount()).toEqual(0);
|
||||
ba.selectAll(dim2);
|
||||
expect(ba.selectionCount).toEqual(defaultTestLength);
|
||||
expect(ba.selectionCount()).toEqual(defaultTestLength);
|
||||
|
||||
for (let i = 0; i < defaultTestLength; i += 1) {
|
||||
ba.deselectOne(dim1, i);
|
||||
expect(ba.selectionCount).toEqual(defaultTestLength - i - 1);
|
||||
expect(ba.selectionCount).toEqual(ba.countAllOnes());
|
||||
expect(ba.selectionCount()).toEqual(defaultTestLength - i - 1);
|
||||
expect(ba.selectionCount()).toEqual(ba.countAllOnes());
|
||||
}
|
||||
|
||||
ba.freeDimension(dim1);
|
||||
|
||||
@@ -119,14 +119,22 @@ function groupReduce(data, valueMap, valueReduce, valueInit) {
|
||||
}
|
||||
|
||||
function groupCount(data, map) {
|
||||
return groupReduce(data, map, (p, v) => p + 1, () => 0);
|
||||
return groupReduce(data, map, p => p + 1, () => 0);
|
||||
}
|
||||
|
||||
function groupSum(data, map) {
|
||||
return groupReduce(data, map, (p, v) => (p += map(v)), () => 0);
|
||||
return groupReduce(
|
||||
data,
|
||||
map,
|
||||
(p, v) => {
|
||||
p += map(v);
|
||||
return p;
|
||||
},
|
||||
() => 0
|
||||
);
|
||||
}
|
||||
|
||||
var payments = null;
|
||||
let payments = null;
|
||||
beforeEach(() => {
|
||||
payments = crossfilter(someData);
|
||||
});
|
||||
@@ -139,7 +147,7 @@ describe("typedCrossfilter", () => {
|
||||
|
||||
const quantity = payments.dimension(
|
||||
crossfilter.ScalarDimension,
|
||||
r => r.quantity,
|
||||
(i, data) => data[i].quantity,
|
||||
Int32Array
|
||||
);
|
||||
expect(quantity).toBeDefined();
|
||||
@@ -154,20 +162,23 @@ describe("typedCrossfilter", () => {
|
||||
expect(payments).toBeDefined();
|
||||
const quantity = payments.dimension(
|
||||
crossfilter.ScalarDimension,
|
||||
r => r.quantity,
|
||||
(i, data) => data[i].quantity,
|
||||
Int32Array
|
||||
);
|
||||
const tip = payments.dimension(
|
||||
crossfilter.ScalarDimension,
|
||||
r => r.tip,
|
||||
(i, data) => data[i].tip,
|
||||
Float32Array
|
||||
);
|
||||
const total = payments.dimension(
|
||||
crossfilter.ScalarDimension,
|
||||
r => r.total,
|
||||
(i, data) => data[i].total,
|
||||
Float32Array
|
||||
);
|
||||
const type = payments.dimension(crossfilter.EnumDimension, r => r.type);
|
||||
const type = payments.dimension(
|
||||
crossfilter.EnumDimension,
|
||||
(i, data) => data[i].type
|
||||
);
|
||||
|
||||
expect(quantity).toBeDefined();
|
||||
expect(tip).toBeDefined();
|
||||
@@ -214,20 +225,18 @@ describe("typedCrossfilter", () => {
|
||||
expect(payments).toBeDefined();
|
||||
const quantity = payments.dimension(
|
||||
crossfilter.ScalarDimension,
|
||||
r => r.quantity,
|
||||
(i, data) => data[i].quantity,
|
||||
Int32Array
|
||||
);
|
||||
const tip = payments.dimension(
|
||||
crossfilter.ScalarDimension,
|
||||
r => r.tip,
|
||||
(i, data) => data[i].tip,
|
||||
Float32Array
|
||||
);
|
||||
const total = payments.dimension(
|
||||
crossfilter.ScalarDimension,
|
||||
r => r.total,
|
||||
Float32Array
|
||||
const type = payments.dimension(
|
||||
crossfilter.EnumDimension,
|
||||
(i, data) => data[i].type
|
||||
);
|
||||
const type = payments.dimension(crossfilter.EnumDimension, r => r.type);
|
||||
|
||||
quantity.filterExact(1);
|
||||
expect(payments.countFiltered()).toEqual(
|
||||
@@ -250,20 +259,23 @@ describe("typedCrossfilter", () => {
|
||||
expect(payments).toBeDefined();
|
||||
const quantity = payments.dimension(
|
||||
crossfilter.ScalarDimension,
|
||||
r => r.quantity,
|
||||
(i, data) => data[i].quantity,
|
||||
Int32Array
|
||||
);
|
||||
const tip = payments.dimension(
|
||||
crossfilter.ScalarDimension,
|
||||
r => r.tip,
|
||||
(i, data) => data[i].tip,
|
||||
Float32Array
|
||||
);
|
||||
const total = payments.dimension(
|
||||
crossfilter.ScalarDimension,
|
||||
r => r.total,
|
||||
(i, data) => data[i].total,
|
||||
Float32Array
|
||||
);
|
||||
const type = payments.dimension(crossfilter.EnumDimension, r => r.type);
|
||||
const type = payments.dimension(
|
||||
crossfilter.EnumDimension,
|
||||
(i, data) => data[i].type
|
||||
);
|
||||
|
||||
tip.filterRange([0, 91]);
|
||||
expect(payments.allFiltered()).toEqual(
|
||||
@@ -291,20 +303,23 @@ describe("typedCrossfilter", () => {
|
||||
expect(payments).toBeDefined();
|
||||
const quantity = payments.dimension(
|
||||
crossfilter.ScalarDimension,
|
||||
r => r.quantity,
|
||||
(i, data) => data[i].quantity,
|
||||
Int32Array
|
||||
);
|
||||
const tip = payments.dimension(
|
||||
crossfilter.ScalarDimension,
|
||||
r => r.tip,
|
||||
(i, data) => data[i].tip,
|
||||
Float32Array
|
||||
);
|
||||
const total = payments.dimension(
|
||||
crossfilter.ScalarDimension,
|
||||
r => r.total,
|
||||
(i, data) => data[i].total,
|
||||
Float32Array
|
||||
);
|
||||
const type = payments.dimension(crossfilter.EnumDimension, r => r.type);
|
||||
const type = payments.dimension(
|
||||
crossfilter.EnumDimension,
|
||||
(i, data) => data[i].type
|
||||
);
|
||||
|
||||
type.filterEnum(["tab", "cash"]);
|
||||
expect(payments.allFiltered()).toEqual(
|
||||
@@ -326,27 +341,30 @@ describe("typedCrossfilter", () => {
|
||||
expect(payments).toBeDefined();
|
||||
const quantity = payments.dimension(
|
||||
crossfilter.ScalarDimension,
|
||||
r => r.quantity,
|
||||
(i, data) => data[i].quantity,
|
||||
Int32Array
|
||||
);
|
||||
const tip = payments.dimension(
|
||||
crossfilter.ScalarDimension,
|
||||
r => r.tip,
|
||||
(i, data) => data[i].tip,
|
||||
Float32Array
|
||||
);
|
||||
const total = payments.dimension(
|
||||
crossfilter.ScalarDimension,
|
||||
r => r.total,
|
||||
(i, data) => data[i].total,
|
||||
Float32Array
|
||||
);
|
||||
const type = payments.dimension(crossfilter.EnumDimension, r => r.type);
|
||||
const type = payments.dimension(
|
||||
crossfilter.EnumDimension,
|
||||
(i, data) => data[i].type
|
||||
);
|
||||
|
||||
// Create a bunch of fake dimensions to ensure we can handle > 32
|
||||
let dimMap = {};
|
||||
for (let i = 0; i < 65; i++) {
|
||||
dimMap[i] = payments.dimension(
|
||||
crossfilter.ScalarDimension,
|
||||
r => Math.random(),
|
||||
() => Math.random(),
|
||||
Float32Array
|
||||
);
|
||||
expect(dimMap[i]).toBeDefined();
|
||||
@@ -372,18 +390,21 @@ describe("typedCrossfilter", () => {
|
||||
|
||||
const quantity = payments.dimension(
|
||||
crossfilter.ScalarDimension,
|
||||
r => r.quantity,
|
||||
(i, data) => data[i].quantity,
|
||||
Int32Array
|
||||
);
|
||||
const tip = payments.dimension(
|
||||
crossfilter.ScalarDimension,
|
||||
r => r.tip,
|
||||
(i, data) => data[i].tip,
|
||||
Int32Array
|
||||
);
|
||||
const type = payments.dimension(crossfilter.EnumDimension, r => r.type);
|
||||
const type = payments.dimension(
|
||||
crossfilter.EnumDimension,
|
||||
(i, data) => data[i].type
|
||||
);
|
||||
const total = payments.dimension(
|
||||
crossfilter.ScalarDimension,
|
||||
r => r.total,
|
||||
(i, data) => data[i].total,
|
||||
Int32Array
|
||||
);
|
||||
|
||||
@@ -411,15 +432,18 @@ describe("typedCrossfilter", () => {
|
||||
|
||||
const tip = payments.dimension(
|
||||
crossfilter.ScalarDimension,
|
||||
r => r.tip,
|
||||
(i, data) => data[i].tip,
|
||||
Int32Array
|
||||
);
|
||||
const totalX10 = payments.dimension(
|
||||
crossfilter.ScalarDimension,
|
||||
r => r.total * 10,
|
||||
(i, data) => data[i].total * 10,
|
||||
Int32Array
|
||||
);
|
||||
const type = payments.dimension(crossfilter.EnumDimension, r => r.type);
|
||||
const type = payments.dimension(
|
||||
crossfilter.EnumDimension,
|
||||
(i, data) => data[i].type
|
||||
);
|
||||
|
||||
const paymentsByTip_A = tip.group();
|
||||
const paymentsByTip_B = tip.group(r => 10 * r);
|
||||
@@ -458,10 +482,13 @@ describe("typedCrossfilter", () => {
|
||||
|
||||
const total = payments.dimension(
|
||||
crossfilter.ScalarDimension,
|
||||
r => r.total,
|
||||
(i, data) => data[i].total,
|
||||
Float32Array
|
||||
);
|
||||
const type = payments.dimension(crossfilter.EnumDimension, r => r.type);
|
||||
const type = payments.dimension(
|
||||
crossfilter.EnumDimension,
|
||||
(i, data) => data[i].type
|
||||
);
|
||||
|
||||
const paymentsByTotal = total.group();
|
||||
const paymentsByType = type.group();
|
||||
@@ -499,15 +526,18 @@ describe("typedCrossfilter", () => {
|
||||
|
||||
const tip = payments.dimension(
|
||||
crossfilter.ScalarDimension,
|
||||
r => r.tip,
|
||||
(i, data) => data[i].tip,
|
||||
Int32Array
|
||||
);
|
||||
const total = payments.dimension(
|
||||
crossfilter.ScalarDimension,
|
||||
r => r.total,
|
||||
(i, data) => data[i].total,
|
||||
Int32Array
|
||||
);
|
||||
const type = payments.dimension(crossfilter.EnumDimension, r => r.type);
|
||||
const type = payments.dimension(
|
||||
crossfilter.EnumDimension,
|
||||
(i, data) => data[i].type
|
||||
);
|
||||
|
||||
const paymentsByTip = tip.group();
|
||||
const paymentsByTotal = total.group();
|
||||
|
||||
@@ -40,7 +40,7 @@ const doInitialDataLoad = () =>
|
||||
/* set config defaults */
|
||||
const config = { ...globals.configDefaults, ...results[0].config };
|
||||
const [, schema, obsAnno, varAnno, obsLayout] = [...results];
|
||||
const universe = Universe.createUniverseFromRestV02Response(
|
||||
const universe = Universe.createUniverseFromResponse(
|
||||
config,
|
||||
schema,
|
||||
obsAnno,
|
||||
@@ -242,12 +242,14 @@ const requestDifferentialExpression = (set1, set2, num_genes = 10) => async (
|
||||
*/
|
||||
const state = getState();
|
||||
const { universe } = state.controls;
|
||||
const set1ByIndex = rangeEncodeIndices(
|
||||
_.map(set1, s => universe.obsNameToIndexMap[s])
|
||||
);
|
||||
const set2ByIndex = rangeEncodeIndices(
|
||||
_.map(set2, s => universe.obsNameToIndexMap[s])
|
||||
);
|
||||
|
||||
// These lines ensure that we convert any TypedArray to an Array.
|
||||
// This is necessary because JSON.stringify() does some very strange
|
||||
// things with TypedArrays (they are marshalled to JSON objects, rather
|
||||
// than being marshalled as a JSON array).
|
||||
const aset1 = Array.isArray(set1) ? set1 : Array.from(set1);
|
||||
const aset2 = Array.isArray(set2) ? set2 : Array.from(set2);
|
||||
|
||||
const res = await fetch(
|
||||
`${globals.API.prefix}${globals.API.version}diffexp/obs`,
|
||||
{
|
||||
@@ -259,8 +261,8 @@ const requestDifferentialExpression = (set1, set2, num_genes = 10) => async (
|
||||
body: JSON.stringify({
|
||||
mode: "topN",
|
||||
count: num_genes,
|
||||
set1: { filter: { obs: { index: set1ByIndex } } },
|
||||
set2: { filter: { obs: { index: set2ByIndex } } }
|
||||
set1: { filter: { obs: { index: aset1 } } },
|
||||
set2: { filter: { obs: { index: aset2 } } }
|
||||
})
|
||||
}
|
||||
);
|
||||
@@ -271,7 +273,9 @@ const requestDifferentialExpression = (set1, set2, num_genes = 10) => async (
|
||||
|
||||
const data = await res.json();
|
||||
// result is [ [varIdx, ...], ... ]
|
||||
const topNGenes = _.map(data, r => universe.varAnnotations[r[0]].name);
|
||||
const topNGenes = _.map(data, r =>
|
||||
universe.varAnnotations.at(r[0], "name")
|
||||
);
|
||||
|
||||
/*
|
||||
Kick off secondary action to fetch all of the expression data for the
|
||||
|
||||
@@ -35,9 +35,11 @@ class HistogramBrush extends React.Component {
|
||||
.scaleLinear()
|
||||
.range([this.height - this.marginBottom, 0]);
|
||||
|
||||
if (obsAnnotations[0][field] !== undefined) {
|
||||
if (obsAnnotations.col(field)) {
|
||||
// recalculate expensive stuff
|
||||
const allValuesForContinuousFieldAsArray = _.map(obsAnnotations, field);
|
||||
const allValuesForContinuousFieldAsArray = obsAnnotations
|
||||
.col(field)
|
||||
.asArray();
|
||||
|
||||
histogramCache.x = d3
|
||||
.scaleLinear()
|
||||
@@ -149,7 +151,7 @@ class HistogramBrush extends React.Component {
|
||||
initializeRanges
|
||||
} = this.props;
|
||||
|
||||
if (obsAnnotations[0][field]) {
|
||||
if (obsAnnotations.col(field)) {
|
||||
dispatch({
|
||||
type: "color by continuous metadata",
|
||||
colorAccessor: field,
|
||||
|
||||
@@ -65,7 +65,11 @@ class CategoryValue extends React.Component {
|
||||
})[0].categories;
|
||||
}
|
||||
|
||||
if (colorAccessor && !isColorBy) {
|
||||
if (
|
||||
colorAccessor &&
|
||||
!isColorBy &&
|
||||
categoricalSelectionState[colorAccessor]
|
||||
) {
|
||||
occupancy = countCategoryValues2D(
|
||||
metadataField,
|
||||
colorAccessor,
|
||||
|
||||
@@ -10,7 +10,6 @@ import HistogramBrush from "../brushableHistogram";
|
||||
|
||||
@connect(state => ({
|
||||
ranges: _.get(state.controls.world, "summary.obs", null),
|
||||
metadata: _.get(state.controls.world, "obsAnnotations", null),
|
||||
colorAccessor: state.controls.colorAccessor,
|
||||
colorScale: state.controls.colorScale,
|
||||
selectionUpdate: _.get(state.controls, "crossfilter.updateTime", null),
|
||||
@@ -39,7 +38,7 @@ class Continuous extends React.Component {
|
||||
}
|
||||
|
||||
render() {
|
||||
const { ranges, obsAnnotations, schema } = this.props;
|
||||
const { ranges, schema } = this.props;
|
||||
if (schema && !this.continuousChecked) {
|
||||
this.hasContinuous = _.some(
|
||||
schema.annotations.obs,
|
||||
@@ -73,7 +72,6 @@ class Continuous extends React.Component {
|
||||
field={key}
|
||||
isObs
|
||||
zebra={zebra % 2 === 0}
|
||||
fieldValues={obsAnnotations}
|
||||
ranges={value.range}
|
||||
handleColorAction={this.handleColorAction(key).bind(this)}
|
||||
/>
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
// jshint esversion: 6
|
||||
import React from "react";
|
||||
import _ from "lodash";
|
||||
import { AnchorButton, Tooltip } from "@blueprintjs/core";
|
||||
import { connect } from "react-redux";
|
||||
import { World } from "../../util/stateManager";
|
||||
|
||||
@connect()
|
||||
class CellSetButton extends React.Component {
|
||||
@@ -14,7 +14,7 @@ class CellSetButton extends React.Component {
|
||||
eitherCellSetOneOrTwo
|
||||
} = this.props;
|
||||
|
||||
const set = _.map(crossfilter.allFiltered(), "name");
|
||||
const set = World.getSelectedByIndex(crossfilter);
|
||||
|
||||
if (!differential.diffExp) {
|
||||
/* diffexp needs to be cleared before we store a new set */
|
||||
|
||||
@@ -29,8 +29,7 @@ const renderGene = (fuzzySortResult, { handleClick, modifiers, query }) => {
|
||||
return null;
|
||||
}
|
||||
/* the fuzzysort wraps the object with other properties, like a score */
|
||||
const gene = fuzzySortResult.obj;
|
||||
const text = gene.name;
|
||||
const geneName = fuzzySortResult.target;
|
||||
|
||||
return (
|
||||
<MenuItem
|
||||
@@ -39,39 +38,34 @@ const renderGene = (fuzzySortResult, { handleClick, modifiers, query }) => {
|
||||
// Use of annotations in this way is incorrect and dataset specific.
|
||||
// See https://github.com/chanzuckerberg/cellxgene/issues/483
|
||||
// label={gene.n_counts}
|
||||
key={gene.name}
|
||||
onClick={g => {
|
||||
key={geneName}
|
||||
onClick={g =>
|
||||
/* this fires when user clicks a menu item */
|
||||
handleClick(g);
|
||||
}}
|
||||
text={text}
|
||||
handleClick(g)
|
||||
}
|
||||
text={geneName}
|
||||
/>
|
||||
);
|
||||
};
|
||||
|
||||
const filterGenes = (query, genes) => {
|
||||
const filterGenes = (query, genes) =>
|
||||
/* fires on load, once, and then for each character typed into the input */
|
||||
return fuzzysort.go(query, genes, {
|
||||
key: "name",
|
||||
fuzzysort.go(query, genes, {
|
||||
limit: 5,
|
||||
threshold: -10000 // don't return bad results
|
||||
});
|
||||
};
|
||||
|
||||
@connect(state => {
|
||||
const metadata = _.get(state.controls.world, "obsAnnotations", null);
|
||||
const ranges = _.get(state.controls.world, "summary.obs", null);
|
||||
const initializeRanges = _.get(state.controls.world, "summary.obs");
|
||||
|
||||
return {
|
||||
ranges,
|
||||
metadata,
|
||||
initializeRanges,
|
||||
userDefinedGenes: state.controls.userDefinedGenes,
|
||||
userDefinedGenesLoading: state.controls.userDefinedGenesLoading,
|
||||
world: state.controls.world,
|
||||
colorAccessor: state.controls.colorAccessor,
|
||||
allGeneNames: state.controls.allGeneNames,
|
||||
differential: state.differential
|
||||
};
|
||||
})
|
||||
@@ -84,6 +78,37 @@ class GeneExpression extends React.Component {
|
||||
};
|
||||
}
|
||||
|
||||
placeholderGeneNames() {
|
||||
/*
|
||||
return a string containing gene name suggestions for use as a user hint.
|
||||
Eg., Apod, Cd74, ...
|
||||
Will return a max of 3 genes, totalling 15 characters in length.
|
||||
Randomly selects gene names.
|
||||
|
||||
NOTE: the random selection means it will re-render constantly.
|
||||
*/
|
||||
const { world } = this.props;
|
||||
const { varAnnotations } = world;
|
||||
const geneNames = varAnnotations.col("name").asArray();
|
||||
if (geneNames.length > 0) {
|
||||
const placeholder = [];
|
||||
let len = geneNames.length;
|
||||
const maxGeneNameCount = 3;
|
||||
const maxStrLength = 15;
|
||||
len = len < maxGeneNameCount ? len : maxGeneNameCount;
|
||||
for (let i = 0, strLen = 0; i < len && strLen < maxStrLength; i += 1) {
|
||||
const deal = Math.floor(Math.random() * geneNames.length);
|
||||
const geneName = geneNames[deal];
|
||||
placeholder.push(geneName);
|
||||
strLen += geneName.length + 2; // '2' is the length of a comma and space
|
||||
}
|
||||
placeholder.push("...");
|
||||
return placeholder.join(", ");
|
||||
}
|
||||
// default - should never happen.
|
||||
return "Apod, Cd74, ...";
|
||||
}
|
||||
|
||||
handleClick(g) {
|
||||
const { world, dispatch, userDefinedGenes } = this.props;
|
||||
const gene = g.target;
|
||||
@@ -93,7 +118,7 @@ class GeneExpression extends React.Component {
|
||||
postUserErrorToast(
|
||||
"That's too many genes, you can have at most 15 user defined genes"
|
||||
);
|
||||
} else if (!_.find(world.varAnnotations, { name: gene })) {
|
||||
} else if (world.varAnnotations.col("name").indexOf(gene) === undefined) {
|
||||
postUserErrorToast("That doesn't appear to be a valid gene name.");
|
||||
} else {
|
||||
dispatch(actions.requestUserDefinedGene(gene));
|
||||
@@ -116,9 +141,13 @@ class GeneExpression extends React.Component {
|
||||
const genes = _.pull(_.uniq(bulkAdd.split(/[ ,]+/)), "");
|
||||
|
||||
genes.forEach(gene => {
|
||||
if (userDefinedGenes.indexOf(gene) !== -1) {
|
||||
if (gene.length === 0) {
|
||||
keepAroundErrorToast("Must enter a gene name.");
|
||||
} else if (userDefinedGenes.indexOf(gene) !== -1) {
|
||||
keepAroundErrorToast("That gene already exists");
|
||||
} else if (!_.find(world.varAnnotations, { name: gene })) {
|
||||
} else if (
|
||||
world.varAnnotations.col("name").indexOf(gene) === undefined
|
||||
) {
|
||||
keepAroundErrorToast(
|
||||
`${gene} doesn't appear to be a valid gene name.`
|
||||
);
|
||||
@@ -214,8 +243,8 @@ class GeneExpression extends React.Component {
|
||||
itemRenderer={renderGene.bind(this)}
|
||||
items={
|
||||
world && world.varAnnotations
|
||||
? world.varAnnotations
|
||||
: [{ name: "No genes" }]
|
||||
? world.varAnnotations.col("name").asArray()
|
||||
: ["No genes"]
|
||||
}
|
||||
popoverProps={{ minimal: true }}
|
||||
/>
|
||||
@@ -245,7 +274,7 @@ class GeneExpression extends React.Component {
|
||||
this.setState({ bulkAdd: e.target.value });
|
||||
}}
|
||||
id="text-input-bulk-add"
|
||||
placeholder="Apod, Cd74, ..."
|
||||
placeholder={this.placeholderGeneNames()}
|
||||
value={bulkAdd}
|
||||
/>
|
||||
<Button
|
||||
@@ -290,8 +319,7 @@ class GeneExpression extends React.Component {
|
||||
<ExpressionButtons />
|
||||
{differential.diffExp
|
||||
? _.map(differential.diffExp, (value, index) => {
|
||||
const annotations = world.varAnnotations[value[0]];
|
||||
const { name } = annotations;
|
||||
const name = world.varAnnotations.at(value[0], "name");
|
||||
const values = world.varDataCache[name];
|
||||
if (!values) {
|
||||
return null;
|
||||
|
||||
@@ -35,7 +35,8 @@ class Graph extends React.Component {
|
||||
this.graphPaddingRight = globals.leftSidebarWidth;
|
||||
this.renderCache = {
|
||||
positions: null,
|
||||
colors: null
|
||||
colors: null,
|
||||
sizes: null
|
||||
};
|
||||
this.state = {
|
||||
svg: null,
|
||||
@@ -83,12 +84,13 @@ class Graph extends React.Component {
|
||||
}
|
||||
|
||||
componentDidUpdate(prevProps) {
|
||||
const { renderCache } = this;
|
||||
const {
|
||||
world,
|
||||
crossfilter,
|
||||
selectionUpdate,
|
||||
colorRGB,
|
||||
responsive
|
||||
responsive,
|
||||
selectionUpdate
|
||||
} = this.props;
|
||||
const {
|
||||
reglRender,
|
||||
@@ -109,35 +111,27 @@ class Graph extends React.Component {
|
||||
|
||||
if (regl && world) {
|
||||
/* update the regl state */
|
||||
const { obsLayout } = world;
|
||||
const cellCount = crossfilter.size();
|
||||
const { obsLayout, nObs } = world;
|
||||
const X = obsLayout.col("X").asArray();
|
||||
const Y = obsLayout.col("Y").asArray();
|
||||
|
||||
// X/Y positions for each point - a cached value that only
|
||||
// changes if we have loaded entirely new cell data
|
||||
//
|
||||
if (
|
||||
!this.renderCache.positions ||
|
||||
selectionUpdate !== prevProps.selectionUpdate
|
||||
) {
|
||||
if (!this.renderCache.positions) {
|
||||
this.renderCache.positions = new Float32Array(2 * cellCount);
|
||||
}
|
||||
if (!renderCache.positions || world !== prevProps.world) {
|
||||
renderCache.positions = new Float32Array(2 * nObs);
|
||||
|
||||
const glScaleX = scaleLinear([0, 1], [-1, 1]);
|
||||
const glScaleY = scaleLinear([0, 1], [1, -1]);
|
||||
|
||||
const offset = [d3.mean(obsLayout.X) - 0.5, d3.mean(obsLayout.Y) - 0.5];
|
||||
const offset = [d3.mean(X) - 0.5, d3.mean(Y) - 0.5];
|
||||
|
||||
for (
|
||||
let i = 0, { positions } = this.renderCache;
|
||||
i < cellCount;
|
||||
i += 1
|
||||
) {
|
||||
positions[2 * i] = glScaleX(obsLayout.X[i] - offset[0]);
|
||||
positions[2 * i + 1] = glScaleY(obsLayout.Y[i] - offset[1]);
|
||||
for (let i = 0, { positions } = renderCache; i < nObs; i += 1) {
|
||||
positions[2 * i] = glScaleX(X[i] - offset[0]);
|
||||
positions[2 * i + 1] = glScaleY(Y[i] - offset[1]);
|
||||
}
|
||||
pointBuffer({
|
||||
data: this.renderCache.positions,
|
||||
data: renderCache.positions,
|
||||
dimension: 2
|
||||
});
|
||||
|
||||
@@ -152,30 +146,28 @@ class Graph extends React.Component {
|
||||
// could have changed for some other reason, but for now color is
|
||||
// the only metadata that changes client-side. If this is problematic,
|
||||
// we could add some sort of color-specific indicator to the app state.
|
||||
if (!this.renderCache.colors || colorRGB !== prevProps.colorRGB) {
|
||||
if (!renderCache.colors || colorRGB !== prevProps.colorRGB) {
|
||||
const rgb = colorRGB;
|
||||
if (!this.renderCache.colors) {
|
||||
this.renderCache.colors = new Float32Array(3 * rgb.length);
|
||||
if (!renderCache.colors) {
|
||||
renderCache.colors = new Float32Array(3 * rgb.length);
|
||||
}
|
||||
for (let i = 0, { colors } = this.renderCache; i < rgb.length; i += 1) {
|
||||
for (let i = 0, { colors } = renderCache; i < rgb.length; i += 1) {
|
||||
colors.set(rgb[i], 3 * i);
|
||||
}
|
||||
colorBuffer({ data: this.renderCache.colors, dimension: 3 });
|
||||
colorBuffer({ data: renderCache.colors, dimension: 3 });
|
||||
}
|
||||
|
||||
// Sizes for each point - this is presumed to change each time the
|
||||
// component receives new props. Almost always a true assumption, as
|
||||
// most property upates are due to changes driving a crossfilter
|
||||
// selection set change.
|
||||
//
|
||||
if (!this.renderCache.sizes) {
|
||||
this.renderCache.sizes = new Float32Array(cellCount);
|
||||
// Sizes for each point - updates are triggered only when selected
|
||||
// obs change
|
||||
if (!renderCache.sizes || selectionUpdate !== prevProps.selectionUpdate) {
|
||||
if (!renderCache.sizes) {
|
||||
renderCache.sizes = new Float32Array(nObs);
|
||||
}
|
||||
crossfilter.fillByIsFiltered(renderCache.sizes, 4, 0.2);
|
||||
sizeBuffer({ data: renderCache.sizes, dimension: 1 });
|
||||
}
|
||||
|
||||
crossfilter.fillByIsFiltered(this.renderCache.sizes, 4, 0.2);
|
||||
sizeBuffer({ data: this.renderCache.sizes, dimension: 1 });
|
||||
|
||||
this.count = cellCount;
|
||||
this.count = nObs;
|
||||
|
||||
regl._refresh();
|
||||
this.reglDraw(
|
||||
|
||||
@@ -65,12 +65,17 @@ class Scatterplot extends React.Component {
|
||||
super(props);
|
||||
this.count = 0;
|
||||
this.axes = false;
|
||||
this.state = {
|
||||
svg: null,
|
||||
minimized: null,
|
||||
this.renderCache = {
|
||||
positions: null,
|
||||
colors: null,
|
||||
sizes: null,
|
||||
xScale: null,
|
||||
yScale: null
|
||||
};
|
||||
this.state = {
|
||||
svg: null,
|
||||
minimized: null
|
||||
};
|
||||
}
|
||||
|
||||
componentDidMount() {
|
||||
@@ -81,6 +86,7 @@ class Scatterplot extends React.Component {
|
||||
if (svg && expressionX && expressionY) {
|
||||
scales = Scatterplot.setupScales(expressionX, expressionY);
|
||||
this.drawAxesSVG(scales.xScale, scales.yScale, svg);
|
||||
this.renderCache = { ...this.renderCache, ...scales };
|
||||
}
|
||||
|
||||
const camera = _camera(this.reglCanvas, { scale: true, rotate: false });
|
||||
@@ -113,8 +119,6 @@ class Scatterplot extends React.Component {
|
||||
pointBuffer,
|
||||
colorBuffer,
|
||||
svg,
|
||||
xScale: scales ? scales.xScale : null,
|
||||
yScale: scales ? scales.yScale : null,
|
||||
reglRender,
|
||||
camera,
|
||||
drawPoints
|
||||
@@ -129,12 +133,11 @@ class Scatterplot extends React.Component {
|
||||
scatterplotYYaccessor,
|
||||
expressionX,
|
||||
expressionY,
|
||||
colorRGB
|
||||
colorRGB,
|
||||
selectionUpdate
|
||||
} = this.props;
|
||||
const {
|
||||
reglRender,
|
||||
xScale,
|
||||
yScale,
|
||||
regl,
|
||||
pointBuffer,
|
||||
colorBuffer,
|
||||
@@ -145,17 +148,12 @@ class Scatterplot extends React.Component {
|
||||
} = this.state;
|
||||
|
||||
if (
|
||||
world &&
|
||||
svg &&
|
||||
xScale &&
|
||||
yScale &&
|
||||
scatterplotXXaccessor &&
|
||||
scatterplotYYaccessor &&
|
||||
(scatterplotXXaccessor !== prevProps.scatterplotXXaccessor || // was CLU now FTH1 etc
|
||||
scatterplotYYaccessor !== prevProps.scatterplotYYaccessor || // was CLU now FTH1 etc
|
||||
!this.axes) // clicked off the tab and back again, rerender
|
||||
scatterplotXXaccessor !== prevProps.scatterplotXXaccessor || // was CLU now FTH1 etc
|
||||
scatterplotYYaccessor !== prevProps.scatterplotYYaccessor // was CLU now FTH1 etc
|
||||
) {
|
||||
this.drawAxesSVG(xScale, yScale, svg);
|
||||
const scales = Scatterplot.setupScales(expressionX, expressionY);
|
||||
this.drawAxesSVG(scales.xScale, scales.yScale, svg);
|
||||
this.renderCache = { ...this.renderCache, ...scales };
|
||||
}
|
||||
|
||||
if (reglRender && this.reglRenderState === "rendering") {
|
||||
@@ -172,35 +170,51 @@ class Scatterplot extends React.Component {
|
||||
expressionX &&
|
||||
expressionY &&
|
||||
scatterplotXXaccessor &&
|
||||
scatterplotYYaccessor &&
|
||||
xScale &&
|
||||
yScale
|
||||
scatterplotYYaccessor
|
||||
) {
|
||||
const { renderCache } = this;
|
||||
const { xScale, yScale } = this.renderCache;
|
||||
const cellCount = expressionX.length;
|
||||
const positionsBuf = new Float32Array(2 * cellCount);
|
||||
const colorsBuf = new Float32Array(3 * cellCount);
|
||||
const sizesBuf = new Float32Array(cellCount);
|
||||
|
||||
const glScaleX = scaleLinear([0, width], [-0.95, 0.95]);
|
||||
const glScaleY = scaleLinear([0, height], [-1, 1]);
|
||||
|
||||
/*
|
||||
Construct Vectors
|
||||
*/
|
||||
for (let i = 0; i < cellCount; i += 1) {
|
||||
positionsBuf[2 * i] = glScaleX(xScale(expressionX[i]));
|
||||
positionsBuf[2 * i + 1] = glScaleY(yScale(expressionY[i]));
|
||||
// Points change when expressionX or expressionY change.
|
||||
if (
|
||||
!renderCache.positions ||
|
||||
expressionX !== prevProps.expressionX ||
|
||||
expressionY !== prevProps.expressionY
|
||||
) {
|
||||
if (!renderCache.positions) {
|
||||
renderCache.positions = new Float32Array(2 * cellCount);
|
||||
}
|
||||
const glScaleX = scaleLinear([0, width], [-0.95, 0.95]);
|
||||
const glScaleY = scaleLinear([0, height], [-1, 1]);
|
||||
for (let i = 0, { positions } = renderCache; i < cellCount; i += 1) {
|
||||
positions[2 * i] = glScaleX(xScale(expressionX[i]));
|
||||
positions[2 * i + 1] = glScaleY(yScale(expressionY[i]));
|
||||
}
|
||||
pointBuffer({ data: renderCache.positions, dimension: 2 });
|
||||
}
|
||||
|
||||
for (let i = 0; i < cellCount; i += 1) {
|
||||
colorsBuf.set(colorRGB[i], 3 * i);
|
||||
// Colors for each point - change only when props.colorsRGB change.
|
||||
if (!renderCache.colors || colorRGB !== prevProps.colorRGB) {
|
||||
if (!renderCache.colors) {
|
||||
renderCache.colors = new Float32Array(3 * cellCount);
|
||||
}
|
||||
for (let i = 0, { colors } = renderCache; i < cellCount; i += 1) {
|
||||
colors.set(colorRGB[i], 3 * i);
|
||||
}
|
||||
colorBuffer({ data: renderCache.colors, dimension: 3 });
|
||||
}
|
||||
|
||||
crossfilter.fillByIsFiltered(sizesBuf, 4, 0.2);
|
||||
// Sizes for each point - updates are triggered only when selected
|
||||
// obs change
|
||||
if (!renderCache.sizes || selectionUpdate !== prevProps.selctionUpdate) {
|
||||
if (!renderCache.sizes) {
|
||||
renderCache.sizes = new Float32Array(cellCount);
|
||||
}
|
||||
crossfilter.fillByIsFiltered(renderCache.sizes, 4, 0.2);
|
||||
sizeBuffer({ data: renderCache.sizes, dimension: 1 });
|
||||
}
|
||||
|
||||
pointBuffer({ data: positionsBuf, dimension: 2 });
|
||||
colorBuffer({ data: colorsBuf, dimension: 3 });
|
||||
sizeBuffer({ data: sizesBuf, dimension: 1 });
|
||||
this.count = cellCount;
|
||||
|
||||
regl._refresh();
|
||||
@@ -213,16 +227,6 @@ class Scatterplot extends React.Component {
|
||||
camera
|
||||
);
|
||||
}
|
||||
|
||||
if (
|
||||
expressionX &&
|
||||
expressionY &&
|
||||
(scatterplotXXaccessor !== prevProps.scatterplotXXaccessor || // was CLU now FTH1 etc
|
||||
scatterplotYYaccessor !== prevProps.scatterplotYYaccessor)
|
||||
) {
|
||||
const scales = Scatterplot.setupScales(expressionX, expressionY);
|
||||
this.setState(scales);
|
||||
}
|
||||
}
|
||||
|
||||
static setupScales(expressionX, expressionY) {
|
||||
|
||||
@@ -41,13 +41,13 @@ const updateCellColorsMiddleware = store => next => action => {
|
||||
action.type === "color by continuous metadata" ||
|
||||
action.type === "color by categorical metadata";
|
||||
|
||||
if (!filterJustChanged || !s.controls.world.obsAnnotations) {
|
||||
const obsAnnotations = _.get(s.controls, "world.obsAnnotations", null);
|
||||
if (!filterJustChanged || !obsAnnotations) {
|
||||
return next(
|
||||
action
|
||||
); /* if the cells haven't loaded or the action wasn't a color change, bail */
|
||||
}
|
||||
|
||||
const { obsAnnotations } = s.controls.world;
|
||||
let colorScale;
|
||||
const colorsByRGB = new Array(obsAnnotations.length);
|
||||
|
||||
@@ -73,9 +73,9 @@ const updateCellColorsMiddleware = store => next => action => {
|
||||
});
|
||||
|
||||
const key = action.colorAccessor;
|
||||
const col = obsAnnotations.col(key).asArray();
|
||||
for (let i = 0, len = obsAnnotations.length; i < len; i += 1) {
|
||||
const obs = obsAnnotations[i];
|
||||
const cat = obs[key];
|
||||
const cat = col[i];
|
||||
colorsByRGB[i] = colors[cat];
|
||||
}
|
||||
}
|
||||
@@ -96,8 +96,9 @@ const updateCellColorsMiddleware = store => next => action => {
|
||||
|
||||
const key = action.colorAccessor;
|
||||
const nonFiniteColor = parseRGB(globals.nonFiniteCellColor);
|
||||
const col = obsAnnotations.col(key).asArray();
|
||||
for (let i = 0, len = obsAnnotations.length; i < len; i += 1) {
|
||||
const val = obsAnnotations[i][key];
|
||||
const val = col[i];
|
||||
if (Number.isFinite(val)) {
|
||||
const c = colorScale(val);
|
||||
colorsByRGB[i] = colors[c];
|
||||
|
||||
152
client/src/reducers/controls.js
vendored
152
client/src/reducers/controls.js
vendored
@@ -100,6 +100,28 @@ function selectedValuesForCategory(categorySelectionState) {
|
||||
return selectedValues;
|
||||
}
|
||||
|
||||
/*
|
||||
build a crossfilter dimension map for all gene expression related dimensions.
|
||||
*/
|
||||
function createGenesDimMap(userDefinedGenes, diffexpGenes, world, crossfilter) {
|
||||
function _createGenesDimMap(genes, nameF) {
|
||||
return genes.reduce((acc, gene) => {
|
||||
acc[nameF(gene)] = World.createVarDimension(
|
||||
world,
|
||||
world.varDataCache,
|
||||
crossfilter,
|
||||
gene
|
||||
);
|
||||
return acc;
|
||||
}, {});
|
||||
}
|
||||
|
||||
return {
|
||||
..._createGenesDimMap(userDefinedGenes, userDefinedDimensionName),
|
||||
..._createGenesDimMap(diffexpGenes, diffexpDimensionName)
|
||||
};
|
||||
}
|
||||
|
||||
const Controls = (
|
||||
state = {
|
||||
// data loading flag
|
||||
@@ -111,6 +133,7 @@ const Controls = (
|
||||
|
||||
// the whole big bang
|
||||
universe: null,
|
||||
fullUniverseCache: null,
|
||||
|
||||
// all of the data + selection state
|
||||
world: null,
|
||||
@@ -164,9 +187,7 @@ const Controls = (
|
||||
case "initial data load start": {
|
||||
return { ...state, loading: true };
|
||||
}
|
||||
case "initial data load complete (universe exists)":
|
||||
case "reset World to eq Universe": {
|
||||
const { userDefinedGenes, diffexpGenes } = state;
|
||||
case "initial data load complete (universe exists)": {
|
||||
/* first light - create world & other data-driven defaults */
|
||||
const { universe } = action;
|
||||
const world = World.createWorldFromEntireUniverse(universe);
|
||||
@@ -181,51 +202,53 @@ const Controls = (
|
||||
const dimensionMap = World.createObsDimensionMap(crossfilter, world);
|
||||
WorldUtil.clearCaches();
|
||||
|
||||
const worldVarDataCache = world.varDataCache;
|
||||
|
||||
// dimensionMap = {
|
||||
// layout_X: dim-for-X,
|
||||
// obsAnno_name: dim for an annotation,
|
||||
// varData_userDefined_genename: dim for user defined expression,
|
||||
// varData_diffexp_genename: dim for diff-exp added gene expression
|
||||
// }
|
||||
/* var dimensions */
|
||||
if (userDefinedGenes.length > 0) {
|
||||
/*
|
||||
verbose & slightly confusing that we also access this as an object
|
||||
in controls rather than an array, should be abstracted into
|
||||
util ie., createDimensionsFromBothListsOfGenes(userGenes, diffExp)
|
||||
*/
|
||||
_.forEach(userDefinedGenes, gene => {
|
||||
dimensionMap[
|
||||
userDefinedDimensionName(gene)
|
||||
] = World.createVarDimension(
|
||||
/* "__var__" + */
|
||||
world,
|
||||
worldVarDataCache,
|
||||
crossfilter,
|
||||
gene
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
if (diffexpGenes.length > 0) {
|
||||
_.forEach(diffexpGenes, gene => {
|
||||
dimensionMap[diffexpDimensionName(gene)] = World.createVarDimension(
|
||||
/* "__var__" + */
|
||||
world,
|
||||
worldVarDataCache,
|
||||
crossfilter,
|
||||
gene
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
...state,
|
||||
loading: false,
|
||||
error: null,
|
||||
universe,
|
||||
fullUniverseCache: { world, crossfilter, dimensionMap },
|
||||
world,
|
||||
colorRGB,
|
||||
categoricalSelectionState,
|
||||
crossfilter,
|
||||
dimensionMap,
|
||||
colorAccessor: null,
|
||||
resettingInterface: false
|
||||
};
|
||||
}
|
||||
case "reset World to eq Universe": {
|
||||
const {
|
||||
userDefinedGenes,
|
||||
diffexpGenes,
|
||||
universe,
|
||||
fullUniverseCache
|
||||
} = state;
|
||||
const { world, crossfilter } = fullUniverseCache;
|
||||
// reset all crossfilter dimensions
|
||||
_.forEach(fullUniverseCache.dimensionMap, dim => dim.filterAll());
|
||||
const colorRGB = new Array(universe.nObs).fill(
|
||||
parseRGB(globals.defaultCellColor)
|
||||
);
|
||||
const categoricalSelectionState = createCategoricalSelectionState(
|
||||
state,
|
||||
world
|
||||
);
|
||||
|
||||
/* free dimensions not in cache (otherwise they leak) */
|
||||
_.forEach(state.dimensionMap, (dim, dimName) => {
|
||||
if (!fullUniverseCache.dimensionMap[dimName]) {
|
||||
dim.dispose();
|
||||
}
|
||||
});
|
||||
const dimensionMap = {
|
||||
...fullUniverseCache.dimensionMap,
|
||||
...createGenesDimMap(userDefinedGenes, diffexpGenes, world, crossfilter)
|
||||
};
|
||||
WorldUtil.clearCaches();
|
||||
|
||||
return {
|
||||
...state,
|
||||
world,
|
||||
colorRGB,
|
||||
categoricalSelectionState,
|
||||
@@ -252,43 +275,12 @@ const Controls = (
|
||||
world
|
||||
);
|
||||
const crossfilter = Crossfilter(world.obsAnnotations);
|
||||
const dimensionMap = World.createObsDimensionMap(crossfilter, world);
|
||||
const dimensionMap = {
|
||||
...World.createObsDimensionMap(crossfilter, world),
|
||||
...createGenesDimMap(userDefinedGenes, diffexpGenes, world, crossfilter)
|
||||
};
|
||||
WorldUtil.clearCaches();
|
||||
|
||||
const worldVarDataCache = world.varDataCache;
|
||||
/* var dimensions */
|
||||
|
||||
if (userDefinedGenes.length > 0) {
|
||||
/*
|
||||
verbose & slightly confusing that we also access this as an object
|
||||
in controls rather than an array, should be abstracted into
|
||||
util ie., createDimensionsFromBothListsOfGenes(userGenes, diffExp)
|
||||
*/
|
||||
_.forEach(userDefinedGenes, gene => {
|
||||
dimensionMap[
|
||||
userDefinedDimensionName(gene)
|
||||
] = World.createVarDimension(
|
||||
/* "__var__" + */
|
||||
world,
|
||||
worldVarDataCache,
|
||||
crossfilter,
|
||||
gene
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
if (diffexpGenes.length > 0) {
|
||||
_.forEach(diffexpGenes, gene => {
|
||||
dimensionMap[diffexpDimensionName(gene)] = World.createVarDimension(
|
||||
/* "__var__" + */
|
||||
world,
|
||||
worldVarDataCache,
|
||||
crossfilter,
|
||||
gene
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
...state,
|
||||
loading: false,
|
||||
@@ -367,7 +359,7 @@ const Controls = (
|
||||
const _diffexpGenes = [];
|
||||
|
||||
action.data.forEach(d => {
|
||||
_diffexpGenes.push(world.varAnnotations[d[0]].name);
|
||||
_diffexpGenes.push(world.varAnnotations.at(d[0], "name"));
|
||||
});
|
||||
|
||||
_.forEach(_diffexpGenes, gene => {
|
||||
@@ -393,7 +385,7 @@ const Controls = (
|
||||
const worldVarDataCache = world.varDataCache;
|
||||
|
||||
_.forEach(action.diffExp, values => {
|
||||
const { name } = world.varAnnotations[values[0]];
|
||||
const name = world.varAnnotations.at(values[0], "name");
|
||||
// clean up crossfilter dimensions
|
||||
const dimension = dimensionMap[diffexpDimensionName(name)];
|
||||
dimension.dispose();
|
||||
|
||||
465
client/src/util/dataframe/dataframe.js
Normal file
465
client/src/util/dataframe/dataframe.js
Normal file
@@ -0,0 +1,465 @@
|
||||
import { IdentityInt32Index } from "./labelIndex";
|
||||
// weird cross-dependency that we should clean up someday...
|
||||
import { sort } from "../typedCrossfilter/sort";
|
||||
import { isTypedArray, isArrayOrTypedArray, callOnceLazy } from "./util";
|
||||
import { summarizeContinuous, summarizeCategorical } from "./summarize";
|
||||
|
||||
/*
|
||||
Dataframe is an immutable 2D matrix similiar to Python Pandas Dataframe,
|
||||
but (currently) without all of the surrounding support functions.
|
||||
Data is stored in column-major layout, and each column is monomorphic.
|
||||
|
||||
It supports:
|
||||
* Relatively efficient creation, cloning and subsetting ("cut")
|
||||
* Very efficient columnar access (eg, sum down a column), and access
|
||||
to the underlying column arrays.
|
||||
* Data access by row/col offset or label. Labels are reasonably well
|
||||
optimized for both numeric lables and arbitrary (eg, sting) labels.
|
||||
|
||||
It does not currently support:
|
||||
* Views on matrix subset - for currently known access patterns,
|
||||
it is more effiicent to copy on subsetting, optimizing for access
|
||||
speed over memory use.
|
||||
* JS iterators - they are too slow. Use explicit iteration over
|
||||
offest or labels.
|
||||
|
||||
Important assumptions embedded in the API:
|
||||
* Columns are implicitly categorical if they are a JS Array and numeric
|
||||
(aka continuous) if they are a TypedArray.
|
||||
|
||||
There are three index types for row/col indexing:
|
||||
* IdentityInt32Index - noop index, where the index label is the offset.
|
||||
* KeyIndex - index arbitrary JS objects.
|
||||
* DenseInt32Index - integer indexing. Optimization over KeyIndex as it uses
|
||||
Int32Array as a back-map to offsets. This means that the index array
|
||||
must be sized to [minLabel, maxLabel), so this is only useful when the label
|
||||
range is relatively close the underlying offset range [minOffset, maxOffset).
|
||||
|
||||
All private functions/methods/fields are prefixed by '__', eg, __compile().
|
||||
Don't use them outside of this file.
|
||||
|
||||
Simple example:
|
||||
|
||||
// default indexing is integer offset.
|
||||
const df = Dataframe.create([2,2], [['a', 'b'], [0, 1]])
|
||||
console.log(df.at(0,0)); // outputs: a
|
||||
console.log(df.col(1).asArray()); // outputs: [0, 1]
|
||||
|
||||
// KeyIndex
|
||||
const df = new Dataframe([1,2], [['a'], ['b']], null, new KeyIndex(['A', 'B']))
|
||||
console.log(df.at(0, 'A')); // outputs: a
|
||||
console.log(df.col('A').asArray(); // outputs: ['a']
|
||||
|
||||
Performance tuning is primarily focused on columnar access patterns, which is the
|
||||
dominant pattern in cellxgene.
|
||||
*/
|
||||
|
||||
/**
|
||||
Dataframe
|
||||
**/
|
||||
|
||||
class Dataframe {
|
||||
/**
|
||||
Constructors & factories
|
||||
**/
|
||||
|
||||
constructor(dims, columnarData, rowIndex = null, colIndex = null) {
|
||||
/*
|
||||
The base constructor is relatively hard to use - as an alternative,
|
||||
see factory methods and clone/slice, below.
|
||||
|
||||
Parameters:
|
||||
* dims - 2D array describing intendend dimensionality: [nRows,nCols].
|
||||
* columnarData - JS array, nCols in length, containing array
|
||||
or TypedArray of length nRows.
|
||||
* rowIndex/colIndex - null (create default index using offsets as key),
|
||||
or a caller-provided index.
|
||||
All columns and indices must have appropriate dimensionality.
|
||||
*/
|
||||
Dataframe.__errorChecks(dims, columnarData, rowIndex, colIndex);
|
||||
const [nRows, nCols] = dims;
|
||||
if (!rowIndex) {
|
||||
rowIndex = new IdentityInt32Index(nRows);
|
||||
}
|
||||
if (!colIndex) {
|
||||
colIndex = new IdentityInt32Index(nCols);
|
||||
}
|
||||
|
||||
this.__columns = Array.from(columnarData);
|
||||
this.dims = dims;
|
||||
this.length = nRows; // convenience accessor for row dimension
|
||||
this.rowIndex = rowIndex;
|
||||
this.colIndex = colIndex;
|
||||
|
||||
this.__compile();
|
||||
}
|
||||
|
||||
static __errorChecks(dims, columnarData) {
|
||||
const [nRows, nCols] = dims;
|
||||
if (nRows < 0 || nCols < 0) {
|
||||
throw new RangeError("Dataframe dimensions must be positive");
|
||||
}
|
||||
if (!Array.isArray(columnarData)) {
|
||||
throw new TypeError("Dataframe constructor requires array of columns");
|
||||
}
|
||||
if (!columnarData.every(c => isArrayOrTypedArray(c))) {
|
||||
throw new TypeError("Dataframe columns must all be Array or TypedArray");
|
||||
}
|
||||
if (
|
||||
nCols !== columnarData.length ||
|
||||
!columnarData.every(c => c.length === nRows)
|
||||
) {
|
||||
throw new RangeError(
|
||||
"Dataframe dimension does not match column data shape"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
__compile() {
|
||||
/*
|
||||
Compile data accessors for each column.
|
||||
|
||||
Each column accessor is a function which will lookup data by
|
||||
index (ie, is equivalent to dataframe.get(row, col), where 'col'
|
||||
is fixed.
|
||||
|
||||
In addition, each column accessor has several functions:
|
||||
|
||||
asArray() -- return the entire column as a native Array or TypedArray.
|
||||
Crucially, this native array only supports label indexing.
|
||||
Example:
|
||||
const arr = df.col('a').asArray();
|
||||
|
||||
has(rlabel) -- return boolean indicating of the row label
|
||||
is contained within the column. Example:
|
||||
const isInColumn = df.col('a').includes(99)
|
||||
For the default offset indexing, this is identical to:
|
||||
const isInColumn = (99 > 0) && (99 < df.nRows);
|
||||
|
||||
ihas(roffset) -- same as has(), but accepts a row offset
|
||||
instead of a row label.
|
||||
|
||||
indexOf(value) -- return the label (not offset) of the first instance of
|
||||
'value' in the column. If you want the offset, just use the builtin JS
|
||||
indexOf() function, available on both Array and TypedArray.
|
||||
|
||||
iget(offset) -- return the value at 'offset'
|
||||
|
||||
*/
|
||||
const { getOffset, getLabel } = this.rowIndex;
|
||||
this.__columnsAccessor = this.__columns.map(column => {
|
||||
const { length } = column;
|
||||
|
||||
/* get value by row label */
|
||||
const get = function get(rlabel) {
|
||||
return column[getOffset(rlabel)];
|
||||
};
|
||||
|
||||
/* get value by row offset */
|
||||
const iget = function iget(roffset) {
|
||||
return column[roffset];
|
||||
};
|
||||
|
||||
/* full column array access */
|
||||
const asArray = function asArray() {
|
||||
return column;
|
||||
};
|
||||
|
||||
/* test for row label inclusion in column */
|
||||
const has = function has(rlabel) {
|
||||
const offset = getOffset(rlabel);
|
||||
return offset >= 0 && offset < length;
|
||||
};
|
||||
|
||||
const ihas = function ihas(offset) {
|
||||
return offset >= 0 && offset < length;
|
||||
};
|
||||
|
||||
/*
|
||||
return first label (index) at which the value is found in this column,
|
||||
or undefined if not found.
|
||||
|
||||
NOTE: not found return is DIFFERENT than the default Array.indexOf as
|
||||
-1 is a plausible Dataframe row/col label.
|
||||
*/
|
||||
const indexOf = function indexOf(value) {
|
||||
const offset = column.indexOf(value);
|
||||
if (offset === -1) {
|
||||
return undefined;
|
||||
}
|
||||
return getLabel(offset);
|
||||
};
|
||||
|
||||
/*
|
||||
Summarize the column data. Lazy eval;
|
||||
*/
|
||||
const summarize = callOnceLazy(() =>
|
||||
isTypedArray(column)
|
||||
? summarizeContinuous(column)
|
||||
: summarizeCategorical(column)
|
||||
);
|
||||
|
||||
get.summarize = summarize;
|
||||
get.asArray = asArray;
|
||||
get.has = has;
|
||||
get.ihas = ihas;
|
||||
get.indexOf = indexOf;
|
||||
get.iget = iget;
|
||||
return get;
|
||||
});
|
||||
}
|
||||
|
||||
clone() {
|
||||
return new this.constructor(
|
||||
this.dims,
|
||||
[...this.__columns],
|
||||
this.rowIndex,
|
||||
this.colIndex
|
||||
);
|
||||
}
|
||||
|
||||
static empty() {
|
||||
return new Dataframe([0, 0], []);
|
||||
}
|
||||
|
||||
static create(dims, columnarData) {
|
||||
/*
|
||||
Create a dataframe from raw columnar data. All column arrays
|
||||
must have the same length. Identity indexing will be used.
|
||||
|
||||
Example:
|
||||
const df = Dataframe.create([2,2], [new Uint32Array(2), new Float32Array(2)]);
|
||||
*/
|
||||
return new Dataframe(dims, columnarData, null, null);
|
||||
}
|
||||
|
||||
__cut(rowOffsets, colOffsets) {
|
||||
const dims = [...this.dims];
|
||||
|
||||
const getSortedLabelAndOffsets = (offsets, index) => {
|
||||
/*
|
||||
Given offsets, return both offsets and associated lables,
|
||||
sorted by offset.
|
||||
*/
|
||||
if (!offsets) {
|
||||
return [null, null];
|
||||
}
|
||||
const sortedOffsets = sort(offsets);
|
||||
const sortedLabels = new Array(sortedOffsets.length);
|
||||
for (let i = 0, l = sortedOffsets.length; i < l; i += 1) {
|
||||
sortedLabels[i] = index.getLabel(sortedOffsets[i]);
|
||||
}
|
||||
return [sortedLabels, sortedOffsets];
|
||||
};
|
||||
|
||||
let { colIndex } = this;
|
||||
if (colOffsets) {
|
||||
let colLabels;
|
||||
[colLabels, colOffsets] = getSortedLabelAndOffsets(
|
||||
colOffsets,
|
||||
this.colIndex
|
||||
);
|
||||
dims[1] = colOffsets.length;
|
||||
colIndex = this.colIndex.cut(colLabels);
|
||||
}
|
||||
|
||||
let { rowIndex } = this;
|
||||
if (rowOffsets) {
|
||||
let rowLabels;
|
||||
[rowLabels, rowOffsets] = getSortedLabelAndOffsets(
|
||||
rowOffsets,
|
||||
this.rowIndex
|
||||
);
|
||||
dims[0] = rowLabels.length;
|
||||
rowIndex = this.rowIndex.cut(rowLabels);
|
||||
}
|
||||
|
||||
/* cut columns */
|
||||
let columns = this.__columns;
|
||||
if (colOffsets) {
|
||||
columns = new Array(colOffsets.length);
|
||||
for (let i = 0, l = colOffsets.length; i < l; i += 1) {
|
||||
columns[i] = this.__columns[colOffsets[i]];
|
||||
}
|
||||
}
|
||||
|
||||
/* cut rows */
|
||||
if (rowOffsets) {
|
||||
columns = columns.map(col => {
|
||||
const newCol = new col.constructor(rowOffsets.length);
|
||||
for (let i = 0, l = rowOffsets.length; i < l; i += 1) {
|
||||
newCol[i] = col[rowOffsets[i]];
|
||||
}
|
||||
return newCol;
|
||||
});
|
||||
}
|
||||
return new Dataframe(dims, columns, rowIndex, colIndex);
|
||||
}
|
||||
|
||||
cutByList(rowLabels, colLabels = null) {
|
||||
const toOffsets = (labels, index) => {
|
||||
if (!labels) {
|
||||
return null;
|
||||
}
|
||||
return labels.map(label => {
|
||||
const off = index.getOffset(label);
|
||||
if (off === undefined) {
|
||||
throw new RangeError(`unknown label: ${label}`);
|
||||
}
|
||||
return off;
|
||||
});
|
||||
};
|
||||
|
||||
const rowOffsets = toOffsets(rowLabels, this.rowIndex);
|
||||
const colOffsets = toOffsets(colLabels, this.colIndex);
|
||||
return this.__cut(rowOffsets, colOffsets);
|
||||
}
|
||||
|
||||
icutByList(rowOffsets, colOffsets = null) {
|
||||
return this.__cut(rowOffsets, colOffsets);
|
||||
}
|
||||
|
||||
icutByMask(rowMask, colMask = null) {
|
||||
/*
|
||||
Cut on row/column based upon a truthy/falsey array.
|
||||
*/
|
||||
const [nRows, nCols] = this.dims;
|
||||
if (
|
||||
(rowMask && rowMask.length !== nRows) ||
|
||||
(colMask && colMask.length !== nCols)
|
||||
) {
|
||||
throw new RangeError("boolean arrays must match row/col dimensions");
|
||||
}
|
||||
|
||||
/* convert masks to lists - method wastes space, but is fast */
|
||||
const toList = (mask, maxSize) => {
|
||||
if (!mask) {
|
||||
return null;
|
||||
}
|
||||
const list = new Int32Array(maxSize);
|
||||
let elems = 0;
|
||||
for (let i = 0, l = mask.length; i < l; i += 1) {
|
||||
if (mask[i]) {
|
||||
list[elems] = i;
|
||||
elems += 1;
|
||||
}
|
||||
}
|
||||
return new Int32Array(list.buffer, 0, elems);
|
||||
};
|
||||
const rowOffsets = toList(rowMask, nRows);
|
||||
const colOffsets = toList(colMask, nCols);
|
||||
return this.__cut(rowOffsets, colOffsets);
|
||||
}
|
||||
|
||||
/**
|
||||
Data access with row/col.
|
||||
**/
|
||||
|
||||
col(columnLabel) {
|
||||
/*
|
||||
Return accessor bound to a column. Allows random row access
|
||||
based upon the row indexing. Returns undefined if the
|
||||
columnLabel is not present in the dataframe.
|
||||
|
||||
Example for a dataframe with string labeled columns, and
|
||||
default (offset) indices for rows (eg, [0, 'foo'])
|
||||
|
||||
const getValue = df.col('foo');
|
||||
for (let r = 0; r < df.nRows; r += 1) {
|
||||
console.log(r, getValue(r));
|
||||
}
|
||||
|
||||
See __compile() for the functions available in a column accessor.
|
||||
*/
|
||||
const coff = this.colIndex.getOffset(columnLabel);
|
||||
return this.__columnsAccessor[coff];
|
||||
}
|
||||
|
||||
icol(columnOffset) {
|
||||
/*
|
||||
Return column accessor by offset.
|
||||
*/
|
||||
return this.__columnsAccessor[columnOffset];
|
||||
}
|
||||
|
||||
at(r, c) {
|
||||
/*
|
||||
Access a single value, for a row/col label pair.
|
||||
|
||||
For performance reasons, there are no bounds or existance
|
||||
checks on labels, and no defined behavior when these are supplied.
|
||||
May return undefined, throw an Error, or do something else for
|
||||
non-existant labels. If you want predictable out-of-bounds
|
||||
behavior, use has(), eg,
|
||||
|
||||
const myVal = df.has(r,l) ? df.at(r,l) : undefined;
|
||||
*/
|
||||
const coff = this.colIndex.getOffset(c);
|
||||
const roff = this.rowIndex.getOffset(r);
|
||||
return this.__columns[coff][roff];
|
||||
}
|
||||
|
||||
iat(r, c) {
|
||||
/*
|
||||
Access a single value, for a row/col offset (integer) position.
|
||||
|
||||
For performance reasons, there are no bounds checks on row/col offsets
|
||||
or other well-defined behavior for out-of-bounds values. If you want
|
||||
well-defined bounds checking, use ihas(), eg,
|
||||
|
||||
const myVal = df.ihas(r, c) ? df.iat(r, c) : undefined;
|
||||
*/
|
||||
return this.__columns[c][r];
|
||||
}
|
||||
|
||||
has(r, c) {
|
||||
/*
|
||||
Test if row/col labels exist in the dataframe - returns true/false
|
||||
*/
|
||||
const [nRows, nCols] = this.dims;
|
||||
const coff = this.colIndex.getOffset(c);
|
||||
const roff = this.rowIndex.getOffset(r);
|
||||
return coff >= 0 && coff < nCols && roff >= 0 && roff < nRows;
|
||||
}
|
||||
|
||||
ihas(r, c) {
|
||||
/*
|
||||
Test if row/col offset (integer) position exists in the
|
||||
dataframe - returns true/false
|
||||
*/
|
||||
const [nRows, nCols] = this.dims;
|
||||
return c >= 0 && c < nCols && r >= 0 && r < nRows;
|
||||
}
|
||||
|
||||
/****
|
||||
Functional (map/reduce/etc) data access
|
||||
|
||||
XXX: not yet implemented, as there is no clear use case. Can easily
|
||||
add these as useful.
|
||||
****/
|
||||
|
||||
/*
|
||||
Map & reduce of column or row
|
||||
|
||||
XXX TODO remainder of map/reduce functions: mapCol, mapRow, reduceRow, ...
|
||||
*/
|
||||
/* comment out until we have a use for this
|
||||
|
||||
reduceCol(clabel, callback, initialValue) {
|
||||
const coff = this.colIndex.getOffset(clabel);
|
||||
const column = this.__columns[coff];
|
||||
let start = 0;
|
||||
let acc = initialValue;
|
||||
if (initialValue === undefined) {
|
||||
acc = column[0];
|
||||
start = 1;
|
||||
}
|
||||
for (let i = start, l = column.length; i < l; i += 1) {
|
||||
acc = callback(acc, column[i]);
|
||||
}
|
||||
return acc;
|
||||
}
|
||||
*/
|
||||
}
|
||||
|
||||
export default Dataframe;
|
||||
2
client/src/util/dataframe/index.js
Normal file
2
client/src/util/dataframe/index.js
Normal file
@@ -0,0 +1,2 @@
|
||||
export { default as Dataframe } from "./dataframe";
|
||||
export { DenseInt32Index, IdentityInt32Index, KeyIndex } from "./labelIndex";
|
||||
188
client/src/util/dataframe/labelIndex.js
Normal file
188
client/src/util/dataframe/labelIndex.js
Normal file
@@ -0,0 +1,188 @@
|
||||
/**
|
||||
Label indexing - map a label to & from an integer offset. See Dataframe
|
||||
for how this is used.
|
||||
**/
|
||||
|
||||
/*
|
||||
Private utility functions
|
||||
*/
|
||||
function extent(tarr) {
|
||||
let min = 0x7fffffff;
|
||||
let max = ~min; // eslint-disable-line no-bitwise
|
||||
for (let i = 0, l = tarr.length; i < l; i += 1) {
|
||||
const v = tarr[i];
|
||||
if (v < min) {
|
||||
min = v;
|
||||
}
|
||||
if (v > max) {
|
||||
max = v;
|
||||
}
|
||||
}
|
||||
return [min, max];
|
||||
}
|
||||
|
||||
function fillRange(arr, start = 0) {
|
||||
const larr = arr;
|
||||
for (let i = 0, l = larr.length; i < l; i += 1) {
|
||||
larr[i] = i + start;
|
||||
}
|
||||
return larr;
|
||||
}
|
||||
|
||||
/* eslint-disable class-methods-use-this */
|
||||
class IdentityInt32Index {
|
||||
/*
|
||||
identity/noop index, with small assumptions that labels are int32
|
||||
*/
|
||||
constructor(maxOffset) {
|
||||
this.maxOffset = maxOffset;
|
||||
}
|
||||
|
||||
keys() {
|
||||
// memoize
|
||||
const k = fillRange(new Int32Array(this.maxOffset));
|
||||
this.keys = function keys() {
|
||||
return k;
|
||||
};
|
||||
return k;
|
||||
}
|
||||
|
||||
getOffset(i) {
|
||||
// label to offset
|
||||
return i;
|
||||
}
|
||||
|
||||
getLabel(i) {
|
||||
// offset to label
|
||||
return i;
|
||||
}
|
||||
|
||||
getMaxOffset() {
|
||||
return this.maxOffset;
|
||||
}
|
||||
|
||||
cut(labelArray) {
|
||||
/*
|
||||
if density of resulting integer
|
||||
*/
|
||||
const [minLabel, maxLabel] = extent(labelArray);
|
||||
const labelSpaceSize = maxLabel - minLabel + 1;
|
||||
const density = labelSpaceSize / this.maxOffset;
|
||||
/* 0.1 is a magic number, that needs testing to optimize */
|
||||
if (density < 0.1) {
|
||||
return new KeyIndex(labelArray);
|
||||
}
|
||||
return new DenseInt32Index(labelArray, [minLabel, maxLabel]);
|
||||
}
|
||||
}
|
||||
/* eslint-enable class-methods-use-this */
|
||||
|
||||
/* eslint-disable class-methods-use-this */
|
||||
class DenseInt32Index {
|
||||
/*
|
||||
DenseInt32Index indexes integer labels, and uses Int32Array typed arrays
|
||||
for both forward and reverse indexing. This means that the min/max range
|
||||
of the forward index labels must be known a priori (so that the index
|
||||
array can be pre-allocated).
|
||||
*/
|
||||
constructor(labels, labelRange = null) {
|
||||
if (labels.constructor !== Int32Array) {
|
||||
labels = new Int32Array(labels);
|
||||
}
|
||||
|
||||
if (!labelRange) {
|
||||
labelRange = extent(labels);
|
||||
}
|
||||
const [minLabel, maxLabel] = labelRange;
|
||||
const labelSpaceSize = maxLabel - minLabel + 1;
|
||||
const index = new Int32Array(labelSpaceSize).fill(-1);
|
||||
for (let i = 0, l = labels.length; i < l; i += 1) {
|
||||
const label = labels[i];
|
||||
index[label - minLabel] = i;
|
||||
}
|
||||
|
||||
this.minLabel = minLabel;
|
||||
this.rindex = labels;
|
||||
this.index = index;
|
||||
this.__compile();
|
||||
}
|
||||
|
||||
__compile() {
|
||||
const { minLabel, index, rindex } = this;
|
||||
this.getOffset = function getOffset(l) {
|
||||
return index[l - minLabel];
|
||||
};
|
||||
this.getLabel = function getLabel(i) {
|
||||
return rindex[i];
|
||||
};
|
||||
}
|
||||
|
||||
keys() {
|
||||
return this.rindex;
|
||||
}
|
||||
|
||||
getMaxOffset() {
|
||||
return this.rindex.length;
|
||||
}
|
||||
|
||||
cut(labelArray) {
|
||||
/*
|
||||
time/space decision - if we are going to use less than 10% of the
|
||||
dense index space, switch to a KeyIndex (which is slower, but uses
|
||||
less memory for sparse label spaces).
|
||||
*/
|
||||
const [minLabel, maxLabel] = extent(labelArray);
|
||||
const labelSpaceSize = maxLabel - minLabel + 1;
|
||||
const density = labelSpaceSize / this.rindex.length;
|
||||
/* 0.1 is a magic number, that needs testing to optimize */
|
||||
if (density < 0.1) {
|
||||
return new KeyIndex(labelArray);
|
||||
}
|
||||
return new DenseInt32Index(labelArray, [minLabel, maxLabel]);
|
||||
}
|
||||
}
|
||||
/* eslint-enable class-methods-use-this */
|
||||
|
||||
/* eslint-disable class-methods-use-this */
|
||||
class KeyIndex {
|
||||
/*
|
||||
KeyIndex indexes arbitrary JS primitive types, and uses a Map()
|
||||
as its core data structure.
|
||||
*/
|
||||
constructor(labels) {
|
||||
const index = new Map();
|
||||
const rindex = labels;
|
||||
labels.forEach((v, i) => {
|
||||
index.set(v, i);
|
||||
});
|
||||
|
||||
this.index = index;
|
||||
this.rindex = rindex;
|
||||
this.__compile();
|
||||
}
|
||||
|
||||
__compile() {
|
||||
const { index, rindex } = this;
|
||||
this.getOffset = function getOffset(k) {
|
||||
return index.get(k);
|
||||
};
|
||||
this.getLabel = function getLabel(i) {
|
||||
return rindex[i];
|
||||
};
|
||||
}
|
||||
|
||||
keys() {
|
||||
return this.rindex;
|
||||
}
|
||||
|
||||
getMaxOffset() {
|
||||
return this.rindex.length;
|
||||
}
|
||||
|
||||
cut(labelArray) {
|
||||
return new KeyIndex(labelArray);
|
||||
}
|
||||
}
|
||||
/* eslint-enable class-methods-use-this */
|
||||
|
||||
export { DenseInt32Index, IdentityInt32Index, KeyIndex };
|
||||
57
client/src/util/dataframe/summarize.js
Normal file
57
client/src/util/dataframe/summarize.js
Normal file
@@ -0,0 +1,57 @@
|
||||
/*
|
||||
Private dataframe support functions
|
||||
*/
|
||||
|
||||
export function summarizeContinuous(col) {
|
||||
let min;
|
||||
let max;
|
||||
let nan = 0;
|
||||
let pinf = 0;
|
||||
let ninf = 0;
|
||||
if (col) {
|
||||
for (let r = 0, l = col.length; r < l; r += 1) {
|
||||
const val = Number(col[r]);
|
||||
if (Number.isFinite(val)) {
|
||||
if (min === undefined) {
|
||||
min = val;
|
||||
max = val;
|
||||
} else {
|
||||
min = val < min ? val : min;
|
||||
max = val > max ? val : max;
|
||||
}
|
||||
} else if (Number.isNaN(val)) {
|
||||
nan += 1;
|
||||
} else if (val > 0) {
|
||||
pinf += 1;
|
||||
} else {
|
||||
ninf += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
return {
|
||||
categorical: false,
|
||||
min,
|
||||
max,
|
||||
nan,
|
||||
pinf,
|
||||
ninf
|
||||
};
|
||||
}
|
||||
|
||||
export function summarizeCategorical(col) {
|
||||
const categoryCounts = new Map();
|
||||
if (col) {
|
||||
for (let r = 0, l = col.length; r < l; r += 1) {
|
||||
const val = col[r];
|
||||
let curCount = categoryCounts.get(val);
|
||||
if (curCount === undefined) curCount = 0;
|
||||
categoryCounts.set(val, curCount + 1);
|
||||
}
|
||||
}
|
||||
return {
|
||||
categorical: true,
|
||||
categories: [...categoryCounts.keys()],
|
||||
categoryCounts,
|
||||
numCategories: categoryCounts.size
|
||||
};
|
||||
}
|
||||
28
client/src/util/dataframe/util.js
Normal file
28
client/src/util/dataframe/util.js
Normal file
@@ -0,0 +1,28 @@
|
||||
/*
|
||||
Private utility code for dataframe
|
||||
*/
|
||||
|
||||
export function isTypedArray(x) {
|
||||
return (
|
||||
ArrayBuffer.isView(x) &&
|
||||
Object.prototype.toString.call(x) !== "[object DataView]"
|
||||
);
|
||||
}
|
||||
|
||||
export function isArrayOrTypedArray(x) {
|
||||
return Array.isArray(x) || isTypedArray(x);
|
||||
}
|
||||
|
||||
export function callOnceLazy(f) {
|
||||
let value;
|
||||
let calledOnce = false;
|
||||
const result = function result(...args) {
|
||||
if (!calledOnce) {
|
||||
value = f(...args);
|
||||
calledOnce = true;
|
||||
}
|
||||
return value;
|
||||
};
|
||||
|
||||
return result;
|
||||
}
|
||||
@@ -53,13 +53,15 @@ Example:
|
||||
NOTE: will not summarize the required 'name' annotation, as that is
|
||||
specified as unique per element.
|
||||
*/
|
||||
function _summarizeAnnotations(_schema, annotations) {
|
||||
function _summarizeAnnotations(_schema, df) {
|
||||
const summary = _(_schema) // lodash wrapping: https://lodash.com/docs/4.17.11#lodash
|
||||
.filter(v => v.name !== "name")
|
||||
.filter(v => v.name !== "name") // don't summarize name
|
||||
.keyBy("name")
|
||||
.mapValues(anno => {
|
||||
const { name, type } = anno;
|
||||
const continuous = type === "int32" || type === "float32";
|
||||
const numRows = df.length;
|
||||
const col = df.col(name) ? df.col(name).asArray() : null;
|
||||
|
||||
if (continuous) {
|
||||
let min;
|
||||
@@ -67,22 +69,24 @@ function _summarizeAnnotations(_schema, annotations) {
|
||||
let nan = 0;
|
||||
let pinf = 0;
|
||||
let ninf = 0;
|
||||
for (let r = 0; r < annotations.length; r += 1) {
|
||||
const val = Number(annotations[r][name]);
|
||||
if (Number.isFinite(val)) {
|
||||
if (min === undefined) {
|
||||
min = val;
|
||||
max = val;
|
||||
if (col) {
|
||||
for (let r = 0; r < numRows; r += 1) {
|
||||
const val = Number(col[r]);
|
||||
if (Number.isFinite(val)) {
|
||||
if (min === undefined) {
|
||||
min = val;
|
||||
max = val;
|
||||
} else {
|
||||
min = val < min ? val : min;
|
||||
max = val > max ? val : max;
|
||||
}
|
||||
} else if (Number.isNaN(val)) {
|
||||
nan += 1;
|
||||
} else if (val > 0) {
|
||||
pinf += 1;
|
||||
} else {
|
||||
min = val < min ? val : min;
|
||||
max = val > max ? val : max;
|
||||
ninf += 1;
|
||||
}
|
||||
} else if (Number.isNaN(val)) {
|
||||
nan += 1;
|
||||
} else if (val > 0) {
|
||||
pinf += 1;
|
||||
} else {
|
||||
ninf += 1;
|
||||
}
|
||||
}
|
||||
return {
|
||||
@@ -93,11 +97,13 @@ function _summarizeAnnotations(_schema, annotations) {
|
||||
|
||||
/* else categorical */
|
||||
const categoryCounts = new Map();
|
||||
for (let r = 0; r < annotations.length; r += 1) {
|
||||
const val = annotations[r][name];
|
||||
let curCount = categoryCounts.get(val);
|
||||
if (curCount === undefined) curCount = 0;
|
||||
categoryCounts.set(val, curCount + 1);
|
||||
if (col) {
|
||||
for (let r = 0; r < numRows; r += 1) {
|
||||
const val = col[r];
|
||||
let curCount = categoryCounts.get(val);
|
||||
if (curCount === undefined) curCount = 0;
|
||||
categoryCounts.set(val, curCount + 1);
|
||||
}
|
||||
}
|
||||
return {
|
||||
categorical: true,
|
||||
|
||||
@@ -5,6 +5,7 @@ import _ from "lodash";
|
||||
import * as kvCache from "./keyvalcache";
|
||||
import summarizeAnnotations from "./summarizeAnnotations";
|
||||
import decodeMatrixFBS from "./matrix";
|
||||
import * as Dataframe from "../dataframe";
|
||||
|
||||
/*
|
||||
Private helper function - create and return a template Universe
|
||||
@@ -27,13 +28,10 @@ function templateUniverse() {
|
||||
/*
|
||||
Annotations
|
||||
*/
|
||||
obsAnnotations: [] /* all obs annotations, by obs index */,
|
||||
varAnnotations: [] /* all var annotations, by var index */,
|
||||
obsNameToIndexMap: {} /* reverse map 'name' to index */,
|
||||
varNameToIndexMap: {} /* reverse map 'name' to index */,
|
||||
summary: null /* derived data summaries XXX: consider exploding in place */,
|
||||
|
||||
obsLayout: { X: [], Y: [] } /* xy layout */,
|
||||
obsAnnotations: null,
|
||||
varAnnotations: null,
|
||||
obsLayout: null,
|
||||
summary: null /* derived data summaries. XXX: consider exploding in place */,
|
||||
|
||||
/*
|
||||
Cache of var data (expression), by var annotation name. Data can be
|
||||
@@ -61,9 +59,8 @@ function finalize(universe) {
|
||||
/* A bit of sanity checking! */
|
||||
const { nObs, nVar } = universe;
|
||||
if (
|
||||
nObs !== universe.obsLayout.length ||
|
||||
nObs !== universe.obsAnnotations.length ||
|
||||
nObs !== universe.obsLayout.X.length ||
|
||||
nObs !== universe.obsLayout.Y.length ||
|
||||
nVar !== universe.varAnnotations.length
|
||||
) {
|
||||
throw new Error("Universe dimensionality mismatch - failed to load");
|
||||
@@ -73,61 +70,33 @@ function finalize(universe) {
|
||||
// - layout has supported number of dimensions
|
||||
// - ...
|
||||
|
||||
/*
|
||||
Create all derived (convenience) data structures.
|
||||
*/
|
||||
universe.obsNameToIndexMap = _.transform(
|
||||
universe.obsAnnotations,
|
||||
(acc, value, idx) => {
|
||||
acc[value.name] = idx;
|
||||
},
|
||||
{}
|
||||
);
|
||||
universe.varNameToIndexMap = _.transform(
|
||||
universe.varAnnotations,
|
||||
(acc, value, idx) => {
|
||||
acc[value.name] = idx;
|
||||
},
|
||||
{}
|
||||
);
|
||||
universe.finalized = true;
|
||||
return universe;
|
||||
}
|
||||
|
||||
function RESTv02AnotationsFBSResponseToInternal(arrayBuffer) {
|
||||
function AnnotationsFBSToDataframe(arrayBuffer) {
|
||||
/*
|
||||
Convert a Matrix FBS to our internal format -- row-major array of
|
||||
observations/cells, stored as an object. Each obs has a key for each
|
||||
annotation, plus __index__ containing its obsIndex.
|
||||
|
||||
Example:
|
||||
[
|
||||
{ __index__: 0, tissue_type: "lung", sex: "F", ... },
|
||||
...
|
||||
]
|
||||
|
||||
XXX TODO: we could make use of the columns in building crossfilter
|
||||
dimensions (they have to be recreated). Future optimization.
|
||||
Convert a Matrix FBS to a Dataframe.
|
||||
*/
|
||||
const fbs = decodeMatrixFBS(arrayBuffer);
|
||||
const keys = fbs.colIdx;
|
||||
const result = Array(fbs.nRows);
|
||||
for (let row = 0; row < fbs.nRows; row += 1) {
|
||||
const rec = { __index__: row };
|
||||
for (let col = 0; col < fbs.nCols; col += 1) {
|
||||
rec[keys[col]] = fbs.columns[col][row];
|
||||
}
|
||||
result[row] = rec;
|
||||
}
|
||||
return result;
|
||||
const df = new Dataframe.Dataframe(
|
||||
[fbs.nRows, fbs.nCols],
|
||||
fbs.columns,
|
||||
null,
|
||||
new Dataframe.KeyIndex(fbs.colIdx)
|
||||
);
|
||||
return df;
|
||||
}
|
||||
|
||||
function RESTv02LayoutFBSResponseToInternal(arrayBuffer) {
|
||||
function LayoutFBSToDataframe(arrayBuffer) {
|
||||
const fbs = decodeMatrixFBS(arrayBuffer, true);
|
||||
return {
|
||||
X: fbs.columns[0],
|
||||
Y: fbs.columns[1]
|
||||
};
|
||||
const df = new Dataframe.Dataframe(
|
||||
[fbs.nRows, fbs.nCols],
|
||||
fbs.columns,
|
||||
null,
|
||||
new Dataframe.KeyIndex(["X", "Y"])
|
||||
);
|
||||
return df;
|
||||
}
|
||||
|
||||
function reconcileSchemaCategoriesWithSummary(universe) {
|
||||
@@ -156,7 +125,7 @@ function reconcileSchemaCategoriesWithSummary(universe) {
|
||||
});
|
||||
}
|
||||
|
||||
export function createUniverseFromRestV02Response(
|
||||
export function createUniverseFromResponse(
|
||||
configResponse,
|
||||
schemaResponse,
|
||||
annotationsObsResponse,
|
||||
@@ -178,15 +147,10 @@ export function createUniverseFromRestV02Response(
|
||||
universe.nVar = schema.dataframe.nVar;
|
||||
|
||||
/* annotations */
|
||||
universe.obsAnnotations = RESTv02AnotationsFBSResponseToInternal(
|
||||
annotationsObsResponse
|
||||
);
|
||||
universe.varAnnotations = RESTv02AnotationsFBSResponseToInternal(
|
||||
annotationsVarResponse
|
||||
);
|
||||
|
||||
universe.obsAnnotations = AnnotationsFBSToDataframe(annotationsObsResponse);
|
||||
universe.varAnnotations = AnnotationsFBSToDataframe(annotationsVarResponse);
|
||||
/* layout */
|
||||
universe.obsLayout = RESTv02LayoutFBSResponseToInternal(layoutFBSResponse);
|
||||
universe.obsLayout = LayoutFBSToDataframe(layoutFBSResponse);
|
||||
|
||||
universe.summary = summarizeAnnotations(
|
||||
universe.schema,
|
||||
@@ -214,8 +178,8 @@ export function convertDataFBStoObject(universe, arrayBuffer) {
|
||||
const result = {};
|
||||
|
||||
for (let c = 0; c < colIdx.length; c += 1) {
|
||||
const gene = universe.varAnnotations[colIdx[c]].name;
|
||||
result[gene] = columns[c];
|
||||
const varName = universe.varAnnotations.at(colIdx[c], "name");
|
||||
result[varName] = columns[c];
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
@@ -8,6 +8,7 @@ import Crossfilter from "../typedCrossfilter";
|
||||
import { sliceByIndex } from "../typedCrossfilter/util";
|
||||
|
||||
/*
|
||||
|
||||
World is a subset of universe. Most code should use world, and should
|
||||
(generally) not use Universe. World contains any per-obs or per-var data
|
||||
that must be consistent acorss the app when we view/manipulate subsets
|
||||
@@ -16,37 +17,32 @@ of Universe.
|
||||
Private API indicated by leading underscore in key name (eg, _foo). Anything else
|
||||
is public.
|
||||
|
||||
World contains several public keys, obsAnnotations, and obsLayout, which are
|
||||
arrays contianing information about an OBS in the same order/offset. In
|
||||
other words, world.obsAnnotations[0] and world.obsLayout.X[0] refer to the same
|
||||
obs/cell.
|
||||
Notable keys in the world object:
|
||||
|
||||
* nObs, nVar: dimensions
|
||||
|
||||
* schema: data schema from the server
|
||||
|
||||
* obsAnnotations:
|
||||
|
||||
obsAnnotations will return an array of objects. Each object contains all annotation
|
||||
values for a given observation/cell, keyed by annotation name, PLUS a key
|
||||
'__cellId__', containing a REST API ID for this obs/cell (referred to as the
|
||||
obsIndex in the REST 0.2 spec or cellIndex in the 0.1 spec.
|
||||
Dataframe containing obs annotations. Columns are indexed by annotation
|
||||
name (eg, 'tissue type'), and rows are indexed by the REST API obsIndex
|
||||
(ie, the offset into the underlying server-side dataframe).
|
||||
|
||||
Example: [ { __cellId__: 99, cluster: 'blue', numReads: 93933 } ]
|
||||
|
||||
NOTE: world.obsAnnotation should be identical to the old state.cells value,
|
||||
EXCEPT that
|
||||
* __cellIndex__ renamed to __index__
|
||||
* __x__ and __y__ are now in world.obsLayout
|
||||
* __color__ and __colorRBG__ should be moved to controls reducer
|
||||
This indexing means that you can access data by _either_ the server's
|
||||
obxIndex, or the offset into the client-side column array . Be careful
|
||||
to know which you want and are using.
|
||||
|
||||
* obsLayout:
|
||||
|
||||
obsLayout will return an object containing two arrays, containing X and Y
|
||||
coordinates respectively.
|
||||
A dataframe containing the X/Y layout for all obs. Columns are named
|
||||
'X' and 'Y', and rows are indexed in the same way as obsAnnotation.
|
||||
|
||||
Example: { X: [ 0.33, 0.23, ... ], Y: [ 0.8, 0.777, ... ]}
|
||||
* summary: summary of each obsAnnotation column (eg, numeric extent for
|
||||
continuous data, category counts for categorical metadata)
|
||||
|
||||
* crossfilter - a crossfilter object across world.obsAnnotations
|
||||
|
||||
* dimensionMap - an object mapping annotation names to dimensions on
|
||||
the crossfilter
|
||||
* varDataCache: expression columns, in a kvCache. TODO: maybe move to a
|
||||
Dataframe in the future.
|
||||
|
||||
*/
|
||||
|
||||
@@ -56,11 +52,6 @@ const VarDataCacheTTLMs = 1000; // min cache time in MS
|
||||
|
||||
function templateWorld() {
|
||||
return {
|
||||
// map from universe obsIndex to world offset.
|
||||
// Undefined / null indicates identity mapping.
|
||||
obsIndex: null,
|
||||
obsBackIndex: null,
|
||||
|
||||
/* schema/version related */
|
||||
api: null,
|
||||
schema: null,
|
||||
@@ -71,7 +62,7 @@ function templateWorld() {
|
||||
obsAnnotations: null,
|
||||
varAnnotations: null,
|
||||
|
||||
/* layout of graph */
|
||||
/* layout of graph. Dataframe. */
|
||||
obsLayout: null,
|
||||
|
||||
/* derived data summaries XXX: consider exploding in place */
|
||||
@@ -91,15 +82,6 @@ export function createWorldFromEntireUniverse(universe) {
|
||||
|
||||
const world = templateWorld();
|
||||
|
||||
// map from the universe obsIndex to our world offset.
|
||||
// undefined/null indicates identity map.
|
||||
// In other words obsBackIndex[universeIdx] -> worldIdx
|
||||
world.obsBackIndex = null;
|
||||
// Map to the universe index for each element in world.
|
||||
// Null indicates identity map (aka world === universe)
|
||||
// In other wrods obsIndex[worldIdx] -> universeIdx
|
||||
world.obsIndex = null;
|
||||
|
||||
/*
|
||||
public interface follows
|
||||
*/
|
||||
@@ -143,35 +125,11 @@ export function createWorldFromCurrentSelection(universe, world, crossfilter) {
|
||||
newWorld.schema = universe.schema;
|
||||
newWorld.varAnnotations = universe.varAnnotations;
|
||||
|
||||
/* build index maps and back maps based upon current selection state */
|
||||
const obsBackIndex = new Uint32Array(universe.nObs);
|
||||
obsBackIndex.fill(-1); // default - aka unused
|
||||
const notSelected = obsBackIndex[0];
|
||||
let nObs = 0;
|
||||
for (let i = 0; i < universe.nObs; i += 1) {
|
||||
if (crossfilter.isElementFiltered(i)) {
|
||||
obsBackIndex[i] = nObs;
|
||||
nObs += 1;
|
||||
}
|
||||
}
|
||||
const obsIndex = new Uint32Array(nObs);
|
||||
for (let i = 0; i < universe.nObs; i += 1) {
|
||||
const worldIdx = obsBackIndex[i];
|
||||
if (worldIdx !== notSelected) {
|
||||
obsIndex[worldIdx] = i;
|
||||
}
|
||||
}
|
||||
|
||||
newWorld.nObs = nObs;
|
||||
newWorld.obsIndex = obsIndex;
|
||||
newWorld.obsBackIndex = obsBackIndex;
|
||||
|
||||
/* now slice */
|
||||
newWorld.obsAnnotations = sliceByIndex(universe.obsAnnotations, obsIndex);
|
||||
newWorld.obsLayout = {
|
||||
X: sliceByIndex(universe.obsLayout.X, obsIndex),
|
||||
Y: sliceByIndex(universe.obsLayout.Y, obsIndex)
|
||||
};
|
||||
/* now subset/cut obs */
|
||||
const mask = crossfilter.allFilteredMask();
|
||||
newWorld.obsAnnotations = world.obsAnnotations.icutByMask(mask);
|
||||
newWorld.obsLayout = world.obsLayout.icutByMask(mask);
|
||||
newWorld.nObs = newWorld.obsAnnotations.dims[0];
|
||||
|
||||
/* derived data & summaries */
|
||||
newWorld.summary = summarizeAnnotations(
|
||||
@@ -245,22 +203,23 @@ export function createObsDimensionMap(crossfilter, world) {
|
||||
create and return a crossfilter dimension for every obs annotation
|
||||
for which we have a supported type.
|
||||
*/
|
||||
const { schema, obsLayout } = world;
|
||||
const { schema, obsLayout, obsAnnotations } = world;
|
||||
|
||||
// Create a crossfilter dimension for all obs annotations *except* 'name'
|
||||
const dimensionMap = _(schema.annotations.obs)
|
||||
.filter(anno => anno.name !== "name")
|
||||
.transform((result, anno) => {
|
||||
const dimType = deduceDimensionType(anno, anno.name);
|
||||
const colData = obsAnnotations.col(anno.name).asArray();
|
||||
if (dimType === "enum") {
|
||||
result[obsAnnoDimensionName(anno.name)] = crossfilter.dimension(
|
||||
Crossfilter.EnumDimension,
|
||||
r => r[anno.name]
|
||||
colData
|
||||
);
|
||||
} else {
|
||||
} else if (dimType) {
|
||||
result[obsAnnoDimensionName(anno.name)] = crossfilter.dimension(
|
||||
Crossfilter.ScalarDimension,
|
||||
r => r[anno.name],
|
||||
colData,
|
||||
dimType
|
||||
);
|
||||
} // else ignore the annotation
|
||||
@@ -272,8 +231,8 @@ export function createObsDimensionMap(crossfilter, world) {
|
||||
*/
|
||||
dimensionMap[layoutDimensionName("XY")] = crossfilter.dimension(
|
||||
Crossfilter.SpatialDimension,
|
||||
obsLayout.X,
|
||||
obsLayout.Y
|
||||
obsLayout.col("X").asArray(),
|
||||
obsLayout.col("Y").asArray()
|
||||
);
|
||||
|
||||
return dimensionMap;
|
||||
@@ -288,5 +247,23 @@ export function subsetVarData(world, universe, varData) {
|
||||
if (worldEqUniverse(world, universe)) {
|
||||
return varData;
|
||||
}
|
||||
return sliceByIndex(varData, world.obsIndex);
|
||||
return sliceByIndex(varData, world.obsAnnotations.rowIndex.keys());
|
||||
}
|
||||
|
||||
export function getSelectedByIndex(crossfilter) {
|
||||
/*
|
||||
return array of obsIndex, containing all selected obs/cells.
|
||||
*/
|
||||
const selected = crossfilter.allFilteredMask(); // array of bool-ish
|
||||
const keys = crossfilter.data.rowIndex.keys(); // row keys, aka universe rowIndex
|
||||
|
||||
const set = new Int32Array(selected.length);
|
||||
let numElems = 0;
|
||||
for (let i = 0, l = selected.length; i < l; i += 1) {
|
||||
if (selected[i]) {
|
||||
set[numElems] = keys[i];
|
||||
numElems += 1;
|
||||
}
|
||||
}
|
||||
return new Int32Array(set.buffer, 0, numElems);
|
||||
}
|
||||
|
||||
@@ -18,13 +18,23 @@ Map {
|
||||
...
|
||||
}
|
||||
|
||||
Parameters are:
|
||||
- dim1: dimension 1 name/label
|
||||
- dim2: dimension 2 name/label
|
||||
- df: dataframe containing dim1 and dim2 on the column axis
|
||||
|
||||
*/
|
||||
function _countCategoryValues2D(dim1, dim2, rows) {
|
||||
function _countCategoryValues2D(dim1, dim2, df) {
|
||||
const dimMap = new Map();
|
||||
for (let r = 0; r < rows.length; r += 1) {
|
||||
const row = rows[r];
|
||||
const val1 = row[dim1];
|
||||
const val2 = row[dim2];
|
||||
const col1 = df.col(dim1) ? df.col(dim1).asArray() : null;
|
||||
const col2 = df.col(dim2) ? df.col(dim2).asArray() : null;
|
||||
if (!col1 || !col2) {
|
||||
return dimMap;
|
||||
}
|
||||
|
||||
for (let r = 0, l = df.length; r < l; r += 1) {
|
||||
const val1 = col1[r];
|
||||
const val2 = col2[r];
|
||||
let d2Map = dimMap.get(val1);
|
||||
if (d2Map === undefined) {
|
||||
d2Map = new Map();
|
||||
|
||||
@@ -40,7 +40,7 @@ class BitArray {
|
||||
// Return the number of records that are selected, ie, have a one bit in
|
||||
// all allocated dimensions.
|
||||
//
|
||||
get selectionCount() {
|
||||
selectionCount() {
|
||||
return this.countAllOnes();
|
||||
}
|
||||
|
||||
@@ -48,16 +48,27 @@ class BitArray {
|
||||
//
|
||||
countAllOnes() {
|
||||
let count = 0;
|
||||
const { bitarray, bitmask, length, width } = this;
|
||||
for (let l = 0; l < length; l += 1) {
|
||||
let dimensionsSet = 0;
|
||||
for (let w = 0; w < width; w += 1) {
|
||||
if (bitarray[w * length + l] === bitmask[w]) {
|
||||
dimensionsSet += 1;
|
||||
const { bitarray, length, width } = this;
|
||||
if (width === 1) {
|
||||
// special case, width === 1, for performance
|
||||
const bitmask = this.bitmask[0];
|
||||
for (let l = 0; l < length; l += 1) {
|
||||
if (bitarray[l] === bitmask) {
|
||||
count += 1;
|
||||
}
|
||||
}
|
||||
if (dimensionsSet === width) {
|
||||
count += 1;
|
||||
} else {
|
||||
const { bitmask } = this;
|
||||
for (let l = 0; l < length; l += 1) {
|
||||
let dimensionsSet = 0;
|
||||
for (let w = 0; w < width; w += 1) {
|
||||
if (bitarray[w * length + l] === bitmask[w]) {
|
||||
dimensionsSet += 1;
|
||||
}
|
||||
}
|
||||
if (dimensionsSet === width) {
|
||||
count += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
return count;
|
||||
@@ -233,12 +244,14 @@ class BitArray {
|
||||
fillBySelection(result, selectedValue, deselectedValue) {
|
||||
// special case (width === 1) for performance
|
||||
if (this.width === 1) {
|
||||
const bitmask = this.bitmask[0];
|
||||
for (let i = 0, len = this.length; i < len; i += 1) {
|
||||
result[i] =
|
||||
bitmask && this.bitarray[i] === bitmask
|
||||
? selectedValue
|
||||
: deselectedValue;
|
||||
const { bitmask, bitarray } = this;
|
||||
const mask = bitmask[0];
|
||||
if (!mask) {
|
||||
result.fill(deselectedValue);
|
||||
} else {
|
||||
for (let i = 0, len = this.length; i < len; i += 1) {
|
||||
result[i] = bitarray[i] === mask ? selectedValue : deselectedValue;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
for (let i = 0, len = this.length; i < len; i += 1) {
|
||||
|
||||
@@ -39,6 +39,14 @@ import {
|
||||
upperBoundIndirect
|
||||
} from "./util";
|
||||
|
||||
function isArrayOrTypedArray(x) {
|
||||
return (
|
||||
Array.isArray(x) ||
|
||||
(ArrayBuffer.isView(x) &&
|
||||
Object.prototype.toString.call(x) !== "[object DataView]")
|
||||
);
|
||||
}
|
||||
|
||||
class NotImplementedError extends Error {
|
||||
constructor(...params) {
|
||||
super(...params);
|
||||
@@ -52,6 +60,11 @@ class NotImplementedError extends Error {
|
||||
|
||||
class TypedCrossfilter {
|
||||
constructor(data) {
|
||||
/*
|
||||
Typically, data is one of:
|
||||
- Array of objects/records
|
||||
- Dataframe (util/dataframe)
|
||||
*/
|
||||
this.data = data;
|
||||
|
||||
// filters: array of { id, dimension }
|
||||
@@ -97,18 +110,32 @@ class TypedCrossfilter {
|
||||
// return array of all records that are selected/filtered
|
||||
// by all dimensions.
|
||||
allFiltered() {
|
||||
const { selection } = this;
|
||||
const res = [];
|
||||
for (let i = 0, len = this.data.length; i < len; i += 1) {
|
||||
if (selection.isSelected(i)) {
|
||||
res.push(this.data[i]);
|
||||
const { data, selection } = this;
|
||||
if (Array.isArray(data)) {
|
||||
const res = [];
|
||||
for (let i = 0, len = data.length; i < len; i += 1) {
|
||||
if (selection.isSelected(i)) {
|
||||
res.push(data[i]);
|
||||
}
|
||||
}
|
||||
return res;
|
||||
}
|
||||
return res;
|
||||
/* else, Dataframe-like */
|
||||
return data.icutByMask(this.allFilteredMask());
|
||||
}
|
||||
|
||||
// return Uint8array containing selection state (truthy/falsey) for each record.
|
||||
//
|
||||
allFilteredMask() {
|
||||
return this.selection.fillBySelection(
|
||||
new Uint8Array(this.data.length),
|
||||
1,
|
||||
0
|
||||
);
|
||||
}
|
||||
|
||||
countFiltered() {
|
||||
return this.selection.selectionCount;
|
||||
return this.selection.selectionCount();
|
||||
}
|
||||
|
||||
isElementFiltered(i) {
|
||||
@@ -161,6 +188,7 @@ class ScalarDimension extends _Dimension {
|
||||
// or a map function which will create it.
|
||||
let array;
|
||||
if (value instanceof ValueArrayType) {
|
||||
// user has provided the final typed array - just use it
|
||||
if (value.length !== this.crossfilter.data.length) {
|
||||
throw new RangeError(
|
||||
"ScalarDimension values length must equal crossfilter data record count"
|
||||
@@ -168,11 +196,18 @@ class ScalarDimension extends _Dimension {
|
||||
}
|
||||
array = value;
|
||||
} else if (value instanceof Function) {
|
||||
// Create value array
|
||||
// Create value array from user-provided map function.
|
||||
array = this._createValueArray(
|
||||
value,
|
||||
new ValueArrayType(this.crossfilter.data.length)
|
||||
);
|
||||
} else if (isArrayOrTypedArray(value)) {
|
||||
// Create value array from user-provided array. Typically used
|
||||
// only by enumerated dimensions
|
||||
array = this._createValueArray(
|
||||
i => value[i],
|
||||
new ValueArrayType(this.crossfilter.data.length)
|
||||
);
|
||||
} else {
|
||||
throw new NotImplementedError(
|
||||
"dimension value must be function or value array type"
|
||||
@@ -190,7 +225,7 @@ class ScalarDimension extends _Dimension {
|
||||
const len = data.length;
|
||||
const larray = array;
|
||||
for (let i = 0; i < len; i += 1) {
|
||||
larray[i] = value(data[i]);
|
||||
larray[i] = value(i, data);
|
||||
}
|
||||
return larray;
|
||||
}
|
||||
@@ -387,7 +422,7 @@ class EnumDimension extends ScalarDimension {
|
||||
// and the enum.
|
||||
const s = new Set();
|
||||
for (let i = 0; i < len; i += 1) {
|
||||
s.add(value(data[i]));
|
||||
s.add(value(i, data));
|
||||
}
|
||||
this.enumIndex = Array.from(s);
|
||||
this.enumIndex.sort();
|
||||
@@ -395,7 +430,7 @@ class EnumDimension extends ScalarDimension {
|
||||
// create dimension value array
|
||||
const enumLen = this.enumIndex.length;
|
||||
for (let i = 0; i < len; i += 1) {
|
||||
const v = value(data[i]);
|
||||
const v = value(i, data);
|
||||
const e = lowerBound(this.enumIndex, v, 0, enumLen);
|
||||
larray[i] = e;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user