fork/cellxgene
1
0
Fork 0
mirror of https://github.com/chanzuckerberg/cellxgene.git synced 2026-10-05 20:58:12 +08:00
Code Issues Packages Projects Releases Wiki Activity

binary wire format with flatbuffers (#509)

Browse Source 
* first flatbuffer schema

* do not lint auto-generated files

* add flatbuffers package

* add flatbuffer module

* wire up /data/X/T route

* use flatbuffers for matrix data fetc

* clarity and comments

* add flatbuffer layout route

* clean up obsolete code

* fix tests

* move flake8 config to setup.cfg

* add comments

* lint

* rework layout routes for fbs

* add more type support to fbs

* lint

* add flatbuffer support for annotations

* function name improvements

* fix botched merge with master

* remove unused import

* route cleanup for flatbuffers

* rename function for clarity

* add missing globals to Jest tests

* fix client JS tests

* fix routes for Python tests

* comments for clarity

* non-finite floating point hardening

* more non-finite number handling

* lint

* fix tests for summarizeAnnotations

* harden diffexp calculation against FP errors

* cleanup unused code

* lint

* add encoding tests for flatbuffers

* application type specified as strings

* fix spelling error

* improve variable names

* add note about documentation gap

* rename FBS DataFrame to Matrix
This commit is contained in:
Bruce Martin
2019-01-09 14:26:05 -08:00
committed by GitHub
parent 42e25a1a1f
commit b90447c387
41 changed files with 2317 additions and 266 deletions
Download Patch File Download Diff File Expand all files Collapse all files
client/__tests__/setupMissingGlobals.js
+9
View File
@@ -0,0 +1,9 @@
/*
Define globals which are present in the client, but not in node (and therefore not in
the jest test environment).
*/
import { TextDecoder, TextEncoder } from "util";
global.TextDecoder = TextDecoder;
global.TextEncoder = TextEncoder;
client/__tests__/util/stateManager/sampleResponses.js
+105 -6
View File
@@ -1,5 +1,7 @@
/* eslint no-bitwise: "off" */
import _ from "lodash";
import { flatbuffers } from "flatbuffers";
import { NetEncoding } from "../../../src/util/stateManager/matrix_generated";
/*
test data mocking REST 0.2 API responses. Used in several tests.
@@ -58,7 +60,7 @@ const aSchemaResponse = {
}
};
const anAnnotationsObsResponse = {
const anAnnotationsObsJSONResponse = {
names: ["name", "field1", "field2", "field3", "field4"],
data: _()
.range(nObs)
@@ -73,7 +75,7 @@ const anAnnotationsObsResponse = {
.value()
};
const anAnnotationsVarResponse = {
const anAnnotationsVarJSONResponse = {
names: ["fieldA", "fieldB", "fieldC", "fieldD", "name"],
data: _()
.range(nVar)
@@ -88,7 +90,74 @@ const anAnnotationsVarResponse = {
.value()
};
const aLayoutResponse = {
function encodeTypedArray(builder, uType, uData) {
const uTypeName = NetEncoding.TypedArray[uType];
const ArrayType = NetEncoding[uTypeName];
const dv = ArrayType.createDataVector(builder, uData);
builder.startObject(1);
builder.addFieldOffset(0, dv, 0);
return builder.endObject();
}
function encodeMatrix(columns, colIndex = undefined) {
const utf8Encoder = new TextEncoder("utf-8");
const builder = new flatbuffers.Builder(1024);
const cols = _.map(columns, carr => {
let uType;
let tarr;
if (_.every(carr, _.isNumber)) {
uType = NetEncoding.TypedArray.Float32Array;
tarr = encodeTypedArray(builder, uType, new Float32Array(carr));
} else {
uType = NetEncoding.TypedArray.JSONEncodedArray;
const json = JSON.stringify(carr);
const jsonUTF8 = utf8Encoder.encode(json);
tarr = encodeTypedArray(builder, uType, jsonUTF8);
}
NetEncoding.Column.startColumn(builder);
NetEncoding.Column.addUType(builder, uType);
NetEncoding.Column.addU(builder, tarr);
return NetEncoding.Column.endColumn(builder);
});
const encColumns = NetEncoding.Matrix.createColumnsVector(builder, cols);
let encColIndex;
if (colIndex) {
encColIndex = encodeTypedArray(
builder,
NetEncoding.TypedArray.JSONEncodedArray,
utf8Encoder.encode(JSON.stringify(colIndex))
);
}
NetEncoding.Matrix.startMatrix(builder);
NetEncoding.Matrix.addNRows(builder, columns[0].length);
NetEncoding.Matrix.addNCols(builder, columns.length);
NetEncoding.Matrix.addColumns(builder, encColumns);
if (colIndex) {
NetEncoding.Matrix.addColIndexType(
builder,
NetEncoding.TypedArray.JSONEncodedArray
);
NetEncoding.Matrix.addColIndex(builder, encColIndex);
}
const root = NetEncoding.Matrix.endMatrix(builder);
builder.finish(root);
return builder.asUint8Array();
}
const anAnnotationsObsFBSResponse = (() => {
const columns = _.zip(...anAnnotationsObsJSONResponse.data).slice(1);
return encodeMatrix(columns, anAnnotationsObsJSONResponse.names);
})();
const anAnnotationsVarFBSResponse = (() => {
const columns = _.zip(...anAnnotationsVarJSONResponse.data).slice(1);
return encodeMatrix(columns, anAnnotationsVarJSONResponse.names);
})();
const aLayoutJSONResponse = {
layout: {
ndims: 2,
coordinates: _()
@@ -98,6 +167,36 @@ const aLayoutResponse = {
}
};
const aLayoutFBSResponse = (() => {
const coords = [
new Float32Array(nObs).fill(Math.random()),
new Float32Array(nObs).fill(Math.random())
];
const builder = new flatbuffers.Builder(1024);
const cols = _.map(coords, carr => {
const cdv = NetEncoding.Float32Array.createDataVector(builder, carr);
NetEncoding.Float32Array.startFloat32Array(builder);
NetEncoding.Float32Array.addData(builder, cdv);
const floatArr = NetEncoding.Float32Array.endFloat32Array(builder);
NetEncoding.Column.startColumn(builder);
NetEncoding.Column.addUType(builder, NetEncoding.TypedArray.Float32Array);
NetEncoding.Column.addU(builder, floatArr);
return NetEncoding.Column.endColumn(builder);
});
const columns = NetEncoding.Matrix.createColumnsVector(builder, cols);
NetEncoding.Matrix.startMatrix(builder);
NetEncoding.Matrix.addNRows(builder, nObs);
NetEncoding.Matrix.addNCols(builder, nVar);
NetEncoding.Matrix.addColumns(builder, columns);
const matrix = NetEncoding.Matrix.endMatrix(builder);
builder.finish(matrix);
return builder.asUint8Array();
})();
const aDataObsResponse = {
var: [2, 4, 29],
obs: _()
@@ -107,10 +206,10 @@ const aDataObsResponse = {
};
export {
aLayoutResponse as layoutObs,
aLayoutFBSResponse as layoutObs,
aDataObsResponse as dataObs,
anAnnotationsVarResponse as annotationsVar,
anAnnotationsObsResponse as annotationsObs,
anAnnotationsVarFBSResponse as annotationsVar,
anAnnotationsObsFBSResponse as annotationsObs,
aSchemaResponse as schema,
aConfigResponse as config
};
client/__tests__/util/stateManager/summarizeAnnotations.test.js
+96 -8
View File
@@ -39,15 +39,21 @@ describe("summarizeAnnotations", () => {
nameFloat32: {
categorical: false,
range: {
max: Number.NEGATIVE_INFINITY,
min: Number.POSITIVE_INFINITY
max: undefined,
min: undefined,
nan: 0,
ninf: 0,
pinf: 0
}
},
nameInt32: {
categorical: false,
range: {
max: Number.NEGATIVE_INFINITY,
min: Number.POSITIVE_INFINITY
max: undefined,
min: undefined,
nan: 0,
ninf: 0,
pinf: 0
}
},
nameCategorical: {
@@ -99,11 +105,11 @@ describe("summarizeAnnotations", () => {
},
nameFloat32: {
categorical: false,
range: { min: 39.3, max: 39.3 }
range: { min: 39.3, max: 39.3, nan: 0, ninf: 0, pinf: 0 }
},
nameInt32: {
categorical: false,
range: { min: 99, max: 99 }
range: { min: 99, max: 99, nan: 0, ninf: 0, pinf: 0 }
},
nameCategorical: {
categorical: true,
@@ -172,11 +178,93 @@ describe("summarizeAnnotations", () => {
},
nameFloat32: {
categorical: false,
range: { min: 0, max: 39.3 }
range: { min: 0, max: 39.3, nan: 0, ninf: 0, pinf: 0 }
},
nameInt32: {
categorical: false,
range: { min: 99, max: 99 }
range: { min: 99, max: 99, nan: 0, ninf: 0, pinf: 0 }
},
nameCategorical: {
categorical: true,
categories: expect.arrayContaining([1, false, "0"]),
categoryCounts: new Map([[1, 1], [false, 1], ["0", 1]]),
numCategories: 3
}
},
var: {}
})
);
});
test("non-finite numbers", () => {
const obsAnnotations = [
{
__index__: 0,
name: "n0",
nameString: "hi",
nameBoolean: false,
nameFloat32: 39.3,
nameInt32: 99,
nameCategorical: 1
},
{
__index__: 1,
name: "n1",
nameString: "hi",
nameBoolean: true,
nameFloat32: Number.NEGATIVE_INFINITY,
nameInt32: 99,
nameCategorical: false
},
{
__index__: 2,
name: "n2",
nameString: "bye",
nameBoolean: true,
nameFloat32: Number.NaN,
nameInt32: 99,
nameCategorical: "0"
},
{
__index__: 3,
name: "n2",
nameString: "bye",
nameBoolean: true,
nameFloat32: Number.POSITIVE_INFINITY,
nameInt32: 99,
nameCategorical: "0"
}
];
const varAnnotations = [];
const summary = summarizeAnnotations(
schema,
obsAnnotations,
varAnnotations
);
expect(summary).toMatchObject(
expect.objectContaining({
obs: {
nameString: {
categorical: true,
categories: expect.arrayContaining(["hi", "bye"]),
categoryCounts: new Map([["hi", 2], ["bye", 1]]),
numCategories: 2
},
nameBoolean: {
categorical: true,
categories: expect.arrayContaining([true, false]),
categoryCounts: new Map([[true, 2], [false, 1]]),
numCategories: 2
},
nameFloat32: {
categorical: false,
range: { min: 39.3, max: 39.3, nan: 1, ninf: 1, pinf: 1 }
},
nameInt32: {
categorical: false,
range: { min: 99, max: 99, nan: 0, ninf: 0, pinf: 0 }
},
nameCategorical: {
categorical: true,
client/__tests__/util/stateManager/universe.test.js
-54
View File
@@ -30,7 +30,6 @@ describe("createUniverseFromRestV02Response", () => {
create a universe from sample data nad validate its shape & contents
*/
const { nObs, nVar } = REST.schema.schema.dataframe;
const universe = Universe.createUniverseFromRestV02Response(
REST.config,
REST.schema,
@@ -66,56 +65,3 @@ describe("createUniverseFromRestV02Response", () => {
expect(_.keys(universe.varNameToIndexMap)).toHaveLength(nVar);
});
});
describe("convertExpressionRESTv02ToObject", () => {
/*
test convertExpressionRESTv02ToObject
convertExpressionRESTv02ToObject(
universe,
response) --> { geneName: Float32Array, geneName: Float32Array, ... }
reponse is a /data/obs response:
{
var: [ varIndices fetched ],
obs: [
[ obsIndex, evalue, ... ],
...
]
}
*/
test("create from response data", () => {
const universe = Universe.createUniverseFromRestV02Response(
REST.config,
REST.schema,
REST.annotationsObs,
REST.annotationsVar,
REST.layoutObs
);
const expression = Universe.convertExpressionRESTv02ToObject(
universe,
REST.dataObs
);
/* Check that the expected keys are present */
const expectedGeneNames = _.map(
REST.dataObs.var,
v => REST.annotationsVar.data[v][5]
);
expect(Object.keys(expression)).toEqual(
expect.arrayContaining(expectedGeneNames)
);
const expectedExpressionValues = _.map(
_.unzip(REST.dataObs.obs),
a => new Float32Array(a)
);
_.forEach(REST.dataObs.var, (varIdx, idx) => {
const varName = universe.varAnnotations[varIdx].name;
expect(varName).toBeDefined();
expect(varIdx).toBe(universe.varNameToIndexMap[varName]);
expect(expression[varName]).toEqual(expectedExpressionValues[idx + 1]);
});
});
});
client/__tests__/util/stateManager/world.test.js
+5 -2
View File
@@ -168,10 +168,13 @@ describe("createObsDimensionMap", () => {
*/
const { dimensionMap } = defaultBigBang();
const annotationNames = _.map(
REST.schema.schema.annotations.obs,
c => c.name
);
const schemaByObsName = _.keyBy(REST.schema.schema.annotations.obs, "name");
expect(dimensionMap).toBeDefined();
REST.annotationsObs.names.forEach(name => {
annotationNames.forEach(name => {
const dim = dimensionMap[obsAnnoDimensionName(name)];
if (name === "name") {
expect(dim).toBeUndefined();