fork/cellxgene
1
0
Fork 0
mirror of https://github.com/chanzuckerberg/cellxgene.git synced 2026-09-15 12:47:56 +08:00
Code Issues Packages Projects Releases Wiki Activity

binary wire format with flatbuffers (#509)

Browse Source 
* first flatbuffer schema

* do not lint auto-generated files

* add flatbuffers package

* add flatbuffer module

* wire up /data/X/T route

* use flatbuffers for matrix data fetc

* clarity and comments

* add flatbuffer layout route

* clean up obsolete code

* fix tests

* move flake8 config to setup.cfg

* add comments

* lint

* rework layout routes for fbs

* add more type support to fbs

* lint

* add flatbuffer support for annotations

* function name improvements

* fix botched merge with master

* remove unused import

* route cleanup for flatbuffers

* rename function for clarity

* add missing globals to Jest tests

* fix client JS tests

* fix routes for Python tests

* comments for clarity

* non-finite floating point hardening

* more non-finite number handling

* lint

* fix tests for summarizeAnnotations

* harden diffexp calculation against FP errors

* cleanup unused code

* lint

* add encoding tests for flatbuffers

* application type specified as strings

* fix spelling error

* improve variable names

* add note about documentation gap

* rename FBS DataFrame to Matrix
This commit is contained in:
Bruce Martin
2019-01-09 14:26:05 -08:00
committed by GitHub
parent 42e25a1a1f
commit b90447c387
41 changed files with 2317 additions and 266 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
client/__tests__/setupMissingGlobals.js Normal file
View File

@@ -0,0 +1,9 @@
/*
Define globals which are present in the client, but not in node (and therefore not in
the jest test environment).
*/
import { TextDecoder, TextEncoder } from "util";
global.TextDecoder = TextDecoder;
global.TextEncoder = TextEncoder;

111
client/__tests__/util/stateManager/sampleResponses.js
View File

@@ -1,5 +1,7 @@
/* eslint no-bitwise: "off" */
import _ from "lodash";
import { flatbuffers } from "flatbuffers";
import { NetEncoding } from "../../../src/util/stateManager/matrix_generated";
/*
test data mocking REST 0.2 API responses. Used in several tests.
@@ -58,7 +60,7 @@ const aSchemaResponse = {
}
};
const anAnnotationsObsResponse = {
const anAnnotationsObsJSONResponse = {
names: ["name", "field1", "field2", "field3", "field4"],
data: _()
.range(nObs)
@@ -73,7 +75,7 @@ const anAnnotationsObsResponse = {
.value()
};
const anAnnotationsVarResponse = {
const anAnnotationsVarJSONResponse = {
names: ["fieldA", "fieldB", "fieldC", "fieldD", "name"],
data: _()
.range(nVar)
@@ -88,7 +90,74 @@ const anAnnotationsVarResponse = {
.value()
};
const aLayoutResponse = {
function encodeTypedArray(builder, uType, uData) {
const uTypeName = NetEncoding.TypedArray[uType];
const ArrayType = NetEncoding[uTypeName];
const dv = ArrayType.createDataVector(builder, uData);
builder.startObject(1);
builder.addFieldOffset(0, dv, 0);
return builder.endObject();
}
function encodeMatrix(columns, colIndex = undefined) {
const utf8Encoder = new TextEncoder("utf-8");
const builder = new flatbuffers.Builder(1024);
const cols = _.map(columns, carr => {
let uType;
let tarr;
if (_.every(carr, _.isNumber)) {
uType = NetEncoding.TypedArray.Float32Array;
tarr = encodeTypedArray(builder, uType, new Float32Array(carr));
} else {
uType = NetEncoding.TypedArray.JSONEncodedArray;
const json = JSON.stringify(carr);
const jsonUTF8 = utf8Encoder.encode(json);
tarr = encodeTypedArray(builder, uType, jsonUTF8);
}
NetEncoding.Column.startColumn(builder);
NetEncoding.Column.addUType(builder, uType);
NetEncoding.Column.addU(builder, tarr);
return NetEncoding.Column.endColumn(builder);
});
const encColumns = NetEncoding.Matrix.createColumnsVector(builder, cols);
let encColIndex;
if (colIndex) {
encColIndex = encodeTypedArray(
builder,
NetEncoding.TypedArray.JSONEncodedArray,
utf8Encoder.encode(JSON.stringify(colIndex))
);
}
NetEncoding.Matrix.startMatrix(builder);
NetEncoding.Matrix.addNRows(builder, columns[0].length);
NetEncoding.Matrix.addNCols(builder, columns.length);
NetEncoding.Matrix.addColumns(builder, encColumns);
if (colIndex) {
NetEncoding.Matrix.addColIndexType(
builder,
NetEncoding.TypedArray.JSONEncodedArray
);
NetEncoding.Matrix.addColIndex(builder, encColIndex);
}
const root = NetEncoding.Matrix.endMatrix(builder);
builder.finish(root);
return builder.asUint8Array();
}
const anAnnotationsObsFBSResponse = (() => {
const columns = _.zip(...anAnnotationsObsJSONResponse.data).slice(1);
return encodeMatrix(columns, anAnnotationsObsJSONResponse.names);
})();
const anAnnotationsVarFBSResponse = (() => {
const columns = _.zip(...anAnnotationsVarJSONResponse.data).slice(1);
return encodeMatrix(columns, anAnnotationsVarJSONResponse.names);
})();
const aLayoutJSONResponse = {
layout: {
ndims: 2,
coordinates: _()
@@ -98,6 +167,36 @@ const aLayoutResponse = {
}
};
const aLayoutFBSResponse = (() => {
const coords = [
new Float32Array(nObs).fill(Math.random()),
new Float32Array(nObs).fill(Math.random())
];
const builder = new flatbuffers.Builder(1024);
const cols = _.map(coords, carr => {
const cdv = NetEncoding.Float32Array.createDataVector(builder, carr);
NetEncoding.Float32Array.startFloat32Array(builder);
NetEncoding.Float32Array.addData(builder, cdv);
const floatArr = NetEncoding.Float32Array.endFloat32Array(builder);
NetEncoding.Column.startColumn(builder);
NetEncoding.Column.addUType(builder, NetEncoding.TypedArray.Float32Array);
NetEncoding.Column.addU(builder, floatArr);
return NetEncoding.Column.endColumn(builder);
});
const columns = NetEncoding.Matrix.createColumnsVector(builder, cols);
NetEncoding.Matrix.startMatrix(builder);
NetEncoding.Matrix.addNRows(builder, nObs);
NetEncoding.Matrix.addNCols(builder, nVar);
NetEncoding.Matrix.addColumns(builder, columns);
const matrix = NetEncoding.Matrix.endMatrix(builder);
builder.finish(matrix);
return builder.asUint8Array();
})();
const aDataObsResponse = {
var: [2, 4, 29],
obs: _()
@@ -107,10 +206,10 @@ const aDataObsResponse = {
};
export {
aLayoutResponse as layoutObs,
aLayoutFBSResponse as layoutObs,
aDataObsResponse as dataObs,
anAnnotationsVarResponse as annotationsVar,
anAnnotationsObsResponse as annotationsObs,
anAnnotationsVarFBSResponse as annotationsVar,
anAnnotationsObsFBSResponse as annotationsObs,
aSchemaResponse as schema,
aConfigResponse as config
};

104
client/__tests__/util/stateManager/summarizeAnnotations.test.js
View File

@@ -39,15 +39,21 @@ describe("summarizeAnnotations", () => {
nameFloat32: {
categorical: false,
range: {
max: Number.NEGATIVE_INFINITY,
min: Number.POSITIVE_INFINITY
max: undefined,
min: undefined,
nan: 0,
ninf: 0,
pinf: 0
}
},
nameInt32: {
categorical: false,
range: {
max: Number.NEGATIVE_INFINITY,
min: Number.POSITIVE_INFINITY
max: undefined,
min: undefined,
nan: 0,
ninf: 0,
pinf: 0
}
},
nameCategorical: {
@@ -99,11 +105,11 @@ describe("summarizeAnnotations", () => {
},
nameFloat32: {
categorical: false,
range: { min: 39.3, max: 39.3 }
range: { min: 39.3, max: 39.3, nan: 0, ninf: 0, pinf: 0 }
},
nameInt32: {
categorical: false,
range: { min: 99, max: 99 }
range: { min: 99, max: 99, nan: 0, ninf: 0, pinf: 0 }
},
nameCategorical: {
categorical: true,
@@ -172,11 +178,93 @@ describe("summarizeAnnotations", () => {
},
nameFloat32: {
categorical: false,
range: { min: 0, max: 39.3 }
range: { min: 0, max: 39.3, nan: 0, ninf: 0, pinf: 0 }
},
nameInt32: {
categorical: false,
range: { min: 99, max: 99 }
range: { min: 99, max: 99, nan: 0, ninf: 0, pinf: 0 }
},
nameCategorical: {
categorical: true,
categories: expect.arrayContaining([1, false, "0"]),
categoryCounts: new Map([[1, 1], [false, 1], ["0", 1]]),
numCategories: 3
}
},
var: {}
})
);
});
test("non-finite numbers", () => {
const obsAnnotations = [
{
__index__: 0,
name: "n0",
nameString: "hi",
nameBoolean: false,
nameFloat32: 39.3,
nameInt32: 99,
nameCategorical: 1
},
{
__index__: 1,
name: "n1",
nameString: "hi",
nameBoolean: true,
nameFloat32: Number.NEGATIVE_INFINITY,
nameInt32: 99,
nameCategorical: false
},
{
__index__: 2,
name: "n2",
nameString: "bye",
nameBoolean: true,
nameFloat32: Number.NaN,
nameInt32: 99,
nameCategorical: "0"
},
{
__index__: 3,
name: "n2",
nameString: "bye",
nameBoolean: true,
nameFloat32: Number.POSITIVE_INFINITY,
nameInt32: 99,
nameCategorical: "0"
}
];
const varAnnotations = [];
const summary = summarizeAnnotations(
schema,
obsAnnotations,
varAnnotations
);
expect(summary).toMatchObject(
expect.objectContaining({
obs: {
nameString: {
categorical: true,
categories: expect.arrayContaining(["hi", "bye"]),
categoryCounts: new Map([["hi", 2], ["bye", 1]]),
numCategories: 2
},
nameBoolean: {
categorical: true,
categories: expect.arrayContaining([true, false]),
categoryCounts: new Map([[true, 2], [false, 1]]),
numCategories: 2
},
nameFloat32: {
categorical: false,
range: { min: 39.3, max: 39.3, nan: 1, ninf: 1, pinf: 1 }
},
nameInt32: {
categorical: false,
range: { min: 99, max: 99, nan: 0, ninf: 0, pinf: 0 }
},
nameCategorical: {
categorical: true,

54
client/__tests__/util/stateManager/universe.test.js
View File

@@ -30,7 +30,6 @@ describe("createUniverseFromRestV02Response", () => {
create a universe from sample data nad validate its shape & contents
*/
const { nObs, nVar } = REST.schema.schema.dataframe;
const universe = Universe.createUniverseFromRestV02Response(
REST.config,
REST.schema,
@@ -66,56 +65,3 @@ describe("createUniverseFromRestV02Response", () => {
expect(_.keys(universe.varNameToIndexMap)).toHaveLength(nVar);
});
});
describe("convertExpressionRESTv02ToObject", () => {
/*
test convertExpressionRESTv02ToObject
convertExpressionRESTv02ToObject(
universe,
response) --> { geneName: Float32Array, geneName: Float32Array, ... }
reponse is a /data/obs response:
{
var: [ varIndices fetched ],
obs: [
[ obsIndex, evalue, ... ],
...
]
}
*/
test("create from response data", () => {
const universe = Universe.createUniverseFromRestV02Response(
REST.config,
REST.schema,
REST.annotationsObs,
REST.annotationsVar,
REST.layoutObs
);
const expression = Universe.convertExpressionRESTv02ToObject(
universe,
REST.dataObs
);
/* Check that the expected keys are present */
const expectedGeneNames = _.map(
REST.dataObs.var,
v => REST.annotationsVar.data[v][5]
);
expect(Object.keys(expression)).toEqual(
expect.arrayContaining(expectedGeneNames)
);
const expectedExpressionValues = _.map(
_.unzip(REST.dataObs.obs),
a => new Float32Array(a)
);
_.forEach(REST.dataObs.var, (varIdx, idx) => {
const varName = universe.varAnnotations[varIdx].name;
expect(varName).toBeDefined();
expect(varIdx).toBe(universe.varNameToIndexMap[varName]);
expect(expression[varName]).toEqual(expectedExpressionValues[idx + 1]);
});
});
});

7
client/__tests__/util/stateManager/world.test.js
View File

@@ -168,10 +168,13 @@ describe("createObsDimensionMap", () => {
*/
const { dimensionMap } = defaultBigBang();
const annotationNames = _.map(
REST.schema.schema.annotations.obs,
c => c.name
);
const schemaByObsName = _.keyBy(REST.schema.schema.annotations.obs, "name");
expect(dimensionMap).toBeDefined();
REST.annotationsObs.names.forEach(name => {
annotationNames.forEach(name => {
const dim = dimensionMap[obsAnnoDimensionName(name)];
if (name === "name") {
expect(dim).toBeUndefined();

5
client/package-lock.json generated
View File

@@ -5279,6 +5279,11 @@
"write": "^0.2.1"
}
},
"flatbuffers": {
"version": "1.10.2",
"resolved": "https://registry.npmjs.org/flatbuffers/-/flatbuffers-1.10.2.tgz",
"integrity": "sha512-VK7lHZF/corkykjXZ0+dqViI8Wk1YpwPCFN2wrnTs+PMCMG5+uHRvkRW14fuA7Smkhkgx+Dj5UdS3YXktJL+qw=="
},
"flatted": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/flatted/-/flatted-2.0.0.tgz",

6
client/package.json
View File

@@ -33,6 +33,7 @@
"canvas-fit": "^1.5.0",
"d3": "^4.10.0",
"d3-scale-chromatic": "^1.3.0",
"flatbuffers": "^1.10.2",
"font-color-contrast": "^1.0.3",
"fuzzysort": "^1.1.4",
"gl-mat4": "^1.1.4",
@@ -111,7 +112,10 @@
"testMatch": [
"**/__tests__/**/?(*.)(spec|test).js?(x)"
],
"testURL": "http://localhost/"
"testURL": "http://localhost/",
"setupFiles": [
"./__tests__/setupMissingGlobals.js"
]
},
"babel": {
"env": {

78
client/src/actions/index.js
View File

@@ -5,6 +5,7 @@ import { Universe, kvCache } from "../util/stateManager";
import {
catchErrorsWrap,
doJsonRequest,
doBinaryRequest,
rangeEncodeIndices,
dispatchNetworkErrorMessageToUser
} from "../util/actionHelpers";
@@ -13,24 +14,28 @@ import {
Bootstrap application with the initial data loading.
* /config - application configuration
* /schema - schema of dataframe
* /annotations/obs - all metadata annotation
* /annotations - all metadata annotation
* /layout - all default layout
*/
const doInitialDataLoad = () =>
catchErrorsWrap(async dispatch => {
dispatch({ type: "initial data load start" });
try {
const requests = _([
"config",
"schema",
const requestJson = _(["config", "schema"])
.map(r => `${globals.API.prefix}${globals.API.version}${r}`)
.map(url => doJsonRequest(url))
.value();
const requestBinary = _([
"annotations/obs",
"annotations/var?annotation-name=name",
"layout/obs"
])
.map(r => `${globals.API.prefix}${globals.API.version}${r}`)
.map(url => doJsonRequest(url))
.map(url => doBinaryRequest(url))
.value();
const results = await Promise.all(requests);
const results = await Promise.all(_.concat(requestJson, requestBinary));
/* set config defaults */
const config = { ...globals.configDefaults, ...results[0].config };
@@ -87,6 +92,38 @@ needs expression data.
Transparently utilizes cached data if it is already present.
*/
async function _doRequestExpressionData(dispatch, getState, genes) {
/* helper for this function only */
const fetchData = async geneNames => {
const res = await fetch(
`${globals.API.prefix}${globals.API.version}data/var`,
{
method: "PUT",
body: JSON.stringify({
filter: {
var: {
annotation_value: [{ name: "name", values: geneNames }]
}
}
}),
headers: new Headers({
accept: "application/octet-stream",
"Content-Type": "application/json"
})
}
);
if (
!res.ok ||
res.headers.get("Content-Type") !== "application/octet-stream"
) {
// WILL throw
return dispatchExpressionErrors(dispatch, res);
}
const data = await res.arrayBuffer();
return Universe.convertDataFBStoObject(universe, data);
};
const state = getState();
const { universe } = state.controls;
/* preload data already in cache */
@@ -108,35 +145,10 @@ async function _doRequestExpressionData(dispatch, getState, genes) {
/* Fetch data for any genes not in cache */
if (genesToFetch.length) {
try {
// XXX: TODO - this could be using /data/var rather than /data/obs,
// as that would simplify the transformation in convertExpressionRESTv02ToObject
const res = await fetch(
`${globals.API.prefix}${globals.API.version}data/obs`,
{
method: "PUT",
body: JSON.stringify({
filter: {
var: {
annotation_value: [{ name: "name", values: genesToFetch }]
}
}
}),
headers: new Headers({
accept: "application/json",
"Content-Type": "application/json"
})
}
);
if (!res.ok || res.headers.get("Content-Type") !== "application/json") {
// WILL throw
return dispatchExpressionErrors(dispatch, res);
}
const data = await res.json();
const newExpressionData = await fetchData(genesToFetch);
expressionData = {
...expressionData,
...Universe.convertExpressionRESTv02ToObject(universe, data)
...newExpressionData
};
} catch (error) {
dispatch({ type: "expression load error", error });

3
client/src/components/brushableHistogram/index.js
View File

@@ -13,6 +13,7 @@ import memoize from "memoize-one";
import { kvCache } from "../../util/stateManager";
import * as globals from "../../globals";
import actions from "../../actions";
import finiteExtent from "../../util/finiteExtent";
@connect(state => ({
world: state.controls.world,
@@ -56,7 +57,7 @@ class HistogramBrush extends React.Component {
histogramCache.x = d3
.scaleLinear()
.domain(
d3.extent(varValues)
finiteExtent(varValues)
) /* replace this if we have ranges for genes back from server like we do for annotations on cells */
.range([0, this.width]);

6
client/src/components/geneExpression/index.js
View File

@@ -3,7 +3,6 @@
import React from "react";
import _ from "lodash";
import * as d3 from "d3";
import fuzzysort from "fuzzysort";
import { connect } from "react-redux";
@@ -14,6 +13,7 @@ import * as globals from "../../globals";
import actions from "../../actions";
import { postUserErrorToast } from "../framework/toasters";
import ExpressionButtons from "./expressionButtons";
import finiteExtent from "../../util/finiteExtent";
const renderGene = (fuzzySortResult, { handleClick, modifiers, query }) => {
if (!modifiers.matchesPredicate) {
@@ -155,7 +155,7 @@ class GeneExpression extends React.Component {
key={geneName}
field={geneName}
zebra={index % 2 === 0}
ranges={d3.extent(values)}
ranges={finiteExtent(values)}
isUserDefined
/>
);
@@ -185,7 +185,7 @@ class GeneExpression extends React.Component {
key={name}
field={name}
zebra={index % 2 === 0}
ranges={d3.extent(values)}
ranges={finiteExtent(values)}
isDiffExp
logFoldChange={value[1]}
pval={value[2]}

5
client/src/components/scatterplot/scatterplot.js
View File

@@ -20,6 +20,7 @@ import scaleLinear from "../../util/scaleLinear";
import { margin, width, height } from "./util";
import { kvCache } from "../../util/stateManager";
import finiteExtent from "../../util/finiteExtent";
@connect(state => {
const {
@@ -227,11 +228,11 @@ class Scatterplot extends React.Component {
static setupScales(expressionX, expressionY) {
const xScale = d3
.scaleLinear()
.domain(d3.extent(expressionX))
.domain(finiteExtent(expressionX))
.range([0, width]);
const yScale = d3
.scaleLinear()
.domain(d3.extent(expressionY))
.domain(finiteExtent(expressionY))
.range([height, 0]);
return {

1
client/src/globals.js
View File

@@ -57,6 +57,7 @@ export const brightBlue = "#4a90e2";
export const brightGreen = "#A2D729";
export const darkGreen = "#448C4D";
export const nonFiniteCellColor = lightGrey;
export const defaultCellColor = "rgb(0,0,0,1)";
/* typography constants */

26
client/src/middleware/updateCellColors.js
View File

@@ -2,7 +2,9 @@
import _ from "lodash";
import * as d3 from "d3";
import { interpolateRainbow, interpolateCool } from "d3-scale-chromatic";
import * as globals from "../globals";
import parseRGB from "../util/parseRGB";
import finiteExtent from "../util/finiteExtent";
/*
https://medium.com/@jacobp100/you-arent-using-redux-middleware-enough-94ffe991e6
@@ -93,10 +95,15 @@ const updateCellColorsMiddleware = store => next => action => {
}
const key = action.colorAccessor;
const nonFiniteColor = parseRGB(globals.nonFiniteCellColor);
for (let i = 0, len = obsAnnotations.length; i < len; i += 1) {
const obs = obsAnnotations[i];
const c = colorScale(obs[key]);
colorsByRGB[i] = colors[c];
const val = obsAnnotations[i][key];
if (Number.isFinite(val)) {
const c = colorScale(val);
colorsByRGB[i] = colors[c];
} else {
colorsByRGB[i] = nonFiniteColor;
}
}
}
@@ -104,8 +111,7 @@ const updateCellColorsMiddleware = store => next => action => {
const { gene, data } = action;
const expression = data[gene]; // Float32Array
const colorBins = 100;
// XXX TODO - replace _.min/_.max with the much faster finiteExtent
const [min, max] = [_.min(expression), _.max(expression)];
const [min, max] = finiteExtent(expression);
colorScale = d3
.scaleQuantile()
.domain([min, max])
@@ -116,10 +122,16 @@ const updateCellColorsMiddleware = store => next => action => {
for (let i = 0; i < colorBins; i += 1) {
colors[i] = parseRGB(interpolateCool(i / colorBins));
}
const nonFiniteColor = parseRGB(globals.nonFiniteCellColor);
for (let i = 0, len = expression.length; i < len; i += 1) {
const c = colorScale(expression[i]);
colorsByRGB[i] = colors[c];
const e = expression[i];
if (Number.isFinite(e)) {
const c = colorScale(e);
colorsByRGB[i] = colors[c];
} else {
colorsByRGB[i] = nonFiniteColor;
}
}
}

27
client/src/util/actionHelpers.js
View File

@@ -25,17 +25,18 @@ export function catchErrorsWrap(fn, dispatchToUser = false) {
}
/*
Wrapper to perform an async fetch and JSON decode response.
Wrapper to perform async fetch with some modest error handling
and decoding.
*/
export const doJsonRequest = async url => {
const doFetch = async (url, acceptType) => {
const res = await fetch(url, {
method: "get",
headers: new Headers({
"Content-Type": "application/json"
Accept: acceptType
})
});
if (res.ok && res.headers.get("Content-Type") === "application/json") {
return res.json();
if (res.ok && res.headers.get("Content-Type") === acceptType) {
return res;
}
// else an error
let msg = `Unexpected HTTP response ${res.status}, ${res.statusText}`;
@@ -47,6 +48,22 @@ export const doJsonRequest = async url => {
throw new Error(msg);
};
/*
Wrapper to perform an async fetch and JSON decode response.
*/
export const doJsonRequest = async url => {
const res = await doFetch(url, "application/json");
return res.json();
};
/*
Wrapper to perform an async fetch for binary data.
*/
export const doBinaryRequest = async url => {
const res = await doFetch(url, "application/octet-stream");
return res.arrayBuffer();
};
/*
This function "packs" filter index lists into the more efficient
"range" form specified in the REST 0.2 spec.

33
client/src/util/finiteExtent.js Normal file
View File

@@ -0,0 +1,33 @@
/*
Return the [minimum, maximum] extent, of the given typed array, ignoring
non-finite values (ie, +Infinity, -Infinity).
If undefined or empty array, or array contains only non-finite numbers,
will return [undefined, undefined]
*/
function finiteExtent(tarr) {
let min;
let max;
let i;
for (i = 0; i < tarr.length; i += 1) {
const val = tarr[i];
if (Number.isFinite(val)) {
min = val;
max = val;
i += 1;
break;
}
}
for (; i < tarr.length; i += 1) {
const val = tarr[i];
if (Number.isFinite(val)) {
if (min > val) min = val;
if (max < val) max = val;
}
}
return [min, max];
}
export default finiteExtent;

2
client/src/util/stateManager/index.js
View File

@@ -3,7 +3,7 @@
/*
Model manager providing an abstraction for the use of the reducer code.
This module provides several buckets of functionality:
- schema and config driven tranformation of the dataframe wire protocol
- schema and config driven tranformation of the wire protocol
into a format that is easy for the UI code to use.
- manage the universe/world abstraction:
+ universe: all of the server-provided, read-only data

75
client/src/util/stateManager/matrix.js Normal file
View File

@@ -0,0 +1,75 @@
import { flatbuffers } from "flatbuffers";
import { NetEncoding } from "./matrix_generated";
const utf8Decoder = new TextDecoder("utf-8");
/*
Matrix flatbuffer decoding support. See fbs/matrix.fbs
*/
/*
Decode NetEncoding.TypedArray
*/
function decodeTypedArray(uType, uValF, inplace = false) {
if (uType === NetEncoding.TypedArray.NONE) {
return null;
}
// Convert to a JS class that supports this type
const TypeClass = NetEncoding[NetEncoding.TypedArray[uType]];
// Create a TypedArray that references the underlying buffer
let arr = uValF(new TypeClass()).dataArray();
if (uType === NetEncoding.TypedArray.JSONEncodedArray) {
const json = utf8Decoder.decode(arr);
arr = JSON.parse(json);
} else if (!inplace) {
/* force copy to release underlying FBS buffer */
arr = new arr.constructor(arr);
}
return arr;
}
/*
Parameter: Uint8Array or ArrayBuffer containing raw flatbuffer Matrix
Returns: object containing decoded Matrix:
{
nRows: num,
nCols: num,
columns: [
each column, which will be a TypedArray or Array
]
colIdx: []|null
}
*/
function decodeMatrixFBS(arrayBuffer, inplace = false) {
const bb = new flatbuffers.ByteBuffer(new Uint8Array(arrayBuffer));
const df = NetEncoding.Matrix.getRootAsMatrix(bb);
const nRows = df.nRows();
const nCols = df.nCols();
/* decode columns */
const columnsLength = df.columnsLength();
const columns = Array(columnsLength).fill(null);
for (let c = 0; c < columnsLength; c += 1) {
const col = df.columns(c);
columns[c] = decodeTypedArray(col.uType(), col.u.bind(col), inplace);
}
/* decode col_idx */
const colIdx = decodeTypedArray(
df.colIndexType(),
df.colIndex.bind(df),
inplace
);
return {
nRows,
nCols,
columns,
colIdx,
rowIdx: null
};
}
export default decodeMatrixFBS;

835
client/src/util/stateManager/matrix_generated.js Normal file
View File

@@ -0,0 +1,835 @@
// automatically generated by the FlatBuffers compiler, do not modify
/**
* @const
* @namespace
*/
var NetEncoding = NetEncoding || {};
/**
* @enum
*/
NetEncoding.TypedArray = {
NONE: 0, 0: 'NONE',
Float32Array: 1, 1: 'Float32Array',
Int32Array: 2, 2: 'Int32Array',
Uint32Array: 3, 3: 'Uint32Array',
Float64Array: 4, 4: 'Float64Array',
JSONEncodedArray: 5, 5: 'JSONEncodedArray'
};
/**
* @constructor
*/
NetEncoding.Float32Array = function() {
/**
* @type {flatbuffers.ByteBuffer}
*/
this.bb = null;
/**
* @type {number}
*/
this.bb_pos = 0;
};
/**
* @param {number} i
* @param {flatbuffers.ByteBuffer} bb
* @returns {NetEncoding.Float32Array}
*/
NetEncoding.Float32Array.prototype.__init = function(i, bb) {
this.bb_pos = i;
this.bb = bb;
return this;
};
/**
* @param {flatbuffers.ByteBuffer} bb
* @param {NetEncoding.Float32Array=} obj
* @returns {NetEncoding.Float32Array}
*/
NetEncoding.Float32Array.getRootAsFloat32Array = function(bb, obj) {
return (obj || new NetEncoding.Float32Array).__init(bb.readInt32(bb.position()) + bb.position(), bb);
};
/**
* @param {number} index
* @returns {number}
*/
NetEncoding.Float32Array.prototype.data = function(index) {
var offset = this.bb.__offset(this.bb_pos, 4);
return offset ? this.bb.readFloat32(this.bb.__vector(this.bb_pos + offset) + index * 4) : 0;
};
/**
* @returns {number}
*/
NetEncoding.Float32Array.prototype.dataLength = function() {
var offset = this.bb.__offset(this.bb_pos, 4);
return offset ? this.bb.__vector_len(this.bb_pos + offset) : 0;
};
/**
* @returns {Float32Array}
*/
NetEncoding.Float32Array.prototype.dataArray = function() {
var offset = this.bb.__offset(this.bb_pos, 4);
return offset ? new Float32Array(this.bb.bytes().buffer, this.bb.bytes().byteOffset + this.bb.__vector(this.bb_pos + offset), this.bb.__vector_len(this.bb_pos + offset)) : null;
};
/**
* @param {flatbuffers.Builder} builder
*/
NetEncoding.Float32Array.startFloat32Array = function(builder) {
builder.startObject(1);
};
/**
* @param {flatbuffers.Builder} builder
* @param {flatbuffers.Offset} dataOffset
*/
NetEncoding.Float32Array.addData = function(builder, dataOffset) {
builder.addFieldOffset(0, dataOffset, 0);
};
/**
* @param {flatbuffers.Builder} builder
* @param {Array.<number>} data
* @returns {flatbuffers.Offset}
*/
NetEncoding.Float32Array.createDataVector = function(builder, data) {
builder.startVector(4, data.length, 4);
for (var i = data.length - 1; i >= 0; i--) {
builder.addFloat32(data[i]);
}
return builder.endVector();
};
/**
* @param {flatbuffers.Builder} builder
* @param {number} numElems
*/
NetEncoding.Float32Array.startDataVector = function(builder, numElems) {
builder.startVector(4, numElems, 4);
};
/**
* @param {flatbuffers.Builder} builder
* @returns {flatbuffers.Offset}
*/
NetEncoding.Float32Array.endFloat32Array = function(builder) {
var offset = builder.endObject();
return offset;
};
/**
* @constructor
*/
NetEncoding.Uint32Array = function() {
/**
* @type {flatbuffers.ByteBuffer}
*/
this.bb = null;
/**
* @type {number}
*/
this.bb_pos = 0;
};
/**
* @param {number} i
* @param {flatbuffers.ByteBuffer} bb
* @returns {NetEncoding.Uint32Array}
*/
NetEncoding.Uint32Array.prototype.__init = function(i, bb) {
this.bb_pos = i;
this.bb = bb;
return this;
};
/**
* @param {flatbuffers.ByteBuffer} bb
* @param {NetEncoding.Uint32Array=} obj
* @returns {NetEncoding.Uint32Array}
*/
NetEncoding.Uint32Array.getRootAsUint32Array = function(bb, obj) {
return (obj || new NetEncoding.Uint32Array).__init(bb.readInt32(bb.position()) + bb.position(), bb);
};
/**
* @param {number} index
* @returns {number}
*/
NetEncoding.Uint32Array.prototype.data = function(index) {
var offset = this.bb.__offset(this.bb_pos, 4);
return offset ? this.bb.readUint32(this.bb.__vector(this.bb_pos + offset) + index * 4) : 0;
};
/**
* @returns {number}
*/
NetEncoding.Uint32Array.prototype.dataLength = function() {
var offset = this.bb.__offset(this.bb_pos, 4);
return offset ? this.bb.__vector_len(this.bb_pos + offset) : 0;
};
/**
* @returns {Uint32Array}
*/
NetEncoding.Uint32Array.prototype.dataArray = function() {
var offset = this.bb.__offset(this.bb_pos, 4);
return offset ? new Uint32Array(this.bb.bytes().buffer, this.bb.bytes().byteOffset + this.bb.__vector(this.bb_pos + offset), this.bb.__vector_len(this.bb_pos + offset)) : null;
};
/**
* @param {flatbuffers.Builder} builder
*/
NetEncoding.Uint32Array.startUint32Array = function(builder) {
builder.startObject(1);
};
/**
* @param {flatbuffers.Builder} builder
* @param {flatbuffers.Offset} dataOffset
*/
NetEncoding.Uint32Array.addData = function(builder, dataOffset) {
builder.addFieldOffset(0, dataOffset, 0);
};
/**
* @param {flatbuffers.Builder} builder
* @param {Array.<number>} data
* @returns {flatbuffers.Offset}
*/
NetEncoding.Uint32Array.createDataVector = function(builder, data) {
builder.startVector(4, data.length, 4);
for (var i = data.length - 1; i >= 0; i--) {
builder.addInt32(data[i]);
}
return builder.endVector();
};
/**
* @param {flatbuffers.Builder} builder
* @param {number} numElems
*/
NetEncoding.Uint32Array.startDataVector = function(builder, numElems) {
builder.startVector(4, numElems, 4);
};
/**
* @param {flatbuffers.Builder} builder
* @returns {flatbuffers.Offset}
*/
NetEncoding.Uint32Array.endUint32Array = function(builder) {
var offset = builder.endObject();
return offset;
};
/**
* @constructor
*/
NetEncoding.Int32Array = function() {
/**
* @type {flatbuffers.ByteBuffer}
*/
this.bb = null;
/**
* @type {number}
*/
this.bb_pos = 0;
};
/**
* @param {number} i
* @param {flatbuffers.ByteBuffer} bb
* @returns {NetEncoding.Int32Array}
*/
NetEncoding.Int32Array.prototype.__init = function(i, bb) {
this.bb_pos = i;
this.bb = bb;
return this;
};
/**
* @param {flatbuffers.ByteBuffer} bb
* @param {NetEncoding.Int32Array=} obj
* @returns {NetEncoding.Int32Array}
*/
NetEncoding.Int32Array.getRootAsInt32Array = function(bb, obj) {
return (obj || new NetEncoding.Int32Array).__init(bb.readInt32(bb.position()) + bb.position(), bb);
};
/**
* @param {number} index
* @returns {number}
*/
NetEncoding.Int32Array.prototype.data = function(index) {
var offset = this.bb.__offset(this.bb_pos, 4);
return offset ? this.bb.readInt32(this.bb.__vector(this.bb_pos + offset) + index * 4) : 0;
};
/**
* @returns {number}
*/
NetEncoding.Int32Array.prototype.dataLength = function() {
var offset = this.bb.__offset(this.bb_pos, 4);
return offset ? this.bb.__vector_len(this.bb_pos + offset) : 0;
};
/**
* @returns {Int32Array}
*/
NetEncoding.Int32Array.prototype.dataArray = function() {
var offset = this.bb.__offset(this.bb_pos, 4);
return offset ? new Int32Array(this.bb.bytes().buffer, this.bb.bytes().byteOffset + this.bb.__vector(this.bb_pos + offset), this.bb.__vector_len(this.bb_pos + offset)) : null;
};
/**
* @param {flatbuffers.Builder} builder
*/
NetEncoding.Int32Array.startInt32Array = function(builder) {
builder.startObject(1);
};
/**
* @param {flatbuffers.Builder} builder
* @param {flatbuffers.Offset} dataOffset
*/
NetEncoding.Int32Array.addData = function(builder, dataOffset) {
builder.addFieldOffset(0, dataOffset, 0);
};
/**
* @param {flatbuffers.Builder} builder
* @param {Array.<number>} data
* @returns {flatbuffers.Offset}
*/
NetEncoding.Int32Array.createDataVector = function(builder, data) {
builder.startVector(4, data.length, 4);
for (var i = data.length - 1; i >= 0; i--) {
builder.addInt32(data[i]);
}
return builder.endVector();
};
/**
* @param {flatbuffers.Builder} builder
* @param {number} numElems
*/
NetEncoding.Int32Array.startDataVector = function(builder, numElems) {
builder.startVector(4, numElems, 4);
};
/**
* @param {flatbuffers.Builder} builder
* @returns {flatbuffers.Offset}
*/
NetEncoding.Int32Array.endInt32Array = function(builder) {
var offset = builder.endObject();
return offset;
};
/**
* @constructor
*/
NetEncoding.Float64Array = function() {
/**
* @type {flatbuffers.ByteBuffer}
*/
this.bb = null;
/**
* @type {number}
*/
this.bb_pos = 0;
};
/**
* @param {number} i
* @param {flatbuffers.ByteBuffer} bb
* @returns {NetEncoding.Float64Array}
*/
NetEncoding.Float64Array.prototype.__init = function(i, bb) {
this.bb_pos = i;
this.bb = bb;
return this;
};
/**
* @param {flatbuffers.ByteBuffer} bb
* @param {NetEncoding.Float64Array=} obj
* @returns {NetEncoding.Float64Array}
*/
NetEncoding.Float64Array.getRootAsFloat64Array = function(bb, obj) {
return (obj || new NetEncoding.Float64Array).__init(bb.readInt32(bb.position()) + bb.position(), bb);
};
/**
* @param {number} index
* @returns {number}
*/
NetEncoding.Float64Array.prototype.data = function(index) {
var offset = this.bb.__offset(this.bb_pos, 4);
return offset ? this.bb.readFloat64(this.bb.__vector(this.bb_pos + offset) + index * 8) : 0;
};
/**
* @returns {number}
*/
NetEncoding.Float64Array.prototype.dataLength = function() {
var offset = this.bb.__offset(this.bb_pos, 4);
return offset ? this.bb.__vector_len(this.bb_pos + offset) : 0;
};
/**
* @returns {Float64Array}
*/
NetEncoding.Float64Array.prototype.dataArray = function() {
var offset = this.bb.__offset(this.bb_pos, 4);
return offset ? new Float64Array(this.bb.bytes().buffer, this.bb.bytes().byteOffset + this.bb.__vector(this.bb_pos + offset), this.bb.__vector_len(this.bb_pos + offset)) : null;
};
/**
* @param {flatbuffers.Builder} builder
*/
NetEncoding.Float64Array.startFloat64Array = function(builder) {
builder.startObject(1);
};
/**
* @param {flatbuffers.Builder} builder
* @param {flatbuffers.Offset} dataOffset
*/
NetEncoding.Float64Array.addData = function(builder, dataOffset) {
builder.addFieldOffset(0, dataOffset, 0);
};
/**
* @param {flatbuffers.Builder} builder
* @param {Array.<number>} data
* @returns {flatbuffers.Offset}
*/
NetEncoding.Float64Array.createDataVector = function(builder, data) {
builder.startVector(8, data.length, 8);
for (var i = data.length - 1; i >= 0; i--) {
builder.addFloat64(data[i]);
}
return builder.endVector();
};
/**
* @param {flatbuffers.Builder} builder
* @param {number} numElems
*/
NetEncoding.Float64Array.startDataVector = function(builder, numElems) {
builder.startVector(8, numElems, 8);
};
/**
* @param {flatbuffers.Builder} builder
* @returns {flatbuffers.Offset}
*/
NetEncoding.Float64Array.endFloat64Array = function(builder) {
var offset = builder.endObject();
return offset;
};
/**
* @constructor
*/
NetEncoding.JSONEncodedArray = function() {
/**
* @type {flatbuffers.ByteBuffer}
*/
this.bb = null;
/**
* @type {number}
*/
this.bb_pos = 0;
};
/**
* @param {number} i
* @param {flatbuffers.ByteBuffer} bb
* @returns {NetEncoding.JSONEncodedArray}
*/
NetEncoding.JSONEncodedArray.prototype.__init = function(i, bb) {
this.bb_pos = i;
this.bb = bb;
return this;
};
/**
* @param {flatbuffers.ByteBuffer} bb
* @param {NetEncoding.JSONEncodedArray=} obj
* @returns {NetEncoding.JSONEncodedArray}
*/
NetEncoding.JSONEncodedArray.getRootAsJSONEncodedArray = function(bb, obj) {
return (obj || new NetEncoding.JSONEncodedArray).__init(bb.readInt32(bb.position()) + bb.position(), bb);
};
/**
* @param {number} index
* @returns {number}
*/
NetEncoding.JSONEncodedArray.prototype.data = function(index) {
var offset = this.bb.__offset(this.bb_pos, 4);
return offset ? this.bb.readUint8(this.bb.__vector(this.bb_pos + offset) + index) : 0;
};
/**
* @returns {number}
*/
NetEncoding.JSONEncodedArray.prototype.dataLength = function() {
var offset = this.bb.__offset(this.bb_pos, 4);
return offset ? this.bb.__vector_len(this.bb_pos + offset) : 0;
};
/**
* @returns {Uint8Array}
*/
NetEncoding.JSONEncodedArray.prototype.dataArray = function() {
var offset = this.bb.__offset(this.bb_pos, 4);
return offset ? new Uint8Array(this.bb.bytes().buffer, this.bb.bytes().byteOffset + this.bb.__vector(this.bb_pos + offset), this.bb.__vector_len(this.bb_pos + offset)) : null;
};
/**
* @param {flatbuffers.Builder} builder
*/
NetEncoding.JSONEncodedArray.startJSONEncodedArray = function(builder) {
builder.startObject(1);
};
/**
* @param {flatbuffers.Builder} builder
* @param {flatbuffers.Offset} dataOffset
*/
NetEncoding.JSONEncodedArray.addData = function(builder, dataOffset) {
builder.addFieldOffset(0, dataOffset, 0);
};
/**
* @param {flatbuffers.Builder} builder
* @param {Array.<number>} data
* @returns {flatbuffers.Offset}
*/
NetEncoding.JSONEncodedArray.createDataVector = function(builder, data) {
builder.startVector(1, data.length, 1);
for (var i = data.length - 1; i >= 0; i--) {
builder.addInt8(data[i]);
}
return builder.endVector();
};
/**
* @param {flatbuffers.Builder} builder
* @param {number} numElems
*/
NetEncoding.JSONEncodedArray.startDataVector = function(builder, numElems) {
builder.startVector(1, numElems, 1);
};
/**
* @param {flatbuffers.Builder} builder
* @returns {flatbuffers.Offset}
*/
NetEncoding.JSONEncodedArray.endJSONEncodedArray = function(builder) {
var offset = builder.endObject();
return offset;
};
/**
* @constructor
*/
NetEncoding.Column = function() {
/**
* @type {flatbuffers.ByteBuffer}
*/
this.bb = null;
/**
* @type {number}
*/
this.bb_pos = 0;
};
/**
* @param {number} i
* @param {flatbuffers.ByteBuffer} bb
* @returns {NetEncoding.Column}
*/
NetEncoding.Column.prototype.__init = function(i, bb) {
this.bb_pos = i;
this.bb = bb;
return this;
};
/**
* @param {flatbuffers.ByteBuffer} bb
* @param {NetEncoding.Column=} obj
* @returns {NetEncoding.Column}
*/
NetEncoding.Column.getRootAsColumn = function(bb, obj) {
return (obj || new NetEncoding.Column).__init(bb.readInt32(bb.position()) + bb.position(), bb);
};
/**
* @returns {NetEncoding.TypedArray}
*/
NetEncoding.Column.prototype.uType = function() {
var offset = this.bb.__offset(this.bb_pos, 4);
return offset ? /** @type {NetEncoding.TypedArray} */ (this.bb.readUint8(this.bb_pos + offset)) : NetEncoding.TypedArray.NONE;
};
/**
* @param {flatbuffers.Table} obj
* @returns {?flatbuffers.Table}
*/
NetEncoding.Column.prototype.u = function(obj) {
var offset = this.bb.__offset(this.bb_pos, 6);
return offset ? this.bb.__union(obj, this.bb_pos + offset) : null;
};
/**
* @param {flatbuffers.Builder} builder
*/
NetEncoding.Column.startColumn = function(builder) {
builder.startObject(2);
};
/**
* @param {flatbuffers.Builder} builder
* @param {NetEncoding.TypedArray} uType
*/
NetEncoding.Column.addUType = function(builder, uType) {
builder.addFieldInt8(0, uType, NetEncoding.TypedArray.NONE);
};
/**
* @param {flatbuffers.Builder} builder
* @param {flatbuffers.Offset} uOffset
*/
NetEncoding.Column.addU = function(builder, uOffset) {
builder.addFieldOffset(1, uOffset, 0);
};
/**
* @param {flatbuffers.Builder} builder
* @returns {flatbuffers.Offset}
*/
NetEncoding.Column.endColumn = function(builder) {
var offset = builder.endObject();
return offset;
};
/**
* @constructor
*/
NetEncoding.Matrix = function() {
/**
* @type {flatbuffers.ByteBuffer}
*/
this.bb = null;
/**
* @type {number}
*/
this.bb_pos = 0;
};
/**
* @param {number} i
* @param {flatbuffers.ByteBuffer} bb
* @returns {NetEncoding.Matrix}
*/
NetEncoding.Matrix.prototype.__init = function(i, bb) {
this.bb_pos = i;
this.bb = bb;
return this;
};
/**
* @param {flatbuffers.ByteBuffer} bb
* @param {NetEncoding.Matrix=} obj
* @returns {NetEncoding.Matrix}
*/
NetEncoding.Matrix.getRootAsMatrix = function(bb, obj) {
return (obj || new NetEncoding.Matrix).__init(bb.readInt32(bb.position()) + bb.position(), bb);
};
/**
* @returns {number}
*/
NetEncoding.Matrix.prototype.nRows = function() {
var offset = this.bb.__offset(this.bb_pos, 4);
return offset ? this.bb.readUint32(this.bb_pos + offset) : 0;
};
/**
* @returns {number}
*/
NetEncoding.Matrix.prototype.nCols = function() {
var offset = this.bb.__offset(this.bb_pos, 6);
return offset ? this.bb.readUint32(this.bb_pos + offset) : 0;
};
/**
* @param {number} index
* @param {NetEncoding.Column=} obj
* @returns {NetEncoding.Column}
*/
NetEncoding.Matrix.prototype.columns = function(index, obj) {
var offset = this.bb.__offset(this.bb_pos, 8);
return offset ? (obj || new NetEncoding.Column).__init(this.bb.__indirect(this.bb.__vector(this.bb_pos + offset) + index * 4), this.bb) : null;
};
/**
* @returns {number}
*/
NetEncoding.Matrix.prototype.columnsLength = function() {
var offset = this.bb.__offset(this.bb_pos, 8);
return offset ? this.bb.__vector_len(this.bb_pos + offset) : 0;
};
/**
* @returns {NetEncoding.TypedArray}
*/
NetEncoding.Matrix.prototype.colIndexType = function() {
var offset = this.bb.__offset(this.bb_pos, 10);
return offset ? /** @type {NetEncoding.TypedArray} */ (this.bb.readUint8(this.bb_pos + offset)) : NetEncoding.TypedArray.NONE;
};
/**
* @param {flatbuffers.Table} obj
* @returns {?flatbuffers.Table}
*/
NetEncoding.Matrix.prototype.colIndex = function(obj) {
var offset = this.bb.__offset(this.bb_pos, 12);
return offset ? this.bb.__union(obj, this.bb_pos + offset) : null;
};
/**
* @returns {NetEncoding.TypedArray}
*/
NetEncoding.Matrix.prototype.rowIndexType = function() {
var offset = this.bb.__offset(this.bb_pos, 14);
return offset ? /** @type {NetEncoding.TypedArray} */ (this.bb.readUint8(this.bb_pos + offset)) : NetEncoding.TypedArray.NONE;
};
/**
* @param {flatbuffers.Table} obj
* @returns {?flatbuffers.Table}
*/
NetEncoding.Matrix.prototype.rowIndex = function(obj) {
var offset = this.bb.__offset(this.bb_pos, 16);
return offset ? this.bb.__union(obj, this.bb_pos + offset) : null;
};
/**
* @param {flatbuffers.Builder} builder
*/
NetEncoding.Matrix.startMatrix = function(builder) {
builder.startObject(7);
};
/**
* @param {flatbuffers.Builder} builder
* @param {number} nRows
*/
NetEncoding.Matrix.addNRows = function(builder, nRows) {
builder.addFieldInt32(0, nRows, 0);
};
/**
* @param {flatbuffers.Builder} builder
* @param {number} nCols
*/
NetEncoding.Matrix.addNCols = function(builder, nCols) {
builder.addFieldInt32(1, nCols, 0);
};
/**
* @param {flatbuffers.Builder} builder
* @param {flatbuffers.Offset} columnsOffset
*/
NetEncoding.Matrix.addColumns = function(builder, columnsOffset) {
builder.addFieldOffset(2, columnsOffset, 0);
};
/**
* @param {flatbuffers.Builder} builder
* @param {Array.<flatbuffers.Offset>} data
* @returns {flatbuffers.Offset}
*/
NetEncoding.Matrix.createColumnsVector = function(builder, data) {
builder.startVector(4, data.length, 4);
for (var i = data.length - 1; i >= 0; i--) {
builder.addOffset(data[i]);
}
return builder.endVector();
};
/**
* @param {flatbuffers.Builder} builder
* @param {number} numElems
*/
NetEncoding.Matrix.startColumnsVector = function(builder, numElems) {
builder.startVector(4, numElems, 4);
};
/**
* @param {flatbuffers.Builder} builder
* @param {NetEncoding.TypedArray} colIndexType
*/
NetEncoding.Matrix.addColIndexType = function(builder, colIndexType) {
builder.addFieldInt8(3, colIndexType, NetEncoding.TypedArray.NONE);
};
/**
* @param {flatbuffers.Builder} builder
* @param {flatbuffers.Offset} colIndexOffset
*/
NetEncoding.Matrix.addColIndex = function(builder, colIndexOffset) {
builder.addFieldOffset(4, colIndexOffset, 0);
};
/**
* @param {flatbuffers.Builder} builder
* @param {NetEncoding.TypedArray} rowIndexType
*/
NetEncoding.Matrix.addRowIndexType = function(builder, rowIndexType) {
builder.addFieldInt8(5, rowIndexType, NetEncoding.TypedArray.NONE);
};
/**
* @param {flatbuffers.Builder} builder
* @param {flatbuffers.Offset} rowIndexOffset
*/
NetEncoding.Matrix.addRowIndex = function(builder, rowIndexOffset) {
builder.addFieldOffset(6, rowIndexOffset, 0);
};
/**
* @param {flatbuffers.Builder} builder
* @returns {flatbuffers.Offset}
*/
NetEncoding.Matrix.endMatrix = function(builder) {
var offset = builder.endObject();
return offset;
};
/**
* @param {flatbuffers.Builder} builder
* @param {flatbuffers.Offset} offset
*/
NetEncoding.Matrix.finishMatrixBuffer = function(builder, offset) {
builder.finish(offset);
};
// Exports for ECMAScript6 Modules
export {NetEncoding};

27
client/src/util/stateManager/summarizeAnnotations.js
View File

@@ -1,4 +1,5 @@
import _ from "lodash";
import finiteExtent from "../finiteExtent";
/*
Build and return obs/var summary using any annotation in the schema
@@ -61,16 +62,32 @@ function _summarizeAnnotations(_schema, annotations) {
const continuous = type === "int32" || type === "float32";
if (continuous) {
let min = Number.POSITIVE_INFINITY;
let max = Number.NEGATIVE_INFINITY;
let min;
let max;
let nan = 0;
let pinf = 0;
let ninf = 0;
for (let r = 0; r < annotations.length; r += 1) {
const val = Number(annotations[r][name]);
min = val < min ? val : min;
max = val > max ? val : max;
if (Number.isFinite(val)) {
if (min === undefined) {
min = val;
max = val;
} else {
min = val < min ? val : min;
max = val > max ? val : max;
}
} else if (Number.isNaN(val)) {
nan += 1;
} else if (val > 0) {
pinf += 1;
} else {
ninf += 1;
}
}
return {
categorical: false,
range: { min, max }
range: { min, max, nan, pinf, ninf }
};
}

127
client/src/util/stateManager/universe.js
View File

@@ -1,8 +1,10 @@
// jshint esversion: 6
import _ from "lodash";
import * as kvCache from "./keyvalcache";
import summarizeAnnotations from "./summarizeAnnotations";
import decodeMatrixFBS from "./matrix";
/*
Private helper function - create and return a template Universe
@@ -92,73 +94,40 @@ function finalize(universe) {
return universe;
}
function RESTv02AnnotationsResponseToInternal(response) {
function RESTv02AnotationsFBSResponseToInternal(arrayBuffer) {
/*
Source per the spec:
{
names: [
'tissue_type', 'sex', 'num_reads', 'clusters'
],
data: [
[ 0, 'lung', 'F', 39844, 99 ],
[ 1, 'heart', 'M', 83, 1 ],
[ 49, 'spleen', null, 2, "unknown cluster" ],
// [ obsOrVarIndex, value, value, value, value ],
// ...
]
}
Convert a Matrix FBS to our internal format -- row-major array of
observations/cells, stored as an object. Each obs has a key for each
annotation, plus __index__ containing its obsIndex.
Internal (target) format:
Example:
[
{ __index__: 0, tissue_type: "lung", sex: "F", ... },
...
]
XXX TODO: we could make use of the columns in building crossfilter
dimensions (they have to be recreated). Future optimization.
*/
const { names, data } = response;
const keys = ["__index__", ...names];
return _(data)
.map(obs => _.zipObject(keys, obs))
.value();
const fbs = decodeMatrixFBS(arrayBuffer);
const keys = fbs.colIdx;
const result = Array(fbs.nRows);
for (let row = 0; row < fbs.nRows; row += 1) {
const rec = { __index__: row };
for (let col = 0; col < fbs.nCols; col += 1) {
rec[keys[col]] = fbs.columns[col][row];
}
result[row] = rec;
}
return result;
}
function RESTv02LayoutResponseToInternal(response) {
/*
Source per the spec:
{
layout: {
ndims: 2,
coordinates: [
[ 0, 0.284483, 0.983744 ],
[ 1, 0.038844, 0.739444 ],
// [ obsOrVarIndex, X_coord, Y_coord ],
// ...
]
}
}
Target (internal) format:
{
X: Float32Array(numObs),
Y: Float32Array(numObs)
}
In the same order as obsAnnotations
*/
const { ndims, coordinates } = response.layout;
if (ndims !== 2) {
throw new Error("Unsupported layout dimensionality");
}
const layout = {
X: new Float32Array(coordinates.length),
Y: new Float32Array(coordinates.length)
function RESTv02LayoutFBSResponseToInternal(arrayBuffer) {
const fbs = decodeMatrixFBS(arrayBuffer, true);
return {
X: fbs.columns[0],
Y: fbs.columns[1]
};
for (let i = 0; i < coordinates.length; i += 1) {
const [idx, x, y] = coordinates[i];
layout.X[idx] = x;
layout.Y[idx] = y;
}
return layout;
}
function reconcileSchemaCategoriesWithSummary(universe) {
@@ -192,7 +161,7 @@ export function createUniverseFromRestV02Response(
schemaResponse,
annotationsObsResponse,
annotationsVarResponse,
layoutObsResponse
layoutFBSResponse
) {
/*
build & return universe from a REST 0.2 /config, /schema and /annotations/obs response
@@ -209,15 +178,15 @@ export function createUniverseFromRestV02Response(
universe.nVar = schema.dataframe.nVar;
/* annotations */
universe.obsAnnotations = RESTv02AnnotationsResponseToInternal(
universe.obsAnnotations = RESTv02AnotationsFBSResponseToInternal(
annotationsObsResponse
);
universe.varAnnotations = RESTv02AnnotationsResponseToInternal(
universe.varAnnotations = RESTv02AnotationsFBSResponseToInternal(
annotationsVarResponse
);
/* layout */
universe.obsLayout = RESTv02LayoutResponseToInternal(layoutObsResponse);
universe.obsLayout = RESTv02LayoutFBSResponseToInternal(layoutFBSResponse);
universe.summary = summarizeAnnotations(
universe.schema,
@@ -229,32 +198,24 @@ export function createUniverseFromRestV02Response(
return finalize(universe);
}
export function convertExpressionRESTv02ToObject(universe, response) {
export function convertDataFBStoObject(universe, arrayBuffer) {
/*
/data/obs response looks like:
{
var: [ varIndices fetched ],
obs: [
[ obsIndex, evalue, ... ],
...
]
}
/data/var returns a flatbuffer (FBS) as described by cellxgene/fbs/matrix.fbs
convert expression toa simple Float32Array, and return
{ geneName: array, geneName: array, ... }
NOTE: geneName, not varIndex
This routine converts the binary wire encoding into a JS object:
{
gene: Float32Array,
...
}
*/
const vars = response.var;
const { obs } = response;
const fbs = decodeMatrixFBS(arrayBuffer);
const { colIdx, columns } = fbs;
const result = {};
// XXX TODO: could this use _.unzip and have less code?
for (let varIdx = 0; varIdx < vars.length; varIdx += 1) {
const gene = universe.varAnnotations[vars[varIdx]].name;
const data = new Float32Array(universe.nObs);
for (let obsIdx = 0; obsIdx < obs.length; obsIdx += 1) {
data[obsIdx] = obs[obsIdx][varIdx + 1];
}
result[gene] = data;
for (let c = 0; c < colIdx.length; c += 1) {
const gene = universe.varAnnotations[colIdx[c]].name;
result[gene] = columns[c];
}
return result;
}

2
dev_docs/REST_API.md
View File

@@ -1,5 +1,7 @@
# cellxgene REST API 0.2 specification
_Note:_ this document lacks any information about the binary encoding utilized by various routes. This will be added at a later date.
Items marked as (_future_) are intended for future implementation, and are included in the design to round out the concept, and highlight what we would do when/if we needed more functionality. The (_future_) items are not currently used by the cellxgene web application, and may be omitted from any backend - see [Current Front-End Dependencies](#current-front-end-dependencies) for more details.
_Caveat emptor, partial spec_: this is a sketch for a spec, not a full spec, and some shortcuts have been taken in the authorship. Best practices for a REST API are assumed but not documented here, such as API versioning, reasonable choices for HTTP response codes, etc. In addition, for clarity the JSON examples will not always have all required quoting (eg, on keys) - the actual implementation should use legal JSON/CSV.

73
fbs/matrix.fbs Normal file
View File

@@ -0,0 +1,73 @@
/*
Flatbuffers schema for use in cellxgene wire-format.
Schema defines a general purpose, polymorphic, 2D matrix. Data is
organized in a columnar layout. Each column is homomorphic, and
several column types are supported:
- IEEE 32 and 64 bit floats
- signed and unsigned 32 bit integers
- JSON/UTF8 encoded array (for other types)
https://github.com/google/flatbuffers
http://google.github.io/flatbuffers/
NOTE: IF YOU MODIFY THIS FILE, YOU MUST RECOMPILE AND COMMIT
RESULTING FILES TO THE REPO:
* server/app/util/fbs/NetEncoding/*
* client/src/util/stateManager/matrix_generated.js
*/
namespace NetEncoding;
table Float32Array {
data: [float32];
}
table Uint32Array {
data: [uint32];
}
table Int32Array {
data: [int32];
}
table Float64Array {
data: [float64];
}
table JSONEncodedArray {
// contains a UTF-8/JSON encoded array. Used to store other
// types (or polymorphic arrays)
data: [uint8];
}
union TypedArray {
Float32Array,
Int32Array,
Uint32Array,
Float64Array,
JSONEncodedArray
}
// Extra level of indirection required because vector of union not yet supported
table Column {
u: TypedArray;
}
// 2D matrix stored in columnar layout
//
table Matrix {
n_rows: uint32; // all columns have this length
n_cols: uint32; // same as columns.length
columns: [Column]; // length n_cols
// optional row and column index, with same length as corresponding dimension.
// If null, defaults to numeric index, ie, [0, n_rows) or [0, n_cols)
col_index: TypedArray;
row_index: TypedArray;
}
root_type Matrix;

14
server/app/driver/driver.py
View File

@@ -66,6 +66,11 @@ class CXGDriver(metaclass=ABCMeta):
"""
pass
@abstractmethod
def annotation_to_fbs_matrix(self, axis, field=None):
""" Same as annotation(), except returns a flatbuffer, and does not support filtering. """
pass
@abstractmethod
def data_frame(self, filter, axis):
"""
@@ -79,6 +84,10 @@ class CXGDriver(metaclass=ABCMeta):
"""
pass
@abstractmethod
def data_frame_to_fbs_matrix(self, filter, axis):
pass
@abstractmethod
def diffexp_topN(self, obsFilter1, obsFilter2, top_n=None, interactive_limit=None):
"""
@@ -104,3 +113,8 @@ class CXGDriver(metaclass=ABCMeta):
:return: [cellid, x, y, ...]
"""
pass
@abstractmethod
def layout_to_fbs_matrix(self, filter):
""" same as layout, except returns a flatbuffer """
pass

108
server/app/rest_api/rest.py
View File

@@ -9,7 +9,6 @@ from werkzeug.datastructures import ImmutableMultiDict
from server.app.util.constants import (
Axis,
DiffExpMode,
JSON_MIMETYPE,
JSON_NaN_to_num_warning_msg,
)
from server.app.util.filter import parse_filter, QueryStringError
@@ -194,11 +193,21 @@ class AnnotationsObsAPI(Resource):
)
def get(self):
fields = request.args.getlist("annotation-name", None)
preferred_mimetype = request.accept_mimetypes.best_match(
["application/json", "application/octet-stream"],
"application/json"
)
try:
annotation_response = current_app.data.annotation({}, "obs", fields)
return make_response(
annotation_response, HTTPStatus.OK, {"Content-Type": JSON_MIMETYPE}
)
if preferred_mimetype == "application/json":
return make_response(
current_app.data.annotation({}, "obs", fields), HTTPStatus.OK, {"Content-Type": "application/json"}
)
elif preferred_mimetype == "application/octet-stream":
return make_response(current_app.data.annotation_to_fbs_matrix("obs", fields),
HTTPStatus.OK,
{"Content-Type": "application/octet-stream"})
else:
return make_response(f"Unsupported MIME type '{request.accept_mimetypes}'", HTTPStatus.NOT_ACCEPTABLE)
except KeyError:
return make_response(f"Error bad key in {fields}", HTTPStatus.BAD_REQUEST)
except JSONEncodingValueError as e:
@@ -254,7 +263,7 @@ class AnnotationsObsAPI(Resource):
request.get_json()["filter"], "obs", fields
)
return make_response(
annotation_response, HTTPStatus.OK, {"Content-Type": JSON_MIMETYPE}
annotation_response, HTTPStatus.OK, {"Content-Type": "application/json"}
)
except KeyError:
return make_response(f"Error bad key in {fields}", HTTPStatus.BAD_REQUEST)
@@ -304,11 +313,21 @@ class AnnotationsVarAPI(Resource):
)
def get(self):
fields = request.args.getlist("annotation-name", None)
preferred_mimetype = request.accept_mimetypes.best_match(
["application/json", "application/octet-stream"],
"application/json"
)
try:
annotation_response = current_app.data.annotation({}, "var", fields)
return make_response(
annotation_response, HTTPStatus.OK, {"Content-Type": JSON_MIMETYPE}
)
if preferred_mimetype == "application/json":
return make_response(current_app.data.annotation({}, "var", fields),
HTTPStatus.OK,
{"Content-Type": "application/json"})
elif preferred_mimetype == "application/octet-stream":
return make_response(current_app.data.annotation_to_fbs_matrix("var", fields),
HTTPStatus.OK,
{"Content-Type": "application/octet-stream"})
else:
return make_response(f"Unsupported MIME type '{request.accept_mimetypes}'", HTTPStatus.NOT_ACCEPTABLE)
except KeyError:
return make_response(f"Error bad key in {fields}", HTTPStatus.BAD_REQUEST)
except JSONEncodingValueError as e:
@@ -364,7 +383,7 @@ class AnnotationsVarAPI(Resource):
request.get_json()["filter"], "var", fields
)
return make_response(
annotation_response, HTTPStatus.OK, {"Content-Type": JSON_MIMETYPE}
annotation_response, HTTPStatus.OK, {"Content-Type": "application/json"}
)
except KeyError:
return make_response(f"Error bad key in {fields}", HTTPStatus.BAD_REQUEST)
@@ -437,7 +456,7 @@ class DataObsAPI(Resource):
return make_response(
current_app.data.data_frame(filter_, axis=Axis.OBS),
HTTPStatus.OK,
{"Content-Type": JSON_MIMETYPE},
{"Content-Type": "application/json"},
)
except FilterError as e:
return make_response(e.message, HTTPStatus.BAD_REQUEST)
@@ -495,7 +514,7 @@ class DataObsAPI(Resource):
)
),
HTTPStatus.OK,
{"Content-Type": JSON_MIMETYPE},
{"Content-Type": "application/json"},
)
except FilterError as e:
return make_response(e.message, HTTPStatus.BAD_REQUEST)
@@ -564,7 +583,7 @@ class DataVarAPI(Resource):
return make_response(
current_app.data.data_frame(filter_, axis=Axis.VAR),
HTTPStatus.OK,
{"Content-Type": JSON_MIMETYPE},
{"Content-Type": "application/json"},
)
except FilterError as e:
return make_response(e.message, HTTPStatus.BAD_REQUEST)
@@ -603,28 +622,30 @@ class DataVarAPI(Resource):
}
)
def put(self):
if not request.accept_mimetypes.best_match(["application/json", "text/csv"]):
return make_response(
f"Unsupported MIME type '{request.accept_mimetypes}'",
HTTPStatus.NOT_ACCEPTABLE,
)
# TODO support CSV
preferred_mimetype = request.accept_mimetypes.best_match(
["application/json", "application/octet-stream"],
"application/json"
)
try:
get_mime_type(
acceptable_types=["application/json"], header=request.accept_mimetypes
)
except MimeTypeError as e:
return make_response(e.message, HTTPStatus.NOT_ACCEPTABLE)
try:
return make_response(
(
current_app.data.data_frame(
if preferred_mimetype == "application/json":
return make_response(
(
current_app.data.data_frame(
request.get_json()["filter"], axis=Axis.VAR
)
),
HTTPStatus.OK,
{"Content-Type": "application/json"},
)
elif preferred_mimetype == "application/octet-stream":
return make_response(
current_app.data.data_frame_to_fbs_matrix(
request.get_json()["filter"], axis=Axis.VAR
)
),
HTTPStatus.OK,
{"Content-Type": JSON_MIMETYPE},
)
),
HTTPStatus.OK,
{"Content-Type": "application/octet-stream"})
else:
return make_response(f"Unsupported MIME type '{request.accept_mimetypes}'", HTTPStatus.NOT_ACCEPTABLE)
except FilterError as e:
return make_response(e.message, HTTPStatus.BAD_REQUEST)
except JSONEncodingValueError as e:
@@ -747,7 +768,7 @@ class DiffExpObsAPI(Resource):
current_app.data.features["diffexp"]["interactiveLimit"],
)
return make_response(
diffexp, HTTPStatus.OK, {"Content-Type": JSON_MIMETYPE}
diffexp, HTTPStatus.OK, {"Content-Type": "application/json"}
)
except (ValueError, FilterError) as e:
return make_response(e.message, HTTPStatus.BAD_REQUEST)
@@ -787,13 +808,22 @@ class LayoutObsAPI(Resource):
}
)
def get(self):
content_type = JSON_MIMETYPE
preferred_mimetype = request.accept_mimetypes.best_match(
["application/json", "application/octet-stream"],
"application/json"
)
try:
layout = current_app.data.layout({})
if preferred_mimetype == "application/json":
return make_response(current_app.data.layout({}), HTTPStatus.OK, {"Content-Type": "application/json"})
elif preferred_mimetype == "application/octet-stream":
return make_response(current_app.data.layout_to_fbs_matrix(),
HTTPStatus.OK,
{"Content-Type": "application/octet-stream"})
else:
return make_response(f"Unsupported MIME type '{request.accept_mimetypes}'", HTTPStatus.NOT_ACCEPTABLE)
except PrepareError as e:
return make_response(e.message, HTTPStatus.INTERNAL_SERVER_ERROR)
try:
return make_response(layout, HTTPStatus.OK, {"Content-Type": content_type})
except JSONEncodingValueError as e:
# JSON encoding failure, usually due to bad data
warnings.warn(JSON_NaN_to_num_warning_msg)

35
server/app/scanpy_engine/diffexp.py
View File

@@ -9,18 +9,31 @@ def _mean_var_n(X):
than naive methods (and same method used by numpy.var())
https://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Two-pass
"""
n = X.shape[0]
if sparse.issparse(X):
mean = X.mean(axis=0).A1
dfm = X - mean
sumsq = np.sum(np.multiply(dfm, dfm), axis=0).A1
v = sumsq / (n - 1)
else:
mean = X.mean(axis=0)
dfm = X - mean
sumsq = np.sum(np.multiply(dfm, dfm), axis=0)
v = sumsq / (n - 1)
# fp_err_occurred is a flag indicating that a floating point error
# occured somewhere in our compute. Used to trigger non-finite
# number handling.
fp_err_occurred = False
def fp_err_set(err, flag):
nonlocal fp_err_occurred
fp_err_occurred = True
with np.errstate(divide="call", invalid="call", call=fp_err_set):
n = X.shape[0]
if sparse.issparse(X):
mean = X.mean(axis=0).A1
dfm = X - mean
sumsq = np.sum(np.multiply(dfm, dfm), axis=0).A1
v = sumsq / (n - 1)
else:
mean = X.mean(axis=0)
dfm = X - mean
sumsq = np.sum(np.multiply(dfm, dfm), axis=0)
v = sumsq / (n - 1)
if fp_err_occurred:
mean[np.isfinite(mean) == False] = 0 # noqa: E712
v[np.isfinite(v) == False] = 0 # noqa: E712
return mean, v, n

51
server/app/scanpy_engine/scanpy_engine.py
View File

@@ -17,6 +17,7 @@ from server.app.util.errors import (
)
from server.app.util.utils import jsonify_scanpy
from server.app.scanpy_engine.diffexp import diffexp_ttest
from server.app.util.fbs.matrix import encode_matrix_fbs
"""
Sort order for methods
@@ -411,6 +412,15 @@ class ScanpyEngine(CXGDriver):
except ValueError:
raise JSONEncodingValueError("Error encoding annotations to JSON")
def annotation_to_fbs_matrix(self, axis, fields=None):
if axis == Axis.OBS:
df = self.data.obs
else:
df = self.data.var
if fields is not None and len(fields) > 0:
df = df[fields]
return encode_matrix_fbs(df, col_idx=df.columns)
def data_frame(self, filter, axis):
"""
Retrieves data for each variable for observations in data frame
@@ -449,6 +459,30 @@ class ScanpyEngine(CXGDriver):
except ValueError:
raise JSONEncodingValueError("Error encoding dataframe to JSON")
def data_frame_to_fbs_matrix(self, filter, axis):
"""
Retrieves data 'X' and returns in a flatbuffer Matrix.
:param filter: filter: dictionary with filter params
:param axis: string obs or var
:return: flatbuffer Matrix
Caveats:
* currently only supports access on VAR axis
* currently only supports filtering on VAR axis
"""
if axis != Axis.VAR:
raise ValueError("Only VAR dimension access is supported")
try:
obs_selector, var_selector = self._filter_to_mask(filter, use_slices=False)
except (KeyError, IndexError) as e:
raise FilterError(f"Error parsing filter: {e}") from e
if obs_selector is not None:
raise FilterError("filtering on obs unsupported")
# Currently only handles VAR dimension
X = self.data._X[:, var_selector]
return encode_matrix_fbs(X, col_idx=np.nonzero(var_selector)[0], row_idx=None)
def diffexp_topN(self, obsFilterA, obsFilterB, top_n=None, interactive_limit=None):
if Axis.VAR in obsFilterA or Axis.VAR in obsFilterB:
raise FilterError("Observation filters may not contain vaiable conditions")
@@ -516,3 +550,20 @@ class ScanpyEngine(CXGDriver):
)
except ValueError:
raise JSONEncodingValueError("Error encoding layout to JSON")
def layout_to_fbs_matrix(self):
"""
Return the default 2-D layout for cells as a FBS Matrix.
Caveats:
* does not support filtering
* only returns Matrix in columnar layout
"""
try:
df_layout = self.data.obsm[f"X_{self.layout_method}"]
except ValueError as e:
raise PrepareError(
f"Layout has not been calculated using {self.layout_method}, "
f"please prepare your datafile and relaunch cellxgene") from e
normalized_layout = (df_layout - df_layout.min()) / (df_layout.max() - df_layout.min())
return encode_matrix_fbs(normalized_layout.astype(dtype=np.float32), col_idx=None, row_idx=None)

3
server/app/util/constants.py
View File

@@ -3,9 +3,6 @@ from enum import Enum
DEFAULT_TOP_N = 10
# response mimetypes
JSON_MIMETYPE = "application/json"
class AugmentedEnum(Enum):
def __hash__(self):

41
server/app/util/fbs/NetEncoding/Column.py Normal file
View File

@@ -0,0 +1,41 @@
# automatically generated by the FlatBuffers compiler, do not modify
# namespace: NetEncoding
import flatbuffers
class Column(object):
__slots__ = ['_tab']
@classmethod
def GetRootAsColumn(cls, buf, offset):
n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, offset)
x = Column()
x.Init(buf, n + offset)
return x
# Column
def Init(self, buf, pos):
self._tab = flatbuffers.table.Table(buf, pos)
# Column
def UType(self):
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
if o != 0:
return self._tab.Get(flatbuffers.number_types.Uint8Flags, o + self._tab.Pos)
return 0
# Column
def U(self):
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(6))
if o != 0:
from flatbuffers.table import Table
obj = Table(bytearray(), 0)
self._tab.Union(obj, o)
return obj
return None
def ColumnStart(builder): builder.StartObject(2)
def ColumnAddUType(builder, uType): builder.PrependUint8Slot(0, uType, 0)
def ColumnAddU(builder, u): builder.PrependUOffsetTRelativeSlot(1, flatbuffers.number_types.UOffsetTFlags.py_type(u), 0)
def ColumnEnd(builder): return builder.EndObject()

46
server/app/util/fbs/NetEncoding/Float32Array.py Normal file
View File

@@ -0,0 +1,46 @@
# automatically generated by the FlatBuffers compiler, do not modify
# namespace: NetEncoding
import flatbuffers
class Float32Array(object):
__slots__ = ['_tab']
@classmethod
def GetRootAsFloat32Array(cls, buf, offset):
n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, offset)
x = Float32Array()
x.Init(buf, n + offset)
return x
# Float32Array
def Init(self, buf, pos):
self._tab = flatbuffers.table.Table(buf, pos)
# Float32Array
def Data(self, j):
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
if o != 0:
a = self._tab.Vector(o)
return self._tab.Get(flatbuffers.number_types.Float32Flags, a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 4))
return 0
# Float32Array
def DataAsNumpy(self):
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
if o != 0:
return self._tab.GetVectorAsNumpy(flatbuffers.number_types.Float32Flags, o)
return 0
# Float32Array
def DataLength(self):
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
if o != 0:
return self._tab.VectorLen(o)
return 0
def Float32ArrayStart(builder): builder.StartObject(1)
def Float32ArrayAddData(builder, data): builder.PrependUOffsetTRelativeSlot(0, flatbuffers.number_types.UOffsetTFlags.py_type(data), 0)
def Float32ArrayStartDataVector(builder, numElems): return builder.StartVector(4, numElems, 4)
def Float32ArrayEnd(builder): return builder.EndObject()

46
server/app/util/fbs/NetEncoding/Float64Array.py Normal file
View File

@@ -0,0 +1,46 @@
# automatically generated by the FlatBuffers compiler, do not modify
# namespace: NetEncoding
import flatbuffers
class Float64Array(object):
__slots__ = ['_tab']
@classmethod
def GetRootAsFloat64Array(cls, buf, offset):
n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, offset)
x = Float64Array()
x.Init(buf, n + offset)
return x
# Float64Array
def Init(self, buf, pos):
self._tab = flatbuffers.table.Table(buf, pos)
# Float64Array
def Data(self, j):
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
if o != 0:
a = self._tab.Vector(o)
return self._tab.Get(flatbuffers.number_types.Float64Flags, a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 8))
return 0
# Float64Array
def DataAsNumpy(self):
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
if o != 0:
return self._tab.GetVectorAsNumpy(flatbuffers.number_types.Float64Flags, o)
return 0
# Float64Array
def DataLength(self):
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
if o != 0:
return self._tab.VectorLen(o)
return 0
def Float64ArrayStart(builder): builder.StartObject(1)
def Float64ArrayAddData(builder, data): builder.PrependUOffsetTRelativeSlot(0, flatbuffers.number_types.UOffsetTFlags.py_type(data), 0)
def Float64ArrayStartDataVector(builder, numElems): return builder.StartVector(8, numElems, 8)
def Float64ArrayEnd(builder): return builder.EndObject()

46
server/app/util/fbs/NetEncoding/Int32Array.py Normal file
View File

@@ -0,0 +1,46 @@
# automatically generated by the FlatBuffers compiler, do not modify
# namespace: NetEncoding
import flatbuffers
class Int32Array(object):
__slots__ = ['_tab']
@classmethod
def GetRootAsInt32Array(cls, buf, offset):
n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, offset)
x = Int32Array()
x.Init(buf, n + offset)
return x
# Int32Array
def Init(self, buf, pos):
self._tab = flatbuffers.table.Table(buf, pos)
# Int32Array
def Data(self, j):
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
if o != 0:
a = self._tab.Vector(o)
return self._tab.Get(flatbuffers.number_types.Int32Flags, a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 4))
return 0
# Int32Array
def DataAsNumpy(self):
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
if o != 0:
return self._tab.GetVectorAsNumpy(flatbuffers.number_types.Int32Flags, o)
return 0
# Int32Array
def DataLength(self):
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
if o != 0:
return self._tab.VectorLen(o)
return 0
def Int32ArrayStart(builder): builder.StartObject(1)
def Int32ArrayAddData(builder, data): builder.PrependUOffsetTRelativeSlot(0, flatbuffers.number_types.UOffsetTFlags.py_type(data), 0)
def Int32ArrayStartDataVector(builder, numElems): return builder.StartVector(4, numElems, 4)
def Int32ArrayEnd(builder): return builder.EndObject()

46
server/app/util/fbs/NetEncoding/JSONEncodedArray.py Normal file
View File

@@ -0,0 +1,46 @@
# automatically generated by the FlatBuffers compiler, do not modify
# namespace: NetEncoding
import flatbuffers
class JSONEncodedArray(object):
__slots__ = ['_tab']
@classmethod
def GetRootAsJSONEncodedArray(cls, buf, offset):
n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, offset)
x = JSONEncodedArray()
x.Init(buf, n + offset)
return x
# JSONEncodedArray
def Init(self, buf, pos):
self._tab = flatbuffers.table.Table(buf, pos)
# JSONEncodedArray
def Data(self, j):
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
if o != 0:
a = self._tab.Vector(o)
return self._tab.Get(flatbuffers.number_types.Uint8Flags, a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 1))
return 0
# JSONEncodedArray
def DataAsNumpy(self):
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
if o != 0:
return self._tab.GetVectorAsNumpy(flatbuffers.number_types.Uint8Flags, o)
return 0
# JSONEncodedArray
def DataLength(self):
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
if o != 0:
return self._tab.VectorLen(o)
return 0
def JSONEncodedArrayStart(builder): builder.StartObject(1)
def JSONEncodedArrayAddData(builder, data): builder.PrependUOffsetTRelativeSlot(0, flatbuffers.number_types.UOffsetTFlags.py_type(data), 0)
def JSONEncodedArrayStartDataVector(builder, numElems): return builder.StartVector(1, numElems, 1)
def JSONEncodedArrayEnd(builder): return builder.EndObject()

98
server/app/util/fbs/NetEncoding/Matrix.py Normal file
View File

@@ -0,0 +1,98 @@
# automatically generated by the FlatBuffers compiler, do not modify
# namespace: NetEncoding
import flatbuffers
class Matrix(object):
__slots__ = ['_tab']
@classmethod
def GetRootAsMatrix(cls, buf, offset):
n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, offset)
x = Matrix()
x.Init(buf, n + offset)
return x
# Matrix
def Init(self, buf, pos):
self._tab = flatbuffers.table.Table(buf, pos)
# Matrix
def NRows(self):
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
if o != 0:
return self._tab.Get(flatbuffers.number_types.Uint32Flags, o + self._tab.Pos)
return 0
# Matrix
def NCols(self):
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(6))
if o != 0:
return self._tab.Get(flatbuffers.number_types.Uint32Flags, o + self._tab.Pos)
return 0
# Matrix
def Columns(self, j):
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(8))
if o != 0:
x = self._tab.Vector(o)
x += flatbuffers.number_types.UOffsetTFlags.py_type(j) * 4
x = self._tab.Indirect(x)
from .Column import Column
obj = Column()
obj.Init(self._tab.Bytes, x)
return obj
return None
# Matrix
def ColumnsLength(self):
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(8))
if o != 0:
return self._tab.VectorLen(o)
return 0
# Matrix
def ColIndexType(self):
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(10))
if o != 0:
return self._tab.Get(flatbuffers.number_types.Uint8Flags, o + self._tab.Pos)
return 0
# Matrix
def ColIndex(self):
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(12))
if o != 0:
from flatbuffers.table import Table
obj = Table(bytearray(), 0)
self._tab.Union(obj, o)
return obj
return None
# Matrix
def RowIndexType(self):
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(14))
if o != 0:
return self._tab.Get(flatbuffers.number_types.Uint8Flags, o + self._tab.Pos)
return 0
# Matrix
def RowIndex(self):
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(16))
if o != 0:
from flatbuffers.table import Table
obj = Table(bytearray(), 0)
self._tab.Union(obj, o)
return obj
return None
def MatrixStart(builder): builder.StartObject(7)
def MatrixAddNRows(builder, nRows): builder.PrependUint32Slot(0, nRows, 0)
def MatrixAddNCols(builder, nCols): builder.PrependUint32Slot(1, nCols, 0)
def MatrixAddColumns(builder, columns): builder.PrependUOffsetTRelativeSlot(2, flatbuffers.number_types.UOffsetTFlags.py_type(columns), 0)
def MatrixStartColumnsVector(builder, numElems): return builder.StartVector(4, numElems, 4)
def MatrixAddColIndexType(builder, colIndexType): builder.PrependUint8Slot(3, colIndexType, 0)
def MatrixAddColIndex(builder, colIndex): builder.PrependUOffsetTRelativeSlot(4, flatbuffers.number_types.UOffsetTFlags.py_type(colIndex), 0)
def MatrixAddRowIndexType(builder, rowIndexType): builder.PrependUint8Slot(5, rowIndexType, 0)
def MatrixAddRowIndex(builder, rowIndex): builder.PrependUOffsetTRelativeSlot(6, flatbuffers.number_types.UOffsetTFlags.py_type(rowIndex), 0)
def MatrixEnd(builder): return builder.EndObject()

12
server/app/util/fbs/NetEncoding/TypedArray.py Normal file
View File

@@ -0,0 +1,12 @@
# automatically generated by the FlatBuffers compiler, do not modify
# namespace: NetEncoding
class TypedArray(object):
NONE = 0
Float32Array = 1
Int32Array = 2
Uint32Array = 3
Float64Array = 4
JSONEncodedArray = 5

46
server/app/util/fbs/NetEncoding/Uint32Array.py Normal file
View File

@@ -0,0 +1,46 @@
# automatically generated by the FlatBuffers compiler, do not modify
# namespace: NetEncoding
import flatbuffers
class Uint32Array(object):
__slots__ = ['_tab']
@classmethod
def GetRootAsUint32Array(cls, buf, offset):
n = flatbuffers.encode.Get(flatbuffers.packer.uoffset, buf, offset)
x = Uint32Array()
x.Init(buf, n + offset)
return x
# Uint32Array
def Init(self, buf, pos):
self._tab = flatbuffers.table.Table(buf, pos)
# Uint32Array
def Data(self, j):
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
if o != 0:
a = self._tab.Vector(o)
return self._tab.Get(flatbuffers.number_types.Uint32Flags, a + flatbuffers.number_types.UOffsetTFlags.py_type(j * 4))
return 0
# Uint32Array
def DataAsNumpy(self):
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
if o != 0:
return self._tab.GetVectorAsNumpy(flatbuffers.number_types.Uint32Flags, o)
return 0
# Uint32Array
def DataLength(self):
o = flatbuffers.number_types.UOffsetTFlags.py_type(self._tab.Offset(4))
if o != 0:
return self._tab.VectorLen(o)
return 0
def Uint32ArrayStart(builder): builder.StartObject(1)
def Uint32ArrayAddData(builder, data): builder.PrependUOffsetTRelativeSlot(0, flatbuffers.number_types.UOffsetTFlags.py_type(data), 0)
def Uint32ArrayStartDataVector(builder, numElems): return builder.StartVector(4, numElems, 4)
def Uint32ArrayEnd(builder): return builder.EndObject()

0
server/app/util/fbs/NetEncoding/__init__.py Normal file
View File

207
server/app/util/fbs/matrix.py Normal file
View File

@@ -0,0 +1,207 @@
import flatbuffers
import numpy as np
from scipy import sparse
import pandas as pd
import server.app.util.fbs.NetEncoding.Column as Column
import server.app.util.fbs.NetEncoding.TypedArray as TypedArray
import server.app.util.fbs.NetEncoding.Matrix as Matrix
# Placeholder until recent enhancements to flatbuffers Python
# runtime are released, at which point we can use the default
# version. This code is a port of the head. See:
#
# https://github.com/google/flatbuffers/pull/4829
#
def CreateNumpyVector(builder, x):
"""CreateNumpyVector writes a numpy array into the buffer."""
if not isinstance(x, np.ndarray):
raise TypeError("non-numpy-ndarray passed to CreateNumpyVector")
if x.dtype.kind not in ['b', 'i', 'u', 'f']:
raise TypeError("numpy-ndarray holds elements of unsupported datatype")
if x.ndim > 1:
raise TypeError("multidimensional-ndarray passed to CreateNumpyVector")
builder.StartVector(x.itemsize, x.size, x.dtype.alignment)
# Ensure little endian byte ordering
if x.dtype.str[0] == "<":
x_little_endian = x
else:
x_little_endian = x.byteswap(inplace=False)
# Calculate total length
len = int(x_little_endian.itemsize * x_little_endian.size)
builder.head = int(builder.Head() - len)
# tobytes ensures c_contiguous ordering
builder.Bytes[builder.Head():builder.Head() + len] = x_little_endian.tobytes(order='C')
return builder.EndVector(x.size)
# Serialization helper
def serialize_column(builder, typed_arr):
""" Serialize NetEncoding.Column """
(u_type, u_value) = typed_arr
Column.ColumnStart(builder)
Column.ColumnAddUType(builder, u_type)
Column.ColumnAddU(builder, u_value)
return Column.ColumnEnd(builder)
# Serialization helper
def serialize_matrix(builder, n_rows, n_cols, columns, col_idx):
""" Serialize NetEncoding.Matrix """
Matrix.MatrixStart(builder)
Matrix.MatrixAddNRows(builder, n_rows)
Matrix.MatrixAddNCols(builder, n_cols)
Matrix.MatrixAddColumns(builder, columns)
if col_idx is not None:
(u_type, u_val) = col_idx
Matrix.MatrixAddColIndexType(builder, u_type)
Matrix.MatrixAddColIndex(builder, u_val)
return Matrix.MatrixEnd(builder)
# Serialization helper
def serialize_typed_array(builder, source_array, encoding_info):
"""
Serialize any of the various typed arrays, eg, Float32Array. Specific
means of serialization and type conversion are provided by type_info.
"""
arr = source_array
(array_type, as_type) = encoding_info(source_array)
if isinstance(arr, pd.Index):
arr = arr.to_series()
# convert to a simple ndarray
if as_type == 'json':
as_json = arr.to_json(orient='records')
arr = np.array(bytearray(as_json, 'utf-8'))
else:
if sparse.issparse(arr):
arr = arr.toarray()
elif isinstance(arr, pd.Series):
arr = arr.get_values()
if arr.dtype != as_type:
arr = arr.astype(as_type)
# serialize the ndarray into a vector
if arr.ndim == 2 and arr.shape[0] == 1:
arr = arr[0]
vec = CreateNumpyVector(builder, arr)
# serialize the typed array table
builder.StartObject(1)
builder.PrependUOffsetTRelativeSlot(0, vec, 0)
array_value = builder.EndObject()
return (array_type, array_value)
def column_encoding(arr):
type_map = {
# dtype: ( array_type, as_type )
np.float64: (TypedArray.TypedArray.Float32Array, np.float32),
np.float32: (TypedArray.TypedArray.Float32Array, np.float32),
np.float16: (TypedArray.TypedArray.Float32Array, np.float32),
np.int8: (TypedArray.TypedArray.Int32Array, np.int32),
np.int16: (TypedArray.TypedArray.Int32Array, np.int32),
np.int32: (TypedArray.TypedArray.Int32Array, np.int32),
np.int64: (TypedArray.TypedArray.Int32Array, np.int32),
np.uint8: (TypedArray.TypedArray.Uint32Array, np.uint32),
np.uint16: (TypedArray.TypedArray.Uint32Array, np.uint32),
np.uint32: (TypedArray.TypedArray.Uint32Array, np.uint32),
np.uint64: (TypedArray.TypedArray.Uint32Array, np.uint32)
}
type_map_default = (TypedArray.TypedArray.JSONEncodedArray, 'json')
return type_map.get(arr.dtype.type, type_map_default)
def index_encoding(arr):
type_map = {
# dtype: ( array_type, as_type )
np.int32: (TypedArray.TypedArray.Int32Array, np.int32),
np.int64: (TypedArray.TypedArray.Int32Array, np.int32),
np.uint32: (TypedArray.TypedArray.Uint32Array, np.uint32),
np.uint64: (TypedArray.TypedArray.Uint32Array, np.uint32)
}
type_map_default = (TypedArray.TypedArray.JSONEncodedArray, 'json')
return type_map.get(arr.dtype.type, type_map_default)
def guess_at_mem_needed(matrix):
(n_rows, n_cols) = matrix.shape
if isinstance(matrix, np.ndarray) or sparse.issparse(matrix):
guess = (n_rows * n_cols * matrix.dtype.itemsize) + 1024
elif isinstance(matrix, pd.DataFrame):
# XXX TODO - DataFrame type estimate
guess = 1
else:
guess = 1
# round up to nearest 1024 bytes
guess = (guess + 0x400) & (~0x3ff)
return guess
def encode_matrix_fbs(matrix, row_idx=None, col_idx=None):
"""
Given a 2D DataFrame, ndarray or sparse equivalent, create and return a
Matrix flatbuffer.
:param matrix: 2D DataFrame, ndarray or sparse equivalent
:param row_idx: index for row dimension, Index or ndarray
:param col_idx: index for col dimension, Index or ndarray
NOTE: row indices are (currently) unsupported and must be None
"""
if row_idx is not None:
raise ValueError("row indexing not supported for FBS Matrix")
if matrix.ndim != 2:
raise ValueError("FBS Matrix must be 2D")
(n_rows, n_cols) = matrix.shape
# estimate size needed, so we don't unnecessarily realloc.
builder = flatbuffers.Builder(guess_at_mem_needed(matrix))
if isinstance(matrix, pd.DataFrame):
matrix_columns = reversed(tuple(matrix[name] for name in matrix))
else:
matrix_columns = reversed(tuple(c for c in matrix.T))
columns = []
# for idx in reversed(np.arange(n_cols)):
for c in matrix_columns:
# serialize the typed array
typed_arr = serialize_typed_array(builder, c, column_encoding)
# serialize the Column union
columns.append(serialize_column(builder, typed_arr))
# Serialize Matrix.columns[]
Matrix.MatrixStartColumnsVector(builder, n_cols)
for c in columns:
builder.PrependUOffsetTRelative(c)
matrix_column_vec = builder.EndVector(n_cols)
# serialize the colIndex if provided
cidx = None
if col_idx is not None:
cidx = serialize_typed_array(builder, col_idx, index_encoding)
# Serialize Matrix
matrix = serialize_matrix(builder, n_rows, n_cols, matrix_column_vec, cidx)
builder.Finish(matrix)
return builder.Output()

1
server/requirements.txt
View File

@@ -6,6 +6,7 @@ Flask-Compress>=1.4.0
Flask-Cors>=3.0.6
Flask-RESTful>=0.3.6
flask-restful-swagger-2>=0.35
flatbuffers>=1.10.0
matplotlib>=2.2
numpy>=1.14.5
pandas>=0.23.1

69
server/test/decode_fbs.py Normal file
View File

@@ -0,0 +1,69 @@
"""
Code to decode, for testing purposes, the flatbuffer encoded blobs.
This code will need to be updated if fbs/matrix.fbs changes.
For more information, see fbs/matrix.fbs and server/app/util/fbs/
"""
import json
import server.app.util.fbs.NetEncoding.TypedArray as TypedArray
import server.app.util.fbs.NetEncoding.Matrix as Matrix
import server.app.util.fbs.NetEncoding.Int32Array as Int32Array
import server.app.util.fbs.NetEncoding.Uint32Array as Uint32Array
import server.app.util.fbs.NetEncoding.Float32Array as Float32Array
import server.app.util.fbs.NetEncoding.Float64Array as Float64Array
import server.app.util.fbs.NetEncoding.JSONEncodedArray as JSONEncodedArray
def decode_typed_array(tarr):
type_map = {
TypedArray.TypedArray.Uint32Array: Uint32Array.Uint32Array,
TypedArray.TypedArray.Int32Array: Int32Array.Int32Array,
TypedArray.TypedArray.Float32Array: Float32Array.Float32Array,
TypedArray.TypedArray.Float64Array: Float64Array.Float64Array,
TypedArray.TypedArray.JSONEncodedArray: JSONEncodedArray.JSONEncodedArray
}
(u_type, u) = tarr
if u_type == TypedArray.TypedArray.NONE:
return None
TarType = type_map.get(u_type, None)
assert(TarType is not None)
arr = TarType()
arr.Init(u.Bytes, u.Pos)
narr = arr.DataAsNumpy()
if u_type == TypedArray.TypedArray.JSONEncodedArray:
narr = json.loads(narr.tostring().decode('utf-8'))
return narr
def decode_matrix_FBS(buf):
"""
Given a FBS Matrix, return an decoded Python dict containing
same info in native format.
NOTE / TODO: row_idx not currently implemented
"""
df = Matrix.Matrix.GetRootAsMatrix(buf, 0)
n_rows = df.NRows()
n_cols = df.NCols()
columns_length = df.ColumnsLength()
decoded_columns = []
for col_idx in range(0, columns_length):
col = df.Columns(col_idx)
tarr = (col.UType(), col.U())
decoded_columns.append(decode_typed_array(tarr))
cidx = decode_typed_array((df.ColIndexType(), df.ColIndex()))
return {
"n_rows": n_rows,
"n_cols": n_cols,
"columns": decoded_columns,
"col_idx": cidx,
"row_idx": None
}

95
server/test/test_api.py
View File

@@ -5,6 +5,8 @@ import time
import requests
import decode_fbs
LOCAL_URL = "http://127.0.0.1:5005/"
VERSION = "v0.2"
URL_BASE = f"{LOCAL_URL}api/{VERSION}/"
@@ -40,6 +42,7 @@ class EndPoints(unittest.TestCase):
url = f"{URL_BASE}{endpoint}"
result = self.session.get(url)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/json")
result_data = result.json()
self.assertEqual(result_data["schema"]["dataframe"]["nObs"], 2638)
self.assertEqual(len(result_data["schema"]["annotations"]["obs"]), 5)
@@ -49,6 +52,7 @@ class EndPoints(unittest.TestCase):
url = f"{URL_BASE}{endpoint}"
result = self.session.get(url)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/json")
result_data = result.json()
self.assertEqual(result_data["config"]["displayNames"]["dataset"], "pbmc3k")
self.assertEqual(len(result_data["config"]["features"]), 4)
@@ -58,10 +62,26 @@ class EndPoints(unittest.TestCase):
url = f"{URL_BASE}{endpoint}"
result = self.session.get(url)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/json")
result_data = result.json()
self.assertEqual(result_data["layout"]["ndims"], 2)
self.assertEqual(len(result_data["layout"]["coordinates"]), 2638)
def test_get_layout_fbs(self):
endpoint = "layout/obs"
url = f"{URL_BASE}{endpoint}"
header = {"Accept": "application/octet-stream"}
result = self.session.get(url, headers=header)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.content)
self.assertEqual(df['n_rows'], 2638)
self.assertEqual(df['n_cols'], 2)
self.assertIsNotNone(df['columns'])
self.assertIsNone(df['col_idx'])
self.assertIsNone(df['row_idx'])
self.assertEqual(len(df['columns']), df['n_cols'])
# def test_put_layout(self):
# endpoint = "layout/obs"
# url = f"{URL_BASE}{endpoint}"
@@ -93,6 +113,7 @@ class EndPoints(unittest.TestCase):
url = f"{URL_BASE}{endpoint}"
result = self.session.get(url)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/json")
result_data = result.json()
self.assertEqual(result_data["names"], ["name", "n_genes", "percent_mito", "n_counts", "louvain"])
self.assertEqual(len(result_data["data"]), 2638)
@@ -103,11 +124,28 @@ class EndPoints(unittest.TestCase):
query = "annotation-name=n_genes&annotation-name=percent_mito"
url = f"{URL_BASE}{endpoint}?{query}"
result = self.session.get(url)
self.assertEqual(result.headers["Content-Type"], "application/json")
self.assertEqual(result.status_code, HTTPStatus.OK)
result_data = result.json()
self.assertEqual(result_data["names"], ["n_genes", "percent_mito"])
self.assertEqual(len(result_data["data"][0]), 3)
def test_get_annotations_obs_fbs(self):
endpoint = "annotations/obs"
url = f"{URL_BASE}{endpoint}"
header = {"Accept": "application/octet-stream"}
result = self.session.get(url, headers=header)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.content)
self.assertEqual(df['n_rows'], 2638)
self.assertEqual(df['n_cols'], 5)
self.assertIsNotNone(df['columns'])
self.assertIsNotNone(df['col_idx'])
self.assertIsNone(df['row_idx'])
self.assertEqual(len(df['columns']), df['n_cols'])
self.assertListEqual(df['col_idx'], ['name', 'n_genes', 'percent_mito', 'n_counts', 'louvain'])
def test_get_annotations_obs_error(self):
endpoint = "annotations/obs"
query = "annotation-name=notakey"
@@ -131,6 +169,7 @@ class EndPoints(unittest.TestCase):
}
result = self.session.put(url, json=obs_filter)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/json")
result_data = result.json()
self.assertEqual(result_data["names"], ["name", "n_genes", "percent_mito", "n_counts", "louvain"])
self.assertEqual(len(result_data["data"]), 15)
@@ -152,6 +191,7 @@ class EndPoints(unittest.TestCase):
}
result = self.session.put(url, json=obs_filter)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/json")
result_data = result.json()
self.assertEqual(result_data["names"], ["n_genes", "percent_mito"])
self.assertEqual(len(result_data["data"][0]), 3)
@@ -168,6 +208,7 @@ class EndPoints(unittest.TestCase):
}
result = self.session.post(url, json=params)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/json")
result_data = result.json()
self.assertEqual(len(result_data), 7)
@@ -182,6 +223,7 @@ class EndPoints(unittest.TestCase):
}
result = self.session.post(url, json=params)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/json")
result_data = result.json()
self.assertEqual(len(result_data), 10)
@@ -190,6 +232,7 @@ class EndPoints(unittest.TestCase):
url = f"{URL_BASE}{endpoint}"
result = self.session.get(url)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/json")
result_data = result.json()
self.assertEqual(result_data["names"], ["name", "n_cells"])
self.assertEqual(len(result_data["data"]), 1838)
@@ -201,10 +244,27 @@ class EndPoints(unittest.TestCase):
url = f"{URL_BASE}{endpoint}?{query}"
result = self.session.get(url)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/json")
result_data = result.json()
self.assertEqual(result_data["names"], ["n_cells"])
self.assertEqual(len(result_data["data"][0]), 2)
def test_get_annotations_var_fbs(self):
endpoint = "annotations/var"
url = f"{URL_BASE}{endpoint}"
header = {"Accept": "application/octet-stream"}
result = self.session.get(url, headers=header)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.content)
self.assertEqual(df['n_rows'], 1838)
self.assertEqual(df['n_cols'], 2)
self.assertIsNotNone(df['columns'])
self.assertIsNotNone(df['col_idx'])
self.assertIsNone(df['row_idx'])
self.assertEqual(len(df['columns']), df['n_cols'])
self.assertListEqual(df['col_idx'], ['name', 'n_cells'])
def test_get_annotations_var_error(self):
endpoint = "annotations/var"
query = "annotation-name=notakey"
@@ -218,6 +278,7 @@ class EndPoints(unittest.TestCase):
var_filter = {"filter": {"var": {"annotation_value": [{"name": "name", "values": ["ATAD3C", "RER1"]}]}}}
result = self.session.put(url, json=var_filter)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/json")
result_data = result.json()
self.assertEqual(result_data["names"], ["name", "n_cells"])
self.assertEqual(len(result_data["data"]), 2)
@@ -229,6 +290,7 @@ class EndPoints(unittest.TestCase):
var_filter = {"filter": {"var": {"annotation_value": [{"name": "name", "values": ["ATAD3C", "RER1"]}]}}}
result = self.session.put(url, json=var_filter)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/json")
result_data = result.json()
self.assertEqual(result_data["names"], ["n_cells"])
self.assertEqual(len(result_data["data"][0]), 2)
@@ -241,6 +303,7 @@ class EndPoints(unittest.TestCase):
url = f"{URL_BASE}{endpoint}?{query}"
result = self.session.get(url)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/json")
result_data = result.json()
self.assertEqual(len(result_data["obs"]), 2638)
@@ -262,6 +325,7 @@ class EndPoints(unittest.TestCase):
url = f"{URL_BASE}{endpoint}"
result = self.session.get(url)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/json")
def test_data_filter(self):
for axis in ["obs", "var"]:
@@ -270,10 +334,11 @@ class EndPoints(unittest.TestCase):
url = f"{URL_BASE}{endpoint}?{query}"
result = self.session.get(url)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/json")
result_data = result.json()
self.assertEqual(len(result_data["obs"]), 38)
def test_data_put(self):
def test_data_json_put(self):
for axis in ["obs", "var"]:
endpoint = f"data/{axis}"
url = f"{URL_BASE}{endpoint}"
@@ -291,9 +356,33 @@ class EndPoints(unittest.TestCase):
}
result = self.session.put(url, headers=header, json=obs_filter)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/json")
result_data = result.json()
self.assertEqual(len(result_data["obs"]), 15)
def test_data_put_fbs(self):
endpoint = f"data/var"
url = f"{URL_BASE}{endpoint}"
header = {"Accept": "application/octet-stream"}
filter = {
"filter": {
"var": {
"index": [0, 1, 4]
}
}
}
result = self.session.put(url, headers=header, json=filter)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
df = decode_fbs.decode_matrix_FBS(result.content)
self.assertEqual(df['n_rows'], 2638)
self.assertEqual(df['n_cols'], 3)
self.assertIsNotNone(df['columns'])
self.assertIsNotNone(df['col_idx'])
self.assertIsNone(df['row_idx'])
self.assertEqual(len(df['columns']), df['n_cols'])
self.assertListEqual(df['col_idx'].tolist(), [0, 1, 4])
def test_data_put_single_var(self):
for axis in ["obs", "var"]:
endpoint = f"data/{axis}"
@@ -302,6 +391,7 @@ class EndPoints(unittest.TestCase):
var_filter = {"filter": {"var": {"annotation_value": [{"name": "name", "values": ["RER1"]}]}}}
result = self.session.put(url, headers=header, json=var_filter)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/json")
result_data = result.json()
if axis == "obs":
self.assertEqual(len(result_data["obs"][0]), 2)
@@ -338,6 +428,7 @@ class EndPoints(unittest.TestCase):
}
result = self.session.put(url, json=f1)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/json")
result_data1 = result.json()
f2 = {
"filter": {
@@ -353,6 +444,7 @@ class EndPoints(unittest.TestCase):
}
result = self.session.put(url, json=f2)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/json")
result_data2 = result.json()
self.assertNotEqual(result_data1, result_data2)
@@ -377,6 +469,7 @@ class EndPoints(unittest.TestCase):
}
result = self.session.put(url, json=f2)
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/json")
result_data2 = result.json()
self.assertNotEqual(result_data1, result_data2)

3
setup.cfg
View File

@@ -1,3 +1,4 @@
[flake8]
max-line-length = 120
ignore = E203, W503
ignore = E203, W503
exclude = server/app/util/fbs/NetEncoding/