binary wire format with flatbuffers (#509)

* first flatbuffer schema * do not lint auto-generated files * add flatbuffers package * add flatbuffer module * wire up /data/X/T route * use flatbuffers for matrix data fetc * clarity and comments * add flatbuffer layout route * clean up obsolete code * fix tests * move flake8 config to setup.cfg * add comments * lint * rework layout routes for fbs * add more type support to fbs * lint * add flatbuffer support for annotations * function name improvements * fix botched merge with master * remove unused import * route cleanup for flatbuffers * rename function for clarity * add missing globals to Jest tests * fix client JS tests * fix routes for Python tests * comments for clarity * non-finite floating point hardening * more non-finite number handling * lint * fix tests for summarizeAnnotations * harden diffexp calculation against FP errors * cleanup unused code * lint * add encoding tests for flatbuffers * application type specified as strings * fix spelling error * improve variable names * add note about documentation gap * rename FBS DataFrame to Matrix
2026-10-03 01:38:11 +08:00 · 2019-01-09 14:26:05 -08:00
parent 42e25a1a1f
commit b90447c387
41 changed files with 2317 additions and 266 deletions
@@ -0,0 +1,75 @@
+import { flatbuffers } from "flatbuffers";
+import { NetEncoding } from "./matrix_generated";
+
+const utf8Decoder = new TextDecoder("utf-8");
+
+/*
+Matrix flatbuffer decoding support.   See fbs/matrix.fbs
+*/
+
+/*
+Decode NetEncoding.TypedArray
+*/
+function decodeTypedArray(uType, uValF, inplace = false) {
+  if (uType === NetEncoding.TypedArray.NONE) {
+    return null;
+  }
+
+  // Convert to a JS class that supports this type
+  const TypeClass = NetEncoding[NetEncoding.TypedArray[uType]];
+  // Create a TypedArray that references the underlying buffer
+  let arr = uValF(new TypeClass()).dataArray();
+  if (uType === NetEncoding.TypedArray.JSONEncodedArray) {
+    const json = utf8Decoder.decode(arr);
+    arr = JSON.parse(json);
+  } else if (!inplace) {
+    /* force copy to release underlying FBS buffer */
+    arr = new arr.constructor(arr);
+  }
+  return arr;
+}
+
+/*
+Parameter: Uint8Array or ArrayBuffer containing raw flatbuffer Matrix
+Returns: object containing decoded Matrix:
+{
+  nRows: num,
+  nCols: num,
+  columns: [
+    each column, which will be a TypedArray or Array
+  ]
+  colIdx: []|null
+}
+*/
+function decodeMatrixFBS(arrayBuffer, inplace = false) {
+  const bb = new flatbuffers.ByteBuffer(new Uint8Array(arrayBuffer));
+  const df = NetEncoding.Matrix.getRootAsMatrix(bb);
+
+  const nRows = df.nRows();
+  const nCols = df.nCols();
+
+  /* decode columns */
+  const columnsLength = df.columnsLength();
+  const columns = Array(columnsLength).fill(null);
+  for (let c = 0; c < columnsLength; c += 1) {
+    const col = df.columns(c);
+    columns[c] = decodeTypedArray(col.uType(), col.u.bind(col), inplace);
+  }
+
+  /* decode col_idx */
+  const colIdx = decodeTypedArray(
+    df.colIndexType(),
+    df.colIndex.bind(df),
+    inplace
+  );
+
+  return {
+    nRows,
+    nCols,
+    columns,
+    colIdx,
+    rowIdx: null
+  };
+}
+
+export default decodeMatrixFBS;