binary wire format with flatbuffers (#509)

* first flatbuffer schema

* do not lint auto-generated files

* add flatbuffers package

* add flatbuffer module

* wire up /data/X/T route

* use flatbuffers for matrix data fetc

* clarity and comments

* add flatbuffer layout route

* clean up obsolete code

* fix tests

* move flake8 config to setup.cfg

* add comments

* lint

* rework layout routes for fbs

* add more type support to fbs

* lint

* add flatbuffer support for annotations

* function name improvements

* fix botched merge with master

* remove unused import

* route cleanup for flatbuffers

* rename function for clarity

* add missing globals to Jest tests

* fix client JS tests

* fix routes for Python tests

* comments for clarity

* non-finite floating point hardening

* more non-finite number handling

* lint

* fix tests for summarizeAnnotations

* harden diffexp calculation against FP errors

* cleanup unused code

* lint

* add encoding tests for flatbuffers

* application type specified as strings

* fix spelling error

* improve variable names

* add note about documentation gap

* rename FBS DataFrame to Matrix

client/src/actions/index.js

@@ -5,6 +5,7 @@ import { Universe, kvCache } from "../util/stateManager";
 import {
   catchErrorsWrap,
   doJsonRequest,
+  doBinaryRequest,
   rangeEncodeIndices,
   dispatchNetworkErrorMessageToUser
 } from "../util/actionHelpers";
@@ -13,24 +14,28 @@ import {
 Bootstrap application with the initial data loading.
   * /config - application configuration
   * /schema - schema of dataframe
-  * /annotations/obs - all metadata annotation
+  * /annotations - all metadata annotation
+  * /layout - all default layout
 */
 const doInitialDataLoad = () =>
   catchErrorsWrap(async dispatch => {
     dispatch({ type: "initial data load start" });
 
     try {
-      const requests = _([
-        "config",
-        "schema",
+      const requestJson = _(["config", "schema"])
+        .map(r => `${globals.API.prefix}${globals.API.version}${r}`)
+        .map(url => doJsonRequest(url))
+        .value();
+      const requestBinary = _([
         "annotations/obs",
         "annotations/var?annotation-name=name",
         "layout/obs"
       ])
         .map(r => `${globals.API.prefix}${globals.API.version}${r}`)
-        .map(url => doJsonRequest(url))
+        .map(url => doBinaryRequest(url))
         .value();
-      const results = await Promise.all(requests);
+
+      const results = await Promise.all(_.concat(requestJson, requestBinary));
 
       /* set config defaults */
       const config = { ...globals.configDefaults, ...results[0].config };
@@ -87,6 +92,38 @@ needs expression data.
 Transparently utilizes cached data if it is already present.
 */
 async function _doRequestExpressionData(dispatch, getState, genes) {
+  /* helper for this function only */
+  const fetchData = async geneNames => {
+    const res = await fetch(
+      `${globals.API.prefix}${globals.API.version}data/var`,
+      {
+        method: "PUT",
+        body: JSON.stringify({
+          filter: {
+            var: {
+              annotation_value: [{ name: "name", values: geneNames }]
+            }
+          }
+        }),
+        headers: new Headers({
+          accept: "application/octet-stream",
+          "Content-Type": "application/json"
+        })
+      }
+    );
+
+    if (
+      !res.ok ||
+      res.headers.get("Content-Type") !== "application/octet-stream"
+    ) {
+      // WILL throw
+      return dispatchExpressionErrors(dispatch, res);
+    }
+
+    const data = await res.arrayBuffer();
+    return Universe.convertDataFBStoObject(universe, data);
+  };
+
   const state = getState();
   const { universe } = state.controls;
   /* preload data already in cache */
@@ -108,35 +145,10 @@ async function _doRequestExpressionData(dispatch, getState, genes) {
   /* Fetch data for any genes not in cache */
   if (genesToFetch.length) {
     try {
-      // XXX: TODO - this could be using /data/var rather than /data/obs,
-      // as that would simplify the transformation in convertExpressionRESTv02ToObject
-      const res = await fetch(
-        `${globals.API.prefix}${globals.API.version}data/obs`,
-        {
-          method: "PUT",
-          body: JSON.stringify({
-            filter: {
-              var: {
-                annotation_value: [{ name: "name", values: genesToFetch }]
-              }
-            }
-          }),
-          headers: new Headers({
-            accept: "application/json",
-            "Content-Type": "application/json"
-          })
-        }
-      );
-
-      if (!res.ok || res.headers.get("Content-Type") !== "application/json") {
-        // WILL throw
-        return dispatchExpressionErrors(dispatch, res);
-      }
-
-      const data = await res.json();
+      const newExpressionData = await fetchData(genesToFetch);
       expressionData = {
         ...expressionData,
-        ...Universe.convertExpressionRESTv02ToObject(universe, data)
+        ...newExpressionData
       };
     } catch (error) {
       dispatch({ type: "expression load error", error });

client/src/components/brushableHistogram/index.js

@@ -13,6 +13,7 @@ import memoize from "memoize-one";
 import { kvCache } from "../../util/stateManager";
 import * as globals from "../../globals";
 import actions from "../../actions";
+import finiteExtent from "../../util/finiteExtent";
 
 @connect(state => ({
   world: state.controls.world,
@@ -56,7 +57,7 @@ class HistogramBrush extends React.Component {
       histogramCache.x = d3
         .scaleLinear()
         .domain(
-          d3.extent(varValues)
+          finiteExtent(varValues)
         ) /* replace this if we have ranges for genes back from server like we do for annotations on cells */
         .range([0, this.width]);
 

client/src/components/geneExpression/index.js

@@ -3,7 +3,6 @@
 
 import React from "react";
 import _ from "lodash";
-import * as d3 from "d3";
 import fuzzysort from "fuzzysort";
 
 import { connect } from "react-redux";
@@ -14,6 +13,7 @@ import * as globals from "../../globals";
 import actions from "../../actions";
 import { postUserErrorToast } from "../framework/toasters";
 import ExpressionButtons from "./expressionButtons";
+import finiteExtent from "../../util/finiteExtent";
 
 const renderGene = (fuzzySortResult, { handleClick, modifiers, query }) => {
   if (!modifiers.matchesPredicate) {
@@ -155,7 +155,7 @@ class GeneExpression extends React.Component {
                     key={geneName}
                     field={geneName}
                     zebra={index % 2 === 0}
-                    ranges={d3.extent(values)}
+                    ranges={finiteExtent(values)}
                     isUserDefined
                   />
                 );
@@ -185,7 +185,7 @@ class GeneExpression extends React.Component {
                     key={name}
                     field={name}
                     zebra={index % 2 === 0}
-                    ranges={d3.extent(values)}
+                    ranges={finiteExtent(values)}
                     isDiffExp
                     logFoldChange={value[1]}
                     pval={value[2]}

client/src/components/scatterplot/scatterplot.js

@@ -20,6 +20,7 @@ import scaleLinear from "../../util/scaleLinear";
 
 import { margin, width, height } from "./util";
 import { kvCache } from "../../util/stateManager";
+import finiteExtent from "../../util/finiteExtent";
 
 @connect(state => {
   const {
@@ -227,11 +228,11 @@ class Scatterplot extends React.Component {
   static setupScales(expressionX, expressionY) {
     const xScale = d3
       .scaleLinear()
-      .domain(d3.extent(expressionX))
+      .domain(finiteExtent(expressionX))
       .range([0, width]);
     const yScale = d3
       .scaleLinear()
-      .domain(d3.extent(expressionY))
+      .domain(finiteExtent(expressionY))
       .range([height, 0]);
 
     return {

client/src/globals.js

@@ -57,6 +57,7 @@ export const brightBlue = "#4a90e2";
 export const brightGreen = "#A2D729";
 export const darkGreen = "#448C4D";
 
+export const nonFiniteCellColor = lightGrey;
 export const defaultCellColor = "rgb(0,0,0,1)";
 
 /* typography constants */

client/src/middleware/updateCellColors.js

@@ -2,7 +2,9 @@
 import _ from "lodash";
 import * as d3 from "d3";
 import { interpolateRainbow, interpolateCool } from "d3-scale-chromatic";
+import * as globals from "../globals";
 import parseRGB from "../util/parseRGB";
+import finiteExtent from "../util/finiteExtent";
 
 /*
   https://medium.com/@jacobp100/you-arent-using-redux-middleware-enough-94ffe991e6
@@ -93,10 +95,15 @@ const updateCellColorsMiddleware = store => next => action => {
     }
 
     const key = action.colorAccessor;
+    const nonFiniteColor = parseRGB(globals.nonFiniteCellColor);
     for (let i = 0, len = obsAnnotations.length; i < len; i += 1) {
-      const obs = obsAnnotations[i];
-      const c = colorScale(obs[key]);
-      colorsByRGB[i] = colors[c];
+      const val = obsAnnotations[i][key];
+      if (Number.isFinite(val)) {
+        const c = colorScale(val);
+        colorsByRGB[i] = colors[c];
+      } else {
+        colorsByRGB[i] = nonFiniteColor;
+      }
     }
   }
 
@@ -104,8 +111,7 @@ const updateCellColorsMiddleware = store => next => action => {
     const { gene, data } = action;
     const expression = data[gene]; // Float32Array
     const colorBins = 100;
-    // XXX TODO - replace _.min/_.max with the much faster finiteExtent
-    const [min, max] = [_.min(expression), _.max(expression)];
+    const [min, max] = finiteExtent(expression);
     colorScale = d3
       .scaleQuantile()
       .domain([min, max])
@@ -116,10 +122,16 @@ const updateCellColorsMiddleware = store => next => action => {
     for (let i = 0; i < colorBins; i += 1) {
       colors[i] = parseRGB(interpolateCool(i / colorBins));
     }
+    const nonFiniteColor = parseRGB(globals.nonFiniteCellColor);
 
     for (let i = 0, len = expression.length; i < len; i += 1) {
-      const c = colorScale(expression[i]);
-      colorsByRGB[i] = colors[c];
+      const e = expression[i];
+      if (Number.isFinite(e)) {
+        const c = colorScale(e);
+        colorsByRGB[i] = colors[c];
+      } else {
+        colorsByRGB[i] = nonFiniteColor;
+      }
     }
   }
 

client/src/util/actionHelpers.js

@@ -25,17 +25,18 @@ export function catchErrorsWrap(fn, dispatchToUser = false) {
 }
 
 /*
-Wrapper to perform an async fetch and JSON decode response.
+Wrapper to perform async fetch with some modest error handling
+and decoding.
 */
-export const doJsonRequest = async url => {
+const doFetch = async (url, acceptType) => {
   const res = await fetch(url, {
     method: "get",
     headers: new Headers({
-      "Content-Type": "application/json"
+      Accept: acceptType
     })
   });
-  if (res.ok && res.headers.get("Content-Type") === "application/json") {
-    return res.json();
+  if (res.ok && res.headers.get("Content-Type") === acceptType) {
+    return res;
   }
   // else an error
   let msg = `Unexpected HTTP response ${res.status}, ${res.statusText}`;
@@ -47,6 +48,22 @@ export const doJsonRequest = async url => {
   throw new Error(msg);
 };
 
+/*
+Wrapper to perform an async fetch and JSON decode response.
+*/
+export const doJsonRequest = async url => {
+  const res = await doFetch(url, "application/json");
+  return res.json();
+};
+
+/*
+Wrapper to perform an async fetch for binary data.
+*/
+export const doBinaryRequest = async url => {
+  const res = await doFetch(url, "application/octet-stream");
+  return res.arrayBuffer();
+};
+
 /*
 This function "packs" filter index lists into the more efficient
 "range" form specified in the REST 0.2 spec.

client/src/util/finiteExtent.js

@@ -0,0 +1,33 @@
+/*
+Return the [minimum, maximum] extent, of the given typed array, ignoring
+non-finite values (ie, +Infinity, -Infinity).
+
+If undefined or empty array, or array contains only non-finite numbers,
+will return [undefined, undefined]
+*/
+
+function finiteExtent(tarr) {
+  let min;
+  let max;
+  let i;
+
+  for (i = 0; i < tarr.length; i += 1) {
+    const val = tarr[i];
+    if (Number.isFinite(val)) {
+      min = val;
+      max = val;
+      i += 1;
+      break;
+    }
+  }
+  for (; i < tarr.length; i += 1) {
+    const val = tarr[i];
+    if (Number.isFinite(val)) {
+      if (min > val) min = val;
+      if (max < val) max = val;
+    }
+  }
+  return [min, max];
+}
+
+export default finiteExtent;

client/src/util/stateManager/index.js

@@ -3,7 +3,7 @@
 /*
 Model manager providing an abstraction for the use of the reducer code.
 This module provides several buckets of functionality:
-  - schema and config driven tranformation of the dataframe wire protocol
+  - schema and config driven tranformation of the wire protocol
     into a format that is easy for the UI code to use.
   - manage the universe/world abstraction:
     + universe: all of the server-provided, read-only data

client/src/util/stateManager/matrix.js

@@ -0,0 +1,75 @@
+import { flatbuffers } from "flatbuffers";
+import { NetEncoding } from "./matrix_generated";
+
+const utf8Decoder = new TextDecoder("utf-8");
+
+/*
+Matrix flatbuffer decoding support.   See fbs/matrix.fbs
+*/
+
+/*
+Decode NetEncoding.TypedArray
+*/
+function decodeTypedArray(uType, uValF, inplace = false) {
+  if (uType === NetEncoding.TypedArray.NONE) {
+    return null;
+  }
+
+  // Convert to a JS class that supports this type
+  const TypeClass = NetEncoding[NetEncoding.TypedArray[uType]];
+  // Create a TypedArray that references the underlying buffer
+  let arr = uValF(new TypeClass()).dataArray();
+  if (uType === NetEncoding.TypedArray.JSONEncodedArray) {
+    const json = utf8Decoder.decode(arr);
+    arr = JSON.parse(json);
+  } else if (!inplace) {
+    /* force copy to release underlying FBS buffer */
+    arr = new arr.constructor(arr);
+  }
+  return arr;
+}
+
+/*
+Parameter: Uint8Array or ArrayBuffer containing raw flatbuffer Matrix
+Returns: object containing decoded Matrix:
+{
+  nRows: num,
+  nCols: num,
+  columns: [
+    each column, which will be a TypedArray or Array
+  ]
+  colIdx: []|null
+}
+*/
+function decodeMatrixFBS(arrayBuffer, inplace = false) {
+  const bb = new flatbuffers.ByteBuffer(new Uint8Array(arrayBuffer));
+  const df = NetEncoding.Matrix.getRootAsMatrix(bb);
+
+  const nRows = df.nRows();
+  const nCols = df.nCols();
+
+  /* decode columns */
+  const columnsLength = df.columnsLength();
+  const columns = Array(columnsLength).fill(null);
+  for (let c = 0; c < columnsLength; c += 1) {
+    const col = df.columns(c);
+    columns[c] = decodeTypedArray(col.uType(), col.u.bind(col), inplace);
+  }
+
+  /* decode col_idx */
+  const colIdx = decodeTypedArray(
+    df.colIndexType(),
+    df.colIndex.bind(df),
+    inplace
+  );
+
+  return {
+    nRows,
+    nCols,
+    columns,
+    colIdx,
+    rowIdx: null
+  };
+}
+
+export default decodeMatrixFBS;

client/src/util/stateManager/matrix_generated.js

@@ -0,0 +1,835 @@
+// automatically generated by the FlatBuffers compiler, do not modify
+
+/**
+ * @const
+ * @namespace
+ */
+var NetEncoding = NetEncoding || {};
+
+/**
+ * @enum
+ */
+NetEncoding.TypedArray = {
+  NONE: 0, 0: 'NONE',
+  Float32Array: 1, 1: 'Float32Array',
+  Int32Array: 2, 2: 'Int32Array',
+  Uint32Array: 3, 3: 'Uint32Array',
+  Float64Array: 4, 4: 'Float64Array',
+  JSONEncodedArray: 5, 5: 'JSONEncodedArray'
+};
+
+/**
+ * @constructor
+ */
+NetEncoding.Float32Array = function() {
+  /**
+   * @type {flatbuffers.ByteBuffer}
+   */
+  this.bb = null;
+
+  /**
+   * @type {number}
+   */
+  this.bb_pos = 0;
+};
+
+/**
+ * @param {number} i
+ * @param {flatbuffers.ByteBuffer} bb
+ * @returns {NetEncoding.Float32Array}
+ */
+NetEncoding.Float32Array.prototype.__init = function(i, bb) {
+  this.bb_pos = i;
+  this.bb = bb;
+  return this;
+};
+
+/**
+ * @param {flatbuffers.ByteBuffer} bb
+ * @param {NetEncoding.Float32Array=} obj
+ * @returns {NetEncoding.Float32Array}
+ */
+NetEncoding.Float32Array.getRootAsFloat32Array = function(bb, obj) {
+  return (obj || new NetEncoding.Float32Array).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+};
+
+/**
+ * @param {number} index
+ * @returns {number}
+ */
+NetEncoding.Float32Array.prototype.data = function(index) {
+  var offset = this.bb.__offset(this.bb_pos, 4);
+  return offset ? this.bb.readFloat32(this.bb.__vector(this.bb_pos + offset) + index * 4) : 0;
+};
+
+/**
+ * @returns {number}
+ */
+NetEncoding.Float32Array.prototype.dataLength = function() {
+  var offset = this.bb.__offset(this.bb_pos, 4);
+  return offset ? this.bb.__vector_len(this.bb_pos + offset) : 0;
+};
+
+/**
+ * @returns {Float32Array}
+ */
+NetEncoding.Float32Array.prototype.dataArray = function() {
+  var offset = this.bb.__offset(this.bb_pos, 4);
+  return offset ? new Float32Array(this.bb.bytes().buffer, this.bb.bytes().byteOffset + this.bb.__vector(this.bb_pos + offset), this.bb.__vector_len(this.bb_pos + offset)) : null;
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ */
+NetEncoding.Float32Array.startFloat32Array = function(builder) {
+  builder.startObject(1);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {flatbuffers.Offset} dataOffset
+ */
+NetEncoding.Float32Array.addData = function(builder, dataOffset) {
+  builder.addFieldOffset(0, dataOffset, 0);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {Array.<number>} data
+ * @returns {flatbuffers.Offset}
+ */
+NetEncoding.Float32Array.createDataVector = function(builder, data) {
+  builder.startVector(4, data.length, 4);
+  for (var i = data.length - 1; i >= 0; i--) {
+    builder.addFloat32(data[i]);
+  }
+  return builder.endVector();
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {number} numElems
+ */
+NetEncoding.Float32Array.startDataVector = function(builder, numElems) {
+  builder.startVector(4, numElems, 4);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @returns {flatbuffers.Offset}
+ */
+NetEncoding.Float32Array.endFloat32Array = function(builder) {
+  var offset = builder.endObject();
+  return offset;
+};
+
+/**
+ * @constructor
+ */
+NetEncoding.Uint32Array = function() {
+  /**
+   * @type {flatbuffers.ByteBuffer}
+   */
+  this.bb = null;
+
+  /**
+   * @type {number}
+   */
+  this.bb_pos = 0;
+};
+
+/**
+ * @param {number} i
+ * @param {flatbuffers.ByteBuffer} bb
+ * @returns {NetEncoding.Uint32Array}
+ */
+NetEncoding.Uint32Array.prototype.__init = function(i, bb) {
+  this.bb_pos = i;
+  this.bb = bb;
+  return this;
+};
+
+/**
+ * @param {flatbuffers.ByteBuffer} bb
+ * @param {NetEncoding.Uint32Array=} obj
+ * @returns {NetEncoding.Uint32Array}
+ */
+NetEncoding.Uint32Array.getRootAsUint32Array = function(bb, obj) {
+  return (obj || new NetEncoding.Uint32Array).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+};
+
+/**
+ * @param {number} index
+ * @returns {number}
+ */
+NetEncoding.Uint32Array.prototype.data = function(index) {
+  var offset = this.bb.__offset(this.bb_pos, 4);
+  return offset ? this.bb.readUint32(this.bb.__vector(this.bb_pos + offset) + index * 4) : 0;
+};
+
+/**
+ * @returns {number}
+ */
+NetEncoding.Uint32Array.prototype.dataLength = function() {
+  var offset = this.bb.__offset(this.bb_pos, 4);
+  return offset ? this.bb.__vector_len(this.bb_pos + offset) : 0;
+};
+
+/**
+ * @returns {Uint32Array}
+ */
+NetEncoding.Uint32Array.prototype.dataArray = function() {
+  var offset = this.bb.__offset(this.bb_pos, 4);
+  return offset ? new Uint32Array(this.bb.bytes().buffer, this.bb.bytes().byteOffset + this.bb.__vector(this.bb_pos + offset), this.bb.__vector_len(this.bb_pos + offset)) : null;
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ */
+NetEncoding.Uint32Array.startUint32Array = function(builder) {
+  builder.startObject(1);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {flatbuffers.Offset} dataOffset
+ */
+NetEncoding.Uint32Array.addData = function(builder, dataOffset) {
+  builder.addFieldOffset(0, dataOffset, 0);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {Array.<number>} data
+ * @returns {flatbuffers.Offset}
+ */
+NetEncoding.Uint32Array.createDataVector = function(builder, data) {
+  builder.startVector(4, data.length, 4);
+  for (var i = data.length - 1; i >= 0; i--) {
+    builder.addInt32(data[i]);
+  }
+  return builder.endVector();
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {number} numElems
+ */
+NetEncoding.Uint32Array.startDataVector = function(builder, numElems) {
+  builder.startVector(4, numElems, 4);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @returns {flatbuffers.Offset}
+ */
+NetEncoding.Uint32Array.endUint32Array = function(builder) {
+  var offset = builder.endObject();
+  return offset;
+};
+
+/**
+ * @constructor
+ */
+NetEncoding.Int32Array = function() {
+  /**
+   * @type {flatbuffers.ByteBuffer}
+   */
+  this.bb = null;
+
+  /**
+   * @type {number}
+   */
+  this.bb_pos = 0;
+};
+
+/**
+ * @param {number} i
+ * @param {flatbuffers.ByteBuffer} bb
+ * @returns {NetEncoding.Int32Array}
+ */
+NetEncoding.Int32Array.prototype.__init = function(i, bb) {
+  this.bb_pos = i;
+  this.bb = bb;
+  return this;
+};
+
+/**
+ * @param {flatbuffers.ByteBuffer} bb
+ * @param {NetEncoding.Int32Array=} obj
+ * @returns {NetEncoding.Int32Array}
+ */
+NetEncoding.Int32Array.getRootAsInt32Array = function(bb, obj) {
+  return (obj || new NetEncoding.Int32Array).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+};
+
+/**
+ * @param {number} index
+ * @returns {number}
+ */
+NetEncoding.Int32Array.prototype.data = function(index) {
+  var offset = this.bb.__offset(this.bb_pos, 4);
+  return offset ? this.bb.readInt32(this.bb.__vector(this.bb_pos + offset) + index * 4) : 0;
+};
+
+/**
+ * @returns {number}
+ */
+NetEncoding.Int32Array.prototype.dataLength = function() {
+  var offset = this.bb.__offset(this.bb_pos, 4);
+  return offset ? this.bb.__vector_len(this.bb_pos + offset) : 0;
+};
+
+/**
+ * @returns {Int32Array}
+ */
+NetEncoding.Int32Array.prototype.dataArray = function() {
+  var offset = this.bb.__offset(this.bb_pos, 4);
+  return offset ? new Int32Array(this.bb.bytes().buffer, this.bb.bytes().byteOffset + this.bb.__vector(this.bb_pos + offset), this.bb.__vector_len(this.bb_pos + offset)) : null;
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ */
+NetEncoding.Int32Array.startInt32Array = function(builder) {
+  builder.startObject(1);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {flatbuffers.Offset} dataOffset
+ */
+NetEncoding.Int32Array.addData = function(builder, dataOffset) {
+  builder.addFieldOffset(0, dataOffset, 0);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {Array.<number>} data
+ * @returns {flatbuffers.Offset}
+ */
+NetEncoding.Int32Array.createDataVector = function(builder, data) {
+  builder.startVector(4, data.length, 4);
+  for (var i = data.length - 1; i >= 0; i--) {
+    builder.addInt32(data[i]);
+  }
+  return builder.endVector();
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {number} numElems
+ */
+NetEncoding.Int32Array.startDataVector = function(builder, numElems) {
+  builder.startVector(4, numElems, 4);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @returns {flatbuffers.Offset}
+ */
+NetEncoding.Int32Array.endInt32Array = function(builder) {
+  var offset = builder.endObject();
+  return offset;
+};
+
+/**
+ * @constructor
+ */
+NetEncoding.Float64Array = function() {
+  /**
+   * @type {flatbuffers.ByteBuffer}
+   */
+  this.bb = null;
+
+  /**
+   * @type {number}
+   */
+  this.bb_pos = 0;
+};
+
+/**
+ * @param {number} i
+ * @param {flatbuffers.ByteBuffer} bb
+ * @returns {NetEncoding.Float64Array}
+ */
+NetEncoding.Float64Array.prototype.__init = function(i, bb) {
+  this.bb_pos = i;
+  this.bb = bb;
+  return this;
+};
+
+/**
+ * @param {flatbuffers.ByteBuffer} bb
+ * @param {NetEncoding.Float64Array=} obj
+ * @returns {NetEncoding.Float64Array}
+ */
+NetEncoding.Float64Array.getRootAsFloat64Array = function(bb, obj) {
+  return (obj || new NetEncoding.Float64Array).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+};
+
+/**
+ * @param {number} index
+ * @returns {number}
+ */
+NetEncoding.Float64Array.prototype.data = function(index) {
+  var offset = this.bb.__offset(this.bb_pos, 4);
+  return offset ? this.bb.readFloat64(this.bb.__vector(this.bb_pos + offset) + index * 8) : 0;
+};
+
+/**
+ * @returns {number}
+ */
+NetEncoding.Float64Array.prototype.dataLength = function() {
+  var offset = this.bb.__offset(this.bb_pos, 4);
+  return offset ? this.bb.__vector_len(this.bb_pos + offset) : 0;
+};
+
+/**
+ * @returns {Float64Array}
+ */
+NetEncoding.Float64Array.prototype.dataArray = function() {
+  var offset = this.bb.__offset(this.bb_pos, 4);
+  return offset ? new Float64Array(this.bb.bytes().buffer, this.bb.bytes().byteOffset + this.bb.__vector(this.bb_pos + offset), this.bb.__vector_len(this.bb_pos + offset)) : null;
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ */
+NetEncoding.Float64Array.startFloat64Array = function(builder) {
+  builder.startObject(1);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {flatbuffers.Offset} dataOffset
+ */
+NetEncoding.Float64Array.addData = function(builder, dataOffset) {
+  builder.addFieldOffset(0, dataOffset, 0);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {Array.<number>} data
+ * @returns {flatbuffers.Offset}
+ */
+NetEncoding.Float64Array.createDataVector = function(builder, data) {
+  builder.startVector(8, data.length, 8);
+  for (var i = data.length - 1; i >= 0; i--) {
+    builder.addFloat64(data[i]);
+  }
+  return builder.endVector();
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {number} numElems
+ */
+NetEncoding.Float64Array.startDataVector = function(builder, numElems) {
+  builder.startVector(8, numElems, 8);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @returns {flatbuffers.Offset}
+ */
+NetEncoding.Float64Array.endFloat64Array = function(builder) {
+  var offset = builder.endObject();
+  return offset;
+};
+
+/**
+ * @constructor
+ */
+NetEncoding.JSONEncodedArray = function() {
+  /**
+   * @type {flatbuffers.ByteBuffer}
+   */
+  this.bb = null;
+
+  /**
+   * @type {number}
+   */
+  this.bb_pos = 0;
+};
+
+/**
+ * @param {number} i
+ * @param {flatbuffers.ByteBuffer} bb
+ * @returns {NetEncoding.JSONEncodedArray}
+ */
+NetEncoding.JSONEncodedArray.prototype.__init = function(i, bb) {
+  this.bb_pos = i;
+  this.bb = bb;
+  return this;
+};
+
+/**
+ * @param {flatbuffers.ByteBuffer} bb
+ * @param {NetEncoding.JSONEncodedArray=} obj
+ * @returns {NetEncoding.JSONEncodedArray}
+ */
+NetEncoding.JSONEncodedArray.getRootAsJSONEncodedArray = function(bb, obj) {
+  return (obj || new NetEncoding.JSONEncodedArray).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+};
+
+/**
+ * @param {number} index
+ * @returns {number}
+ */
+NetEncoding.JSONEncodedArray.prototype.data = function(index) {
+  var offset = this.bb.__offset(this.bb_pos, 4);
+  return offset ? this.bb.readUint8(this.bb.__vector(this.bb_pos + offset) + index) : 0;
+};
+
+/**
+ * @returns {number}
+ */
+NetEncoding.JSONEncodedArray.prototype.dataLength = function() {
+  var offset = this.bb.__offset(this.bb_pos, 4);
+  return offset ? this.bb.__vector_len(this.bb_pos + offset) : 0;
+};
+
+/**
+ * @returns {Uint8Array}
+ */
+NetEncoding.JSONEncodedArray.prototype.dataArray = function() {
+  var offset = this.bb.__offset(this.bb_pos, 4);
+  return offset ? new Uint8Array(this.bb.bytes().buffer, this.bb.bytes().byteOffset + this.bb.__vector(this.bb_pos + offset), this.bb.__vector_len(this.bb_pos + offset)) : null;
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ */
+NetEncoding.JSONEncodedArray.startJSONEncodedArray = function(builder) {
+  builder.startObject(1);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {flatbuffers.Offset} dataOffset
+ */
+NetEncoding.JSONEncodedArray.addData = function(builder, dataOffset) {
+  builder.addFieldOffset(0, dataOffset, 0);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {Array.<number>} data
+ * @returns {flatbuffers.Offset}
+ */
+NetEncoding.JSONEncodedArray.createDataVector = function(builder, data) {
+  builder.startVector(1, data.length, 1);
+  for (var i = data.length - 1; i >= 0; i--) {
+    builder.addInt8(data[i]);
+  }
+  return builder.endVector();
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {number} numElems
+ */
+NetEncoding.JSONEncodedArray.startDataVector = function(builder, numElems) {
+  builder.startVector(1, numElems, 1);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @returns {flatbuffers.Offset}
+ */
+NetEncoding.JSONEncodedArray.endJSONEncodedArray = function(builder) {
+  var offset = builder.endObject();
+  return offset;
+};
+
+/**
+ * @constructor
+ */
+NetEncoding.Column = function() {
+  /**
+   * @type {flatbuffers.ByteBuffer}
+   */
+  this.bb = null;
+
+  /**
+   * @type {number}
+   */
+  this.bb_pos = 0;
+};
+
+/**
+ * @param {number} i
+ * @param {flatbuffers.ByteBuffer} bb
+ * @returns {NetEncoding.Column}
+ */
+NetEncoding.Column.prototype.__init = function(i, bb) {
+  this.bb_pos = i;
+  this.bb = bb;
+  return this;
+};
+
+/**
+ * @param {flatbuffers.ByteBuffer} bb
+ * @param {NetEncoding.Column=} obj
+ * @returns {NetEncoding.Column}
+ */
+NetEncoding.Column.getRootAsColumn = function(bb, obj) {
+  return (obj || new NetEncoding.Column).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+};
+
+/**
+ * @returns {NetEncoding.TypedArray}
+ */
+NetEncoding.Column.prototype.uType = function() {
+  var offset = this.bb.__offset(this.bb_pos, 4);
+  return offset ? /** @type {NetEncoding.TypedArray} */ (this.bb.readUint8(this.bb_pos + offset)) : NetEncoding.TypedArray.NONE;
+};
+
+/**
+ * @param {flatbuffers.Table} obj
+ * @returns {?flatbuffers.Table}
+ */
+NetEncoding.Column.prototype.u = function(obj) {
+  var offset = this.bb.__offset(this.bb_pos, 6);
+  return offset ? this.bb.__union(obj, this.bb_pos + offset) : null;
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ */
+NetEncoding.Column.startColumn = function(builder) {
+  builder.startObject(2);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {NetEncoding.TypedArray} uType
+ */
+NetEncoding.Column.addUType = function(builder, uType) {
+  builder.addFieldInt8(0, uType, NetEncoding.TypedArray.NONE);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {flatbuffers.Offset} uOffset
+ */
+NetEncoding.Column.addU = function(builder, uOffset) {
+  builder.addFieldOffset(1, uOffset, 0);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @returns {flatbuffers.Offset}
+ */
+NetEncoding.Column.endColumn = function(builder) {
+  var offset = builder.endObject();
+  return offset;
+};
+
+/**
+ * @constructor
+ */
+NetEncoding.Matrix = function() {
+  /**
+   * @type {flatbuffers.ByteBuffer}
+   */
+  this.bb = null;
+
+  /**
+   * @type {number}
+   */
+  this.bb_pos = 0;
+};
+
+/**
+ * @param {number} i
+ * @param {flatbuffers.ByteBuffer} bb
+ * @returns {NetEncoding.Matrix}
+ */
+NetEncoding.Matrix.prototype.__init = function(i, bb) {
+  this.bb_pos = i;
+  this.bb = bb;
+  return this;
+};
+
+/**
+ * @param {flatbuffers.ByteBuffer} bb
+ * @param {NetEncoding.Matrix=} obj
+ * @returns {NetEncoding.Matrix}
+ */
+NetEncoding.Matrix.getRootAsMatrix = function(bb, obj) {
+  return (obj || new NetEncoding.Matrix).__init(bb.readInt32(bb.position()) + bb.position(), bb);
+};
+
+/**
+ * @returns {number}
+ */
+NetEncoding.Matrix.prototype.nRows = function() {
+  var offset = this.bb.__offset(this.bb_pos, 4);
+  return offset ? this.bb.readUint32(this.bb_pos + offset) : 0;
+};
+
+/**
+ * @returns {number}
+ */
+NetEncoding.Matrix.prototype.nCols = function() {
+  var offset = this.bb.__offset(this.bb_pos, 6);
+  return offset ? this.bb.readUint32(this.bb_pos + offset) : 0;
+};
+
+/**
+ * @param {number} index
+ * @param {NetEncoding.Column=} obj
+ * @returns {NetEncoding.Column}
+ */
+NetEncoding.Matrix.prototype.columns = function(index, obj) {
+  var offset = this.bb.__offset(this.bb_pos, 8);
+  return offset ? (obj || new NetEncoding.Column).__init(this.bb.__indirect(this.bb.__vector(this.bb_pos + offset) + index * 4), this.bb) : null;
+};
+
+/**
+ * @returns {number}
+ */
+NetEncoding.Matrix.prototype.columnsLength = function() {
+  var offset = this.bb.__offset(this.bb_pos, 8);
+  return offset ? this.bb.__vector_len(this.bb_pos + offset) : 0;
+};
+
+/**
+ * @returns {NetEncoding.TypedArray}
+ */
+NetEncoding.Matrix.prototype.colIndexType = function() {
+  var offset = this.bb.__offset(this.bb_pos, 10);
+  return offset ? /** @type {NetEncoding.TypedArray} */ (this.bb.readUint8(this.bb_pos + offset)) : NetEncoding.TypedArray.NONE;
+};
+
+/**
+ * @param {flatbuffers.Table} obj
+ * @returns {?flatbuffers.Table}
+ */
+NetEncoding.Matrix.prototype.colIndex = function(obj) {
+  var offset = this.bb.__offset(this.bb_pos, 12);
+  return offset ? this.bb.__union(obj, this.bb_pos + offset) : null;
+};
+
+/**
+ * @returns {NetEncoding.TypedArray}
+ */
+NetEncoding.Matrix.prototype.rowIndexType = function() {
+  var offset = this.bb.__offset(this.bb_pos, 14);
+  return offset ? /** @type {NetEncoding.TypedArray} */ (this.bb.readUint8(this.bb_pos + offset)) : NetEncoding.TypedArray.NONE;
+};
+
+/**
+ * @param {flatbuffers.Table} obj
+ * @returns {?flatbuffers.Table}
+ */
+NetEncoding.Matrix.prototype.rowIndex = function(obj) {
+  var offset = this.bb.__offset(this.bb_pos, 16);
+  return offset ? this.bb.__union(obj, this.bb_pos + offset) : null;
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ */
+NetEncoding.Matrix.startMatrix = function(builder) {
+  builder.startObject(7);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {number} nRows
+ */
+NetEncoding.Matrix.addNRows = function(builder, nRows) {
+  builder.addFieldInt32(0, nRows, 0);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {number} nCols
+ */
+NetEncoding.Matrix.addNCols = function(builder, nCols) {
+  builder.addFieldInt32(1, nCols, 0);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {flatbuffers.Offset} columnsOffset
+ */
+NetEncoding.Matrix.addColumns = function(builder, columnsOffset) {
+  builder.addFieldOffset(2, columnsOffset, 0);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {Array.<flatbuffers.Offset>} data
+ * @returns {flatbuffers.Offset}
+ */
+NetEncoding.Matrix.createColumnsVector = function(builder, data) {
+  builder.startVector(4, data.length, 4);
+  for (var i = data.length - 1; i >= 0; i--) {
+    builder.addOffset(data[i]);
+  }
+  return builder.endVector();
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {number} numElems
+ */
+NetEncoding.Matrix.startColumnsVector = function(builder, numElems) {
+  builder.startVector(4, numElems, 4);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {NetEncoding.TypedArray} colIndexType
+ */
+NetEncoding.Matrix.addColIndexType = function(builder, colIndexType) {
+  builder.addFieldInt8(3, colIndexType, NetEncoding.TypedArray.NONE);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {flatbuffers.Offset} colIndexOffset
+ */
+NetEncoding.Matrix.addColIndex = function(builder, colIndexOffset) {
+  builder.addFieldOffset(4, colIndexOffset, 0);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {NetEncoding.TypedArray} rowIndexType
+ */
+NetEncoding.Matrix.addRowIndexType = function(builder, rowIndexType) {
+  builder.addFieldInt8(5, rowIndexType, NetEncoding.TypedArray.NONE);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {flatbuffers.Offset} rowIndexOffset
+ */
+NetEncoding.Matrix.addRowIndex = function(builder, rowIndexOffset) {
+  builder.addFieldOffset(6, rowIndexOffset, 0);
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @returns {flatbuffers.Offset}
+ */
+NetEncoding.Matrix.endMatrix = function(builder) {
+  var offset = builder.endObject();
+  return offset;
+};
+
+/**
+ * @param {flatbuffers.Builder} builder
+ * @param {flatbuffers.Offset} offset
+ */
+NetEncoding.Matrix.finishMatrixBuffer = function(builder, offset) {
+  builder.finish(offset);
+};
+
+// Exports for ECMAScript6 Modules
+export {NetEncoding};

client/src/util/stateManager/summarizeAnnotations.js

@@ -1,4 +1,5 @@
 import _ from "lodash";
+import finiteExtent from "../finiteExtent";
 
 /*
 Build and return obs/var summary using any annotation in the schema
@@ -61,16 +62,32 @@ function _summarizeAnnotations(_schema, annotations) {
       const continuous = type === "int32" || type === "float32";
 
       if (continuous) {
-        let min = Number.POSITIVE_INFINITY;
-        let max = Number.NEGATIVE_INFINITY;
+        let min;
+        let max;
+        let nan = 0;
+        let pinf = 0;
+        let ninf = 0;
         for (let r = 0; r < annotations.length; r += 1) {
           const val = Number(annotations[r][name]);
-          min = val < min ? val : min;
-          max = val > max ? val : max;
+          if (Number.isFinite(val)) {
+            if (min === undefined) {
+              min = val;
+              max = val;
+            } else {
+              min = val < min ? val : min;
+              max = val > max ? val : max;
+            }
+          } else if (Number.isNaN(val)) {
+            nan += 1;
+          } else if (val > 0) {
+            pinf += 1;
+          } else {
+            ninf += 1;
+          }
         }
         return {
           categorical: false,
-          range: { min, max }
+          range: { min, max, nan, pinf, ninf }
         };
       }
 

client/src/util/stateManager/universe.js

@@ -1,8 +1,10 @@
 // jshint esversion: 6
 
 import _ from "lodash";
+
 import * as kvCache from "./keyvalcache";
 import summarizeAnnotations from "./summarizeAnnotations";
+import decodeMatrixFBS from "./matrix";
 
 /*
 Private helper function - create and return a template Universe
@@ -92,73 +94,40 @@ function finalize(universe) {
   return universe;
 }
 
-function RESTv02AnnotationsResponseToInternal(response) {
+function RESTv02AnotationsFBSResponseToInternal(arrayBuffer) {
   /*
-  Source per the spec:
-  {
-    names: [
-      'tissue_type', 'sex', 'num_reads', 'clusters'
-    ],
-    data: [
-      [ 0, 'lung', 'F', 39844, 99 ],
-      [ 1, 'heart', 'M', 83, 1 ],
-      [ 49, 'spleen', null, 2, "unknown cluster" ],
-      // [ obsOrVarIndex, value, value, value, value ],
-      // ...
-    ]
-  }
+  Convert a Matrix FBS to our internal format -- row-major array of
+  observations/cells, stored as an object.  Each obs has a key for each
+  annotation, plus __index__ containing its obsIndex.
 
-  Internal (target) format:
+  Example:
   [
     { __index__: 0, tissue_type: "lung", sex: "F", ... },
     ...
   ]
+
+  XXX TODO: we could make use of the columns in building crossfilter
+  dimensions (they have to be recreated).  Future optimization.
   */
-  const { names, data } = response;
-  const keys = ["__index__", ...names];
-  return _(data)
-    .map(obs => _.zipObject(keys, obs))
-    .value();
+  const fbs = decodeMatrixFBS(arrayBuffer);
+  const keys = fbs.colIdx;
+  const result = Array(fbs.nRows);
+  for (let row = 0; row < fbs.nRows; row += 1) {
+    const rec = { __index__: row };
+    for (let col = 0; col < fbs.nCols; col += 1) {
+      rec[keys[col]] = fbs.columns[col][row];
+    }
+    result[row] = rec;
+  }
+  return result;
 }
 
-function RESTv02LayoutResponseToInternal(response) {
-  /*
-  Source per the spec:
-  {
-    layout: {
-      ndims: 2,
-      coordinates: [
-        [ 0, 0.284483, 0.983744 ],
-        [ 1, 0.038844, 0.739444 ],
-        // [ obsOrVarIndex, X_coord, Y_coord ],
-        // ...
-      ]
-    }
-  }
-
-  Target (internal) format:
-  {
-    X: Float32Array(numObs),
-    Y: Float32Array(numObs)
-  }
-  In the same order as obsAnnotations
-  */
-  const { ndims, coordinates } = response.layout;
-  if (ndims !== 2) {
-    throw new Error("Unsupported layout dimensionality");
-  }
-
-  const layout = {
-    X: new Float32Array(coordinates.length),
-    Y: new Float32Array(coordinates.length)
+function RESTv02LayoutFBSResponseToInternal(arrayBuffer) {
+  const fbs = decodeMatrixFBS(arrayBuffer, true);
+  return {
+    X: fbs.columns[0],
+    Y: fbs.columns[1]
   };
-
-  for (let i = 0; i < coordinates.length; i += 1) {
-    const [idx, x, y] = coordinates[i];
-    layout.X[idx] = x;
-    layout.Y[idx] = y;
-  }
-  return layout;
 }
 
 function reconcileSchemaCategoriesWithSummary(universe) {
@@ -192,7 +161,7 @@ export function createUniverseFromRestV02Response(
   schemaResponse,
   annotationsObsResponse,
   annotationsVarResponse,
-  layoutObsResponse
+  layoutFBSResponse
 ) {
   /*
   build & return universe from a REST 0.2 /config, /schema and /annotations/obs response
@@ -209,15 +178,15 @@ export function createUniverseFromRestV02Response(
   universe.nVar = schema.dataframe.nVar;
 
   /* annotations */
-  universe.obsAnnotations = RESTv02AnnotationsResponseToInternal(
+  universe.obsAnnotations = RESTv02AnotationsFBSResponseToInternal(
     annotationsObsResponse
   );
-  universe.varAnnotations = RESTv02AnnotationsResponseToInternal(
+  universe.varAnnotations = RESTv02AnotationsFBSResponseToInternal(
     annotationsVarResponse
   );
 
   /* layout */
-  universe.obsLayout = RESTv02LayoutResponseToInternal(layoutObsResponse);
+  universe.obsLayout = RESTv02LayoutFBSResponseToInternal(layoutFBSResponse);
 
   universe.summary = summarizeAnnotations(
     universe.schema,
@@ -229,32 +198,24 @@ export function createUniverseFromRestV02Response(
   return finalize(universe);
 }
 
-export function convertExpressionRESTv02ToObject(universe, response) {
+export function convertDataFBStoObject(universe, arrayBuffer) {
   /*
-  /data/obs response looks like:
-  {
-    var: [ varIndices fetched ],
-    obs: [
-      [ obsIndex, evalue, ... ],
-      ...
-    ]
-  }
+  /data/var returns a flatbuffer (FBS) as described by cellxgene/fbs/matrix.fbs
 
-  convert expression toa simple Float32Array, and return
-  { geneName: array, geneName: array, ... }
-  NOTE: geneName, not varIndex
+  This routine converts the binary wire encoding into a JS object:
+
+  {
+    gene: Float32Array,
+    ...
+  }
   */
-  const vars = response.var;
-  const { obs } = response;
+  const fbs = decodeMatrixFBS(arrayBuffer);
+  const { colIdx, columns } = fbs;
   const result = {};
-  // XXX TODO: could this use _.unzip and have less code?
-  for (let varIdx = 0; varIdx < vars.length; varIdx += 1) {
-    const gene = universe.varAnnotations[vars[varIdx]].name;
-    const data = new Float32Array(universe.nObs);
-    for (let obsIdx = 0; obsIdx < obs.length; obsIdx += 1) {
-      data[obsIdx] = obs[obsIdx][varIdx + 1];
-    }
-    result[gene] = data;
+
+  for (let c = 0; c < colIdx.length; c += 1) {
+    const gene = universe.varAnnotations[colIdx[c]].name;
+    result[gene] = columns[c];
   }
   return result;
 }