binary wire format with flatbuffers (#509)

* first flatbuffer schema

* do not lint auto-generated files

* add flatbuffers package

* add flatbuffer module

* wire up /data/X/T route

* use flatbuffers for matrix data fetc

* clarity and comments

* add flatbuffer layout route

* clean up obsolete code

* fix tests

* move flake8 config to setup.cfg

* add comments

* lint

* rework layout routes for fbs

* add more type support to fbs

* lint

* add flatbuffer support for annotations

* function name improvements

* fix botched merge with master

* remove unused import

* route cleanup for flatbuffers

* rename function for clarity

* add missing globals to Jest tests

* fix client JS tests

* fix routes for Python tests

* comments for clarity

* non-finite floating point hardening

* more non-finite number handling

* lint

* fix tests for summarizeAnnotations

* harden diffexp calculation against FP errors

* cleanup unused code

* lint

* add encoding tests for flatbuffers

* application type specified as strings

* fix spelling error

* improve variable names

* add note about documentation gap

* rename FBS DataFrame to Matrix

client/src/actions/index.js

@@ -5,6 +5,7 @@ import { Universe, kvCache } from "../util/stateManager";
 import {
   catchErrorsWrap,
   doJsonRequest,
+  doBinaryRequest,
   rangeEncodeIndices,
   dispatchNetworkErrorMessageToUser
 } from "../util/actionHelpers";
@@ -13,24 +14,28 @@ import {
 Bootstrap application with the initial data loading.
   * /config - application configuration
   * /schema - schema of dataframe
-  * /annotations/obs - all metadata annotation
+  * /annotations - all metadata annotation
+  * /layout - all default layout
 */
 const doInitialDataLoad = () =>
   catchErrorsWrap(async dispatch => {
     dispatch({ type: "initial data load start" });
 
     try {
-      const requests = _([
-        "config",
-        "schema",
+      const requestJson = _(["config", "schema"])
+        .map(r => `${globals.API.prefix}${globals.API.version}${r}`)
+        .map(url => doJsonRequest(url))
+        .value();
+      const requestBinary = _([
         "annotations/obs",
         "annotations/var?annotation-name=name",
         "layout/obs"
       ])
         .map(r => `${globals.API.prefix}${globals.API.version}${r}`)
-        .map(url => doJsonRequest(url))
+        .map(url => doBinaryRequest(url))
         .value();
-      const results = await Promise.all(requests);
+
+      const results = await Promise.all(_.concat(requestJson, requestBinary));
 
       /* set config defaults */
       const config = { ...globals.configDefaults, ...results[0].config };
@@ -87,6 +92,38 @@ needs expression data.
 Transparently utilizes cached data if it is already present.
 */
 async function _doRequestExpressionData(dispatch, getState, genes) {
+  /* helper for this function only */
+  const fetchData = async geneNames => {
+    const res = await fetch(
+      `${globals.API.prefix}${globals.API.version}data/var`,
+      {
+        method: "PUT",
+        body: JSON.stringify({
+          filter: {
+            var: {
+              annotation_value: [{ name: "name", values: geneNames }]
+            }
+          }
+        }),
+        headers: new Headers({
+          accept: "application/octet-stream",
+          "Content-Type": "application/json"
+        })
+      }
+    );
+
+    if (
+      !res.ok ||
+      res.headers.get("Content-Type") !== "application/octet-stream"
+    ) {
+      // WILL throw
+      return dispatchExpressionErrors(dispatch, res);
+    }
+
+    const data = await res.arrayBuffer();
+    return Universe.convertDataFBStoObject(universe, data);
+  };
+
   const state = getState();
   const { universe } = state.controls;
   /* preload data already in cache */
@@ -108,35 +145,10 @@ async function _doRequestExpressionData(dispatch, getState, genes) {
   /* Fetch data for any genes not in cache */
   if (genesToFetch.length) {
     try {
-      // XXX: TODO - this could be using /data/var rather than /data/obs,
-      // as that would simplify the transformation in convertExpressionRESTv02ToObject
-      const res = await fetch(
-        `${globals.API.prefix}${globals.API.version}data/obs`,
-        {
-          method: "PUT",
-          body: JSON.stringify({
-            filter: {
-              var: {
-                annotation_value: [{ name: "name", values: genesToFetch }]
-              }
-            }
-          }),
-          headers: new Headers({
-            accept: "application/json",
-            "Content-Type": "application/json"
-          })
-        }
-      );
-
-      if (!res.ok || res.headers.get("Content-Type") !== "application/json") {
-        // WILL throw
-        return dispatchExpressionErrors(dispatch, res);
-      }
-
-      const data = await res.json();
+      const newExpressionData = await fetchData(genesToFetch);
       expressionData = {
         ...expressionData,
-        ...Universe.convertExpressionRESTv02ToObject(universe, data)
+        ...newExpressionData
       };
     } catch (error) {
       dispatch({ type: "expression load error", error });