initial bug fixes and test improvements for the matrix refactor (#1503)

* initial bug fixes and test improvements for the matrix refactor

* lint

client/src/util/stateManager/annotationsHelpers.js

@@ -100,7 +100,7 @@ export function setLabelByValue(df, colName, fromLabel, toLabel) {
   /* 
 	in the dataframe column `colName`, set any value of `fromLabel` to `toLabel`
 	*/
-  const keys = df.colIndex.keys();
+  const keys = df.colIndex.labels();
   const ndf = df.mapColumns((col, colIdx) => {
     if (colName !== keys[colIdx]) return col;
 
@@ -118,7 +118,7 @@ export function setLabelByMask(df, colName, mask, label) {
   /* 
 	in the dataframe column `colName`, set the masked rows to 'label'
 	*/
-  const keys = df.colIndex.keys();
+  const keys = df.colIndex.labels();
   const ndf = df.mapColumns((col, colIdx) => {
     if (colName !== keys[colIdx]) return col;
 

client/src/util/stateManager/controlsHelpers.js

@@ -187,7 +187,7 @@ export function pruneVarDataCache(varData, needed) {
   if (numOverWatermark <= 0) return varData;
 
   const { colIndex } = varData;
-  const all = colIndex.keys();
+  const all = colIndex.labels();
   const unused = _.difference(all, needed);
   if (unused.length > 0) {
     // sort by offset in the dataframe - ie, psuedo-LRU

client/src/util/stateManager/matrix.js

@@ -1,7 +1,12 @@
 import { flatbuffers } from "flatbuffers";
 import { NetEncoding } from "./matrix_generated";
-import { isTypedArray } from "../typeHelpers";
-import { IdentityInt32Index, DenseInt32Index, KeyIndex } from "../dataframe";
+import { isTypedArray, isFpTypedArray } from "../typeHelpers";
+import {
+  Dataframe,
+  IdentityInt32Index,
+  DenseInt32Index,
+  KeyIndex,
+} from "../dataframe";
 
 const utf8Decoder = new TextDecoder("utf-8");
 
@@ -133,14 +138,14 @@ export function encodeMatrixFBS(df) {
         encColIndex = encodeTypedArray(
           builder,
           encColIndexUType,
-          df.colIndex.keys()
+          df.colIndex.labels()
         );
       } else if (colIndexType === KeyIndex) {
         encColIndexUType = NetEncoding.TypedArray.JSONEncodedArray;
         encColIndex = encodeTypedArray(
           builder,
           encColIndexUType,
-          utf8Encoder.encode(JSON.stringify(df.colIndex.keys()))
+          utf8Encoder.encode(JSON.stringify(df.colIndex.labels()))
         );
       } else {
         throw new Error("Index type FBS encoding unsupported");
@@ -162,3 +167,79 @@ export function encodeMatrixFBS(df) {
   builder.finish(root);
   return builder.asUint8Array();
 }
+
+function promoteTypedArray(o) {
+  /*
+  Decide what internal data type to use for the data returned from 
+  the server.
+
+  TODO - future optimization: not all int32/uint32 data series require
+  promotion to float64.  We COULD simply look at the data to decide. 
+  */
+  if (isFpTypedArray(o) || Array.isArray(o)) return o;
+
+  let TyepdArrayCtor;
+  switch (o.constructor) {
+    case Int8Array:
+    case Uint8Array:
+    case Uint8ClampedArray:
+    case Int16Array:
+    case Uint16Array:
+      TyepdArrayCtor = Float32Array;
+      break;
+
+    case Int32Array:
+    case Uint32Array:
+      TyepdArrayCtor = Float64Array;
+      break;
+
+    default:
+      throw new Error("Unexpected data type returned from server.");
+  }
+  if (o.constructor === TyepdArrayCtor) return o;
+  return new TyepdArrayCtor(o);
+}
+
+export function matrixFBSToDataframe(arrayBuffers) {
+  /*
+  Convert array of Matrix FBS to a Dataframe.
+
+  The application has strong assumptions that all scalar data will be
+  stored as a float32 or float64 (regardless of underlying data types).
+  For example, clipping of value ranges (eg, user-selected percentiles)
+  depends on the ability to use NaN in any numeric type.
+
+  All float data from the server is left as is.  All non-float is promoted
+  to an appropriate float.
+  */
+  if (!Array.isArray(arrayBuffers)) {
+    arrayBuffers = [arrayBuffers];
+  }
+  if (arrayBuffers.length === 0) {
+    return Dataframe.Dataframe.empty();
+  }
+
+  const fbs = arrayBuffers.map((ab) => decodeMatrixFBS(ab, true)); // leave in place
+  /* check that all FBS have same row dimensionality */
+  const { nRows } = fbs[0];
+  fbs.forEach((b) => {
+    if (b.nRows !== nRows)
+      throw new Error("FBS with inconsistent dimensionality");
+  });
+  const columns = fbs
+    .map((fb) =>
+      fb.columns.map((c) => {
+        if (isFpTypedArray(c) || Array.isArray(c)) return c;
+        return promoteTypedArray(c);
+      })
+    )
+    .flat();
+  // colIdx may be TypedArray or Array
+  const colIdx = fbs
+    .map((b) => (Array.isArray(b.colIdx) ? b.colIdx : Array.from(b.colIdx)))
+    .flat();
+  const nCols = columns.length;
+
+  const df = new Dataframe([nRows, nCols], columns, null, new KeyIndex(colIdx));
+  return df;
+}

client/src/util/stateManager/universe.js

@@ -40,84 +40,6 @@ These functions are used exclusively by the actions and reducers to
 build an internal POJO for use by the rendering components.
 */
 
-function promoteTypedArray(o) {
-  /*
-  Decide what internal data type to use for the data returned from 
-  the server.
-
-  TODO - future optimization: not all int32/uint32 data series require
-  promotion to float64.  We COULD simply look at the data to decide. 
-  */
-  if (isFpTypedArray(o) || Array.isArray(o)) return o;
-
-  let TyepdArrayCtor;
-  switch (o.constructor) {
-    case Int8Array:
-    case Uint8Array:
-    case Uint8ClampedArray:
-    case Int16Array:
-    case Uint16Array:
-      TyepdArrayCtor = Float32Array;
-      break;
-
-    case Int32Array:
-    case Uint32Array:
-      TyepdArrayCtor = Float64Array;
-      break;
-
-    default:
-      throw new Error("Unexpected data type returned from server.");
-  }
-  if (o.constructor === TyepdArrayCtor) return o;
-  return new TyepdArrayCtor(o);
-}
-
-export function matrixFBSToDataframe(arrayBuffers) {
-  /*
-  Convert array of Matrix FBS to a Dataframe.
-
-  The application has strong assumptions that all scalar data will be
-  stored as a float32 or float64 (regardless of underlying data types).
-  For example, clipping of value ranges (eg, user-selected percentiles)
-  depends on the ability to use NaN in any numeric type.
-
-  All float data from the server is left as is.  All non-float is promoted
-  to an appropriate float.
-  */
-  if (!Array.isArray(arrayBuffers)) {
-    arrayBuffers = [arrayBuffers];
-  }
-  if (arrayBuffers.length === 0) {
-    return Dataframe.Dataframe.empty();
-  }
-
-  const fbs = arrayBuffers.map((ab) => decodeMatrixFBS(ab, true)); // leave in place
-  /* check that all FBS have same row dimensionality */
-  const { nRows } = fbs[0];
-  fbs.forEach((b) => {
-    if (b.nRows !== nRows)
-      throw new Error("FBS with inconsistent dimensionality");
-  });
-  const columns = fbs
-    .map((fb) =>
-      fb.columns.map((c) => {
-        if (isFpTypedArray(c) || Array.isArray(c)) return c;
-        return promoteTypedArray(c);
-      })
-    )
-    .flat();
-  const colIdx = fbs.map((b) => b.colIdx).flat();
-  const nCols = columns.length;
-
-  const df = new Dataframe.Dataframe(
-    [nRows, nCols],
-    columns,
-    null,
-    new Dataframe.KeyIndex(colIdx)
-  );
-  return df;
-}
-
 export function createUniverseFromResponse(configResponse, schemaResponse) {
   /*
   build & return universe from a REST 0.2 /config, /schema and /annotations/obs response
@@ -178,7 +100,7 @@ export function addObsAnnotations(universe, df) {
 
   // for all of the new data, reconcile with schema and sort categories.
   const dfs = Array.isArray(df) ? df : [df];
-  const keys = dfs.map((d) => d.colIndex.keys()).flat();
+  const keys = dfs.map((d) => d.colIndex.labels()).flat();
   const { schema } = universe;
   keys.forEach((k) => {
     const colSchema = schema.annotations.obsByName[k];

client/src/util/stateManager/world.js

@@ -99,7 +99,7 @@ function clipDataframe(
   if (upperQuantile > 1) upperQuantile = 1;
   if (lowerQuantile === 0 && upperQuantile === 1) return df;
 
-  const keys = df.colIndex.keys();
+  const keys = df.colIndex.labels();
   return df.mapColumns((col, colIdx) => {
     const colLabel = keys[colIdx];
     if (!clipPredicate(df, colIdx, colLabel)) return col;
@@ -277,7 +277,7 @@ export function addObsDimensions(crossfilter, world) {
   but not yet in the crossfilter
   */
   const schema = world.schema.annotations.obsByName;
-  const dimsWeNeed = world.obsAnnotations.colIndex.keys();
+  const dimsWeNeed = world.obsAnnotations.colIndex.labels();
   crossfilter = dimsWeNeed.reduce((xfltr, name) => {
     const dimName = obsAnnoDimensionName(name);
     if (xfltr.hasDimension(dimName)) return xfltr;
@@ -321,7 +321,7 @@ export function getSelectedByIndex(crossfilter) {
   return array of obsIndex, containing all selected obs/cells.
   */
   const selected = crossfilter.allSelectedMask(); // array of bool-ish
-  const keys = crossfilter.data.rowIndex.keys(); // row keys, aka universe rowIndex
+  const keys = crossfilter.data.rowIndex.labels(); // row keys, aka universe rowIndex
 
   const set = new Int32Array(selected.length);
   let numElems = 0;