initial bug fixes and test improvements for the matrix refactor (#1503)

* initial bug fixes and test improvements for the matrix refactor

* lint

client/src/actions/index.js

@@ -29,7 +29,7 @@ async function obsAnnotationFetchAndLoad(dispatch, schema) {
         fetchBinary(
           `annotations/obs?annotation-name=${encodeURIComponent(col.name)}`
         )
-          .then((buffer) => Universe.matrixFBSToDataframe(buffer))
+          .then((buffer) => MatrixFBS.matrixFBSToDataframe(buffer))
           .then((df) =>
             dispatch({
               type: "universe: column load success",
@@ -52,7 +52,7 @@ async function varAnnotationFetchAndLoad(dispatch, schema) {
   return Promise.all(
     names.map((name) =>
       fetchBinary(`annotations/var?annotation-name=${encodeURIComponent(name)}`)
-        .then((buffer) => Universe.matrixFBSToDataframe(buffer))
+        .then((buffer) => MatrixFBS.matrixFBSToDataframe(buffer))
         .then((df) =>
           dispatch({
             type: "universe: column load success",
@@ -77,7 +77,7 @@ function layoutFetchAndLoad(dispatch, schema) {
       plimit.add(() =>
         fetchBinary(
           `layout/obs?layout-name=${encodeURIComponent(e)}`
-        ).then((buffer) => Universe.matrixFBSToDataframe(buffer))
+        ).then((buffer) => MatrixFBS.matrixFBSToDataframe(buffer))
       )
     )
   ).then((dfs) =>

client/src/actions/reembed.js

@@ -1,5 +1,5 @@
 import { API } from "../globals";
-import { Universe } from "../util/stateManager";
+import { MatrixFBS } from "../util/stateManager";
 import {
   postNetworkErrorToast,
   postAsyncSuccessToast,
@@ -24,7 +24,7 @@ function abortableFetch(request, opts, timeout = 0) {
 
 async function doReembedFetch(dispatch, getState) {
   const state = getState();
-  let cells = state.world.obsAnnotations.rowIndex.keys();
+  let cells = state.world.obsAnnotations.rowIndex.labels();
 
   // These lines ensure that we convert any TypedArray to an Array.
   // This is necessary because JSON.stringify() does some very strange
@@ -80,7 +80,7 @@ export function requestReembed() {
       const res = await doReembedFetch(dispatch, getState);
       const schema = JSON.parse(res.headers.get("CxG-Schema"));
       const buffer = await res.arrayBuffer();
-      const df = Universe.matrixFBSToDataframe(buffer);
+      const df = MatrixFBS.matrixFBSToDataframe(buffer);
       dispatch({
         type: "reembed: request completed",
       });

client/src/reducers/categoricalSelection.js

@@ -31,7 +31,7 @@ const CategoricalSelection = (
       const names = CH.selectableCategoryNames(
         world.schema,
         CH.maxCategoryItems(prevSharedState.config),
-        dataframe.colIndex.keys()
+        dataframe.colIndex.labels()
       );
       if (names.length === 0) return state;
       return {

client/src/reducers/world.js

@@ -93,7 +93,7 @@ const WorldReducer = (
           let worldValSlice = val;
           if (!World.worldEqUniverse(state, universe)) {
             worldValSlice = universeVarData
-              .subset(state.obsAnnotations.rowIndex.keys(), [key], null)
+              .subset(state.obsAnnotations.rowIndex.labels(), [key], null)
               .icol(0)
               .asArray();
           }
@@ -129,10 +129,10 @@ const WorldReducer = (
       //
       let clippedVarData = state.varData;
       const keysToDrop = clippedVarData.colIndex
-        .keys()
+        .labels()
         .filter((k) => !unclippedVarData.hasCol(k));
       const keysToAdd = unclippedVarData.colIndex
-        .keys()
+        .labels()
         .filter((k) => !clippedVarData.hasCol(k));
       keysToDrop.forEach((k) => {
         clippedVarData = clippedVarData.dropCol(k);
@@ -171,7 +171,7 @@ const WorldReducer = (
       let newAnnotation = null;
       if (!World.worldEqUniverse(state, universe)) {
         newAnnotation = universe.obsAnnotations
-          .subset(state.obsAnnotations.rowIndex.keys(), [name], null)
+          .subset(state.obsAnnotations.rowIndex.labels(), [name], null)
           .icol(0)
           .asArray();
       } else {
@@ -303,7 +303,7 @@ const WorldReducer = (
       let schema = origSchema;
 
       // alias the names the server sent us, in case they were not the same as the schema
-      const embedingLabels = embedding.colIndex.keys();
+      const embedingLabels = embedding.colIndex.labels();
       const labels = {
         [embedingLabels[0]]: dims[0],
         [embedingLabels[1]]: dims[1],

client/src/util/dataframe/dataframe.js

@@ -1,6 +1,5 @@
 import { IdentityInt32Index, isLabelIndex } from "./labelIndex";
 // weird cross-dependency that we should clean up someday...
-import { sortArray } from "../typedCrossfilter/sort";
 import {
   isTypedArray,
   isArrayOrTypedArray,
@@ -389,7 +388,7 @@ class Dataframe {
     let dstLabels;
     if (!labels) {
       // combine all columns
-      dstLabels = dataframe.colIndex.keys();
+      dstLabels = dataframe.colIndex.labels();
       srcLabels = dstLabels;
     } else if (Array.isArray(labels)) {
       // combine subset of keys with no aliasing
@@ -537,7 +536,12 @@ class Dataframe {
   }
 
   static empty(rowIndex = null, colIndex = null) {
-    return new Dataframe([0, 0], [], rowIndex, colIndex);
+    const dims = [
+      rowIndex ? rowIndex.size() : 0,
+      colIndex ? colIndex.size() : 0,
+    ];
+    if (dims[0] && dims[1]) throw new Error("not an empty dataframe");
+    return new Dataframe(dims, new Array(dims[1]), rowIndex, colIndex);
   }
 
   static create(dims, columnarData) {
@@ -551,97 +555,59 @@ class Dataframe {
     return new Dataframe(dims, columnarData, null, null);
   }
 
-  __subset(rowOffsets, colOffsets, withRowIndex) {
+  __subset(newRowIndex, newColIndex) {
     const dims = [...this.dims];
 
-    const getSortedLabelAndOffsets = (offsets, index) => {
-      /*
-      Given offsets, return both offsets and associated lables,
-      sorted by offset.
-      */
-      if (!offsets) {
-        return [null, null];
+    /* subset columns */
+    let { __columns, colIndex } = this;
+    if (newColIndex) {
+      const colOffsets = this.colIndex.getOffsets(newColIndex.labels());
+      __columns = new Array(colOffsets.length);
+      for (let i = 0, l = colOffsets.length; i < l; i += 1) {
+        __columns[i] = this.__columns[colOffsets[i]];
       }
-      const sortedOffsets = sortArray(offsets);
-      const sortedLabels = new Array(sortedOffsets.length);
-      for (let i = 0, l = sortedOffsets.length; i < l; i += 1) {
-        sortedLabels[i] = index.getLabel(sortedOffsets[i]);
-      }
-      return [sortedLabels, sortedOffsets];
-    };
-
-    let { colIndex } = this;
-    if (colOffsets) {
-      let colLabels;
-      [colLabels, colOffsets] = getSortedLabelAndOffsets(
-        colOffsets,
-        this.colIndex
-      );
+      colIndex = newColIndex;
       dims[1] = colOffsets.length;
-      colIndex = this.colIndex.subsetLabels(colLabels);
     }
 
     let { rowIndex } = this;
-    if (withRowIndex) rowIndex = withRowIndex;
-    if (rowOffsets) {
-      let rowLabels;
-      [rowLabels, rowOffsets] = getSortedLabelAndOffsets(
-        rowOffsets,
-        this.rowIndex
-      );
-      dims[0] = rowLabels.length;
-      if (!withRowIndex) rowIndex = this.rowIndex.subsetLabels(rowLabels);
-    }
-
-    /* subset columns */
-    let columns = this.__columns;
-    if (colOffsets) {
-      columns = new Array(colOffsets.length);
-      for (let i = 0, l = colOffsets.length; i < l; i += 1) {
-        columns[i] = this.__columns[colOffsets[i]];
-      }
-    }
-
-    /* subset rows */
-    if (rowOffsets) {
-      columns = columns.map((col) => {
+    if (newRowIndex) {
+      const rowOffsets = this.rowIndex.getOffsets(newRowIndex.labels());
+      __columns = __columns.map((col) => {
         const newCol = new col.constructor(rowOffsets.length);
         for (let i = 0, l = rowOffsets.length; i < l; i += 1) {
           newCol[i] = col[rowOffsets[i]];
         }
         return newCol;
       });
+      rowIndex = newRowIndex;
+      dims[0] = rowOffsets.length;
     }
 
     if (dims[0] === 0 || dims[1] === 0) return Dataframe.empty();
-    return new Dataframe(dims, columns, rowIndex, colIndex);
+    return new Dataframe(dims, __columns, rowIndex, colIndex);
   }
 
   subset(rowLabels, colLabels = null, withRowIndex = null) {
     /*
     Subset by row/col labels.
 
-    withRowIndex allows assignment of new row index during subset operation.
-    If withRowIndex === null, it will reset the index to identity (offset)
-    indexing.  if withRowIndex is a label index object, it will be used
-    for the new dataframe.
+    withRowIndex allows subset with an index, rather than rowLabels.
+    If withRowIndex is specified, rowLabels is ignored.
     */
-    const toOffsets = (labels, index) => {
-      if (!labels) {
-        return null;
-      }
-      return labels.map((label) => {
-        const off = index.getOffset(label);
-        if (off === undefined) {
-          throw new RangeError(`unknown label: ${label}`);
-        }
-        return off;
-      });
-    };
+    let rowIndex = null;
+    if (withRowIndex) {
+      rowIndex = withRowIndex;
+    } else if (rowLabels) {
+      rowIndex = this.rowIndex.subset(rowLabels);
+    }
 
-    const rowOffsets = toOffsets(rowLabels, this.rowIndex);
-    const colOffsets = toOffsets(colLabels, this.colIndex);
-    return this.__subset(rowOffsets, colOffsets, withRowIndex);
+    let colIndex = null;
+    if (colLabels) {
+      colIndex = this.colIndex.subset(colLabels);
+    }
+
+    return this.__subset(rowIndex, colIndex);
   }
 
   isubset(rowOffsets, colOffsets = null, withRowIndex = null) {
@@ -653,7 +619,19 @@ class Dataframe {
     indexing. If withRowIndex is a label index object, it will be used
     for the new dataframe.
     */
-    return this.__subset(rowOffsets, colOffsets, withRowIndex);
+    let rowIndex = null;
+    if (withRowIndex) {
+      rowIndex = withRowIndex;
+    } else if (rowOffsets) {
+      rowIndex = this.rowIndex.isubset(rowOffsets);
+    }
+
+    let colIndex = null;
+    if (colOffsets) {
+      colIndex = this.colIndex.isubset(colOffsets);
+    }
+
+    return this.__subset(rowIndex, colIndex);
   }
 
   isubsetMask(rowMask, colMask = null, withRowIndex = null) {
@@ -690,7 +668,7 @@ class Dataframe {
     };
     const rowOffsets = toList(rowMask, nRows);
     const colOffsets = toList(colMask, nCols);
-    return this.__subset(rowOffsets, colOffsets, withRowIndex);
+    return this.isubset(rowOffsets, colOffsets, withRowIndex);
   }
 
   /**
@@ -790,7 +768,7 @@ class Dataframe {
     Return true if this is an empty dataframe, ie, has dimensions [0,0]
     */
     const [rows, cols] = this.dims;
-    return rows === 0 && cols === 0;
+    return rows === 0 || cols === 0;
   }
 
   /****

client/src/util/dataframe/index.js

@@ -1,2 +1,7 @@
 export { default as Dataframe } from "./dataframe";
-export { DenseInt32Index, IdentityInt32Index, KeyIndex } from "./labelIndex";
+export {
+  DenseInt32Index,
+  IdentityInt32Index,
+  KeyIndex,
+  isLabelIndex,
+} from "./labelIndex";

client/src/util/dataframe/labelIndex.js

@@ -32,10 +32,10 @@ class IdentityInt32Index {
     this.maxOffset = maxOffset;
   }
 
-  keys() {
+  labels() {
     // memoize
     const k = fillRange(new Int32Array(this.maxOffset));
-    this.keys = function keys() {
+    this.labels = function labels() {
       return k;
     };
     return k;
@@ -47,12 +47,24 @@ class IdentityInt32Index {
     return i;
   }
 
+  // eslint-disable-next-line class-methods-use-this
+  getOffsets(arr) {
+    // labels to offsets
+    return arr;
+  }
+
   // eslint-disable-next-line class-methods-use-this
   getLabel(i) {
     // offset to label
     return i;
   }
 
+  // eslint-disable-next-line class-methods-use-this
+  getLabels(arr) {
+    // offsets to labels
+    return arr;
+  }
+
   size() {
     return this.maxOffset;
   }
@@ -62,6 +74,9 @@ class IdentityInt32Index {
     time/space decision - based on the resulting density
     */
     const [minLabel, maxLabel] = extent(labelArray);
+    if (minLabel === 0 && maxLabel === labelArray.length - 1)
+      return new IdentityInt32Index(labelArray.length);
+
     const labelSpaceSize = maxLabel - minLabel + 1;
     const density = labelSpaceSize / this.maxOffset;
     /* 0.1 is a magic number, that needs testing to optimize */
@@ -71,30 +86,43 @@ class IdentityInt32Index {
     return new DenseInt32Index(labelArray, [minLabel, maxLabel]);
   }
 
-  subsetLabels(labelArray) {
-    return this.__promote(labelArray);
+  subset(labels) {
+    /* validate subset */
+    const { maxOffset } = this;
+    for (let i = 0, l = labels.length; i < l; i += 1) {
+      const label = labels[i];
+      if (label < 0 || label >= maxOffset)
+        throw new RangeError(`offset or label: ${label}`);
+    }
+    return this.__promote(labels);
+  }
+
+  /* identity index - labels are offsets */
+  isubset(offsets) {
+    return this.subset(offsets);
   }
 
   withLabel(label) {
     if (label === this.maxOffset) {
       return new IdentityInt32Index(label + 1);
     }
-    return this.__promote([...this.keys(), label]);
+    return this.__promote([...this.labels(), label]);
   }
 
   withLabels(labels) {
-    return this.__promote([...this.keys(), ...labels]);
+    return this.__promote([...this.labels(), ...labels]);
   }
 
   dropLabel(label) {
     if (label === this.maxOffset - 1) {
       return new IdentityInt32Index(label);
     }
-    const labelArray = [...this.keys()];
+    const labelArray = [...this.labels()];
     labelArray.splice(labelArray.indexOf(label), 1);
     return this.__promote(labelArray);
   }
 }
+
 class DenseInt32Index {
   /*
   DenseInt32Index indexes integer labels, and uses Int32Array typed arrays
@@ -129,12 +157,29 @@ class DenseInt32Index {
     this.getOffset = function getOffset(l) {
       return index[l - minLabel];
     };
+
+    this.getOffsets = function getOffsets(arr) {
+      const res = new arr.constructor(arr.length);
+      for (let i = 0, len = arr.length; i < len; i += 1) {
+        res[i] = index[arr[i] - minLabel];
+      }
+      return res;
+    };
+
     this.getLabel = function getLabel(i) {
       return rindex[i];
     };
+
+    this.getLabels = function getLabels(arr) {
+      const res = new arr.constructor(arr.length);
+      for (let i = 0, len = arr.length; i < len; i += 1) {
+        res[i] = rindex[arr[i]];
+      }
+      return res;
+    };
   }
 
-  keys() {
+  labels() {
     return this.rindex;
   }
 
@@ -158,20 +203,44 @@ class DenseInt32Index {
     return new DenseInt32Index(labelArray, [minLabel, maxLabel]);
   }
 
-  subsetLabels(labelArray) {
-    return this.__promote(labelArray);
+  subset(labels) {
+    /* validate subset */
+    for (let i = 0, l = labels.length; i < l; i += 1) {
+      const label = labels[i];
+      const offset = this.getOffset(label);
+      if (offset === undefined || offset === -1)
+        throw new RangeError(`unknown label: ${label}`);
+    }
+
+    return this.__promote(labels);
+  }
+
+  // eslint-disable-next-line class-methods-use-this
+  isubset(offsets) {
+    /* validate subset */
+    const { rindex } = this;
+    const maxOffset = rindex.length;
+    const labels = new Int32Array(offsets.length);
+    for (let i = 0, l = offsets.length; i < l; i += 1) {
+      const offset = offsets[i];
+      if (offset < 0 || offset >= maxOffset)
+        throw new RangeError(`out of bounds offset: ${offset}`);
+      labels[i] = rindex[offset];
+    }
+
+    return this.__promote(labels);
   }
 
   withLabel(label) {
-    return this.__promote([...this.keys(), label]);
+    return this.__promote([...this.labels(), label]);
   }
 
   withLabels(labels) {
-    return this.__promote([...this.keys(), ...labels]);
+    return this.__promote([...this.labels(), ...labels]);
   }
 
   dropLabel(label) {
-    const labelArray = [...this.keys()];
+    const labelArray = [...this.labels()];
     labelArray.splice(labelArray.indexOf(label), 1);
     return this.__promote(labelArray);
   }
@@ -207,12 +276,29 @@ class KeyIndex {
     this.getOffset = function getOffset(k) {
       return index.get(k);
     };
+
+    this.getOffsets = function getOffsets(arr) {
+      const res = new arr.constructor(arr.length);
+      for (let i = 0, len = arr.length; i < len; i += 1) {
+        res[i] = index.get(arr[i]);
+      }
+      return res;
+    };
+
     this.getLabel = function getLabel(i) {
       return rindex[i];
     };
+
+    this.getLabels = function getLabels(arr) {
+      const res = new arr.constructor(arr.length);
+      for (let i = 0, len = arr.length; i < len; i += 1) {
+        res[i] = rindex[arr[i]];
+      }
+      return res;
+    };
   }
 
-  keys() {
+  labels() {
     return this.rindex;
   }
 
@@ -220,9 +306,30 @@ class KeyIndex {
     return this.rindex.length;
   }
 
+  subset(labels) {
+    /* validate subset */
+    for (let i = 0, l = labels.length; i < l; i += 1) {
+      const label = labels[i];
+      const offset = this.getOffset(label);
+      if (offset === undefined) throw new RangeError(`unknown label: ${label}`);
+    }
+
+    return new KeyIndex(labels);
+  }
+
   // eslint-disable-next-line class-methods-use-this
-  subsetLabels(labelArray) {
-    return new KeyIndex(labelArray);
+  isubset(offsets) {
+    const { rindex } = this;
+    const maxOffset = rindex.length;
+    const labels = new Array(offsets.length);
+    for (let i = 0, l = offsets.length; i < l; i += 1) {
+      const offset = offsets[i];
+      if (offset < 0 || offset >= maxOffset)
+        throw new RangeError(`out of bounds offset: ${offset}`);
+      labels[i] = rindex[offset];
+    }
+
+    return new KeyIndex(labels);
   }
 
   withLabel(label) {

client/src/util/stateManager/annotationsHelpers.js

@@ -100,7 +100,7 @@ export function setLabelByValue(df, colName, fromLabel, toLabel) {
   /* 
 	in the dataframe column `colName`, set any value of `fromLabel` to `toLabel`
 	*/
-  const keys = df.colIndex.keys();
+  const keys = df.colIndex.labels();
   const ndf = df.mapColumns((col, colIdx) => {
     if (colName !== keys[colIdx]) return col;
 
@@ -118,7 +118,7 @@ export function setLabelByMask(df, colName, mask, label) {
   /* 
 	in the dataframe column `colName`, set the masked rows to 'label'
 	*/
-  const keys = df.colIndex.keys();
+  const keys = df.colIndex.labels();
   const ndf = df.mapColumns((col, colIdx) => {
     if (colName !== keys[colIdx]) return col;
 

client/src/util/stateManager/controlsHelpers.js

@@ -187,7 +187,7 @@ export function pruneVarDataCache(varData, needed) {
   if (numOverWatermark <= 0) return varData;
 
   const { colIndex } = varData;
-  const all = colIndex.keys();
+  const all = colIndex.labels();
   const unused = _.difference(all, needed);
   if (unused.length > 0) {
     // sort by offset in the dataframe - ie, psuedo-LRU

client/src/util/stateManager/matrix.js

@@ -1,7 +1,12 @@
 import { flatbuffers } from "flatbuffers";
 import { NetEncoding } from "./matrix_generated";
-import { isTypedArray } from "../typeHelpers";
-import { IdentityInt32Index, DenseInt32Index, KeyIndex } from "../dataframe";
+import { isTypedArray, isFpTypedArray } from "../typeHelpers";
+import {
+  Dataframe,
+  IdentityInt32Index,
+  DenseInt32Index,
+  KeyIndex,
+} from "../dataframe";
 
 const utf8Decoder = new TextDecoder("utf-8");
 
@@ -133,14 +138,14 @@ export function encodeMatrixFBS(df) {
         encColIndex = encodeTypedArray(
           builder,
           encColIndexUType,
-          df.colIndex.keys()
+          df.colIndex.labels()
         );
       } else if (colIndexType === KeyIndex) {
         encColIndexUType = NetEncoding.TypedArray.JSONEncodedArray;
         encColIndex = encodeTypedArray(
           builder,
           encColIndexUType,
-          utf8Encoder.encode(JSON.stringify(df.colIndex.keys()))
+          utf8Encoder.encode(JSON.stringify(df.colIndex.labels()))
         );
       } else {
         throw new Error("Index type FBS encoding unsupported");
@@ -162,3 +167,79 @@ export function encodeMatrixFBS(df) {
   builder.finish(root);
   return builder.asUint8Array();
 }
+
+function promoteTypedArray(o) {
+  /*
+  Decide what internal data type to use for the data returned from 
+  the server.
+
+  TODO - future optimization: not all int32/uint32 data series require
+  promotion to float64.  We COULD simply look at the data to decide. 
+  */
+  if (isFpTypedArray(o) || Array.isArray(o)) return o;
+
+  let TyepdArrayCtor;
+  switch (o.constructor) {
+    case Int8Array:
+    case Uint8Array:
+    case Uint8ClampedArray:
+    case Int16Array:
+    case Uint16Array:
+      TyepdArrayCtor = Float32Array;
+      break;
+
+    case Int32Array:
+    case Uint32Array:
+      TyepdArrayCtor = Float64Array;
+      break;
+
+    default:
+      throw new Error("Unexpected data type returned from server.");
+  }
+  if (o.constructor === TyepdArrayCtor) return o;
+  return new TyepdArrayCtor(o);
+}
+
+export function matrixFBSToDataframe(arrayBuffers) {
+  /*
+  Convert array of Matrix FBS to a Dataframe.
+
+  The application has strong assumptions that all scalar data will be
+  stored as a float32 or float64 (regardless of underlying data types).
+  For example, clipping of value ranges (eg, user-selected percentiles)
+  depends on the ability to use NaN in any numeric type.
+
+  All float data from the server is left as is.  All non-float is promoted
+  to an appropriate float.
+  */
+  if (!Array.isArray(arrayBuffers)) {
+    arrayBuffers = [arrayBuffers];
+  }
+  if (arrayBuffers.length === 0) {
+    return Dataframe.Dataframe.empty();
+  }
+
+  const fbs = arrayBuffers.map((ab) => decodeMatrixFBS(ab, true)); // leave in place
+  /* check that all FBS have same row dimensionality */
+  const { nRows } = fbs[0];
+  fbs.forEach((b) => {
+    if (b.nRows !== nRows)
+      throw new Error("FBS with inconsistent dimensionality");
+  });
+  const columns = fbs
+    .map((fb) =>
+      fb.columns.map((c) => {
+        if (isFpTypedArray(c) || Array.isArray(c)) return c;
+        return promoteTypedArray(c);
+      })
+    )
+    .flat();
+  // colIdx may be TypedArray or Array
+  const colIdx = fbs
+    .map((b) => (Array.isArray(b.colIdx) ? b.colIdx : Array.from(b.colIdx)))
+    .flat();
+  const nCols = columns.length;
+
+  const df = new Dataframe([nRows, nCols], columns, null, new KeyIndex(colIdx));
+  return df;
+}

client/src/util/stateManager/universe.js

@@ -40,84 +40,6 @@ These functions are used exclusively by the actions and reducers to
 build an internal POJO for use by the rendering components.
 */
 
-function promoteTypedArray(o) {
-  /*
-  Decide what internal data type to use for the data returned from 
-  the server.
-
-  TODO - future optimization: not all int32/uint32 data series require
-  promotion to float64.  We COULD simply look at the data to decide. 
-  */
-  if (isFpTypedArray(o) || Array.isArray(o)) return o;
-
-  let TyepdArrayCtor;
-  switch (o.constructor) {
-    case Int8Array:
-    case Uint8Array:
-    case Uint8ClampedArray:
-    case Int16Array:
-    case Uint16Array:
-      TyepdArrayCtor = Float32Array;
-      break;
-
-    case Int32Array:
-    case Uint32Array:
-      TyepdArrayCtor = Float64Array;
-      break;
-
-    default:
-      throw new Error("Unexpected data type returned from server.");
-  }
-  if (o.constructor === TyepdArrayCtor) return o;
-  return new TyepdArrayCtor(o);
-}
-
-export function matrixFBSToDataframe(arrayBuffers) {
-  /*
-  Convert array of Matrix FBS to a Dataframe.
-
-  The application has strong assumptions that all scalar data will be
-  stored as a float32 or float64 (regardless of underlying data types).
-  For example, clipping of value ranges (eg, user-selected percentiles)
-  depends on the ability to use NaN in any numeric type.
-
-  All float data from the server is left as is.  All non-float is promoted
-  to an appropriate float.
-  */
-  if (!Array.isArray(arrayBuffers)) {
-    arrayBuffers = [arrayBuffers];
-  }
-  if (arrayBuffers.length === 0) {
-    return Dataframe.Dataframe.empty();
-  }
-
-  const fbs = arrayBuffers.map((ab) => decodeMatrixFBS(ab, true)); // leave in place
-  /* check that all FBS have same row dimensionality */
-  const { nRows } = fbs[0];
-  fbs.forEach((b) => {
-    if (b.nRows !== nRows)
-      throw new Error("FBS with inconsistent dimensionality");
-  });
-  const columns = fbs
-    .map((fb) =>
-      fb.columns.map((c) => {
-        if (isFpTypedArray(c) || Array.isArray(c)) return c;
-        return promoteTypedArray(c);
-      })
-    )
-    .flat();
-  const colIdx = fbs.map((b) => b.colIdx).flat();
-  const nCols = columns.length;
-
-  const df = new Dataframe.Dataframe(
-    [nRows, nCols],
-    columns,
-    null,
-    new Dataframe.KeyIndex(colIdx)
-  );
-  return df;
-}
-
 export function createUniverseFromResponse(configResponse, schemaResponse) {
   /*
   build & return universe from a REST 0.2 /config, /schema and /annotations/obs response
@@ -178,7 +100,7 @@ export function addObsAnnotations(universe, df) {
 
   // for all of the new data, reconcile with schema and sort categories.
   const dfs = Array.isArray(df) ? df : [df];
-  const keys = dfs.map((d) => d.colIndex.keys()).flat();
+  const keys = dfs.map((d) => d.colIndex.labels()).flat();
   const { schema } = universe;
   keys.forEach((k) => {
     const colSchema = schema.annotations.obsByName[k];

client/src/util/stateManager/world.js

@@ -99,7 +99,7 @@ function clipDataframe(
   if (upperQuantile > 1) upperQuantile = 1;
   if (lowerQuantile === 0 && upperQuantile === 1) return df;
 
-  const keys = df.colIndex.keys();
+  const keys = df.colIndex.labels();
   return df.mapColumns((col, colIdx) => {
     const colLabel = keys[colIdx];
     if (!clipPredicate(df, colIdx, colLabel)) return col;
@@ -277,7 +277,7 @@ export function addObsDimensions(crossfilter, world) {
   but not yet in the crossfilter
   */
   const schema = world.schema.annotations.obsByName;
-  const dimsWeNeed = world.obsAnnotations.colIndex.keys();
+  const dimsWeNeed = world.obsAnnotations.colIndex.labels();
   crossfilter = dimsWeNeed.reduce((xfltr, name) => {
     const dimName = obsAnnoDimensionName(name);
     if (xfltr.hasDimension(dimName)) return xfltr;
@@ -321,7 +321,7 @@ export function getSelectedByIndex(crossfilter) {
   return array of obsIndex, containing all selected obs/cells.
   */
   const selected = crossfilter.allSelectedMask(); // array of bool-ish
-  const keys = crossfilter.data.rowIndex.keys(); // row keys, aka universe rowIndex
+  const keys = crossfilter.data.rowIndex.labels(); // row keys, aka universe rowIndex
 
   const set = new Int32Array(selected.length);
   let numElems = 0;

client/src/util/typedCrossfilter/crossfilter.js

@@ -60,9 +60,7 @@ export default class ImmutableTypedCrossfilter {
   }
 
   setData(data) {
-    const { selectionCache } = this;
-    this.selectionCache = {};
-    return new ImmutableTypedCrossfilter(data, this.dimensions, selectionCache);
+    return new ImmutableTypedCrossfilter(data, this.dimensions);
   }
 
   dimensionNames() {