initial bug fixes and test improvements for the matrix refactor (#1503)

* initial bug fixes and test improvements for the matrix refactor

* lint

client/src/util/dataframe/dataframe.js

@@ -1,6 +1,5 @@
 import { IdentityInt32Index, isLabelIndex } from "./labelIndex";
 // weird cross-dependency that we should clean up someday...
-import { sortArray } from "../typedCrossfilter/sort";
 import {
   isTypedArray,
   isArrayOrTypedArray,
@@ -389,7 +388,7 @@ class Dataframe {
     let dstLabels;
     if (!labels) {
       // combine all columns
-      dstLabels = dataframe.colIndex.keys();
+      dstLabels = dataframe.colIndex.labels();
       srcLabels = dstLabels;
     } else if (Array.isArray(labels)) {
       // combine subset of keys with no aliasing
@@ -537,7 +536,12 @@ class Dataframe {
   }
 
   static empty(rowIndex = null, colIndex = null) {
-    return new Dataframe([0, 0], [], rowIndex, colIndex);
+    const dims = [
+      rowIndex ? rowIndex.size() : 0,
+      colIndex ? colIndex.size() : 0,
+    ];
+    if (dims[0] && dims[1]) throw new Error("not an empty dataframe");
+    return new Dataframe(dims, new Array(dims[1]), rowIndex, colIndex);
   }
 
   static create(dims, columnarData) {
@@ -551,97 +555,59 @@ class Dataframe {
     return new Dataframe(dims, columnarData, null, null);
   }
 
-  __subset(rowOffsets, colOffsets, withRowIndex) {
+  __subset(newRowIndex, newColIndex) {
     const dims = [...this.dims];
 
-    const getSortedLabelAndOffsets = (offsets, index) => {
-      /*
-      Given offsets, return both offsets and associated lables,
-      sorted by offset.
-      */
-      if (!offsets) {
-        return [null, null];
+    /* subset columns */
+    let { __columns, colIndex } = this;
+    if (newColIndex) {
+      const colOffsets = this.colIndex.getOffsets(newColIndex.labels());
+      __columns = new Array(colOffsets.length);
+      for (let i = 0, l = colOffsets.length; i < l; i += 1) {
+        __columns[i] = this.__columns[colOffsets[i]];
       }
-      const sortedOffsets = sortArray(offsets);
-      const sortedLabels = new Array(sortedOffsets.length);
-      for (let i = 0, l = sortedOffsets.length; i < l; i += 1) {
-        sortedLabels[i] = index.getLabel(sortedOffsets[i]);
-      }
-      return [sortedLabels, sortedOffsets];
-    };
-
-    let { colIndex } = this;
-    if (colOffsets) {
-      let colLabels;
-      [colLabels, colOffsets] = getSortedLabelAndOffsets(
-        colOffsets,
-        this.colIndex
-      );
+      colIndex = newColIndex;
       dims[1] = colOffsets.length;
-      colIndex = this.colIndex.subsetLabels(colLabels);
     }
 
     let { rowIndex } = this;
-    if (withRowIndex) rowIndex = withRowIndex;
-    if (rowOffsets) {
-      let rowLabels;
-      [rowLabels, rowOffsets] = getSortedLabelAndOffsets(
-        rowOffsets,
-        this.rowIndex
-      );
-      dims[0] = rowLabels.length;
-      if (!withRowIndex) rowIndex = this.rowIndex.subsetLabels(rowLabels);
-    }
-
-    /* subset columns */
-    let columns = this.__columns;
-    if (colOffsets) {
-      columns = new Array(colOffsets.length);
-      for (let i = 0, l = colOffsets.length; i < l; i += 1) {
-        columns[i] = this.__columns[colOffsets[i]];
-      }
-    }
-
-    /* subset rows */
-    if (rowOffsets) {
-      columns = columns.map((col) => {
+    if (newRowIndex) {
+      const rowOffsets = this.rowIndex.getOffsets(newRowIndex.labels());
+      __columns = __columns.map((col) => {
         const newCol = new col.constructor(rowOffsets.length);
         for (let i = 0, l = rowOffsets.length; i < l; i += 1) {
           newCol[i] = col[rowOffsets[i]];
         }
         return newCol;
       });
+      rowIndex = newRowIndex;
+      dims[0] = rowOffsets.length;
     }
 
     if (dims[0] === 0 || dims[1] === 0) return Dataframe.empty();
-    return new Dataframe(dims, columns, rowIndex, colIndex);
+    return new Dataframe(dims, __columns, rowIndex, colIndex);
   }
 
   subset(rowLabels, colLabels = null, withRowIndex = null) {
     /*
     Subset by row/col labels.
 
-    withRowIndex allows assignment of new row index during subset operation.
-    If withRowIndex === null, it will reset the index to identity (offset)
-    indexing.  if withRowIndex is a label index object, it will be used
-    for the new dataframe.
+    withRowIndex allows subset with an index, rather than rowLabels.
+    If withRowIndex is specified, rowLabels is ignored.
     */
-    const toOffsets = (labels, index) => {
-      if (!labels) {
-        return null;
-      }
-      return labels.map((label) => {
-        const off = index.getOffset(label);
-        if (off === undefined) {
-          throw new RangeError(`unknown label: ${label}`);
-        }
-        return off;
-      });
-    };
+    let rowIndex = null;
+    if (withRowIndex) {
+      rowIndex = withRowIndex;
+    } else if (rowLabels) {
+      rowIndex = this.rowIndex.subset(rowLabels);
+    }
 
-    const rowOffsets = toOffsets(rowLabels, this.rowIndex);
-    const colOffsets = toOffsets(colLabels, this.colIndex);
-    return this.__subset(rowOffsets, colOffsets, withRowIndex);
+    let colIndex = null;
+    if (colLabels) {
+      colIndex = this.colIndex.subset(colLabels);
+    }
+
+    return this.__subset(rowIndex, colIndex);
   }
 
   isubset(rowOffsets, colOffsets = null, withRowIndex = null) {
@@ -653,7 +619,19 @@ class Dataframe {
     indexing. If withRowIndex is a label index object, it will be used
     for the new dataframe.
     */
-    return this.__subset(rowOffsets, colOffsets, withRowIndex);
+    let rowIndex = null;
+    if (withRowIndex) {
+      rowIndex = withRowIndex;
+    } else if (rowOffsets) {
+      rowIndex = this.rowIndex.isubset(rowOffsets);
+    }
+
+    let colIndex = null;
+    if (colOffsets) {
+      colIndex = this.colIndex.isubset(colOffsets);
+    }
+
+    return this.__subset(rowIndex, colIndex);
   }
 
   isubsetMask(rowMask, colMask = null, withRowIndex = null) {
@@ -690,7 +668,7 @@ class Dataframe {
     };
     const rowOffsets = toList(rowMask, nRows);
     const colOffsets = toList(colMask, nCols);
-    return this.__subset(rowOffsets, colOffsets, withRowIndex);
+    return this.isubset(rowOffsets, colOffsets, withRowIndex);
   }
 
   /**
@@ -790,7 +768,7 @@ class Dataframe {
     Return true if this is an empty dataframe, ie, has dimensions [0,0]
     */
     const [rows, cols] = this.dims;
-    return rows === 0 && cols === 0;
+    return rows === 0 || cols === 0;
   }
 
   /****

client/src/util/dataframe/index.js

@@ -1,2 +1,7 @@
 export { default as Dataframe } from "./dataframe";
-export { DenseInt32Index, IdentityInt32Index, KeyIndex } from "./labelIndex";
+export {
+  DenseInt32Index,
+  IdentityInt32Index,
+  KeyIndex,
+  isLabelIndex,
+} from "./labelIndex";

client/src/util/dataframe/labelIndex.js

@@ -32,10 +32,10 @@ class IdentityInt32Index {
     this.maxOffset = maxOffset;
   }
 
-  keys() {
+  labels() {
     // memoize
     const k = fillRange(new Int32Array(this.maxOffset));
-    this.keys = function keys() {
+    this.labels = function labels() {
       return k;
     };
     return k;
@@ -47,12 +47,24 @@ class IdentityInt32Index {
     return i;
   }
 
+  // eslint-disable-next-line class-methods-use-this
+  getOffsets(arr) {
+    // labels to offsets
+    return arr;
+  }
+
   // eslint-disable-next-line class-methods-use-this
   getLabel(i) {
     // offset to label
     return i;
   }
 
+  // eslint-disable-next-line class-methods-use-this
+  getLabels(arr) {
+    // offsets to labels
+    return arr;
+  }
+
   size() {
     return this.maxOffset;
   }
@@ -62,6 +74,9 @@ class IdentityInt32Index {
     time/space decision - based on the resulting density
     */
     const [minLabel, maxLabel] = extent(labelArray);
+    if (minLabel === 0 && maxLabel === labelArray.length - 1)
+      return new IdentityInt32Index(labelArray.length);
+
     const labelSpaceSize = maxLabel - minLabel + 1;
     const density = labelSpaceSize / this.maxOffset;
     /* 0.1 is a magic number, that needs testing to optimize */
@@ -71,30 +86,43 @@ class IdentityInt32Index {
     return new DenseInt32Index(labelArray, [minLabel, maxLabel]);
   }
 
-  subsetLabels(labelArray) {
-    return this.__promote(labelArray);
+  subset(labels) {
+    /* validate subset */
+    const { maxOffset } = this;
+    for (let i = 0, l = labels.length; i < l; i += 1) {
+      const label = labels[i];
+      if (label < 0 || label >= maxOffset)
+        throw new RangeError(`offset or label: ${label}`);
+    }
+    return this.__promote(labels);
+  }
+
+  /* identity index - labels are offsets */
+  isubset(offsets) {
+    return this.subset(offsets);
   }
 
   withLabel(label) {
     if (label === this.maxOffset) {
       return new IdentityInt32Index(label + 1);
     }
-    return this.__promote([...this.keys(), label]);
+    return this.__promote([...this.labels(), label]);
   }
 
   withLabels(labels) {
-    return this.__promote([...this.keys(), ...labels]);
+    return this.__promote([...this.labels(), ...labels]);
   }
 
   dropLabel(label) {
     if (label === this.maxOffset - 1) {
       return new IdentityInt32Index(label);
     }
-    const labelArray = [...this.keys()];
+    const labelArray = [...this.labels()];
     labelArray.splice(labelArray.indexOf(label), 1);
     return this.__promote(labelArray);
   }
 }
+
 class DenseInt32Index {
   /*
   DenseInt32Index indexes integer labels, and uses Int32Array typed arrays
@@ -129,12 +157,29 @@ class DenseInt32Index {
     this.getOffset = function getOffset(l) {
       return index[l - minLabel];
     };
+
+    this.getOffsets = function getOffsets(arr) {
+      const res = new arr.constructor(arr.length);
+      for (let i = 0, len = arr.length; i < len; i += 1) {
+        res[i] = index[arr[i] - minLabel];
+      }
+      return res;
+    };
+
     this.getLabel = function getLabel(i) {
       return rindex[i];
     };
+
+    this.getLabels = function getLabels(arr) {
+      const res = new arr.constructor(arr.length);
+      for (let i = 0, len = arr.length; i < len; i += 1) {
+        res[i] = rindex[arr[i]];
+      }
+      return res;
+    };
   }
 
-  keys() {
+  labels() {
     return this.rindex;
   }
 
@@ -158,20 +203,44 @@ class DenseInt32Index {
     return new DenseInt32Index(labelArray, [minLabel, maxLabel]);
   }
 
-  subsetLabels(labelArray) {
-    return this.__promote(labelArray);
+  subset(labels) {
+    /* validate subset */
+    for (let i = 0, l = labels.length; i < l; i += 1) {
+      const label = labels[i];
+      const offset = this.getOffset(label);
+      if (offset === undefined || offset === -1)
+        throw new RangeError(`unknown label: ${label}`);
+    }
+
+    return this.__promote(labels);
+  }
+
+  // eslint-disable-next-line class-methods-use-this
+  isubset(offsets) {
+    /* validate subset */
+    const { rindex } = this;
+    const maxOffset = rindex.length;
+    const labels = new Int32Array(offsets.length);
+    for (let i = 0, l = offsets.length; i < l; i += 1) {
+      const offset = offsets[i];
+      if (offset < 0 || offset >= maxOffset)
+        throw new RangeError(`out of bounds offset: ${offset}`);
+      labels[i] = rindex[offset];
+    }
+
+    return this.__promote(labels);
   }
 
   withLabel(label) {
-    return this.__promote([...this.keys(), label]);
+    return this.__promote([...this.labels(), label]);
   }
 
   withLabels(labels) {
-    return this.__promote([...this.keys(), ...labels]);
+    return this.__promote([...this.labels(), ...labels]);
   }
 
   dropLabel(label) {
-    const labelArray = [...this.keys()];
+    const labelArray = [...this.labels()];
     labelArray.splice(labelArray.indexOf(label), 1);
     return this.__promote(labelArray);
   }
@@ -207,12 +276,29 @@ class KeyIndex {
     this.getOffset = function getOffset(k) {
       return index.get(k);
     };
+
+    this.getOffsets = function getOffsets(arr) {
+      const res = new arr.constructor(arr.length);
+      for (let i = 0, len = arr.length; i < len; i += 1) {
+        res[i] = index.get(arr[i]);
+      }
+      return res;
+    };
+
     this.getLabel = function getLabel(i) {
       return rindex[i];
     };
+
+    this.getLabels = function getLabels(arr) {
+      const res = new arr.constructor(arr.length);
+      for (let i = 0, len = arr.length; i < len; i += 1) {
+        res[i] = rindex[arr[i]];
+      }
+      return res;
+    };
   }
 
-  keys() {
+  labels() {
     return this.rindex;
   }
 
@@ -220,9 +306,30 @@ class KeyIndex {
     return this.rindex.length;
   }
 
+  subset(labels) {
+    /* validate subset */
+    for (let i = 0, l = labels.length; i < l; i += 1) {
+      const label = labels[i];
+      const offset = this.getOffset(label);
+      if (offset === undefined) throw new RangeError(`unknown label: ${label}`);
+    }
+
+    return new KeyIndex(labels);
+  }
+
   // eslint-disable-next-line class-methods-use-this
-  subsetLabels(labelArray) {
-    return new KeyIndex(labelArray);
+  isubset(offsets) {
+    const { rindex } = this;
+    const maxOffset = rindex.length;
+    const labels = new Array(offsets.length);
+    for (let i = 0, l = offsets.length; i < l; i += 1) {
+      const offset = offsets[i];
+      if (offset < 0 || offset >= maxOffset)
+        throw new RangeError(`out of bounds offset: ${offset}`);
+      labels[i] = rindex[offset];
+    }
+
+    return new KeyIndex(labels);
   }
 
   withLabel(label) {