Dataframe (#576)

* initial dataframe commit * initial dataframe port of core app * rename variables for clarity * remove unused import * comment out unused code * fix array handling bug in crossfilter dimension creation * allow creation of empty dataframes * handle non-existent columns * handle non-existent columns * revise tests for new dataframe * comments for clarity * comments for clarity * generate bulk add placeholder with real gene names * fix bug in gene name adding * more dataframe unit tests * fix bug - subset from current world, not universe * put cut and pasted code into a single function * improve caching of crossfilter * remove cascading update bug from graph * more performance work * improve state handling for scatterplot * performance optimization of critical path * add column summarization * dataframe utils * add callOnceLazy * fix tests * minor updates found during review * fix misspelling * remove RESTv02 from function names * comment cleanup * cut/icut col parameter defaults to null * break up large test * improve tests and comments on dataframe at/has functions
2026-09-27 01:38:11 +08:00 · 2019-02-22 11:31:34 -08:00
parent 57c4e9ff33
commit 6b33315cbe
29 changed files with 1798 additions and 611 deletions
@@ -0,0 +1,465 @@
+import { IdentityInt32Index } from "./labelIndex";
+// weird cross-dependency that we should clean up someday...
+import { sort } from "../typedCrossfilter/sort";
+import { isTypedArray, isArrayOrTypedArray, callOnceLazy } from "./util";
+import { summarizeContinuous, summarizeCategorical } from "./summarize";
+
+/*
+Dataframe is an immutable 2D matrix similiar to Python Pandas Dataframe,
+but (currently) without all of the surrounding support functions.
+Data is stored in column-major layout, and each column is monomorphic.
+
+It supports:
+* Relatively efficient creation, cloning and subsetting ("cut")
+* Very efficient columnar access (eg, sum down a column), and access
+  to the underlying column arrays.
+* Data access by row/col offset or label.  Labels are reasonably well
+  optimized for both numeric lables and arbitrary (eg, sting) labels.
+
+It does not currently support:
+* Views on matrix subset - for currently known access patterns,
+  it is more effiicent to copy on subsetting, optimizing for access
+  speed over memory use.
+* JS iterators - they are too slow.  Use explicit iteration over
+  offest or labels.
+
+Important assumptions embedded in the API:
+* Columns are implicitly categorical if they are a JS Array and numeric
+  (aka continuous) if they are a TypedArray.
+
+There are three index types for row/col indexing:
+* IdentityInt32Index - noop index, where the index label is the offset.
+* KeyIndex - index arbitrary JS objects.
+* DenseInt32Index - integer indexing.  Optimization over KeyIndex as it uses
+  Int32Array as a back-map to offsets.  This means that the index array
+  must be sized to [minLabel, maxLabel), so this is only useful when the label
+  range is relatively close the underlying offset range [minOffset, maxOffset).
+
+All private functions/methods/fields are prefixed by '__', eg, __compile().
+Don't use them outside of this file.
+
+Simple example:
+
+  // default indexing is integer offset.
+  const df = Dataframe.create([2,2], [['a', 'b'], [0, 1]])
+  console.log(df.at(0,0));  // outputs: a
+  console.log(df.col(1).asArray());   // outputs: [0, 1]
+
+  // KeyIndex
+  const df = new Dataframe([1,2], [['a'], ['b']], null, new KeyIndex(['A', 'B']))
+  console.log(df.at(0, 'A')); // outputs: a
+  console.log(df.col('A').asArray(); // outputs: ['a']
+
+Performance tuning is primarily focused on columnar access patterns, which is the
+dominant pattern in cellxgene.
+*/
+
+/**
+Dataframe
+**/
+
+class Dataframe {
+  /**
+  Constructors & factories
+  **/
+
+  constructor(dims, columnarData, rowIndex = null, colIndex = null) {
+    /*
+    The base constructor is relatively hard to use - as an alternative,
+    see factory methods and clone/slice, below.
+
+    Parameters:
+      * dims - 2D array describing intendend dimensionality: [nRows,nCols].
+      * columnarData - JS array, nCols in length, containing array
+        or TypedArray of length nRows.
+      * rowIndex/colIndex - null (create default index using offsets as key),
+        or a caller-provided index.
+    All columns and indices must have appropriate dimensionality.
+    */
+    Dataframe.__errorChecks(dims, columnarData, rowIndex, colIndex);
+    const [nRows, nCols] = dims;
+    if (!rowIndex) {
+      rowIndex = new IdentityInt32Index(nRows);
+    }
+    if (!colIndex) {
+      colIndex = new IdentityInt32Index(nCols);
+    }
+
+    this.__columns = Array.from(columnarData);
+    this.dims = dims;
+    this.length = nRows; // convenience accessor for row dimension
+    this.rowIndex = rowIndex;
+    this.colIndex = colIndex;
+
+    this.__compile();
+  }
+
+  static __errorChecks(dims, columnarData) {
+    const [nRows, nCols] = dims;
+    if (nRows < 0 || nCols < 0) {
+      throw new RangeError("Dataframe dimensions must be positive");
+    }
+    if (!Array.isArray(columnarData)) {
+      throw new TypeError("Dataframe constructor requires array of columns");
+    }
+    if (!columnarData.every(c => isArrayOrTypedArray(c))) {
+      throw new TypeError("Dataframe columns must all be Array or TypedArray");
+    }
+    if (
+      nCols !== columnarData.length ||
+      !columnarData.every(c => c.length === nRows)
+    ) {
+      throw new RangeError(
+        "Dataframe dimension does not match column data shape"
+      );
+    }
+  }
+
+  __compile() {
+    /*
+    Compile data accessors for each column.
+
+    Each column accessor is a function which will lookup data by
+    index (ie, is equivalent to dataframe.get(row, col), where 'col'
+    is fixed.
+
+    In addition, each column accessor has several functions:
+
+    asArray() -- return the entire column as a native Array or TypedArray.
+      Crucially, this native array only supports label indexing.
+      Example:
+        const arr = df.col('a').asArray();
+
+    has(rlabel) -- return boolean indicating of the row label
+      is contained within the column.  Example:
+        const isInColumn = df.col('a').includes(99)
+      For the default offset indexing, this is identical to:
+        const isInColumn = (99 > 0) && (99 < df.nRows);
+
+    ihas(roffset) -- same as has(), but accepts a row offset
+      instead of a row label.
+
+    indexOf(value) -- return the label (not offset) of the first instance of
+      'value' in the column.  If you want the offset, just use the builtin JS
+      indexOf() function, available on both Array and TypedArray.
+
+    iget(offset) -- return the value at 'offset'
+
+    */
+    const { getOffset, getLabel } = this.rowIndex;
+    this.__columnsAccessor = this.__columns.map(column => {
+      const { length } = column;
+
+      /* get value by row label */
+      const get = function get(rlabel) {
+        return column[getOffset(rlabel)];
+      };
+
+      /* get value by row offset */
+      const iget = function iget(roffset) {
+        return column[roffset];
+      };
+
+      /* full column array access */
+      const asArray = function asArray() {
+        return column;
+      };
+
+      /* test for row label inclusion in column */
+      const has = function has(rlabel) {
+        const offset = getOffset(rlabel);
+        return offset >= 0 && offset < length;
+      };
+
+      const ihas = function ihas(offset) {
+        return offset >= 0 && offset < length;
+      };
+
+      /*
+      return first label (index) at which the value is found in this column,
+      or undefined if not found.
+
+      NOTE: not found return is DIFFERENT than the default Array.indexOf as
+      -1 is a plausible Dataframe row/col label.
+      */
+      const indexOf = function indexOf(value) {
+        const offset = column.indexOf(value);
+        if (offset === -1) {
+          return undefined;
+        }
+        return getLabel(offset);
+      };
+
+      /*
+      Summarize the column data. Lazy eval;
+      */
+      const summarize = callOnceLazy(() =>
+        isTypedArray(column)
+          ? summarizeContinuous(column)
+          : summarizeCategorical(column)
+      );
+
+      get.summarize = summarize;
+      get.asArray = asArray;
+      get.has = has;
+      get.ihas = ihas;
+      get.indexOf = indexOf;
+      get.iget = iget;
+      return get;
+    });
+  }
+
+  clone() {
+    return new this.constructor(
+      this.dims,
+      [...this.__columns],
+      this.rowIndex,
+      this.colIndex
+    );
+  }
+
+  static empty() {
+    return new Dataframe([0, 0], []);
+  }
+
+  static create(dims, columnarData) {
+    /*
+    Create a dataframe from raw columnar data.  All column arrays
+    must have the same length.   Identity indexing will be used.
+
+    Example:
+    const df = Dataframe.create([2,2], [new Uint32Array(2), new Float32Array(2)]);
+    */
+    return new Dataframe(dims, columnarData, null, null);
+  }
+
+  __cut(rowOffsets, colOffsets) {
+    const dims = [...this.dims];
+
+    const getSortedLabelAndOffsets = (offsets, index) => {
+      /*
+      Given offsets, return both offsets and associated lables,
+      sorted by offset.
+      */
+      if (!offsets) {
+        return [null, null];
+      }
+      const sortedOffsets = sort(offsets);
+      const sortedLabels = new Array(sortedOffsets.length);
+      for (let i = 0, l = sortedOffsets.length; i < l; i += 1) {
+        sortedLabels[i] = index.getLabel(sortedOffsets[i]);
+      }
+      return [sortedLabels, sortedOffsets];
+    };
+
+    let { colIndex } = this;
+    if (colOffsets) {
+      let colLabels;
+      [colLabels, colOffsets] = getSortedLabelAndOffsets(
+        colOffsets,
+        this.colIndex
+      );
+      dims[1] = colOffsets.length;
+      colIndex = this.colIndex.cut(colLabels);
+    }
+
+    let { rowIndex } = this;
+    if (rowOffsets) {
+      let rowLabels;
+      [rowLabels, rowOffsets] = getSortedLabelAndOffsets(
+        rowOffsets,
+        this.rowIndex
+      );
+      dims[0] = rowLabels.length;
+      rowIndex = this.rowIndex.cut(rowLabels);
+    }
+
+    /* cut columns */
+    let columns = this.__columns;
+    if (colOffsets) {
+      columns = new Array(colOffsets.length);
+      for (let i = 0, l = colOffsets.length; i < l; i += 1) {
+        columns[i] = this.__columns[colOffsets[i]];
+      }
+    }
+
+    /* cut rows */
+    if (rowOffsets) {
+      columns = columns.map(col => {
+        const newCol = new col.constructor(rowOffsets.length);
+        for (let i = 0, l = rowOffsets.length; i < l; i += 1) {
+          newCol[i] = col[rowOffsets[i]];
+        }
+        return newCol;
+      });
+    }
+    return new Dataframe(dims, columns, rowIndex, colIndex);
+  }
+
+  cutByList(rowLabels, colLabels = null) {
+    const toOffsets = (labels, index) => {
+      if (!labels) {
+        return null;
+      }
+      return labels.map(label => {
+        const off = index.getOffset(label);
+        if (off === undefined) {
+          throw new RangeError(`unknown label: ${label}`);
+        }
+        return off;
+      });
+    };
+
+    const rowOffsets = toOffsets(rowLabels, this.rowIndex);
+    const colOffsets = toOffsets(colLabels, this.colIndex);
+    return this.__cut(rowOffsets, colOffsets);
+  }
+
+  icutByList(rowOffsets, colOffsets = null) {
+    return this.__cut(rowOffsets, colOffsets);
+  }
+
+  icutByMask(rowMask, colMask = null) {
+    /*
+    Cut on row/column based upon a truthy/falsey array.
+    */
+    const [nRows, nCols] = this.dims;
+    if (
+      (rowMask && rowMask.length !== nRows) ||
+      (colMask && colMask.length !== nCols)
+    ) {
+      throw new RangeError("boolean arrays must match row/col dimensions");
+    }
+
+    /* convert masks to lists - method wastes space, but is fast */
+    const toList = (mask, maxSize) => {
+      if (!mask) {
+        return null;
+      }
+      const list = new Int32Array(maxSize);
+      let elems = 0;
+      for (let i = 0, l = mask.length; i < l; i += 1) {
+        if (mask[i]) {
+          list[elems] = i;
+          elems += 1;
+        }
+      }
+      return new Int32Array(list.buffer, 0, elems);
+    };
+    const rowOffsets = toList(rowMask, nRows);
+    const colOffsets = toList(colMask, nCols);
+    return this.__cut(rowOffsets, colOffsets);
+  }
+
+  /**
+  Data access with row/col.
+  **/
+
+  col(columnLabel) {
+    /*
+    Return accessor bound to a column.  Allows random row access
+    based upon the row indexing.  Returns undefined if the
+    columnLabel is not present in the dataframe.
+
+    Example for a dataframe with string labeled columns, and
+    default (offset) indices for rows (eg, [0, 'foo'])
+
+      const getValue = df.col('foo');
+      for (let r = 0; r < df.nRows; r += 1) {
+        console.log(r, getValue(r));
+      }
+
+    See __compile() for the functions available in a column accessor.
+    */
+    const coff = this.colIndex.getOffset(columnLabel);
+    return this.__columnsAccessor[coff];
+  }
+
+  icol(columnOffset) {
+    /*
+    Return column accessor by offset.
+    */
+    return this.__columnsAccessor[columnOffset];
+  }
+
+  at(r, c) {
+    /*
+    Access a single value, for a row/col label pair.
+
+    For performance reasons, there are no bounds or existance
+    checks on labels, and no defined behavior when these are supplied.
+    May return undefined, throw an Error, or do something else for
+    non-existant labels.  If you want predictable out-of-bounds
+    behavior, use has(), eg,
+
+    const myVal = df.has(r,l) ? df.at(r,l) : undefined;
+    */
+    const coff = this.colIndex.getOffset(c);
+    const roff = this.rowIndex.getOffset(r);
+    return this.__columns[coff][roff];
+  }
+
+  iat(r, c) {
+    /*
+    Access a single value, for a row/col offset (integer) position.
+
+    For performance reasons, there are no bounds checks on row/col offsets
+    or other well-defined behavior for out-of-bounds values.  If you want
+    well-defined bounds checking, use ihas(), eg,
+
+    const myVal = df.ihas(r, c) ? df.iat(r, c) : undefined;
+    */
+    return this.__columns[c][r];
+  }
+
+  has(r, c) {
+    /*
+    Test if row/col labels exist in the dataframe - returns true/false
+    */
+    const [nRows, nCols] = this.dims;
+    const coff = this.colIndex.getOffset(c);
+    const roff = this.rowIndex.getOffset(r);
+    return coff >= 0 && coff < nCols && roff >= 0 && roff < nRows;
+  }
+
+  ihas(r, c) {
+    /*
+    Test if row/col offset (integer) position exists in the
+    dataframe - returns true/false
+    */
+    const [nRows, nCols] = this.dims;
+    return c >= 0 && c < nCols && r >= 0 && r < nRows;
+  }
+
+  /****
+  Functional (map/reduce/etc) data access
+
+  XXX: not yet implemented, as there is no clear use case.   Can easily
+  add these as useful.
+  ****/
+
+  /*
+  Map & reduce of column or row
+
+  XXX TODO remainder of map/reduce functions:  mapCol, mapRow, reduceRow, ...
+  */
+  /* comment out until we have a use for this
+
+  reduceCol(clabel, callback, initialValue) {
+    const coff = this.colIndex.getOffset(clabel);
+    const column = this.__columns[coff];
+    let start = 0;
+    let acc = initialValue;
+    if (initialValue === undefined) {
+      acc = column[0];
+      start = 1;
+    }
+    for (let i = start, l = column.length; i < l; i += 1) {
+      acc = callback(acc, column[i]);
+    }
+    return acc;
+  }
+  */
+}
+
+export default Dataframe;
@@ -0,0 +1,2 @@
+export { default as Dataframe } from "./dataframe";
+export { DenseInt32Index, IdentityInt32Index, KeyIndex } from "./labelIndex";
@@ -0,0 +1,188 @@
+/**
+Label indexing - map a label to & from an integer offset.  See Dataframe
+for how this is used.
+**/
+
+/*
+Private utility functions
+*/
+function extent(tarr) {
+  let min = 0x7fffffff;
+  let max = ~min; // eslint-disable-line no-bitwise
+  for (let i = 0, l = tarr.length; i < l; i += 1) {
+    const v = tarr[i];
+    if (v < min) {
+      min = v;
+    }
+    if (v > max) {
+      max = v;
+    }
+  }
+  return [min, max];
+}
+
+function fillRange(arr, start = 0) {
+  const larr = arr;
+  for (let i = 0, l = larr.length; i < l; i += 1) {
+    larr[i] = i + start;
+  }
+  return larr;
+}
+
+/* eslint-disable class-methods-use-this */
+class IdentityInt32Index {
+  /*
+  identity/noop index, with small assumptions that labels are int32
+  */
+  constructor(maxOffset) {
+    this.maxOffset = maxOffset;
+  }
+
+  keys() {
+    // memoize
+    const k = fillRange(new Int32Array(this.maxOffset));
+    this.keys = function keys() {
+      return k;
+    };
+    return k;
+  }
+
+  getOffset(i) {
+    // label to offset
+    return i;
+  }
+
+  getLabel(i) {
+    // offset to label
+    return i;
+  }
+
+  getMaxOffset() {
+    return this.maxOffset;
+  }
+
+  cut(labelArray) {
+    /*
+    if density of resulting integer
+    */
+    const [minLabel, maxLabel] = extent(labelArray);
+    const labelSpaceSize = maxLabel - minLabel + 1;
+    const density = labelSpaceSize / this.maxOffset;
+    /* 0.1 is a magic number, that needs testing to optimize */
+    if (density < 0.1) {
+      return new KeyIndex(labelArray);
+    }
+    return new DenseInt32Index(labelArray, [minLabel, maxLabel]);
+  }
+}
+/* eslint-enable class-methods-use-this */
+
+/* eslint-disable class-methods-use-this */
+class DenseInt32Index {
+  /*
+  DenseInt32Index indexes integer labels, and uses Int32Array typed arrays
+  for both forward and reverse indexing.   This means that the min/max range
+  of the forward index labels must be known a priori (so that the index
+  array can be pre-allocated).
+  */
+  constructor(labels, labelRange = null) {
+    if (labels.constructor !== Int32Array) {
+      labels = new Int32Array(labels);
+    }
+
+    if (!labelRange) {
+      labelRange = extent(labels);
+    }
+    const [minLabel, maxLabel] = labelRange;
+    const labelSpaceSize = maxLabel - minLabel + 1;
+    const index = new Int32Array(labelSpaceSize).fill(-1);
+    for (let i = 0, l = labels.length; i < l; i += 1) {
+      const label = labels[i];
+      index[label - minLabel] = i;
+    }
+
+    this.minLabel = minLabel;
+    this.rindex = labels;
+    this.index = index;
+    this.__compile();
+  }
+
+  __compile() {
+    const { minLabel, index, rindex } = this;
+    this.getOffset = function getOffset(l) {
+      return index[l - minLabel];
+    };
+    this.getLabel = function getLabel(i) {
+      return rindex[i];
+    };
+  }
+
+  keys() {
+    return this.rindex;
+  }
+
+  getMaxOffset() {
+    return this.rindex.length;
+  }
+
+  cut(labelArray) {
+    /*
+    time/space decision - if we are going to use less than 10% of the
+    dense index space, switch to a KeyIndex (which is slower, but uses
+    less memory for sparse label spaces).
+    */
+    const [minLabel, maxLabel] = extent(labelArray);
+    const labelSpaceSize = maxLabel - minLabel + 1;
+    const density = labelSpaceSize / this.rindex.length;
+    /* 0.1 is a magic number, that needs testing to optimize */
+    if (density < 0.1) {
+      return new KeyIndex(labelArray);
+    }
+    return new DenseInt32Index(labelArray, [minLabel, maxLabel]);
+  }
+}
+/* eslint-enable class-methods-use-this */
+
+/* eslint-disable class-methods-use-this */
+class KeyIndex {
+  /*
+  KeyIndex indexes arbitrary JS primitive types, and uses a Map()
+  as its core data structure.
+  */
+  constructor(labels) {
+    const index = new Map();
+    const rindex = labels;
+    labels.forEach((v, i) => {
+      index.set(v, i);
+    });
+
+    this.index = index;
+    this.rindex = rindex;
+    this.__compile();
+  }
+
+  __compile() {
+    const { index, rindex } = this;
+    this.getOffset = function getOffset(k) {
+      return index.get(k);
+    };
+    this.getLabel = function getLabel(i) {
+      return rindex[i];
+    };
+  }
+
+  keys() {
+    return this.rindex;
+  }
+
+  getMaxOffset() {
+    return this.rindex.length;
+  }
+
+  cut(labelArray) {
+    return new KeyIndex(labelArray);
+  }
+}
+/* eslint-enable class-methods-use-this */
+
+export { DenseInt32Index, IdentityInt32Index, KeyIndex };
@@ -0,0 +1,57 @@
+/*
+Private dataframe support functions
+*/
+
+export function summarizeContinuous(col) {
+  let min;
+  let max;
+  let nan = 0;
+  let pinf = 0;
+  let ninf = 0;
+  if (col) {
+    for (let r = 0, l = col.length; r < l; r += 1) {
+      const val = Number(col[r]);
+      if (Number.isFinite(val)) {
+        if (min === undefined) {
+          min = val;
+          max = val;
+        } else {
+          min = val < min ? val : min;
+          max = val > max ? val : max;
+        }
+      } else if (Number.isNaN(val)) {
+        nan += 1;
+      } else if (val > 0) {
+        pinf += 1;
+      } else {
+        ninf += 1;
+      }
+    }
+  }
+  return {
+    categorical: false,
+    min,
+    max,
+    nan,
+    pinf,
+    ninf
+  };
+}
+
+export function summarizeCategorical(col) {
+  const categoryCounts = new Map();
+  if (col) {
+    for (let r = 0, l = col.length; r < l; r += 1) {
+      const val = col[r];
+      let curCount = categoryCounts.get(val);
+      if (curCount === undefined) curCount = 0;
+      categoryCounts.set(val, curCount + 1);
+    }
+  }
+  return {
+    categorical: true,
+    categories: [...categoryCounts.keys()],
+    categoryCounts,
+    numCategories: categoryCounts.size
+  };
+}
@@ -0,0 +1,28 @@
+/*
+Private utility code for dataframe
+*/
+
+export function isTypedArray(x) {
+  return (
+    ArrayBuffer.isView(x) &&
+    Object.prototype.toString.call(x) !== "[object DataView]"
+  );
+}
+
+export function isArrayOrTypedArray(x) {
+  return Array.isArray(x) || isTypedArray(x);
+}
+
+export function callOnceLazy(f) {
+  let value;
+  let calledOnce = false;
+  const result = function result(...args) {
+    if (!calledOnce) {
+      value = f(...args);
+      calledOnce = true;
+    }
+    return value;
+  };
+
+  return result;
+}
@@ -53,13 +53,15 @@ Example:
 NOTE: will not summarize the required 'name' annotation, as that is
 specified as unique per element.
 */
-function _summarizeAnnotations(_schema, annotations) {
+function _summarizeAnnotations(_schema, df) {
  const summary = _(_schema) // lodash wrapping: https://lodash.com/docs/4.17.11#lodash
-    .filter(v => v.name !== "name")
+    .filter(v => v.name !== "name") // don't summarize name
    .keyBy("name")
    .mapValues(anno => {
      const { name, type } = anno;
      const continuous = type === "int32" || type === "float32";
+      const numRows = df.length;
+      const col = df.col(name) ? df.col(name).asArray() : null;

      if (continuous) {
        let min;
@@ -67,22 +69,24 @@ function _summarizeAnnotations(_schema, annotations) {
        let nan = 0;
        let pinf = 0;
        let ninf = 0;
-        for (let r = 0; r < annotations.length; r += 1) {
-          const val = Number(annotations[r][name]);
-          if (Number.isFinite(val)) {
-            if (min === undefined) {
-              min = val;
-              max = val;
+        if (col) {
+          for (let r = 0; r < numRows; r += 1) {
+            const val = Number(col[r]);
+            if (Number.isFinite(val)) {
+              if (min === undefined) {
+                min = val;
+                max = val;
+              } else {
+                min = val < min ? val : min;
+                max = val > max ? val : max;
+              }
+            } else if (Number.isNaN(val)) {
+              nan += 1;
+            } else if (val > 0) {
+              pinf += 1;
            } else {
-              min = val < min ? val : min;
-              max = val > max ? val : max;
+              ninf += 1;
            }
-          } else if (Number.isNaN(val)) {
-            nan += 1;
-          } else if (val > 0) {
-            pinf += 1;
-          } else {
-            ninf += 1;
          }
        }
        return {
@@ -93,11 +97,13 @@ function _summarizeAnnotations(_schema, annotations) {

      /* else categorical */
      const categoryCounts = new Map();
-      for (let r = 0; r < annotations.length; r += 1) {
-        const val = annotations[r][name];
-        let curCount = categoryCounts.get(val);
-        if (curCount === undefined) curCount = 0;
-        categoryCounts.set(val, curCount + 1);
+      if (col) {
+        for (let r = 0; r < numRows; r += 1) {
+          const val = col[r];
+          let curCount = categoryCounts.get(val);
+          if (curCount === undefined) curCount = 0;
+          categoryCounts.set(val, curCount + 1);
+        }
      }
      return {
        categorical: true,
@@ -5,6 +5,7 @@ import _ from "lodash";
 import * as kvCache from "./keyvalcache";
 import summarizeAnnotations from "./summarizeAnnotations";
 import decodeMatrixFBS from "./matrix";
+import * as Dataframe from "../dataframe";

 /*
 Private helper function - create and return a template Universe
@@ -27,13 +28,10 @@ function templateUniverse() {
    /*
    Annotations
    */
-    obsAnnotations: [] /* all obs annotations, by obs index */,
-    varAnnotations: [] /* all var annotations, by var index */,
-    obsNameToIndexMap: {} /* reverse map 'name' to index */,
-    varNameToIndexMap: {} /* reverse map 'name' to index */,
-    summary: null /* derived data summaries XXX: consider exploding in place */,
-
-    obsLayout: { X: [], Y: [] } /* xy layout */,
+    obsAnnotations: null,
+    varAnnotations: null,
+    obsLayout: null,
+    summary: null /* derived data summaries. XXX: consider exploding in place */,

    /*
    Cache of var data (expression), by var annotation name.   Data can be
@@ -61,9 +59,8 @@ function finalize(universe) {
  /* A bit of sanity checking! */
  const { nObs, nVar } = universe;
  if (
+    nObs !== universe.obsLayout.length ||
    nObs !== universe.obsAnnotations.length ||
-    nObs !== universe.obsLayout.X.length ||
-    nObs !== universe.obsLayout.Y.length ||
    nVar !== universe.varAnnotations.length
  ) {
    throw new Error("Universe dimensionality mismatch - failed to load");
@@ -73,61 +70,33 @@ function finalize(universe) {
  //  - layout has supported number of dimensions
  //  - ...

-  /*
-  Create all derived (convenience) data structures.
-  */
-  universe.obsNameToIndexMap = _.transform(
-    universe.obsAnnotations,
-    (acc, value, idx) => {
-      acc[value.name] = idx;
-    },
-    {}
-  );
-  universe.varNameToIndexMap = _.transform(
-    universe.varAnnotations,
-    (acc, value, idx) => {
-      acc[value.name] = idx;
-    },
-    {}
-  );
  universe.finalized = true;
  return universe;
 }

-function RESTv02AnotationsFBSResponseToInternal(arrayBuffer) {
+function AnnotationsFBSToDataframe(arrayBuffer) {
  /*
-  Convert a Matrix FBS to our internal format -- row-major array of
-  observations/cells, stored as an object.  Each obs has a key for each
-  annotation, plus __index__ containing its obsIndex.
-
-  Example:
-  [
-    { __index__: 0, tissue_type: "lung", sex: "F", ... },
-    ...
-  ]
-
-  XXX TODO: we could make use of the columns in building crossfilter
-  dimensions (they have to be recreated).  Future optimization.
+  Convert a Matrix FBS to a Dataframe.
  */
  const fbs = decodeMatrixFBS(arrayBuffer);
-  const keys = fbs.colIdx;
-  const result = Array(fbs.nRows);
-  for (let row = 0; row < fbs.nRows; row += 1) {
-    const rec = { __index__: row };
-    for (let col = 0; col < fbs.nCols; col += 1) {
-      rec[keys[col]] = fbs.columns[col][row];
-    }
-    result[row] = rec;
-  }
-  return result;
+  const df = new Dataframe.Dataframe(
+    [fbs.nRows, fbs.nCols],
+    fbs.columns,
+    null,
+    new Dataframe.KeyIndex(fbs.colIdx)
+  );
+  return df;
 }

-function RESTv02LayoutFBSResponseToInternal(arrayBuffer) {
+function LayoutFBSToDataframe(arrayBuffer) {
  const fbs = decodeMatrixFBS(arrayBuffer, true);
-  return {
-    X: fbs.columns[0],
-    Y: fbs.columns[1]
-  };
+  const df = new Dataframe.Dataframe(
+    [fbs.nRows, fbs.nCols],
+    fbs.columns,
+    null,
+    new Dataframe.KeyIndex(["X", "Y"])
+  );
+  return df;
 }

 function reconcileSchemaCategoriesWithSummary(universe) {
@@ -156,7 +125,7 @@ function reconcileSchemaCategoriesWithSummary(universe) {
  });
 }

-export function createUniverseFromRestV02Response(
+export function createUniverseFromResponse(
  configResponse,
  schemaResponse,
  annotationsObsResponse,
@@ -178,15 +147,10 @@ export function createUniverseFromRestV02Response(
  universe.nVar = schema.dataframe.nVar;

  /* annotations */
-  universe.obsAnnotations = RESTv02AnotationsFBSResponseToInternal(
-    annotationsObsResponse
-  );
-  universe.varAnnotations = RESTv02AnotationsFBSResponseToInternal(
-    annotationsVarResponse
-  );
-
+  universe.obsAnnotations = AnnotationsFBSToDataframe(annotationsObsResponse);
+  universe.varAnnotations = AnnotationsFBSToDataframe(annotationsVarResponse);
  /* layout */
-  universe.obsLayout = RESTv02LayoutFBSResponseToInternal(layoutFBSResponse);
+  universe.obsLayout = LayoutFBSToDataframe(layoutFBSResponse);

  universe.summary = summarizeAnnotations(
    universe.schema,
@@ -214,8 +178,8 @@ export function convertDataFBStoObject(universe, arrayBuffer) {
  const result = {};

  for (let c = 0; c < colIdx.length; c += 1) {
-    const gene = universe.varAnnotations[colIdx[c]].name;
-    result[gene] = columns[c];
+    const varName = universe.varAnnotations.at(colIdx[c], "name");
+    result[varName] = columns[c];
  }
  return result;
 }
@@ -8,6 +8,7 @@ import Crossfilter from "../typedCrossfilter";
 import { sliceByIndex } from "../typedCrossfilter/util";

 /*
+
 World is a subset of universe.   Most code should use world, and should
 (generally) not use Universe.   World contains any per-obs or per-var data
 that must be consistent acorss the app when we view/manipulate subsets
@@ -16,37 +17,32 @@ of Universe.
 Private API indicated by leading underscore in key name (eg, _foo).  Anything else
 is public.

-World contains several public keys, obsAnnotations, and obsLayout, which are
-arrays contianing information about an OBS in the same order/offset.  In
-other words, world.obsAnnotations[0] and world.obsLayout.X[0] refer to the same
-obs/cell.
+Notable keys in the world object:
+
+* nObs, nVar: dimensions
+
+* schema: data schema from the server

 * obsAnnotations:

-  obsAnnotations will return an array of objects.  Each object contains all annotation
-  values for a given observation/cell, keyed by annotation name, PLUS a key
-  '__cellId__', containing a REST API ID for this obs/cell (referred to as the
-  obsIndex in the REST 0.2 spec or cellIndex in the 0.1 spec.
+  Dataframe containing obs annotations.  Columns are indexed by annotation
+  name (eg, 'tissue type'), and rows are indexed by the REST API obsIndex
+  (ie, the offset into the underlying server-side dataframe).

-  Example:  [ { __cellId__: 99, cluster: 'blue', numReads: 93933 } ]
-
-  NOTE: world.obsAnnotation should be identical to the old state.cells value,
-  EXCEPT that
-    * __cellIndex__ renamed to __index__
-    * __x__ and __y__ are now in world.obsLayout
-    * __color__ and __colorRBG__ should be moved to controls reducer
+  This indexing means that you can access data by _either_ the server's
+  obxIndex, or the offset into the client-side column array .  Be careful
+  to know which you want and are using.

 * obsLayout:

-  obsLayout will return an object containing two arrays, containing X and Y
-  coordinates respectively.
+  A dataframe containing the X/Y layout for all obs.  Columns are named
+  'X' and 'Y', and rows are indexed in the same way as obsAnnotation.

-  Example: { X: [ 0.33, 0.23, ... ], Y: [ 0.8, 0.777, ... ]}
+* summary: summary of each obsAnnotation column (eg, numeric extent for
+  continuous data, category counts for categorical metadata)

-* crossfilter - a crossfilter object across world.obsAnnotations
-
-* dimensionMap - an object mapping annotation names to dimensions on
-  the crossfilter
+* varDataCache: expression columns, in a kvCache.   TODO: maybe move to a
+  Dataframe in the future.

 */

@@ -56,11 +52,6 @@ const VarDataCacheTTLMs = 1000; // min cache time in MS

 function templateWorld() {
  return {
-    // map from universe obsIndex to world offset.
-    // Undefined / null indicates identity mapping.
-    obsIndex: null,
-    obsBackIndex: null,
-
    /* schema/version related */
    api: null,
    schema: null,
@@ -71,7 +62,7 @@ function templateWorld() {
    obsAnnotations: null,
    varAnnotations: null,

-    /* layout of graph */
+    /* layout of graph. Dataframe. */
    obsLayout: null,

    /* derived data summaries XXX: consider exploding in place */
@@ -91,15 +82,6 @@ export function createWorldFromEntireUniverse(universe) {

  const world = templateWorld();

-  // map from the universe obsIndex to our world offset.
-  // undefined/null indicates identity map.
-  // In other words   obsBackIndex[universeIdx] -> worldIdx
-  world.obsBackIndex = null;
-  // Map to the universe index for each element in world.
-  // Null indicates identity map (aka world === universe)
-  // In other wrods  obsIndex[worldIdx]  ->  universeIdx
-  world.obsIndex = null;
-
  /*
  public interface follows
  */
@@ -143,35 +125,11 @@ export function createWorldFromCurrentSelection(universe, world, crossfilter) {
  newWorld.schema = universe.schema;
  newWorld.varAnnotations = universe.varAnnotations;

-  /* build index maps and back maps based upon current selection state */
-  const obsBackIndex = new Uint32Array(universe.nObs);
-  obsBackIndex.fill(-1); // default - aka unused
-  const notSelected = obsBackIndex[0];
-  let nObs = 0;
-  for (let i = 0; i < universe.nObs; i += 1) {
-    if (crossfilter.isElementFiltered(i)) {
-      obsBackIndex[i] = nObs;
-      nObs += 1;
-    }
-  }
-  const obsIndex = new Uint32Array(nObs);
-  for (let i = 0; i < universe.nObs; i += 1) {
-    const worldIdx = obsBackIndex[i];
-    if (worldIdx !== notSelected) {
-      obsIndex[worldIdx] = i;
-    }
-  }
-
-  newWorld.nObs = nObs;
-  newWorld.obsIndex = obsIndex;
-  newWorld.obsBackIndex = obsBackIndex;
-
-  /* now slice */
-  newWorld.obsAnnotations = sliceByIndex(universe.obsAnnotations, obsIndex);
-  newWorld.obsLayout = {
-    X: sliceByIndex(universe.obsLayout.X, obsIndex),
-    Y: sliceByIndex(universe.obsLayout.Y, obsIndex)
-  };
+  /* now subset/cut obs */
+  const mask = crossfilter.allFilteredMask();
+  newWorld.obsAnnotations = world.obsAnnotations.icutByMask(mask);
+  newWorld.obsLayout = world.obsLayout.icutByMask(mask);
+  newWorld.nObs = newWorld.obsAnnotations.dims[0];

  /* derived data & summaries */
  newWorld.summary = summarizeAnnotations(
@@ -245,22 +203,23 @@ export function createObsDimensionMap(crossfilter, world) {
  create and return a crossfilter dimension for every obs annotation
  for which we have a supported type.
  */
-  const { schema, obsLayout } = world;
+  const { schema, obsLayout, obsAnnotations } = world;

  // Create a crossfilter dimension for all obs annotations *except* 'name'
  const dimensionMap = _(schema.annotations.obs)
    .filter(anno => anno.name !== "name")
    .transform((result, anno) => {
      const dimType = deduceDimensionType(anno, anno.name);
+      const colData = obsAnnotations.col(anno.name).asArray();
      if (dimType === "enum") {
        result[obsAnnoDimensionName(anno.name)] = crossfilter.dimension(
          Crossfilter.EnumDimension,
-          r => r[anno.name]
+          colData
        );
-      } else {
+      } else if (dimType) {
        result[obsAnnoDimensionName(anno.name)] = crossfilter.dimension(
          Crossfilter.ScalarDimension,
-          r => r[anno.name],
+          colData,
          dimType
        );
      } // else ignore the annotation
@@ -272,8 +231,8 @@ export function createObsDimensionMap(crossfilter, world) {
  */
  dimensionMap[layoutDimensionName("XY")] = crossfilter.dimension(
    Crossfilter.SpatialDimension,
-    obsLayout.X,
-    obsLayout.Y
+    obsLayout.col("X").asArray(),
+    obsLayout.col("Y").asArray()
  );

  return dimensionMap;
@@ -288,5 +247,23 @@ export function subsetVarData(world, universe, varData) {
  if (worldEqUniverse(world, universe)) {
    return varData;
  }
-  return sliceByIndex(varData, world.obsIndex);
+  return sliceByIndex(varData, world.obsAnnotations.rowIndex.keys());
+}
+
+export function getSelectedByIndex(crossfilter) {
+  /*
+  return array of obsIndex, containing all selected obs/cells.
+  */
+  const selected = crossfilter.allFilteredMask(); // array of bool-ish
+  const keys = crossfilter.data.rowIndex.keys(); // row keys, aka universe rowIndex
+
+  const set = new Int32Array(selected.length);
+  let numElems = 0;
+  for (let i = 0, l = selected.length; i < l; i += 1) {
+    if (selected[i]) {
+      set[numElems] = keys[i];
+      numElems += 1;
+    }
+  }
+  return new Int32Array(set.buffer, 0, numElems);
 }
@@ -18,13 +18,23 @@ Map {
   ...
 }

+Parameters are:
+- dim1: dimension 1 name/label
+- dim2: dimension 2 name/label
+- df: dataframe containing dim1 and dim2 on the column axis
+
 */
-function _countCategoryValues2D(dim1, dim2, rows) {
+function _countCategoryValues2D(dim1, dim2, df) {
  const dimMap = new Map();
-  for (let r = 0; r < rows.length; r += 1) {
-    const row = rows[r];
-    const val1 = row[dim1];
-    const val2 = row[dim2];
+  const col1 = df.col(dim1) ? df.col(dim1).asArray() : null;
+  const col2 = df.col(dim2) ? df.col(dim2).asArray() : null;
+  if (!col1 || !col2) {
+    return dimMap;
+  }
+
+  for (let r = 0, l = df.length; r < l; r += 1) {
+    const val1 = col1[r];
+    const val2 = col2[r];
    let d2Map = dimMap.get(val1);
    if (d2Map === undefined) {
      d2Map = new Map();
@@ -40,7 +40,7 @@ class BitArray {
  // Return the number of records that are selected, ie, have a one bit in
  // all allocated dimensions.
  //
-  get selectionCount() {
+  selectionCount() {
    return this.countAllOnes();
  }

@@ -48,16 +48,27 @@ class BitArray {
  //
  countAllOnes() {
    let count = 0;
-    const { bitarray, bitmask, length, width } = this;
-    for (let l = 0; l < length; l += 1) {
-      let dimensionsSet = 0;
-      for (let w = 0; w < width; w += 1) {
-        if (bitarray[w * length + l] === bitmask[w]) {
-          dimensionsSet += 1;
+    const { bitarray, length, width } = this;
+    if (width === 1) {
+      // special case, width === 1, for performance
+      const bitmask = this.bitmask[0];
+      for (let l = 0; l < length; l += 1) {
+        if (bitarray[l] === bitmask) {
+          count += 1;
        }
      }
-      if (dimensionsSet === width) {
-        count += 1;
+    } else {
+      const { bitmask } = this;
+      for (let l = 0; l < length; l += 1) {
+        let dimensionsSet = 0;
+        for (let w = 0; w < width; w += 1) {
+          if (bitarray[w * length + l] === bitmask[w]) {
+            dimensionsSet += 1;
+          }
+        }
+        if (dimensionsSet === width) {
+          count += 1;
+        }
      }
    }
    return count;
@@ -233,12 +244,14 @@ class BitArray {
  fillBySelection(result, selectedValue, deselectedValue) {
    // special case (width === 1) for performance
    if (this.width === 1) {
-      const bitmask = this.bitmask[0];
-      for (let i = 0, len = this.length; i < len; i += 1) {
-        result[i] =
-          bitmask && this.bitarray[i] === bitmask
-            ? selectedValue
-            : deselectedValue;
+      const { bitmask, bitarray } = this;
+      const mask = bitmask[0];
+      if (!mask) {
+        result.fill(deselectedValue);
+      } else {
+        for (let i = 0, len = this.length; i < len; i += 1) {
+          result[i] = bitarray[i] === mask ? selectedValue : deselectedValue;
+        }
      }
    } else {
      for (let i = 0, len = this.length; i < len; i += 1) {
@@ -39,6 +39,14 @@ import {
  upperBoundIndirect
 } from "./util";

+function isArrayOrTypedArray(x) {
+  return (
+    Array.isArray(x) ||
+    (ArrayBuffer.isView(x) &&
+      Object.prototype.toString.call(x) !== "[object DataView]")
+  );
+}
+
 class NotImplementedError extends Error {
  constructor(...params) {
    super(...params);
@@ -52,6 +60,11 @@ class NotImplementedError extends Error {

 class TypedCrossfilter {
  constructor(data) {
+    /*
+    Typically, data is one of:
+      - Array of objects/records
+      - Dataframe (util/dataframe)
+    */
    this.data = data;

    // filters: array of { id, dimension }
@@ -97,18 +110,32 @@ class TypedCrossfilter {
  // return array of all records that are selected/filtered
  // by all dimensions.
  allFiltered() {
-    const { selection } = this;
-    const res = [];
-    for (let i = 0, len = this.data.length; i < len; i += 1) {
-      if (selection.isSelected(i)) {
-        res.push(this.data[i]);
+    const { data, selection } = this;
+    if (Array.isArray(data)) {
+      const res = [];
+      for (let i = 0, len = data.length; i < len; i += 1) {
+        if (selection.isSelected(i)) {
+          res.push(data[i]);
+        }
      }
+      return res;
    }
-    return res;
+    /* else, Dataframe-like */
+    return data.icutByMask(this.allFilteredMask());
+  }
+
+  // return Uint8array containing selection state (truthy/falsey) for each record.
+  //
+  allFilteredMask() {
+    return this.selection.fillBySelection(
+      new Uint8Array(this.data.length),
+      1,
+      0
+    );
  }

  countFiltered() {
-    return this.selection.selectionCount;
+    return this.selection.selectionCount();
  }

  isElementFiltered(i) {
@@ -161,6 +188,7 @@ class ScalarDimension extends _Dimension {
    // or a map function which will create it.
    let array;
    if (value instanceof ValueArrayType) {
+      // user has provided the final typed array - just use it
      if (value.length !== this.crossfilter.data.length) {
        throw new RangeError(
          "ScalarDimension values length must equal crossfilter data record count"
@@ -168,11 +196,18 @@ class ScalarDimension extends _Dimension {
      }
      array = value;
    } else if (value instanceof Function) {
-      // Create value array
+      // Create value array from user-provided map function.
      array = this._createValueArray(
        value,
        new ValueArrayType(this.crossfilter.data.length)
      );
+    } else if (isArrayOrTypedArray(value)) {
+      // Create value array from user-provided array.  Typically used
+      // only by enumerated dimensions
+      array = this._createValueArray(
+        i => value[i],
+        new ValueArrayType(this.crossfilter.data.length)
+      );
    } else {
      throw new NotImplementedError(
        "dimension value must be function or value array type"
@@ -190,7 +225,7 @@ class ScalarDimension extends _Dimension {
    const len = data.length;
    const larray = array;
    for (let i = 0; i < len; i += 1) {
-      larray[i] = value(data[i]);
+      larray[i] = value(i, data);
    }
    return larray;
  }
@@ -387,7 +422,7 @@ class EnumDimension extends ScalarDimension {
    // and the enum.
    const s = new Set();
    for (let i = 0; i < len; i += 1) {
-      s.add(value(data[i]));
+      s.add(value(i, data));
    }
    this.enumIndex = Array.from(s);
    this.enumIndex.sort();
@@ -395,7 +430,7 @@ class EnumDimension extends ScalarDimension {
    // create dimension value array
    const enumLen = this.enumIndex.length;
    for (let i = 0; i < len; i += 1) {
-      const v = value(data[i]);
+      const v = value(i, data);
      const e = lowerBound(this.enumIndex, v, 0, enumLen);
      larray[i] = e;
    }