load annotations incrementally (#1107)

* load annotations individually

* fix type check to be more general

* update node CI version from 10 to 12

* node 11

* debug print node version

* travis node version to latest

* try nvm

* remove extraneous node_js statement

* remove node version debugging printf

* incrementally load all annotations and layout

* process annotations and layout as they are loaded

* fix tests

* sort categories incrementally

* incrementally build category view summary; add category loading spinner

* add spinner to continuous metadata

* configure undoable reducer

* incremental crossfilter creation

* improve busy layout

* more layout cleanup

* correctly reconcile categories in schema

* refine layout of lsb spinners

* more spinner layout work

* more spinner layout

* always load layout before obs annotations

client/src/util/dataframe/dataframe.js

@@ -351,11 +351,26 @@ class Dataframe {
   withColsFrom(dataframe) {
     /*
     return a new dataframe containing all columns from both `this` and the
-    provided dataframe.
+    provided of dataframe.
 
-    The row index from `this` will be used.  Both dataframes must have identical
+    The row index from `this` will be used.  All dataframes must have identical
     dimensionality, and no overlapping columns labels.
+
+    Special case, if either dataframe is empty, the other is returned unchanged.
      */
+    if (this.isEmpty()) {
+      return dataframe;
+    }
+    if (dataframe.isEmpty()) {
+      return this;
+    }
+
+    this.colIndex.keys().forEach(key => {
+      if (dataframe.has(key)) {
+        throw new Error("duplicate key collision");
+      }
+    });
+
     const dims = [this.dims[0], this.dims[1] + dataframe.dims[1]];
     const { rowIndex } = this;
     const columns = [...this.__columns, ...dataframe.__columns];
@@ -373,6 +388,11 @@ class Dataframe {
     );
   }
 
+  withColsFromAll(dataframes = []) {
+    dataframes = Array.isArray(dataframes) ? dataframes : [dataframes];
+    return dataframes.reduce((acc, df) => acc.withColsFrom(df), this);
+  }
+
   dropCol(label) {
     /*
     Create a new dataframe, omitting one columns.

client/src/util/stateManager/controlsHelpers.js

@@ -11,6 +11,13 @@ import {
   diffexpDimensionName
 } from "../nameCreators";
 
+export function maxCategoryItems(config) {
+  return (
+    config.parameters?.["max-category-items"] ??
+    globals.configDefaults.parameters["max-category-items"]
+  );
+}
+
 /*
 Selection state for categoricals are tracked in an Object that
 has two main components for each category:
@@ -62,15 +69,23 @@ function topNCategories(colSchema, summary, N) {
   return [topNCategories, topNCounts];
 }
 
-export function selectableCategoryNames(world, maxCategoryItems) {
-  const { schema } = world;
+export function selectableCategoryNames(schema, maxCategoryItems, names) {
+  /*
+  return all obs annotation names that are categorical AND have a
+  "reasonably" small number of categories AND are not the index column.
+
+  If the initial name list not provided, use everything in the schema.
+  */
+  if (!schema) return [];
   const { index, columns } = schema.annotations.obs;
+
   return columns
+    .filter(colSchema => !names || names.indexOf(colSchema.name) !== -1)
     .filter(colSchema => {
-      const { name, categories } = colSchema;
-      return (
-        categories && categories.length < maxCategoryItems && name !== index
-      );
+      const { type, name } = colSchema;
+      const isSelectableType =
+        type === "string" || type === "boolean" || type === "categorical";
+      return isSelectableType && name !== index;
     })
     .map(v => v.name);
 }

client/src/util/stateManager/schemaHelpers.js

@@ -31,15 +31,6 @@ export function indexEntireSchema(schema) {
   return schema;
 }
 
-export function sortAllCategorical(schema) {
-  /* UI relies on OBS annotation categories being in presentation sort order */
-  schema.annotations.obs.columns.forEach(c => {
-    if (c.categories) {
-      c.categories = catLabelSort(c.writable, c.categories);
-    }
-  });
-}
-
 function _copy(schema) {
   /* redux copy conventions - WARNING, only for modifyign obs annotations */
   return {

client/src/util/stateManager/universe.js

@@ -4,8 +4,9 @@ import { unassignedCategoryLabel } from "../../globals";
 import { decodeMatrixFBS } from "./matrix";
 import * as Dataframe from "../dataframe";
 import { isFpTypedArray } from "../typeHelpers";
-import { indexEntireSchema, sortAllCategorical } from "./schemaHelpers";
+import { indexEntireSchema } from "./schemaHelpers";
 import { isCategoricalAnnotation } from "./annotationsHelpers";
+import catLabelSort from "../catLabelSort";
 
 /*
 Private helper function - create and return a template Universe
@@ -74,9 +75,9 @@ function promoteTypedArray(o) {
   return new TyepdArrayCtor(o);
 }
 
-function AnnotationsFBSToDataframe(arrayBuffer) {
+export function matrixFBSToDataframe(arrayBuffers) {
   /*
-  Convert a Matrix FBS to a Dataframe.
+  Convert array of Matrix FBS to a Dataframe.
 
   The application has strong assumptions that all scalar data will be
   stored as a float32 or float64 (regardless of underlying data types).
@@ -86,77 +87,41 @@ function AnnotationsFBSToDataframe(arrayBuffer) {
   All float data from the server is left as is.  All non-float is promoted
   to an appropriate float.
   */
-  const fbs = decodeMatrixFBS(arrayBuffer, true); // leave in place
-  const columns = fbs.columns.map(c => {
-    if (isFpTypedArray(c) || Array.isArray(c)) return c;
-    return promoteTypedArray(c);
-  });
-  const df = new Dataframe.Dataframe(
-    [fbs.nRows, fbs.nCols],
-    columns,
-    null,
-    new Dataframe.KeyIndex(fbs.colIdx)
-  );
-  return df;
-}
-
-function LayoutFBSToDataframe(arrayBuffer) {
-  const fbs = decodeMatrixFBS(arrayBuffer, true);
-  if (fbs.columns.length < 2 || !fbs.columns.every(isFpTypedArray)) {
-    // We have strong assumptions about the shape & type of layout data.
-    throw new Error("Unexpected layout data type returned from server");
+  if (!Array.isArray(arrayBuffers)) {
+    arrayBuffers = [arrayBuffers];
+  }
+  if (arrayBuffers.length === 0) {
+    return Dataframe.Dataframe.empty();
   }
 
+  const fbs = arrayBuffers.map(ab => decodeMatrixFBS(ab, true)); // leave in place
+  /* check that all FBS have same row dimensionality */
+  const nRows = fbs[0].nRows;
+  fbs.forEach(b => {
+    if (b.nRows !== nRows)
+      throw new Error("FBS with inconsistent dimensionality");
+  });
+  const columns = fbs
+    .map(fb =>
+      fb.columns.map(c => {
+        if (isFpTypedArray(c) || Array.isArray(c)) return c;
+        return promoteTypedArray(c);
+      })
+    )
+    .flat();
+  const colIdx = fbs.map(b => b.colIdx).flat();
+  const nCols = columns.length;
+
   const df = new Dataframe.Dataframe(
-    [fbs.nRows, fbs.nCols],
-    fbs.columns,
+    [nRows, nCols],
+    columns,
     null,
-    new Dataframe.KeyIndex(fbs.colIdx)
+    new Dataframe.KeyIndex(colIdx)
   );
   return df;
 }
 
-function reconcileSchemaCategoriesWithSummary(universe) {
-  /*
-  where we treat types as (essentially) categorical metadata, update
-  the schema with data-derived categories (in addition to those in
-  the server declared schema).
-
-  For example, boolean defined fields in the schema do not contain
-  explicit declaration of categories (nor do string fields).  In these
-  cases, add a 'categories' field to the schema so it is accessible.
-
-  In addition, we have a client-side convention (UI) that all writable
-  annotations must have an 'unassigned' category, even if it is not currently
-  in use.
-  */
-
-  universe.schema.annotations.obs.columns.forEach(s => {
-    if (
-      s.type === "string" ||
-      s.type === "boolean" ||
-      s.type === "categorical"
-    ) {
-      const categories = _.union(
-        s.categories ?? [],
-        universe.obsAnnotations.col(s.name).summarize().categories ?? []
-      );
-      s.categories = categories;
-    }
-
-    if (s.writable && s.categories.indexOf(unassignedCategoryLabel) === -1) {
-      s.categories = s.categories.concat(unassignedCategoryLabel);
-    }
-  });
-}
-
-export function createUniverseFromResponse(
-  configResponse,
-  schemaResponse,
-  annotationsObsResponse,
-  annotationsVarResponse,
-  layoutFBSResponse
-) {
+export function createUniverseFromResponse(configResponse, schemaResponse) {
   /*
   build & return universe from a REST 0.2 /config, /schema and /annotations/obs response
   */
@@ -167,41 +132,80 @@ export function createUniverseFromResponse(
   universe.schema = schema;
   universe.nObs = schema.dataframe.nObs;
   universe.nVar = schema.dataframe.nVar;
+
   /* add defaults, as we can't assume back-end will fully populate schema */
   if (!schema.layout.var) schema.layout.var = [];
   if (!schema.layout.obs) schema.layout.obs = [];
-
-  /* annotations */
-  universe.obsAnnotations = AnnotationsFBSToDataframe(annotationsObsResponse);
-  universe.varAnnotations = AnnotationsFBSToDataframe(annotationsVarResponse);
-  /* layout */
-  universe.obsLayout = LayoutFBSToDataframe(layoutFBSResponse);
-
-  /* sanity checks */
-  if (
-    universe.nObs !== universe.obsLayout.length ||
-    universe.nObs !== universe.obsAnnotations.length ||
-    universe.nVar !== universe.varAnnotations.length
-  ) {
-    throw new Error("Universe dimensionality mismatch - failed to load");
-  }
-
-  reconcileSchemaCategoriesWithSummary(universe);
-  sortAllCategorical(universe.schema);
   indexEntireSchema(universe.schema);
+  normalizeEntireSchema(universe.schema);
 
-  /* sanity checks */
-  if (
-    schema.annotations.obs.columns.some(
-      s => s.writable && !isCategoricalAnnotation(schema, s.name)
-    )
-  ) {
+  return universe;
+}
+
+function normalizeSchemaCategory(colSchema, col = undefined) {
+  const { type, writable } = colSchema;
+  if (type === "string" || type === "boolean" || type === "categorical") {
+    let categories = [
+      ...new Set([
+        ...(colSchema.categories ?? []),
+        ...(col?.summarize?.().categories ?? [])
+      ])
+    ];
+    if (writable && categories.indexOf(unassignedCategoryLabel) === -1) {
+      categories = categories.concat(unassignedCategoryLabel);
+    }
+    colSchema.categories = categories;
+  } else if (writable) {
     throw new Error(
       "Writable continuous obs annotations are not supported - failed to load"
     );
   }
 
-  return universe;
+  if (colSchema.categories) {
+    colSchema.categories = catLabelSort(writable, colSchema.categories);
+  }
+}
+
+function normalizeEntireSchema(schema) {
+  // currently only needed for obsAnnotations
+  schema.annotations.obs.columns.forEach(colSchema =>
+    normalizeSchemaCategory(colSchema)
+  );
+}
+
+export function addObsAnnotations(universe, df) {
+  const obsAnnotations = universe.obsAnnotations.withColsFromAll(df);
+  if (universe.nObs !== obsAnnotations.length) {
+    throw new Error("Universe dimensionality mismatch - failed to load");
+  }
+
+  // for all of the new data, reconcile with schema and sort categories.
+  const dfs = Array.isArray(df) ? df : [df];
+  const keys = dfs.map(df => df.colIndex.keys()).flat();
+  const { schema } = universe;
+  keys.forEach(k => {
+    const colSchema = schema.annotations.obsByName[k];
+    const col = obsAnnotations.col(k);
+    normalizeSchemaCategory(colSchema, col);
+  });
+
+  return { obsAnnotations, schema };
+}
+
+export function addVarAnnotations(universe, df) {
+  const varAnnotations = universe.varAnnotations.withColsFromAll(df);
+  if (universe.nVar !== varAnnotations.length) {
+    throw new Error("Universe dimensionality mismatch - failed to load");
+  }
+  return { varAnnotations };
+}
+
+export function addObsLayout(universe, df) {
+  const obsLayout = universe.obsLayout.withColsFromAll(df);
+  if (universe.nObs !== obsLayout.length) {
+    throw new Error("Universe dimensionality mismatch - failed to load");
+  }
+  return { obsLayout };
 }
 
 export function convertDataFBStoObject(universe, arrayBuffer) {

client/src/util/stateManager/world.js

@@ -113,7 +113,6 @@ function clipDataframe(
 
 /*
 Create World with contents eq entire universe.   Commonly used to initialize World.
-If clipQuantiles
 */
 export function createWorldFromEntireUniverse(universe) {
   const world = templateWorld();
@@ -253,6 +252,40 @@ function deduceDimensionType(attributes, fieldName) {
   return dimensionType;
 }
 
+function addObsDimension(crossfilter, world, anno) {
+  /*
+  add single dimension to the crosfilter
+  */
+  const { obsAnnotations } = world;
+  if (obsAnnotations.hasCol(anno.name)) {
+    const dimType = deduceDimensionType(anno, anno.name);
+    const colData = obsAnnotations.col(anno.name).asArray();
+    const name = obsAnnoDimensionName(anno.name);
+    if (dimType === "enum") {
+      return crossfilter.addDimension(name, "enum", colData);
+    }
+    if (dimType) {
+      return crossfilter.addDimension(name, "scalar", colData, dimType);
+    }
+  }
+  return crossfilter;
+}
+
+export function addObsDimensions(crossfilter, world) {
+  /*
+  Add to crossfilter any dimension present in world.obsAnnotations
+  but not yet in the crossfilter
+  */
+  const schema = world.schema.annotations.obsByName;
+  const dimsWeNeed = world.obsAnnotations.colIndex.keys();
+  crossfilter = dimsWeNeed.reduce((xfltr, name) => {
+    const dimName = obsAnnoDimensionName(name);
+    if (xfltr.hasDimension(dimName)) return xfltr;
+    return addObsDimension(xfltr, world, schema[name]);
+  }, crossfilter);
+  return crossfilter;
+}
+
 export function createObsDimensions(crossfilter, world, XYdimNames) {
   /*
   create and return a crossfilter with a dimension for every obs annotation
@@ -265,16 +298,7 @@ export function createObsDimensions(crossfilter, world, XYdimNames) {
     anno => anno.name !== indexName
   );
   crossfilter = annoList.reduce((xfltr, anno) => {
-    const dimType = deduceDimensionType(anno, anno.name);
-    const colData = obsAnnotations.col(anno.name).asArray();
-    const name = obsAnnoDimensionName(anno.name);
-    if (dimType === "enum") {
-      return xfltr.addDimension(name, "enum", colData);
-    }
-    if (dimType) {
-      return xfltr.addDimension(name, "scalar", colData, dimType);
-    }
-    return xfltr;
+    return addObsDimension(xfltr, world, anno);
   }, crossfilter);
 
   return crossfilter.addDimension(