load annotations incrementally (#1107)

* load annotations individually

* fix type check to be more general

* update node CI version from 10 to 12

* node 11

* debug print node version

* travis node version to latest

* try nvm

* remove extraneous node_js statement

* remove node version debugging printf

* incrementally load all annotations and layout

* process annotations and layout as they are loaded

* fix tests

* sort categories incrementally

* incrementally build category view summary; add category loading spinner

* add spinner to continuous metadata

* configure undoable reducer

* incremental crossfilter creation

* improve busy layout

* more layout cleanup

* correctly reconcile categories in schema

* refine layout of lsb spinners

* more spinner layout work

* more spinner layout

* always load layout before obs annotations

client/__tests__/util/dataframe/dataframe.test.js

@@ -481,6 +481,7 @@ describe("dataframe factories", () => {
 
     test("simple", () => {
       /* simple test that it works as expected in common case */
+      const dfEmpty = Dataframe.Dataframe.empty();
       const dfA = new Dataframe.Dataframe(
         [2, 1],
         [["red", "blue"]],
@@ -494,6 +495,22 @@ describe("dataframe factories", () => {
         new Dataframe.KeyIndex(["bools"])
       );
 
+      const dfLikeA = dfEmpty.withColsFrom(dfA);
+      expect(dfLikeA).toBeDefined();
+      expect(dfLikeA.dims).toEqual(dfA.dims);
+      expect(dfLikeA.colIndex.keys()).toEqual(dfA.colIndex.keys());
+      expect(dfLikeA.rowIndex).toEqual(dfA.rowIndex);
+      expect(dfLikeA.rowIndex.keys()).toEqual(dfA.rowIndex.keys());
+      expect(dfLikeA.icol(0).asArray()).toEqual(dfA.icol(0).asArray());
+
+      const dfAlsoLikeA = dfA.withColsFrom(dfEmpty);
+      expect(dfAlsoLikeA).toBeDefined();
+      expect(dfAlsoLikeA.dims).toEqual(dfA.dims);
+      expect(dfAlsoLikeA.colIndex.keys()).toEqual(dfA.colIndex.keys());
+      expect(dfAlsoLikeA.rowIndex).toEqual(dfA.rowIndex);
+      expect(dfAlsoLikeA.rowIndex.keys()).toEqual(dfA.rowIndex.keys());
+      expect(dfAlsoLikeA.icol(0).asArray()).toEqual(dfA.icol(0).asArray());
+
       const dfC = dfA.withColsFrom(dfB);
       expect(dfC).toBeDefined();
       expect(dfC.dims).toEqual([2, 2]);

client/__tests__/util/stateManager/universe.test.js

@@ -30,15 +30,39 @@ describe("createUniverseFromResponse", () => {
     create a universe from sample data nad validate its shape & contents
     */
     const { nObs, nVar } = REST.schema.schema.dataframe;
-    const universe = Universe.createUniverseFromResponse(
+    let universe = Universe.createUniverseFromResponse(
       REST.config,
-      REST.schema,
-      REST.annotationsObs,
-      REST.annotationsVar,
-      REST.layoutObs
+      REST.schema
+    );
+    expect(universe).toBeDefined();
+    expect(universe).toMatchObject(
+      expect.objectContaining({
+        nObs,
+        nVar,
+        schema: REST.schema.schema,
+        obsAnnotations: expect.any(Dataframe.Dataframe),
+        varAnnotations: expect.any(Dataframe.Dataframe),
+        obsLayout: expect.any(Dataframe.Dataframe),
+        varData: expect.any(Dataframe.Dataframe)
+      })
     );
 
-    expect(universe).toBeDefined();
+    universe = {
+      ...universe,
+      ...Universe.addObsAnnotations(
+        universe,
+        Universe.matrixFBSToDataframe(REST.annotationsObs)
+      ),
+      ...Universe.addVarAnnotations(
+        universe,
+        Universe.matrixFBSToDataframe(REST.annotationsVar)
+      ),
+      ...Universe.addObsLayout(
+        universe,
+        Universe.matrixFBSToDataframe(REST.layoutObs)
+      )
+    };
+
     expect(universe).toMatchObject(
       expect.objectContaining({
         nObs,

client/__tests__/util/stateManager/world.test.js

@@ -17,13 +17,27 @@ the default REST test response.
 const defaultBigBang = () => {
   /* create unverse, world, crossfilter and dimensionMap */
   /* create universe */
-  const universe = Universe.createUniverseFromResponse(
+  let universe = Universe.createUniverseFromResponse(
     _.cloneDeep(REST.config),
-    _.cloneDeep(REST.schema),
-    _.cloneDeep(REST.annotationsObs),
-    _.cloneDeep(REST.annotationsVar),
-    _.cloneDeep(REST.layoutObs)
+    _.cloneDeep(REST.schema)
   );
+
+  universe = {
+    ...universe,
+    ...Universe.addObsAnnotations(
+      universe,
+      Universe.matrixFBSToDataframe(REST.annotationsObs)
+    ),
+    ...Universe.addVarAnnotations(
+      universe,
+      Universe.matrixFBSToDataframe(REST.annotationsVar)
+    ),
+    ...Universe.addObsLayout(
+      universe,
+      Universe.matrixFBSToDataframe(REST.layoutObs)
+    )
+  };
+
   /* create world */
   const world = World.createWorldFromEntireUniverse(universe);
   /* create crossfilter */
@@ -45,9 +59,9 @@ describe("createWorldFromEntireUniverse", () => {
     const universe = Universe.createUniverseFromResponse(
       _.cloneDeep(REST.config),
       _.cloneDeep(REST.schema),
-      _.cloneDeep(REST.annotationsObs),
-      _.cloneDeep(REST.annotationsVar),
-      _.cloneDeep(REST.layoutObs)
+      Universe.matrixFBSToDataframe(_.cloneDeep(REST.annotationsObs)),
+      Universe.matrixFBSToDataframe(_.cloneDeep(REST.annotationsVar)),
+      Universe.matrixFBSToDataframe(_.cloneDeep(REST.layoutObs))
     );
     expect(universe).toBeDefined();
 

client/src/actions/index.js

@@ -1,4 +1,3 @@
-// jshint esversion: 6
 import _ from "lodash";
 import * as globals from "../globals";
 import { Universe, MatrixFBS } from "../util/stateManager";
@@ -9,6 +8,89 @@ import {
   dispatchNetworkErrorMessageToUser
 } from "../util/actionHelpers";
 
+/*
+return promise to fetch the OBS annotations we need to load.  Omit anything
+we don't need.
+*/
+function obsAnnotationFetchAndLoad(dispatch, schema, universe) {
+  const obsAnnotations = schema?.schema?.annotations?.obs ?? {};
+  const columns = obsAnnotations.columns ?? [];
+  const index = obsAnnotations.index ?? false;
+  return Promise.all(
+    columns
+      .filter(col => col.name !== index)
+      .map(col => {
+        const path = `annotations/obs?annotation-name=${encodeURIComponent(
+          col.name
+        )}`;
+        const url = `${globals.API.prefix}${globals.API.version}${path}`;
+        return doBinaryRequest(url);
+      })
+      .map(rqst => rqst.then(buffer => Universe.matrixFBSToDataframe(buffer)))
+      .map(resp =>
+        resp.then(df =>
+          dispatch({
+            type: "universe: column load success",
+            dim: "obsAnnotations",
+            dataframe: df
+          })
+        )
+      )
+  );
+}
+
+/*
+return promise fetching VAR annotations we need to load.  Only index is currently used.
+*/
+function varAnnotationFetchAndLoad(dispatch, schema, universe) {
+  const varAnnotations = schema?.schema?.annotations?.var ?? {};
+  const index = varAnnotations.index ?? false;
+  const names = index ? [index] : [];
+  return Promise.all(
+    names
+      .map(name => {
+        const path = `annotations/var?annotation-name=${encodeURIComponent(
+          name
+        )}`;
+        const url = `${globals.API.prefix}${globals.API.version}${path}`;
+        return doBinaryRequest(url);
+      })
+      .map(rqst => rqst.then(buffer => Universe.matrixFBSToDataframe(buffer)))
+      .map(resp =>
+        resp.then(df =>
+          dispatch({
+            type: "universe: column load success",
+            dim: "varAnnotations",
+            dataframe: df
+          })
+        )
+      )
+  );
+}
+
+/*
+return promise fetching layout we need
+*/
+function layoutFetchAndLoad(dispatch, schema, universe) {
+  return Promise.all(
+    ["layout/obs"]
+      .map(path => {
+        const url = `${globals.API.prefix}${globals.API.version}${path}`;
+        return doBinaryRequest(url);
+      })
+      .map(rqst => rqst.then(buffer => Universe.matrixFBSToDataframe(buffer)))
+      .map(resp =>
+        resp.then(df =>
+          dispatch({
+            type: "universe: column load success",
+            dim: "obsLayout",
+            dataframe: df
+          })
+        )
+      )
+  );
+}
+
 /*
 Bootstrap application with the initial data loading.
   * /config - application configuration
@@ -31,34 +113,29 @@ const doInitialDataLoad = () =>
       /* set config defaults */
       const config = { ...globals.configDefaults, ...stepOneResults[0].config };
       const schema = stepOneResults[1];
-
-      /*
-      Step 2 - dataframes, all binary.  NOTE: uses results of step 1.
-      */
-      /* only load names for var annotations, if possible*/
-      const varIndexName = schema?.schema?.annotations?.var?.index;
-      const varAnnotationsQuery = varIndexName
-        ? `?annotation-name=${encodeURIComponent(varIndexName)}`
-        : "";
-      const varAnnotationsURL = `annotations/var${varAnnotationsQuery}`;
-      const requestBinary = ["annotations/obs", varAnnotationsURL, "layout/obs"]
-        .map(r => `${globals.API.prefix}${globals.API.version}${r}`)
-        .map(url => doBinaryRequest(url));
-      const stepTwoResults = await Promise.all(requestBinary);
-      const [obsAnno, varAnno, obsLayout] = [...stepTwoResults];
-
-      const universe = Universe.createUniverseFromResponse(
-        config,
-        schema,
-        obsAnno,
-        varAnno,
-        obsLayout
-      );
-
+      const universe = Universe.createUniverseFromResponse(config, schema);
+      dispatch({
+        type: "universe exists, but loading is still in progress",
+        universe
+      });
       dispatch({
         type: "configuration load complete",
         config
       });
+
+      /*
+      Step 2 - load the minimum stuff required to display.
+      */
+      await Promise.all([
+        layoutFetchAndLoad(dispatch, schema, universe),
+        varAnnotationFetchAndLoad(dispatch, schema, universe)
+      ]);
+
+      /*
+      Step 3 - load everything else
+      */
+      await obsAnnotationFetchAndLoad(dispatch, schema, universe);
+
       dispatch({
         type: "initial data load complete (universe exists)",
         universe

client/src/components/brushableHistogram/index.js

@@ -351,8 +351,8 @@ class HistogramBrush extends React.PureComponent {
     const brushX = d3
       .brushX()
       .extent([
-          [x.range()[0], y.range()[1]],
-          [x.range()[1], this.marginTop + this.height + this.marginBottom]
+        [x.range()[0], y.range()[1]],
+        [x.range()[1], this.marginTop + this.height + this.marginBottom]
       ])
       /*
       emit start so that the Undoable history can save an undo point

client/src/components/categorical/categorical.js

@@ -4,15 +4,15 @@ import { Button } from "@blueprintjs/core";
 import { connect } from "react-redux";
 import * as globals from "../../globals";
 import Category from "./category";
-import { AnnotationsHelpers } from "../../util/stateManager";
+import { AnnotationsHelpers, ControlsHelpers } from "../../util/stateManager";
 import AnnoDialog from "./annoDialog";
 import AnnoInputs from "./annoInputs";
 import AnnoSelect from "./annoSelect";
 
 @connect(state => ({
-  categoricalSelection: state.categoricalSelection,
   writableCategoriesEnabled: state.config?.parameters?.["annotations"] ?? false,
-  schema: state.world?.schema
+  schema: state.world?.schema,
+  config: state.config
 }))
 class Categories extends React.Component {
   constructor(props) {
@@ -123,12 +123,15 @@ class Categories extends React.Component {
     const {
       categoricalSelection,
       writableCategoriesEnabled,
-      schema
+      schema,
+      config
     } = this.props;
-    if (!categoricalSelection) return null;
 
     /* all names, sorted in display order.  Will be rendered in this order */
-    const allCategoryNames = Object.keys(categoricalSelection).sort();
+    const allCategoryNames = ControlsHelpers.selectableCategoryNames(
+      schema,
+      ControlsHelpers.maxCategoryItems(config)
+    ).sort();
 
     return (
       <div

client/src/components/categorical/category.js

@@ -2,7 +2,7 @@ import React from "react";
 import _ from "lodash";
 import { connect } from "react-redux";
 import { FaChevronRight, FaChevronDown } from "react-icons/fa";
-import { Button, Tooltip, Icon } from "@blueprintjs/core";
+import { Button, Tooltip, Icon, Spinner } from "@blueprintjs/core";
 import CategoryFlipperLayout from "./categoryFlipperLayout";
 import AnnoMenu from "./annoMenuCategory";
 import AnnoDialogEditCategoryName from "./annoDialogEditCategoryName";
@@ -31,8 +31,12 @@ class Category extends React.Component {
 
   componentDidUpdate(prevProps) {
     const { categoricalSelection, metadataField } = this.props;
-    if (categoricalSelection !== prevProps.categoricalSelection) {
-      const cat = categoricalSelection[metadataField];
+    const cat = categoricalSelection?.[metadataField];
+    if (
+      categoricalSelection !== prevProps.categoricalSelection &&
+      !!cat &&
+      !!this.checkbox
+    ) {
       const categoryCount = {
         // total number of categories in this dimension
         totalCatCount: cat.numCategoryValues,
@@ -95,15 +99,66 @@ class Category extends React.Component {
     }
   }
 
+  renderIsStillLoading(metadataField) {
+    /*
+    We are still loading this category, so render a "busy" signal.
+    */
+    return (
+      <div
+        style={{
+          maxWidth: globals.maxControlsWidth
+        }}
+      >
+        <div
+          style={{
+            display: "flex",
+            justifyContent: "space-between",
+            alignItems: "baseline"
+          }}
+        >
+          <div
+            style={{
+              display: "flex",
+              justifyContent: "flex-start",
+              alignItems: "flex-start"
+            }}
+          >
+            <label className="bp3-control bp3-checkbox">
+              <input disabled checked={true} type="checkbox" />
+              <span className="bp3-control-indicator" />
+            </label>
+            <span
+              style={{
+                cursor: "pointer",
+                display: "inline-block"
+              }}
+            >
+              {metadataField}
+            </span>
+          </div>
+          <div>
+            <Button minimal loading intent="primary" />
+          </div>
+        </div>
+      </div>
+    );
+  }
+
   render() {
     const { isExpanded, isChecked } = this.state;
     const {
       metadataField,
+      categoricalSelection,
       colorAccessor,
       isUserAnno,
       annotations
     } = this.props;
 
+    const isStillLoading = !(categoricalSelection?.[metadataField] ?? false);
+    if (isStillLoading) {
+      return this.renderIsStillLoading(metadataField);
+    }
+
     return (
       <CategoryFlipperLayout
         metadataField={metadataField}

client/src/components/continuous/continuous.js

@@ -5,6 +5,7 @@ import React from "react";
 import _ from "lodash";
 import { connect } from "react-redux";
 import * as globals from "../../globals";
+import { Button } from "@blueprintjs/core";
 
 import HistogramBrush from "../brushableHistogram";
 
@@ -14,17 +15,7 @@ import HistogramBrush from "../brushableHistogram";
   colorScale: state.colors.scale,
   schema: state.world?.schema
 }))
-class Continuous extends React.Component {
-  constructor(props) {
-    super(props);
-    this.hasContinuous = false;
-    this.continuousChecked = false;
-
-    this.state = {};
-  }
-
-  componentDidUpdate() {}
-
+class Continuous extends React.PureComponent {
   handleColorAction = key => {
     return () => {
       const { dispatch, obsAnnotations } = this.props;
@@ -37,61 +28,82 @@ class Continuous extends React.Component {
     };
   };
 
+  renderIsStillLoading(zebra, key) {
+    return (
+      <div
+        key={key}
+        style={{
+          padding: globals.leftSidebarSectionPadding,
+          backgroundColor: zebra % 2 === 0 ? globals.lightestGrey : "white"
+        }}
+      >
+        <div
+          style={{
+            display: "flex",
+            justifyContent: "space-between",
+            justifyItems: "center",
+            alignItems: "center"
+          }}
+        >
+          <div style={{ minWidth: 30 }}></div>
+          <div style={{ display: "flex", alignSelf: "center" }}>
+            <span style={{ fontStyle: "italic" }}>{key}</span>
+          </div>
+          <div
+            style={{
+              display: "flex",
+              justifyContent: "flex-end"
+            }}
+          >
+            <Button minimal loading intent="primary" />
+          </div>
+        </div>
+      </div>
+    );
+  }
+
   render() {
     const { obsAnnotations, schema } = this.props;
-    if (schema && !this.continuousChecked) {
-      this.hasContinuous = _.some(
-        schema.annotations.obs,
-        d => d.type === "int32" || d.type === "float32"
-      );
-      this.continuousChecked = true; /* only do this once */
-    }
+
+    const obsIndex = schema.annotations.obs.index;
+    const allContinuousNames = schema.annotations.obs.columns
+      .filter(col => col.type === "int32" || col.type === "float32")
+      .filter(col => col.name != obsIndex)
+      .map(col => col.name);
 
     /* initial value for iterator to simulate index, ranges is an object */
     let zebra = 0;
 
     return (
       <div>
-        {this.hasContinuous ? (
-          <p
-            style={{
-              ...globals.leftSidebarSectionHeading,
-              marginTop: 40,
-              paddingLeft: globals.leftSidebarSectionPadding
-            }}
-          >
-            Continuous metadata
-          </p>
-        ) : null}
-        {obsAnnotations
-          ? _.map(obsAnnotations.colIndex.keys(), key => {
-              const isColorField =
-                key.includes("color") || key.includes("Color");
-              if (key === schema.annotations.obs.index || isColorField)
-                return null;
-
-              const summary = obsAnnotations.col(key).summarize();
-              const nonFiniteExtent =
-                summary.min === undefined ||
-                summary.max === undefined ||
-                Number.isNaN(summary.min) ||
-                Number.isNaN(summary.max);
-              if (!summary.categorical && !nonFiniteExtent) {
-                zebra += 1;
-                return (
-                  <HistogramBrush
-                    key={key}
-                    field={key}
-                    isObs
-                    zebra={zebra % 2 === 0}
-                    ranges={summary}
-                    handleColorAction={this.handleColorAction(key)}
-                  />
-                );
-              }
-              return null;
-            })
-          : null}
+        {allContinuousNames.map(key => {
+          if (!obsAnnotations.hasCol(key)) {
+            // still loading!
+            zebra += 1;
+            return this.renderIsStillLoading(zebra, key);
+          } else {
+            // data loaded and available
+            const summary = obsAnnotations.col(key).summarize();
+            const nonFiniteExtent =
+              summary.min === undefined ||
+              summary.max === undefined ||
+              Number.isNaN(summary.min) ||
+              Number.isNaN(summary.max);
+            if (!summary.categorical && !nonFiniteExtent) {
+              zebra += 1;
+              return (
+                <HistogramBrush
+                  key={key}
+                  field={key}
+                  isObs
+                  zebra={zebra % 2 === 0}
+                  ranges={summary}
+                  handleColorAction={this.handleColorAction(key)}
+                />
+              );
+            }
+          }
+        })}
       </div>
     );
   }

client/src/components/geneExpression/index.js

@@ -103,7 +103,8 @@ class GeneExpression extends React.Component {
       if (genes.length === 0) {
         return keepAroundErrorToast("Must enter a gene name.");
       }
-      const worldGenes = world.varAnnotations.col(varIndexName).asArray();
+      const worldGenes =
+        world.varAnnotations?.col(varIndexName)?.asArray() || [];
 
       // These gene lists are unique enough where memoization is useless
       const upperGenes = this._genesToUpper(genes);
@@ -206,8 +207,12 @@ class GeneExpression extends React.Component {
       differential
     } = this.props;
     const varIndexName = world?.schema?.annotations?.var?.index;
+    const varIndex = world?.varAnnotations?.col(varIndexName)?.asArray();
     const { tab, bulkAdd, activeItem } = this.state;
 
+    // may still be loading!
+    if (!varIndex) return null;
+
     return (
       <div>
         <div>
@@ -268,11 +273,7 @@ class GeneExpression extends React.Component {
                 itemListPredicate={filterGenes}
                 onActiveItemChange={item => this.setState({ activeItem: item })}
                 itemRenderer={renderGene.bind(this)}
-                items={
-                  world && world.varAnnotations
-                    ? world.varAnnotations.col(varIndexName).asArray()
-                    : ["No genes"]
-                }
+                items={varIndex || ["No genes"]}
                 popoverProps={{ minimal: true }}
               />
               <Button

client/src/components/graph/graph.js

@@ -255,7 +255,7 @@ class Graph extends React.Component {
     const { regl, toolSVG, camera, modelTF } = this.state;
     let stateChanges = {};
 
-    if (regl && world) {
+    if (regl && world && crossfilter) {
       /* update the regl and point rendering state */
       const { obsLayout, nObs } = world;
       const { drawPoints, pointBuffer, colorBuffer, flagBuffer } = this.state;

client/src/components/menubar/index.js

@@ -23,7 +23,6 @@ import * as globals from "../../globals";
 @connect(state => ({
   universe: state.universe,
   world: state.world,
-  loading: state.controls.loading,
   crossfilter: state.crossfilter,
   differential: state.differential,
   resettingInterface: state.controls.resettingInterface,

client/src/reducers/categoricalSelection.js

@@ -1,13 +1,6 @@
 import { ControlsHelpers as CH } from "../util/stateManager";
 import * as globals from "../globals";
 
-function maxCategoryItems(state) {
-  return (
-    state.config.parameters?.["max-category-items"] ??
-    globals.configDefaults.parameters["max-category-items"]
-  );
-}
-
 const CategoricalSelection = (
   state,
   action,
@@ -22,11 +15,32 @@ const CategoricalSelection = (
       const { world } = nextSharedState;
       const newState = CH.createCategoricalSelection(
         world,
-        CH.selectableCategoryNames(world, maxCategoryItems(prevSharedState))
+        CH.selectableCategoryNames(
+          world.schema,
+          CH.maxCategoryItems(prevSharedState.config)
+        )
       );
       return newState;
     }
 
+    case "universe: column load success": {
+      const { dim } = action;
+      if (dim !== "obsAnnotations") return state;
+
+      const { dataframe } = action;
+      const { world } = nextSharedState;
+      const names = CH.selectableCategoryNames(
+        world.schema,
+        CH.maxCategoryItems(prevSharedState.config),
+        dataframe.colIndex.keys()
+      );
+      if (names.length === 0) return state;
+      return {
+        ...state,
+        ...CH.createCategoricalSelection(world, names)
+      };
+    }
+
     case "categorical metadata filter select": {
       /*
       Set the specific category in this field to false

client/src/reducers/colors.js

@@ -12,7 +12,7 @@ const ColorsReducer = (
   prevSharedState
 ) => {
   switch (action.type) {
-    case "initial data load complete (universe exists)":
+    case "universe exists, but loading is still in progress":
     case "reset World to eq Universe": {
       const { world } = nextSharedState;
       const colorMode = null;

client/src/reducers/controls.js

@@ -41,8 +41,31 @@ const Controls = (
     case "initial data load start": {
       return { ...state, loading: true };
     }
+    case "universe: column load success": {
+      /* 
+      we are in a loading state until the following are true:
+        * universe exists (if partially)
+        * embeddings (obsLayout) are loaded
+        * varAnnotations index is loaded
+      */
+      const universeExists = !!nextSharedState.universe;
+      const embeddingsExist =
+        !nextSharedState.universe?.obsLayout?.isEmpty() ?? false;
+      const varIndex = nextSharedState.universe.schema.annotations.var.index;
+      const varAnnotationsIndexExists =
+        universeExists &&
+        nextSharedState.universe.varAnnotations.hasCol(varIndex);
+      return {
+        ...state,
+        loading: !(
+          universeExists &&
+          embeddingsExist &&
+          varAnnotationsIndexExists
+        )
+      };
+    }
     case "initial data load complete (universe exists)": {
-      /* first light - create world & other data-driven defaults */
+      /* now fully loaded */
       return {
         ...state,
         loading: false,

client/src/reducers/crossfilter.js

@@ -23,13 +23,38 @@ const CrossfilterReducerBase = (
   prevSharedState
 ) => {
   switch (action.type) {
-    case "initial data load complete (universe exists)": {
-      const { world, layoutChoice } = nextSharedState;
-      const crossfilter = World.createObsDimensions(
-        new Crossfilter(world.obsAnnotations),
-        world,
-        layoutChoice.currentDimNames
-      );
+    case "universe: column load success": {
+      const { schema, world, layoutChoice } = nextSharedState;
+      const { obsAnnotations, obsLayout } = world;
+      const { dim, dataframe } = action;
+
+      // ignore var dimension loads as these are not currently selectable
+      if (action.dim === "varAnnotations") return state;
+
+      /* 
+      during bootstrap loading, we don't know if obsLayout or obsAnnotations
+      will load first. Take whichever arrives and is not empty (so that our
+      crossfilter has the right dimensionality).
+      */
+      let crossfilter =
+        state ??
+        new Crossfilter(obsAnnotations.isEmpty() ? obsLayout : obsAnnotations);
+
+      // add layout dimension, if not already present
+      if (
+        obsLayout.hasCol(layoutChoice.currentDimNames[0]) &&
+        !crossfilter.hasDimension(XYDimName)
+      ) {
+        crossfilter = crossfilter.addDimension(
+          XYDimName,
+          "spatial",
+          obsLayout.col(layoutChoice.currentDimNames[0]).asArray(),
+          obsLayout.col(layoutChoice.currentDimNames[1]).asArray()
+        );
+      }
+
+      // add any missing obsAnnotations
+      crossfilter = World.addObsDimensions(crossfilter, world);
       return crossfilter;
     }
 
@@ -278,10 +303,19 @@ const CrossfilterReducer = (
     nextSharedState,
     prevSharedState
   );
-  if (!nextState || nextState.all() === nextSharedState.world.obsAnnotations) {
+  /*
+  update the data in the crossfilter to point at the current obsAnnotations, IF
+  they are not empty.  If empty, leave it alone (can occur during boostrap loading).
+  */
+  const nextObsAnnotations = nextSharedState.world?.obsAnnotations;
+  if (
+    !nextState ||
+    nextState.all() === nextObsAnnotations ||
+    nextObsAnnotations.isEmpty()
+  ) {
     return nextState;
   }
-  return nextState.setData(nextSharedState.world.obsAnnotations);
+  return nextState.setData(nextObsAnnotations);
 };
 
 export default CrossfilterReducer;

client/src/reducers/layoutChoice.js

@@ -24,7 +24,7 @@ const LayoutChoice = (
   nextSharedState
 ) => {
   switch (action.type) {
-    case "initial data load complete (universe exists)":
+    case "universe exists, but loading is still in progress":
     case "reset World to eq Universe": {
       // set default to default
       const { schema } = nextSharedState.world;

client/src/reducers/undoableConfig.js

@@ -11,6 +11,8 @@ const skipOnActions = new Set([
   "url changed",
   "interface reset started",
   "initial data load start",
+  "universe: column load success",
+  "universe exists, but loading is still in progress",
   "configuration load complete",
   "increment graph render counter",
   "window resize",

client/src/reducers/universe.js

@@ -1,4 +1,9 @@
 import { unassignedCategoryLabel } from "../globals";
+import {
+  addObsAnnotations,
+  addVarAnnotations,
+  addObsLayout
+} from "../util/stateManager/universe";
 import {
   World,
   ControlsHelpers,
@@ -7,11 +12,38 @@ import {
 
 const Universe = (state = null, action, nextSharedState, prevSharedState) => {
   switch (action.type) {
-    case "initial data load complete (universe exists)": {
+    case "universe exists, but loading is still in progress": {
       const { universe } = action;
       return universe;
     }
 
+    case "universe: column load success": {
+      const { dim, dataframe } = action;
+      switch (dim) {
+        case "obsAnnotations": {
+          return {
+            ...state,
+            ...addObsAnnotations(state, dataframe)
+          };
+        }
+        case "varAnnotations": {
+          return {
+            ...state,
+            ...addVarAnnotations(state, dataframe)
+          };
+        }
+        case "obsLayout": {
+          return {
+            ...state,
+            ...addObsLayout(state, dataframe)
+          };
+        }
+        default: {
+          throw new Error("action handler not implemented");
+        }
+      }
+    }
+
     case "expression load success": {
       let { varData } = state;
 

client/src/reducers/world.js

@@ -24,13 +24,27 @@ const WorldReducer = (
   prevSharedState
 ) => {
   switch (action.type) {
-    case "initial data load complete (universe exists)":
+    case "universe exists, but loading is still in progress":
     case "reset World to eq Universe": {
       const { universe } = nextSharedState;
       const world = World.createWorldFromEntireUniverse(universe);
       return world;
     }
 
+    case "universe: column load success": {
+      const { universe } = nextSharedState;
+      const { dim } = action;
+      return {
+        ...state,
+        schema: universe.schema,
+        [dim]: universe[dim].clone(),
+        unclipped: {
+          ...state.unclipped,
+          [dim]: universe[dim].clone()
+        }
+      };
+    }
+
     case "set World to current selection": {
       /* Set viewable world to be the currently selected data */
       const world = World.createWorldBySelection(

client/src/util/dataframe/dataframe.js

@@ -351,11 +351,26 @@ class Dataframe {
   withColsFrom(dataframe) {
     /*
     return a new dataframe containing all columns from both `this` and the
-    provided dataframe.
+    provided of dataframe.
 
-    The row index from `this` will be used.  Both dataframes must have identical
+    The row index from `this` will be used.  All dataframes must have identical
     dimensionality, and no overlapping columns labels.
+
+    Special case, if either dataframe is empty, the other is returned unchanged.
      */
+    if (this.isEmpty()) {
+      return dataframe;
+    }
+    if (dataframe.isEmpty()) {
+      return this;
+    }
+
+    this.colIndex.keys().forEach(key => {
+      if (dataframe.has(key)) {
+        throw new Error("duplicate key collision");
+      }
+    });
+
     const dims = [this.dims[0], this.dims[1] + dataframe.dims[1]];
     const { rowIndex } = this;
     const columns = [...this.__columns, ...dataframe.__columns];
@@ -373,6 +388,11 @@ class Dataframe {
     );
   }
 
+  withColsFromAll(dataframes = []) {
+    dataframes = Array.isArray(dataframes) ? dataframes : [dataframes];
+    return dataframes.reduce((acc, df) => acc.withColsFrom(df), this);
+  }
+
   dropCol(label) {
     /*
     Create a new dataframe, omitting one columns.

client/src/util/stateManager/controlsHelpers.js

@@ -11,6 +11,13 @@ import {
   diffexpDimensionName
 } from "../nameCreators";
 
+export function maxCategoryItems(config) {
+  return (
+    config.parameters?.["max-category-items"] ??
+    globals.configDefaults.parameters["max-category-items"]
+  );
+}
+
 /*
 Selection state for categoricals are tracked in an Object that
 has two main components for each category:
@@ -62,15 +69,23 @@ function topNCategories(colSchema, summary, N) {
   return [topNCategories, topNCounts];
 }
 
-export function selectableCategoryNames(world, maxCategoryItems) {
-  const { schema } = world;
+export function selectableCategoryNames(schema, maxCategoryItems, names) {
+  /*
+  return all obs annotation names that are categorical AND have a
+  "reasonably" small number of categories AND are not the index column.
+
+  If the initial name list not provided, use everything in the schema.
+  */
+  if (!schema) return [];
   const { index, columns } = schema.annotations.obs;
+
   return columns
+    .filter(colSchema => !names || names.indexOf(colSchema.name) !== -1)
     .filter(colSchema => {
-      const { name, categories } = colSchema;
-      return (
-        categories && categories.length < maxCategoryItems && name !== index
-      );
+      const { type, name } = colSchema;
+      const isSelectableType =
+        type === "string" || type === "boolean" || type === "categorical";
+      return isSelectableType && name !== index;
     })
     .map(v => v.name);
 }

client/src/util/stateManager/schemaHelpers.js

@@ -31,15 +31,6 @@ export function indexEntireSchema(schema) {
   return schema;
 }
 
-export function sortAllCategorical(schema) {
-  /* UI relies on OBS annotation categories being in presentation sort order */
-  schema.annotations.obs.columns.forEach(c => {
-    if (c.categories) {
-      c.categories = catLabelSort(c.writable, c.categories);
-    }
-  });
-}
-
 function _copy(schema) {
   /* redux copy conventions - WARNING, only for modifyign obs annotations */
   return {

client/src/util/stateManager/universe.js

@@ -4,8 +4,9 @@ import { unassignedCategoryLabel } from "../../globals";
 import { decodeMatrixFBS } from "./matrix";
 import * as Dataframe from "../dataframe";
 import { isFpTypedArray } from "../typeHelpers";
-import { indexEntireSchema, sortAllCategorical } from "./schemaHelpers";
+import { indexEntireSchema } from "./schemaHelpers";
 import { isCategoricalAnnotation } from "./annotationsHelpers";
+import catLabelSort from "../catLabelSort";
 
 /*
 Private helper function - create and return a template Universe
@@ -74,9 +75,9 @@ function promoteTypedArray(o) {
   return new TyepdArrayCtor(o);
 }
 
-function AnnotationsFBSToDataframe(arrayBuffer) {
+export function matrixFBSToDataframe(arrayBuffers) {
   /*
-  Convert a Matrix FBS to a Dataframe.
+  Convert array of Matrix FBS to a Dataframe.
 
   The application has strong assumptions that all scalar data will be
   stored as a float32 or float64 (regardless of underlying data types).
@@ -86,77 +87,41 @@ function AnnotationsFBSToDataframe(arrayBuffer) {
   All float data from the server is left as is.  All non-float is promoted
   to an appropriate float.
   */
-  const fbs = decodeMatrixFBS(arrayBuffer, true); // leave in place
-  const columns = fbs.columns.map(c => {
-    if (isFpTypedArray(c) || Array.isArray(c)) return c;
-    return promoteTypedArray(c);
-  });
-  const df = new Dataframe.Dataframe(
-    [fbs.nRows, fbs.nCols],
-    columns,
-    null,
-    new Dataframe.KeyIndex(fbs.colIdx)
-  );
-  return df;
-}
-
-function LayoutFBSToDataframe(arrayBuffer) {
-  const fbs = decodeMatrixFBS(arrayBuffer, true);
-  if (fbs.columns.length < 2 || !fbs.columns.every(isFpTypedArray)) {
-    // We have strong assumptions about the shape & type of layout data.
-    throw new Error("Unexpected layout data type returned from server");
+  if (!Array.isArray(arrayBuffers)) {
+    arrayBuffers = [arrayBuffers];
+  }
+  if (arrayBuffers.length === 0) {
+    return Dataframe.Dataframe.empty();
   }
 
+  const fbs = arrayBuffers.map(ab => decodeMatrixFBS(ab, true)); // leave in place
+  /* check that all FBS have same row dimensionality */
+  const nRows = fbs[0].nRows;
+  fbs.forEach(b => {
+    if (b.nRows !== nRows)
+      throw new Error("FBS with inconsistent dimensionality");
+  });
+  const columns = fbs
+    .map(fb =>
+      fb.columns.map(c => {
+        if (isFpTypedArray(c) || Array.isArray(c)) return c;
+        return promoteTypedArray(c);
+      })
+    )
+    .flat();
+  const colIdx = fbs.map(b => b.colIdx).flat();
+  const nCols = columns.length;
+
   const df = new Dataframe.Dataframe(
-    [fbs.nRows, fbs.nCols],
-    fbs.columns,
+    [nRows, nCols],
+    columns,
     null,
-    new Dataframe.KeyIndex(fbs.colIdx)
+    new Dataframe.KeyIndex(colIdx)
   );
   return df;
 }
 
-function reconcileSchemaCategoriesWithSummary(universe) {
-  /*
-  where we treat types as (essentially) categorical metadata, update
-  the schema with data-derived categories (in addition to those in
-  the server declared schema).
-
-  For example, boolean defined fields in the schema do not contain
-  explicit declaration of categories (nor do string fields).  In these
-  cases, add a 'categories' field to the schema so it is accessible.
-
-  In addition, we have a client-side convention (UI) that all writable
-  annotations must have an 'unassigned' category, even if it is not currently
-  in use.
-  */
-
-  universe.schema.annotations.obs.columns.forEach(s => {
-    if (
-      s.type === "string" ||
-      s.type === "boolean" ||
-      s.type === "categorical"
-    ) {
-      const categories = _.union(
-        s.categories ?? [],
-        universe.obsAnnotations.col(s.name).summarize().categories ?? []
-      );
-      s.categories = categories;
-    }
-
-    if (s.writable && s.categories.indexOf(unassignedCategoryLabel) === -1) {
-      s.categories = s.categories.concat(unassignedCategoryLabel);
-    }
-  });
-}
-
-export function createUniverseFromResponse(
-  configResponse,
-  schemaResponse,
-  annotationsObsResponse,
-  annotationsVarResponse,
-  layoutFBSResponse
-) {
+export function createUniverseFromResponse(configResponse, schemaResponse) {
   /*
   build & return universe from a REST 0.2 /config, /schema and /annotations/obs response
   */
@@ -167,41 +132,80 @@ export function createUniverseFromResponse(
   universe.schema = schema;
   universe.nObs = schema.dataframe.nObs;
   universe.nVar = schema.dataframe.nVar;
+
   /* add defaults, as we can't assume back-end will fully populate schema */
   if (!schema.layout.var) schema.layout.var = [];
   if (!schema.layout.obs) schema.layout.obs = [];
-
-  /* annotations */
-  universe.obsAnnotations = AnnotationsFBSToDataframe(annotationsObsResponse);
-  universe.varAnnotations = AnnotationsFBSToDataframe(annotationsVarResponse);
-  /* layout */
-  universe.obsLayout = LayoutFBSToDataframe(layoutFBSResponse);
-
-  /* sanity checks */
-  if (
-    universe.nObs !== universe.obsLayout.length ||
-    universe.nObs !== universe.obsAnnotations.length ||
-    universe.nVar !== universe.varAnnotations.length
-  ) {
-    throw new Error("Universe dimensionality mismatch - failed to load");
-  }
-
-  reconcileSchemaCategoriesWithSummary(universe);
-  sortAllCategorical(universe.schema);
   indexEntireSchema(universe.schema);
+  normalizeEntireSchema(universe.schema);
 
-  /* sanity checks */
-  if (
-    schema.annotations.obs.columns.some(
-      s => s.writable && !isCategoricalAnnotation(schema, s.name)
-    )
-  ) {
+  return universe;
+}
+
+function normalizeSchemaCategory(colSchema, col = undefined) {
+  const { type, writable } = colSchema;
+  if (type === "string" || type === "boolean" || type === "categorical") {
+    let categories = [
+      ...new Set([
+        ...(colSchema.categories ?? []),
+        ...(col?.summarize?.().categories ?? [])
+      ])
+    ];
+    if (writable && categories.indexOf(unassignedCategoryLabel) === -1) {
+      categories = categories.concat(unassignedCategoryLabel);
+    }
+    colSchema.categories = categories;
+  } else if (writable) {
     throw new Error(
       "Writable continuous obs annotations are not supported - failed to load"
     );
   }
 
-  return universe;
+  if (colSchema.categories) {
+    colSchema.categories = catLabelSort(writable, colSchema.categories);
+  }
+}
+
+function normalizeEntireSchema(schema) {
+  // currently only needed for obsAnnotations
+  schema.annotations.obs.columns.forEach(colSchema =>
+    normalizeSchemaCategory(colSchema)
+  );
+}
+
+export function addObsAnnotations(universe, df) {
+  const obsAnnotations = universe.obsAnnotations.withColsFromAll(df);
+  if (universe.nObs !== obsAnnotations.length) {
+    throw new Error("Universe dimensionality mismatch - failed to load");
+  }
+
+  // for all of the new data, reconcile with schema and sort categories.
+  const dfs = Array.isArray(df) ? df : [df];
+  const keys = dfs.map(df => df.colIndex.keys()).flat();
+  const { schema } = universe;
+  keys.forEach(k => {
+    const colSchema = schema.annotations.obsByName[k];
+    const col = obsAnnotations.col(k);
+    normalizeSchemaCategory(colSchema, col);
+  });
+
+  return { obsAnnotations, schema };
+}
+
+export function addVarAnnotations(universe, df) {
+  const varAnnotations = universe.varAnnotations.withColsFromAll(df);
+  if (universe.nVar !== varAnnotations.length) {
+    throw new Error("Universe dimensionality mismatch - failed to load");
+  }
+  return { varAnnotations };
+}
+
+export function addObsLayout(universe, df) {
+  const obsLayout = universe.obsLayout.withColsFromAll(df);
+  if (universe.nObs !== obsLayout.length) {
+    throw new Error("Universe dimensionality mismatch - failed to load");
+  }
+  return { obsLayout };
 }
 
 export function convertDataFBStoObject(universe, arrayBuffer) {

client/src/util/stateManager/world.js

@@ -113,7 +113,6 @@ function clipDataframe(
 
 /*
 Create World with contents eq entire universe.   Commonly used to initialize World.
-If clipQuantiles
 */
 export function createWorldFromEntireUniverse(universe) {
   const world = templateWorld();
@@ -253,6 +252,40 @@ function deduceDimensionType(attributes, fieldName) {
   return dimensionType;
 }
 
+function addObsDimension(crossfilter, world, anno) {
+  /*
+  add single dimension to the crosfilter
+  */
+  const { obsAnnotations } = world;
+  if (obsAnnotations.hasCol(anno.name)) {
+    const dimType = deduceDimensionType(anno, anno.name);
+    const colData = obsAnnotations.col(anno.name).asArray();
+    const name = obsAnnoDimensionName(anno.name);
+    if (dimType === "enum") {
+      return crossfilter.addDimension(name, "enum", colData);
+    }
+    if (dimType) {
+      return crossfilter.addDimension(name, "scalar", colData, dimType);
+    }
+  }
+  return crossfilter;
+}
+
+export function addObsDimensions(crossfilter, world) {
+  /*
+  Add to crossfilter any dimension present in world.obsAnnotations
+  but not yet in the crossfilter
+  */
+  const schema = world.schema.annotations.obsByName;
+  const dimsWeNeed = world.obsAnnotations.colIndex.keys();
+  crossfilter = dimsWeNeed.reduce((xfltr, name) => {
+    const dimName = obsAnnoDimensionName(name);
+    if (xfltr.hasDimension(dimName)) return xfltr;
+    return addObsDimension(xfltr, world, schema[name]);
+  }, crossfilter);
+  return crossfilter;
+}
+
 export function createObsDimensions(crossfilter, world, XYdimNames) {
   /*
   create and return a crossfilter with a dimension for every obs annotation
@@ -265,16 +298,7 @@ export function createObsDimensions(crossfilter, world, XYdimNames) {
     anno => anno.name !== indexName
   );
   crossfilter = annoList.reduce((xfltr, anno) => {
-    const dimType = deduceDimensionType(anno, anno.name);
-    const colData = obsAnnotations.col(anno.name).asArray();
-    const name = obsAnnoDimensionName(anno.name);
-    if (dimType === "enum") {
-      return xfltr.addDimension(name, "enum", colData);
-    }
-    if (dimType) {
-      return xfltr.addDimension(name, "scalar", colData, dimType);
-    }
-    return xfltr;
+    return addObsDimension(xfltr, world, anno);
   }, crossfilter);
 
   return crossfilter.addDimension(