load annotations incrementally (#1107)

* load annotations individually

* fix type check to be more general

* update node CI version from 10 to 12

* node 11

* debug print node version

* travis node version to latest

* try nvm

* remove extraneous node_js statement

* remove node version debugging printf

* incrementally load all annotations and layout

* process annotations and layout as they are loaded

* fix tests

* sort categories incrementally

* incrementally build category view summary; add category loading spinner

* add spinner to continuous metadata

* configure undoable reducer

* incremental crossfilter creation

* improve busy layout

* more layout cleanup

* correctly reconcile categories in schema

* refine layout of lsb spinners

* more spinner layout work

* more spinner layout

* always load layout before obs annotations

client/src/reducers/categoricalSelection.js

@@ -1,13 +1,6 @@
 import { ControlsHelpers as CH } from "../util/stateManager";
 import * as globals from "../globals";
 
-function maxCategoryItems(state) {
-  return (
-    state.config.parameters?.["max-category-items"] ??
-    globals.configDefaults.parameters["max-category-items"]
-  );
-}
-
 const CategoricalSelection = (
   state,
   action,
@@ -22,11 +15,32 @@ const CategoricalSelection = (
       const { world } = nextSharedState;
       const newState = CH.createCategoricalSelection(
         world,
-        CH.selectableCategoryNames(world, maxCategoryItems(prevSharedState))
+        CH.selectableCategoryNames(
+          world.schema,
+          CH.maxCategoryItems(prevSharedState.config)
+        )
       );
       return newState;
     }
 
+    case "universe: column load success": {
+      const { dim } = action;
+      if (dim !== "obsAnnotations") return state;
+
+      const { dataframe } = action;
+      const { world } = nextSharedState;
+      const names = CH.selectableCategoryNames(
+        world.schema,
+        CH.maxCategoryItems(prevSharedState.config),
+        dataframe.colIndex.keys()
+      );
+      if (names.length === 0) return state;
+      return {
+        ...state,
+        ...CH.createCategoricalSelection(world, names)
+      };
+    }
+
     case "categorical metadata filter select": {
       /*
       Set the specific category in this field to false

client/src/reducers/colors.js

@@ -12,7 +12,7 @@ const ColorsReducer = (
   prevSharedState
 ) => {
   switch (action.type) {
-    case "initial data load complete (universe exists)":
+    case "universe exists, but loading is still in progress":
     case "reset World to eq Universe": {
       const { world } = nextSharedState;
       const colorMode = null;

client/src/reducers/controls.js

@@ -41,8 +41,31 @@ const Controls = (
     case "initial data load start": {
       return { ...state, loading: true };
     }
+    case "universe: column load success": {
+      /* 
+      we are in a loading state until the following are true:
+        * universe exists (if partially)
+        * embeddings (obsLayout) are loaded
+        * varAnnotations index is loaded
+      */
+      const universeExists = !!nextSharedState.universe;
+      const embeddingsExist =
+        !nextSharedState.universe?.obsLayout?.isEmpty() ?? false;
+      const varIndex = nextSharedState.universe.schema.annotations.var.index;
+      const varAnnotationsIndexExists =
+        universeExists &&
+        nextSharedState.universe.varAnnotations.hasCol(varIndex);
+      return {
+        ...state,
+        loading: !(
+          universeExists &&
+          embeddingsExist &&
+          varAnnotationsIndexExists
+        )
+      };
+    }
     case "initial data load complete (universe exists)": {
-      /* first light - create world & other data-driven defaults */
+      /* now fully loaded */
       return {
         ...state,
         loading: false,

client/src/reducers/crossfilter.js

@@ -23,13 +23,38 @@ const CrossfilterReducerBase = (
   prevSharedState
 ) => {
   switch (action.type) {
-    case "initial data load complete (universe exists)": {
-      const { world, layoutChoice } = nextSharedState;
-      const crossfilter = World.createObsDimensions(
-        new Crossfilter(world.obsAnnotations),
-        world,
-        layoutChoice.currentDimNames
-      );
+    case "universe: column load success": {
+      const { schema, world, layoutChoice } = nextSharedState;
+      const { obsAnnotations, obsLayout } = world;
+      const { dim, dataframe } = action;
+
+      // ignore var dimension loads as these are not currently selectable
+      if (action.dim === "varAnnotations") return state;
+
+      /* 
+      during bootstrap loading, we don't know if obsLayout or obsAnnotations
+      will load first. Take whichever arrives and is not empty (so that our
+      crossfilter has the right dimensionality).
+      */
+      let crossfilter =
+        state ??
+        new Crossfilter(obsAnnotations.isEmpty() ? obsLayout : obsAnnotations);
+
+      // add layout dimension, if not already present
+      if (
+        obsLayout.hasCol(layoutChoice.currentDimNames[0]) &&
+        !crossfilter.hasDimension(XYDimName)
+      ) {
+        crossfilter = crossfilter.addDimension(
+          XYDimName,
+          "spatial",
+          obsLayout.col(layoutChoice.currentDimNames[0]).asArray(),
+          obsLayout.col(layoutChoice.currentDimNames[1]).asArray()
+        );
+      }
+
+      // add any missing obsAnnotations
+      crossfilter = World.addObsDimensions(crossfilter, world);
       return crossfilter;
     }
 
@@ -278,10 +303,19 @@ const CrossfilterReducer = (
     nextSharedState,
     prevSharedState
   );
-  if (!nextState || nextState.all() === nextSharedState.world.obsAnnotations) {
+  /*
+  update the data in the crossfilter to point at the current obsAnnotations, IF
+  they are not empty.  If empty, leave it alone (can occur during boostrap loading).
+  */
+  const nextObsAnnotations = nextSharedState.world?.obsAnnotations;
+  if (
+    !nextState ||
+    nextState.all() === nextObsAnnotations ||
+    nextObsAnnotations.isEmpty()
+  ) {
     return nextState;
   }
-  return nextState.setData(nextSharedState.world.obsAnnotations);
+  return nextState.setData(nextObsAnnotations);
 };
 
 export default CrossfilterReducer;

client/src/reducers/layoutChoice.js

@@ -24,7 +24,7 @@ const LayoutChoice = (
   nextSharedState
 ) => {
   switch (action.type) {
-    case "initial data load complete (universe exists)":
+    case "universe exists, but loading is still in progress":
     case "reset World to eq Universe": {
       // set default to default
       const { schema } = nextSharedState.world;

client/src/reducers/undoableConfig.js

@@ -11,6 +11,8 @@ const skipOnActions = new Set([
   "url changed",
   "interface reset started",
   "initial data load start",
+  "universe: column load success",
+  "universe exists, but loading is still in progress",
   "configuration load complete",
   "increment graph render counter",
   "window resize",

client/src/reducers/universe.js

@@ -1,4 +1,9 @@
 import { unassignedCategoryLabel } from "../globals";
+import {
+  addObsAnnotations,
+  addVarAnnotations,
+  addObsLayout
+} from "../util/stateManager/universe";
 import {
   World,
   ControlsHelpers,
@@ -7,11 +12,38 @@ import {
 
 const Universe = (state = null, action, nextSharedState, prevSharedState) => {
   switch (action.type) {
-    case "initial data load complete (universe exists)": {
+    case "universe exists, but loading is still in progress": {
       const { universe } = action;
       return universe;
     }
 
+    case "universe: column load success": {
+      const { dim, dataframe } = action;
+      switch (dim) {
+        case "obsAnnotations": {
+          return {
+            ...state,
+            ...addObsAnnotations(state, dataframe)
+          };
+        }
+        case "varAnnotations": {
+          return {
+            ...state,
+            ...addVarAnnotations(state, dataframe)
+          };
+        }
+        case "obsLayout": {
+          return {
+            ...state,
+            ...addObsLayout(state, dataframe)
+          };
+        }
+        default: {
+          throw new Error("action handler not implemented");
+        }
+      }
+    }
+
     case "expression load success": {
       let { varData } = state;
 

client/src/reducers/world.js

@@ -24,13 +24,27 @@ const WorldReducer = (
   prevSharedState
 ) => {
   switch (action.type) {
-    case "initial data load complete (universe exists)":
+    case "universe exists, but loading is still in progress":
     case "reset World to eq Universe": {
       const { universe } = nextSharedState;
       const world = World.createWorldFromEntireUniverse(universe);
       return world;
     }
 
+    case "universe: column load success": {
+      const { universe } = nextSharedState;
+      const { dim } = action;
+      return {
+        ...state,
+        schema: universe.schema,
+        [dim]: universe[dim].clone(),
+        unclipped: {
+          ...state.unclipped,
+          [dim]: universe[dim].clone()
+        }
+      };
+    }
+
     case "set World to current selection": {
       /* Set viewable world to be the currently selected data */
       const world = World.createWorldBySelection(