load annotations incrementally (#1107)

* load annotations individually

* fix type check to be more general

* update node CI version from 10 to 12

* node 11

* debug print node version

* travis node version to latest

* try nvm

* remove extraneous node_js statement

* remove node version debugging printf

* incrementally load all annotations and layout

* process annotations and layout as they are loaded

* fix tests

* sort categories incrementally

* incrementally build category view summary; add category loading spinner

* add spinner to continuous metadata

* configure undoable reducer

* incremental crossfilter creation

* improve busy layout

* more layout cleanup

* correctly reconcile categories in schema

* refine layout of lsb spinners

* more spinner layout work

* more spinner layout

* always load layout before obs annotations

client/__tests__/util/dataframe/dataframe.test.js

@@ -481,6 +481,7 @@ describe("dataframe factories", () => {
 
     test("simple", () => {
       /* simple test that it works as expected in common case */
+      const dfEmpty = Dataframe.Dataframe.empty();
       const dfA = new Dataframe.Dataframe(
         [2, 1],
         [["red", "blue"]],
@@ -494,6 +495,22 @@ describe("dataframe factories", () => {
         new Dataframe.KeyIndex(["bools"])
       );
 
+      const dfLikeA = dfEmpty.withColsFrom(dfA);
+      expect(dfLikeA).toBeDefined();
+      expect(dfLikeA.dims).toEqual(dfA.dims);
+      expect(dfLikeA.colIndex.keys()).toEqual(dfA.colIndex.keys());
+      expect(dfLikeA.rowIndex).toEqual(dfA.rowIndex);
+      expect(dfLikeA.rowIndex.keys()).toEqual(dfA.rowIndex.keys());
+      expect(dfLikeA.icol(0).asArray()).toEqual(dfA.icol(0).asArray());
+
+      const dfAlsoLikeA = dfA.withColsFrom(dfEmpty);
+      expect(dfAlsoLikeA).toBeDefined();
+      expect(dfAlsoLikeA.dims).toEqual(dfA.dims);
+      expect(dfAlsoLikeA.colIndex.keys()).toEqual(dfA.colIndex.keys());
+      expect(dfAlsoLikeA.rowIndex).toEqual(dfA.rowIndex);
+      expect(dfAlsoLikeA.rowIndex.keys()).toEqual(dfA.rowIndex.keys());
+      expect(dfAlsoLikeA.icol(0).asArray()).toEqual(dfA.icol(0).asArray());
+
       const dfC = dfA.withColsFrom(dfB);
       expect(dfC).toBeDefined();
       expect(dfC.dims).toEqual([2, 2]);

client/__tests__/util/stateManager/universe.test.js

@@ -30,15 +30,39 @@ describe("createUniverseFromResponse", () => {
     create a universe from sample data nad validate its shape & contents
     */
     const { nObs, nVar } = REST.schema.schema.dataframe;
-    const universe = Universe.createUniverseFromResponse(
+    let universe = Universe.createUniverseFromResponse(
       REST.config,
-      REST.schema,
-      REST.annotationsObs,
-      REST.annotationsVar,
-      REST.layoutObs
+      REST.schema
+    );
+    expect(universe).toBeDefined();
+    expect(universe).toMatchObject(
+      expect.objectContaining({
+        nObs,
+        nVar,
+        schema: REST.schema.schema,
+        obsAnnotations: expect.any(Dataframe.Dataframe),
+        varAnnotations: expect.any(Dataframe.Dataframe),
+        obsLayout: expect.any(Dataframe.Dataframe),
+        varData: expect.any(Dataframe.Dataframe)
+      })
     );
 
-    expect(universe).toBeDefined();
+    universe = {
+      ...universe,
+      ...Universe.addObsAnnotations(
+        universe,
+        Universe.matrixFBSToDataframe(REST.annotationsObs)
+      ),
+      ...Universe.addVarAnnotations(
+        universe,
+        Universe.matrixFBSToDataframe(REST.annotationsVar)
+      ),
+      ...Universe.addObsLayout(
+        universe,
+        Universe.matrixFBSToDataframe(REST.layoutObs)
+      )
+    };
+
     expect(universe).toMatchObject(
       expect.objectContaining({
         nObs,

client/__tests__/util/stateManager/world.test.js

@@ -17,13 +17,27 @@ the default REST test response.
 const defaultBigBang = () => {
   /* create unverse, world, crossfilter and dimensionMap */
   /* create universe */
-  const universe = Universe.createUniverseFromResponse(
+  let universe = Universe.createUniverseFromResponse(
     _.cloneDeep(REST.config),
-    _.cloneDeep(REST.schema),
-    _.cloneDeep(REST.annotationsObs),
-    _.cloneDeep(REST.annotationsVar),
-    _.cloneDeep(REST.layoutObs)
+    _.cloneDeep(REST.schema)
   );
+
+  universe = {
+    ...universe,
+    ...Universe.addObsAnnotations(
+      universe,
+      Universe.matrixFBSToDataframe(REST.annotationsObs)
+    ),
+    ...Universe.addVarAnnotations(
+      universe,
+      Universe.matrixFBSToDataframe(REST.annotationsVar)
+    ),
+    ...Universe.addObsLayout(
+      universe,
+      Universe.matrixFBSToDataframe(REST.layoutObs)
+    )
+  };
+
   /* create world */
   const world = World.createWorldFromEntireUniverse(universe);
   /* create crossfilter */
@@ -45,9 +59,9 @@ describe("createWorldFromEntireUniverse", () => {
     const universe = Universe.createUniverseFromResponse(
       _.cloneDeep(REST.config),
       _.cloneDeep(REST.schema),
-      _.cloneDeep(REST.annotationsObs),
-      _.cloneDeep(REST.annotationsVar),
-      _.cloneDeep(REST.layoutObs)
+      Universe.matrixFBSToDataframe(_.cloneDeep(REST.annotationsObs)),
+      Universe.matrixFBSToDataframe(_.cloneDeep(REST.annotationsVar)),
+      Universe.matrixFBSToDataframe(_.cloneDeep(REST.layoutObs))
     );
     expect(universe).toBeDefined();