load annotations incrementally (#1107)

* load annotations individually

* fix type check to be more general

* update node CI version from 10 to 12

* node 11

* debug print node version

* travis node version to latest

* try nvm

* remove extraneous node_js statement

* remove node version debugging printf

* incrementally load all annotations and layout

* process annotations and layout as they are loaded

* fix tests

* sort categories incrementally

* incrementally build category view summary; add category loading spinner

* add spinner to continuous metadata

* configure undoable reducer

* incremental crossfilter creation

* improve busy layout

* more layout cleanup

* correctly reconcile categories in schema

* refine layout of lsb spinners

* more spinner layout work

* more spinner layout

* always load layout before obs annotations

client/src/components/brushableHistogram/index.js

@@ -351,8 +351,8 @@ class HistogramBrush extends React.PureComponent {
     const brushX = d3
       .brushX()
       .extent([
-          [x.range()[0], y.range()[1]],
-          [x.range()[1], this.marginTop + this.height + this.marginBottom]
+        [x.range()[0], y.range()[1]],
+        [x.range()[1], this.marginTop + this.height + this.marginBottom]
       ])
       /*
       emit start so that the Undoable history can save an undo point

client/src/components/categorical/categorical.js

@@ -4,15 +4,15 @@ import { Button } from "@blueprintjs/core";
 import { connect } from "react-redux";
 import * as globals from "../../globals";
 import Category from "./category";
-import { AnnotationsHelpers } from "../../util/stateManager";
+import { AnnotationsHelpers, ControlsHelpers } from "../../util/stateManager";
 import AnnoDialog from "./annoDialog";
 import AnnoInputs from "./annoInputs";
 import AnnoSelect from "./annoSelect";
 
 @connect(state => ({
-  categoricalSelection: state.categoricalSelection,
   writableCategoriesEnabled: state.config?.parameters?.["annotations"] ?? false,
-  schema: state.world?.schema
+  schema: state.world?.schema,
+  config: state.config
 }))
 class Categories extends React.Component {
   constructor(props) {
@@ -123,12 +123,15 @@ class Categories extends React.Component {
     const {
       categoricalSelection,
       writableCategoriesEnabled,
-      schema
+      schema,
+      config
     } = this.props;
-    if (!categoricalSelection) return null;
 
     /* all names, sorted in display order.  Will be rendered in this order */
-    const allCategoryNames = Object.keys(categoricalSelection).sort();
+    const allCategoryNames = ControlsHelpers.selectableCategoryNames(
+      schema,
+      ControlsHelpers.maxCategoryItems(config)
+    ).sort();
 
     return (
       <div

client/src/components/categorical/category.js

@@ -2,7 +2,7 @@ import React from "react";
 import _ from "lodash";
 import { connect } from "react-redux";
 import { FaChevronRight, FaChevronDown } from "react-icons/fa";
-import { Button, Tooltip, Icon } from "@blueprintjs/core";
+import { Button, Tooltip, Icon, Spinner } from "@blueprintjs/core";
 import CategoryFlipperLayout from "./categoryFlipperLayout";
 import AnnoMenu from "./annoMenuCategory";
 import AnnoDialogEditCategoryName from "./annoDialogEditCategoryName";
@@ -31,8 +31,12 @@ class Category extends React.Component {
 
   componentDidUpdate(prevProps) {
     const { categoricalSelection, metadataField } = this.props;
-    if (categoricalSelection !== prevProps.categoricalSelection) {
-      const cat = categoricalSelection[metadataField];
+    const cat = categoricalSelection?.[metadataField];
+    if (
+      categoricalSelection !== prevProps.categoricalSelection &&
+      !!cat &&
+      !!this.checkbox
+    ) {
       const categoryCount = {
         // total number of categories in this dimension
         totalCatCount: cat.numCategoryValues,
@@ -95,15 +99,66 @@ class Category extends React.Component {
     }
   }
 
+  renderIsStillLoading(metadataField) {
+    /*
+    We are still loading this category, so render a "busy" signal.
+    */
+    return (
+      <div
+        style={{
+          maxWidth: globals.maxControlsWidth
+        }}
+      >
+        <div
+          style={{
+            display: "flex",
+            justifyContent: "space-between",
+            alignItems: "baseline"
+          }}
+        >
+          <div
+            style={{
+              display: "flex",
+              justifyContent: "flex-start",
+              alignItems: "flex-start"
+            }}
+          >
+            <label className="bp3-control bp3-checkbox">
+              <input disabled checked={true} type="checkbox" />
+              <span className="bp3-control-indicator" />
+            </label>
+            <span
+              style={{
+                cursor: "pointer",
+                display: "inline-block"
+              }}
+            >
+              {metadataField}
+            </span>
+          </div>
+          <div>
+            <Button minimal loading intent="primary" />
+          </div>
+        </div>
+      </div>
+    );
+  }
+
   render() {
     const { isExpanded, isChecked } = this.state;
     const {
       metadataField,
+      categoricalSelection,
       colorAccessor,
       isUserAnno,
       annotations
     } = this.props;
 
+    const isStillLoading = !(categoricalSelection?.[metadataField] ?? false);
+    if (isStillLoading) {
+      return this.renderIsStillLoading(metadataField);
+    }
+
     return (
       <CategoryFlipperLayout
         metadataField={metadataField}

client/src/components/continuous/continuous.js

@@ -5,6 +5,7 @@ import React from "react";
 import _ from "lodash";
 import { connect } from "react-redux";
 import * as globals from "../../globals";
+import { Button } from "@blueprintjs/core";
 
 import HistogramBrush from "../brushableHistogram";
 
@@ -14,17 +15,7 @@ import HistogramBrush from "../brushableHistogram";
   colorScale: state.colors.scale,
   schema: state.world?.schema
 }))
-class Continuous extends React.Component {
-  constructor(props) {
-    super(props);
-    this.hasContinuous = false;
-    this.continuousChecked = false;
-
-    this.state = {};
-  }
-
-  componentDidUpdate() {}
-
+class Continuous extends React.PureComponent {
   handleColorAction = key => {
     return () => {
       const { dispatch, obsAnnotations } = this.props;
@@ -37,61 +28,82 @@ class Continuous extends React.Component {
     };
   };
 
+  renderIsStillLoading(zebra, key) {
+    return (
+      <div
+        key={key}
+        style={{
+          padding: globals.leftSidebarSectionPadding,
+          backgroundColor: zebra % 2 === 0 ? globals.lightestGrey : "white"
+        }}
+      >
+        <div
+          style={{
+            display: "flex",
+            justifyContent: "space-between",
+            justifyItems: "center",
+            alignItems: "center"
+          }}
+        >
+          <div style={{ minWidth: 30 }}></div>
+          <div style={{ display: "flex", alignSelf: "center" }}>
+            <span style={{ fontStyle: "italic" }}>{key}</span>
+          </div>
+          <div
+            style={{
+              display: "flex",
+              justifyContent: "flex-end"
+            }}
+          >
+            <Button minimal loading intent="primary" />
+          </div>
+        </div>
+      </div>
+    );
+  }
+
   render() {
     const { obsAnnotations, schema } = this.props;
-    if (schema && !this.continuousChecked) {
-      this.hasContinuous = _.some(
-        schema.annotations.obs,
-        d => d.type === "int32" || d.type === "float32"
-      );
-      this.continuousChecked = true; /* only do this once */
-    }
+
+    const obsIndex = schema.annotations.obs.index;
+    const allContinuousNames = schema.annotations.obs.columns
+      .filter(col => col.type === "int32" || col.type === "float32")
+      .filter(col => col.name != obsIndex)
+      .map(col => col.name);
 
     /* initial value for iterator to simulate index, ranges is an object */
     let zebra = 0;
 
     return (
       <div>
-        {this.hasContinuous ? (
-          <p
-            style={{
-              ...globals.leftSidebarSectionHeading,
-              marginTop: 40,
-              paddingLeft: globals.leftSidebarSectionPadding
-            }}
-          >
-            Continuous metadata
-          </p>
-        ) : null}
-        {obsAnnotations
-          ? _.map(obsAnnotations.colIndex.keys(), key => {
-              const isColorField =
-                key.includes("color") || key.includes("Color");
-              if (key === schema.annotations.obs.index || isColorField)
-                return null;
-
-              const summary = obsAnnotations.col(key).summarize();
-              const nonFiniteExtent =
-                summary.min === undefined ||
-                summary.max === undefined ||
-                Number.isNaN(summary.min) ||
-                Number.isNaN(summary.max);
-              if (!summary.categorical && !nonFiniteExtent) {
-                zebra += 1;
-                return (
-                  <HistogramBrush
-                    key={key}
-                    field={key}
-                    isObs
-                    zebra={zebra % 2 === 0}
-                    ranges={summary}
-                    handleColorAction={this.handleColorAction(key)}
-                  />
-                );
-              }
-              return null;
-            })
-          : null}
+        {allContinuousNames.map(key => {
+          if (!obsAnnotations.hasCol(key)) {
+            // still loading!
+            zebra += 1;
+            return this.renderIsStillLoading(zebra, key);
+          } else {
+            // data loaded and available
+            const summary = obsAnnotations.col(key).summarize();
+            const nonFiniteExtent =
+              summary.min === undefined ||
+              summary.max === undefined ||
+              Number.isNaN(summary.min) ||
+              Number.isNaN(summary.max);
+            if (!summary.categorical && !nonFiniteExtent) {
+              zebra += 1;
+              return (
+                <HistogramBrush
+                  key={key}
+                  field={key}
+                  isObs
+                  zebra={zebra % 2 === 0}
+                  ranges={summary}
+                  handleColorAction={this.handleColorAction(key)}
+                />
+              );
+            }
+          }
+        })}
       </div>
     );
   }

client/src/components/geneExpression/index.js

@@ -103,7 +103,8 @@ class GeneExpression extends React.Component {
       if (genes.length === 0) {
         return keepAroundErrorToast("Must enter a gene name.");
       }
-      const worldGenes = world.varAnnotations.col(varIndexName).asArray();
+      const worldGenes =
+        world.varAnnotations?.col(varIndexName)?.asArray() || [];
 
       // These gene lists are unique enough where memoization is useless
       const upperGenes = this._genesToUpper(genes);
@@ -206,8 +207,12 @@ class GeneExpression extends React.Component {
       differential
     } = this.props;
     const varIndexName = world?.schema?.annotations?.var?.index;
+    const varIndex = world?.varAnnotations?.col(varIndexName)?.asArray();
     const { tab, bulkAdd, activeItem } = this.state;
 
+    // may still be loading!
+    if (!varIndex) return null;
+
     return (
       <div>
         <div>
@@ -268,11 +273,7 @@ class GeneExpression extends React.Component {
                 itemListPredicate={filterGenes}
                 onActiveItemChange={item => this.setState({ activeItem: item })}
                 itemRenderer={renderGene.bind(this)}
-                items={
-                  world && world.varAnnotations
-                    ? world.varAnnotations.col(varIndexName).asArray()
-                    : ["No genes"]
-                }
+                items={varIndex || ["No genes"]}
                 popoverProps={{ minimal: true }}
               />
               <Button

client/src/components/graph/graph.js

@@ -255,7 +255,7 @@ class Graph extends React.Component {
     const { regl, toolSVG, camera, modelTF } = this.state;
     let stateChanges = {};
 
-    if (regl && world) {
+    if (regl && world && crossfilter) {
       /* update the regl and point rendering state */
       const { obsLayout, nObs } = world;
       const { drawPoints, pointBuffer, colorBuffer, flagBuffer } = this.state;

client/src/components/menubar/index.js

@@ -23,7 +23,6 @@ import * as globals from "../../globals";
 @connect(state => ({
   universe: state.universe,
   world: state.world,
-  loading: state.controls.loading,
   crossfilter: state.crossfilter,
   differential: state.differential,
   resettingInterface: state.controls.resettingInterface,