Merge branch 'main' into colinmegill/geneset-prototype

client/src/components/app.js

@@ -64,10 +64,10 @@ class App extends React.Component {
               left: window.innerWidth / 2 - 50,
             }}
           >
-            error loading
+            error loading cellxgene
           </div>
         ) : null}
-        {loading ? null : (
+        {loading || error ? null : (
           <Layout>
             <LeftSideBar />
             {(viewportRef) => (

client/src/components/autosave/filenameDialog.js

@@ -11,13 +11,8 @@ import {
 } from "@blueprintjs/core";
 
 @connect((state) => ({
-  universe: state.universe,
   idhash: state.config?.parameters?.["annotations-user-data-idhash"] ?? null,
   annotations: state.annotations,
-  obsAnnotations: state.universe.obsAnnotations,
-  saveInProgress: state.autosave?.saveInProgress ?? false,
-  lastSavedObsAnnotations: state.autosave?.lastSavedObsAnnotations,
-  error: state.autosave?.error,
   writableCategoriesEnabled: state.config?.parameters?.annotations ?? false,
 }))
 class FilenameDialog extends React.Component {
@@ -113,7 +108,7 @@ class FilenameDialog extends React.Component {
             this.handleCreateFilename();
           }}
         >
-          <div className={Classes.DIALOG_BODY}>
+          <div className={Classes.DIALOG_BODY} data-testid="annotation-dialog">
             <div style={{ marginBottom: 20 }}>
               <p>Name your annotations collection:</p>
               <InputGroup
@@ -124,6 +119,7 @@ class FilenameDialog extends React.Component {
                   this.setState({ filenameText: e.target.value })
                 }
                 leftIcon="tag"
+                data-testid="new-annotation-name"
               />
               <p
                 style={{
@@ -159,6 +155,7 @@ class FilenameDialog extends React.Component {
                 onClick={this.handleCreateFilename}
                 intent="primary"
                 type="submit"
+                data-testid="submit-annotation"
               >
                 Create annotations collection
               </Button>

client/src/components/autosave/index.js

@@ -4,14 +4,12 @@ import actions from "../../actions";
 import FilenameDialog from "./filenameDialog";
 
 @connect((state) => ({
-  universe: state.universe,
   annotations: state.annotations,
-  obsAnnotations: state.universe.obsAnnotations,
   saveInProgress: state.autosave?.saveInProgress ?? false,
-  lastSavedObsAnnotations: state.autosave?.lastSavedObsAnnotations,
   error: state.autosave?.error,
   writableCategoriesEnabled: state.config?.parameters?.annotations ?? false,
-  initialDataLoadComplete: state.autosave?.initialDataLoadComplete,
+  annoMatrix: state.annoMatrix,
+  lastSavedAnnoMatrix: state.autosave?.lastSavedAnnoMatrix,
 }))
 class Autosave extends React.Component {
   constructor(props) {
@@ -42,16 +40,14 @@ class Autosave extends React.Component {
   tick = () => {
     const { dispatch, saveInProgress } = this.props;
     if (this.needToSave() && !saveInProgress) {
-      dispatch(actions.saveObsAnnotations());
+      dispatch(actions.saveObsAnnotationsAction());
     }
   };
 
   needToSave = () => {
     /* return true if we need to save, false if we don't */
-    const { obsAnnotations, lastSavedObsAnnotations } = this.props;
-    return (
-      lastSavedObsAnnotations && obsAnnotations !== lastSavedObsAnnotations
-    );
+    const { annoMatrix, lastSavedAnnoMatrix } = this.props;
+    return actions.needToSaveObsAnnotations(annoMatrix, lastSavedAnnoMatrix);
   };
 
   statusMessage() {
@@ -66,9 +62,13 @@ class Autosave extends React.Component {
     const {
       writableCategoriesEnabled,
       saveInProgress,
-      initialDataLoadComplete,
+      lastSavedAnnoMatrix,
     } = this.props;
-    return writableCategoriesEnabled ? (
+    const initialDataLoadComplete = lastSavedAnnoMatrix;
+
+    if (!writableCategoriesEnabled) return null;
+
+    return (
       <div
         id="autosave"
         data-testclass={
@@ -89,7 +89,7 @@ class Autosave extends React.Component {
         {this.statusMessage()}
         <FilenameDialog />
       </div>
-    ) : null;
+    );
   }
 }
 

client/src/components/categorical/annoDialog.js

@@ -1,13 +1,6 @@
 import React from "react";
-import { connect } from "react-redux";
 import { Button, Tooltip, Dialog, Classes, Colors } from "@blueprintjs/core";
 
-@connect((state) => ({
-  colorAccessor: state.colors.colorAccessor,
-  categoricalSelection: state.categoricalSelection,
-  annotations: state.annotations,
-  universe: state.universe,
-}))
 class AnnoDialog extends React.PureComponent {
   constructor(props) {
     super(props);
@@ -72,7 +65,7 @@ class AnnoDialog extends React.PureComponent {
                 </Button>
               ) : null}
               <Button
-                {...primaryButtonProps} // eslint-disable-line react/jsx-props-no-spreading
+                {...primaryButtonProps} // eslint-disable-line react/jsx-props-no-spreading -- Spreading props allows for modularity
                 onClick={handleSubmit}
                 disabled={!text || validationError}
                 intent="primary"

client/src/components/categorical/annoSelect.js

@@ -1,14 +1,7 @@
 import React from "react";
-import { connect } from "react-redux";
 import { Button, MenuItem } from "@blueprintjs/core";
 import { Select } from "@blueprintjs/select";
 
-@connect((state) => ({
-  colorAccessor: state.colors.colorAccessor,
-  categoricalSelection: state.categoricalSelection,
-  annotations: state.annotations,
-  universe: state.universe,
-}))
 class DuplicateCategorySelect extends React.PureComponent {
   constructor(props) {
     super(props);
@@ -34,7 +27,14 @@ class DuplicateCategorySelect extends React.PureComponent {
           }
           filterable={false}
           itemRenderer={(d, { handleClick }) => {
-            return <MenuItem onClick={handleClick} key={d} text={d} />;
+            return (
+              <MenuItem
+                data-testclass="duplicate-category-dropdown-option"
+                onClick={handleClick}
+                key={d}
+                text={d}
+              />
+            );
           }}
           noResults={<MenuItem disabled text="No results." />}
           onItemSelect={(d) => {
@@ -43,6 +43,7 @@ class DuplicateCategorySelect extends React.PureComponent {
         >
           {/* children become the popover target; render value here */}
           <Button
+            data-testid="duplicate-category-dropdown"
             text={categoryToDuplicate || "None (all cells 'unassigned')"}
             rightIcon="double-caret-vertical"
           />

client/src/components/categorical/category/annoDialogAddLabel.js

@@ -3,14 +3,13 @@ import { connect } from "react-redux";
 import AnnoDialog from "../annoDialog";
 import LabelInput from "../labelInput";
 import { labelPrompt, isLabelErroneous } from "../labelUtil";
+import actions from "../../../actions";
 
 @connect((state) => ({
-  colorAccessor: state.colors.colorAccessor,
-  categoricalSelection: state.categoricalSelection,
   annotations: state.annotations,
-  universe: state.universe,
+  schema: state.annoMatrix?.schema,
   ontology: state.ontology,
-  crossfilter: state.crossfilter,
+  obsCrossfilter: state.obsCrossfilter,
 }))
 class Category extends React.PureComponent {
   constructor(props) {
@@ -36,12 +35,13 @@ class Category extends React.PureComponent {
     const { newLabelText } = this.state;
 
     this.disableAddNewLabelMode();
-    dispatch({
-      type: "annotation: add new label to category",
-      metadataField,
-      newLabelText,
-      assignSelectedCells: false,
-    });
+    dispatch(
+      actions.annotationCreateLabelInCategory(
+        metadataField,
+        newLabelText,
+        false
+      )
+    );
     e.preventDefault();
   };
 
@@ -50,18 +50,15 @@ class Category extends React.PureComponent {
     const { newLabelText } = this.state;
 
     this.disableAddNewLabelMode();
-    dispatch({
-      type: "annotation: add new label to category",
-      metadataField,
-      newLabelText,
-      assignSelectedCells: true,
-    });
+    dispatch(
+      actions.annotationCreateLabelInCategory(metadataField, newLabelText, true)
+    );
     e.preventDefault();
   };
 
   labelNameError = (name) => {
-    const { metadataField, ontology, universe } = this.props;
-    return isLabelErroneous(name, metadataField, ontology, universe.schema);
+    const { metadataField, ontology, schema } = this.props;
+    return isLabelErroneous(name, metadataField, ontology, schema);
   };
 
   instruction = (label) => {
@@ -74,7 +71,7 @@ class Category extends React.PureComponent {
 
   render() {
     const { newLabelText } = this.state;
-    const { metadataField, annotations, ontology, crossfilter } = this.props;
+    const { metadataField, annotations, ontology, obsCrossfilter } = this.props;
     const ontologyEnabled = ontology?.enabled ?? false;
 
     return (
@@ -92,7 +89,7 @@ class Category extends React.PureComponent {
           instruction={this.instruction(newLabelText)}
           cancelTooltipContent="Close this dialog without adding a label."
           primaryButtonText="Add label"
-          secondaryButtonText={`Add label & assign ${crossfilter.countSelected()} selected cells`}
+          secondaryButtonText={`Add label & assign ${obsCrossfilter.countSelected()} selected cells`}
           handleSecondaryButtonSubmit={this.addLabelAndAssignCells}
           text={newLabelText}
           validationError={this.labelNameError(newLabelText)}

client/src/components/categorical/category/annoDialogEditCategoryName.js

@@ -1,16 +1,15 @@
 import React from "react";
-import _ from "lodash";
 import { connect } from "react-redux";
 import AnnoDialog from "../annoDialog";
 import LabelInput from "../labelInput";
 import { labelPrompt } from "../labelUtil";
 
 import { AnnotationsHelpers } from "../../../util/stateManager";
+import actions from "../../../actions";
 
 @connect((state) => ({
-  categoricalSelection: state.categoricalSelection,
   annotations: state.annotations,
-  universe: state.universe,
+  schema: state.annoMatrix?.schema,
   ontology: state.ontology,
 }))
 class AnnoDialogEditCategoryName extends React.PureComponent {
@@ -36,10 +35,15 @@ class AnnoDialogEditCategoryName extends React.PureComponent {
   };
 
   handleEditCategory = (e) => {
-    const { dispatch, metadataField, categoricalSelection } = this.props;
+    const { dispatch, metadataField } = this.props;
     const { newCategoryText } = this.state;
 
-    const allCategoryNames = _.keys(categoricalSelection);
+    /*
+    test for uniqueness against *all* annotation names, not just the subset
+    we render as categorical.
+    */
+    const { schema } = this.props;
+    const allCategoryNames = schema.annotations.obs.columns.map((c) => c.name);
 
     if (
       (allCategoryNames.indexOf(newCategoryText) > -1 &&
@@ -50,17 +54,16 @@ class AnnoDialogEditCategoryName extends React.PureComponent {
     }
 
     this.disableEditCategoryMode();
-    dispatch({
-      type: "annotation: category edited",
-      metadataField,
-      newCategoryText,
-      data: newCategoryText,
-    });
+
+    if (metadataField !== newCategoryText)
+      dispatch(
+        actions.annotationRenameCategoryAction(metadataField, newCategoryText)
+      );
     e.preventDefault();
   };
 
   editedCategoryNameError = (name) => {
-    const { metadataField, categoricalSelection } = this.props;
+    const { metadataField } = this.props;
 
     /* check for syntax errors in category name */
     const error = AnnotationsHelpers.annotationNameIsErroneous(name);
@@ -69,7 +72,14 @@ class AnnoDialogEditCategoryName extends React.PureComponent {
     }
 
     /* check for duplicative categories */
-    const allCategoryNames = _.keys(categoricalSelection);
+
+    /*
+    test for uniqueness against *all* annotation names, not just the subset
+    we render as categorical.
+    */
+    const { schema } = this.props;
+    const allCategoryNames = schema.annotations.obs.columns.map((c) => c.name);
+
     const categoryNameAlreadyExists = allCategoryNames.indexOf(name) > -1;
     const sameName = name === metadataField;
     if (categoryNameAlreadyExists && !sameName) {
@@ -88,6 +98,11 @@ class AnnoDialogEditCategoryName extends React.PureComponent {
     );
   };
 
+  allCategoryNames() {
+    const { schema } = this.props;
+    return schema.annotations.obs.columns.map((c) => c.name);
+  }
+
   render() {
     const { newCategoryText } = this.state;
     const { metadataField, annotations, ontology } = this.props;

client/src/components/categorical/category/annoMenuCategory.js

@@ -12,6 +12,7 @@ import {
 } from "@blueprintjs/core";
 
 import * as globals from "../../../globals";
+import actions from "../../../actions";
 
 @connect((state) => ({
   annotations: state.annotations,
@@ -41,10 +42,7 @@ class AnnoMenuCategory extends React.PureComponent {
 
   handleDeleteCategory = () => {
     const { dispatch, metadataField } = this.props;
-    dispatch({
-      type: "annotation: delete category",
-      metadataField,
-    });
+    dispatch(actions.annotationDeleteCategoryAction(metadataField));
   };
 
   render() {

client/src/components/categorical/category/categoryFlipperLayout.js

@@ -1,84 +0,0 @@
-import React from "react";
-import _ from "lodash";
-import { connect } from "react-redux";
-import { Flipper, Flipped } from "react-flip-toolkit";
-
-import * as globals from "../../../globals";
-import Value from "../value";
-
-@connect((state) => ({
-  categoricalSelection: state.categoricalSelection,
-}))
-class Category extends React.Component {
-  constructor(props) {
-    super(props);
-    this.state = {};
-  }
-
-  renderCategoryItems(optTuples) {
-    const { metadataField, isUserAnno } = this.props;
-
-    return _.map(optTuples, (tuple, i) => {
-      return (
-        <Flipped key={tuple[1]} flipId={tuple[1]}>
-          {(flippedProps) => (
-            <Value
-              isUserAnno={isUserAnno}
-              optTuples={optTuples}
-              key={tuple[1]}
-              metadataField={metadataField}
-              categoryIndex={tuple[1]}
-              i={i}
-              flippedProps={flippedProps}
-            />
-          )}
-        </Flipped>
-      );
-    });
-  }
-
-  render() {
-    const {
-      metadataField,
-      categoricalSelection,
-      children,
-      isExpanded,
-    } = this.props;
-    const { isTruncated } = categoricalSelection[metadataField];
-    const cat = categoricalSelection[metadataField];
-    const optTuples = [...cat.categoryValueIndices];
-    const optTuplesAsKey = _.map(optTuples, (t) => t[0]).join(""); // animation
-
-    return (
-      <div
-        style={{
-          maxWidth: globals.maxControlsWidth,
-        }}
-        data-testclass="category"
-        data-testid={`category-${metadataField}`}
-      >
-        <div
-          style={{
-            display: "flex",
-            justifyContent: "space-between",
-            alignItems: "baseline",
-          }}
-        >
-          {children}
-        </div>
-        <div style={{ marginLeft: 26 }}>
-          <Flipper spring="veryGentle" flipKey={optTuplesAsKey}>
-            {isExpanded ? this.renderCategoryItems(optTuples) : null}
-          </Flipper>
-        </div>
-        <div>
-          {isExpanded && isTruncated ? (
-            <p style={{ paddingLeft: 15 }}>... truncated list ...</p>
-          ) : null}
-        </div>
-      </div>
-    );
-  }
-}
-
-export default Category;

client/src/components/categorical/category/index.js

@@ -1,70 +1,78 @@
-import React from "react";
-import _ from "lodash";
-import { connect } from "react-redux";
+import React, { useRef, useEffect } from "react";
+import { connect, shallowEqual } from "react-redux";
 import { FaChevronRight, FaChevronDown } from "react-icons/fa";
 import { AnchorButton, Button, Tooltip } from "@blueprintjs/core";
-import CategoryFlipperLayout from "./categoryFlipperLayout";
+import { Flipper, Flipped } from "react-flip-toolkit";
+import Async from "react-async";
+import memoize from "memoize-one";
+
+import Value from "../value";
 import AnnoMenu from "./annoMenuCategory";
 import AnnoDialogEditCategoryName from "./annoDialogEditCategoryName";
 import AnnoDialogAddLabel from "./annoDialogAddLabel";
 import Truncate from "../../util/truncate";
+import { CategoryCrossfilterContext } from "../categoryContext";
 
 import * as globals from "../../../globals";
+import { createCategorySummaryFromDfCol } from "../../../util/stateManager/controlsHelpers";
+import {
+  createColorTable,
+  createColorQuery,
+} from "../../../util/stateManager/colorHelpers";
+import actions from "../../../actions";
 
 const LABEL_WIDTH = globals.leftSidebarWidth - 100;
 const ANNO_BUTTON_WIDTH = 50;
 const LABEL_WIDTH_ANNO = LABEL_WIDTH - ANNO_BUTTON_WIDTH;
 
 @connect((state, ownProps) => {
+  const schema = state.annoMatrix?.schema;
   const { metadataField } = ownProps;
+  const isUserAnno = schema?.annotations?.obsByName[metadataField]?.writable;
+  const categoricalSelection = state.categoricalSelection?.[metadataField];
   return {
-    isColorAccessor: state.colors.colorAccessor === metadataField,
-    categoricalSelection: state.categoricalSelection,
+    colors: state.colors,
+    categoricalSelection,
     annotations: state.annotations,
-    universe: state.universe,
-    schema: state.world?.schema,
+    annoMatrix: state.annoMatrix,
+    schema,
+    crossfilter: state.obsCrossfilter,
+    isUserAnno,
   };
 })
-class Category extends React.Component {
-  constructor(props) {
-    super(props);
-    this.state = {
-      isChecked: true,
-    };
+class Category extends React.PureComponent {
+  static getSelectionState(
+    categoricalSelection,
+    metadataField,
+    categorySummary
+  ) {
+    // total number of categories in this dimension
+    const totalCatCount = categorySummary.numCategoryValues;
+    // number of selected options in this category
+    const selectedCatCount = categorySummary.categoryValues.reduce(
+      (res, label) => (categoricalSelection.get(label) ?? true ? res + 1 : res),
+      0
+    );
+    return selectedCatCount === totalCatCount
+      ? "all"
+      : selectedCatCount === 0
+      ? "none"
+      : "some";
   }
 
-  componentDidUpdate(prevProps) {
+  static watchAsync(props, prevProps) {
+    return !shallowEqual(props.watchProps, prevProps.watchProps);
+  }
+
+  createCategorySummaryFromDfCol = memoize(createCategorySummaryFromDfCol);
+
+  getSelectionState(categorySummary) {
     const { categoricalSelection, metadataField } = this.props;
-    const cat = categoricalSelection?.[metadataField];
-    if (
-      categoricalSelection !== prevProps.categoricalSelection &&
-      !!cat &&
-      !!this.checkbox
-    ) {
-      const categoryCount = {
-        // total number of categories in this dimension
-        totalCatCount: cat.numCategoryValues,
-        // number of selected options in this category
-        selectedCatCount: _.reduce(
-          cat.categoryValueSelected,
-          (res, cond) => (cond ? res + 1 : res),
-          0
-        ),
-      };
-      if (categoryCount.selectedCatCount === categoryCount.totalCatCount) {
-        /* everything is on, so not indeterminate */
-        this.checkbox.indeterminate = false;
-        this.setState({ isChecked: true }); // eslint-disable-line react/no-did-update-set-state
-      } else if (categoryCount.selectedCatCount === 0) {
-        /* nothing is on, so no */
-        this.checkbox.indeterminate = false;
-        this.setState({ isChecked: false }); // eslint-disable-line react/no-did-update-set-state
-      } else if (categoryCount.selectedCatCount < categoryCount.totalCatCount) {
-        /* to be explicit... */
-        this.checkbox.indeterminate = true;
-        this.setState({ isChecked: false }); // eslint-disable-line react/no-did-update-set-state
-      }
-    }
+    return Category.getSelectionState(
+      categoricalSelection,
+      metadataField,
+      categorySummary
+    );
   }
 
   handleColorChange = () => {
@@ -85,137 +93,281 @@ class Category extends React.Component {
     }
   };
 
-  toggleNone() {
-    const { dispatch, metadataField } = this.props;
-    dispatch({
-      type: "categorical metadata filter none of these",
-      metadataField,
-    });
-    this.setState({ isChecked: false });
-  }
-
-  toggleAll() {
-    const { dispatch, metadataField } = this.props;
-    dispatch({
-      type: "categorical metadata filter all of these",
-      metadataField,
-    });
-    this.setState({ isChecked: true });
-  }
-
-  handleToggleAllClick() {
-    const { isChecked } = this.state;
-    // || this.checkbox.indeterminate === false
-    if (isChecked) {
-      this.toggleNone();
-    } else {
-      this.toggleAll();
+  handleCategoryKeyPress = (e) => {
+    if (e.key === "Enter") {
+      this.handleCategoryClick();
     }
+  };
+
+  handleToggleAllClick = (categorySummary) => {
+    const isChecked = this.getSelectionState(categorySummary);
+    if (isChecked === "all") {
+      this.toggleNone(categorySummary);
+    } else {
+      this.toggleAll(categorySummary);
+    }
+  };
+
+  fetchAsyncProps = async (props) => {
+    const { annoMatrix, metadataField, colors } = props.watchProps;
+    const { crossfilter } = this.props;
+
+    const [categoryData, categorySummary, colorData] = await this.fetchData(
+      annoMatrix,
+      metadataField,
+      colors
+    );
+
+    return {
+      categoryData,
+      categorySummary,
+      colorData,
+      crossfilter,
+      ...this.updateColorTable(colorData),
+      handleCategoryToggleAllClick: () =>
+        this.handleToggleAllClick(categorySummary),
+    };
+  };
+
+  async fetchData(annoMatrix, metadataField, colors) {
+    /*
+    fetch our data and the color-by data if appropriate, and then build a summary
+    of our category and a color table for the color-by annotation.
+    */
+    const { schema } = annoMatrix;
+    const { colorAccessor, colorMode } = colors;
+    let colorDataPromise = Promise.resolve(null);
+    if (colorAccessor) {
+      const query = createColorQuery(colorMode, colorAccessor, schema);
+      if (query) colorDataPromise = annoMatrix.fetch(...query);
+    }
+    const [categoryData, colorData] = await Promise.all([
+      annoMatrix.fetch("obs", metadataField),
+      colorDataPromise,
+    ]);
+
+    // our data
+    const column = categoryData.icol(0);
+    const colSchema = schema.annotations.obsByName[metadataField];
+    const categorySummary = this.createCategorySummaryFromDfCol(
+      column,
+      colSchema
+    );
+    return [categoryData, categorySummary, colorData];
   }
 
-  renderIsStillLoading() {
-    /*
-    We are still loading this category, so render a "busy" signal.
-    */
-    const { metadataField } = this.props;
+  updateColorTable(colorData) {
+    // color table, which may be null
+    const { schema, colors, metadataField } = this.props;
+    const { colorAccessor, userColors, colorMode } = colors;
+    return {
+      isColorAccessor: colorAccessor === metadataField,
+      colorAccessor,
+      colorMode,
+      colorTable: createColorTable(
+        colorMode,
+        colorAccessor,
+        colorData,
+        schema,
+        userColors
+      ),
+    };
+  }
+
+  toggleNone(categorySummary) {
+    const { dispatch, metadataField } = this.props;
+    dispatch(
+      actions.selectCategoricalAllMetadataAction(
+        "categorical metadata filter none of these",
+        metadataField,
+        categorySummary.allCategoryValues,
+        false
+      )
+    );
+  }
+
+  toggleAll(categorySummary) {
+    const { dispatch, metadataField } = this.props;
+    dispatch(
+      actions.selectCategoricalAllMetadataAction(
+        "categorical metadata filter all of these",
+        metadataField,
+        categorySummary.allCategoryValues,
+        true
+      )
+    );
+  }
+
+  render() {
+    const {
+      metadataField,
+      isExpanded,
+      categoricalSelection,
+      crossfilter,
+      colors,
+      annoMatrix,
+      isUserAnno,
+    } = this.props;
 
     const checkboxID = `category-select-${metadataField}`;
 
     return (
+      <CategoryCrossfilterContext.Provider value={crossfilter}>
+        <Async
+          watchFn={Category.watchAsync}
+          promiseFn={this.fetchAsyncProps}
+          watchProps={{
+            metadataField,
+            annoMatrix,
+            categoricalSelection,
+            colors,
+          }}
+        >
+          <Async.Pending initial>
+            <StillLoading
+              metadataField={metadataField}
+              checkboxID={checkboxID}
+            />
+          </Async.Pending>
+          <Async.Rejected>
+            {(error) => (
+              <ErrorLoading metadataField={metadataField} error={error} />
+            )}
+          </Async.Rejected>
+          <Async.Fulfilled persist>
+            {(asyncProps) => {
+              const {
+                colorAccessor,
+                colorTable,
+                colorData,
+                categoryData,
+                categorySummary,
+                isColorAccessor,
+                handleCategoryToggleAllClick,
+              } = asyncProps;
+              const isTruncated = !!categorySummary?.isTruncated;
+              const selectionState = this.getSelectionState(categorySummary);
+              return (
+                <CategoryRender
+                  metadataField={metadataField}
+                  checkboxID={checkboxID}
+                  isUserAnno={isUserAnno}
+                  isTruncated={isTruncated}
+                  isExpanded={isExpanded}
+                  isColorAccessor={isColorAccessor}
+                  selectionState={selectionState}
+                  categoryData={categoryData}
+                  categorySummary={categorySummary}
+                  colorAccessor={colorAccessor}
+                  colorData={colorData}
+                  colorTable={colorTable}
+                  onColorChangeClick={this.handleColorChange}
+                  onCategoryToggleAllClick={handleCategoryToggleAllClick}
+                  onCategoryMenuClick={this.handleCategoryClick}
+                  onCategoryMenuKeyPress={this.handleCategoryKeyPress}
+                />
+              );
+            }}
+          </Async.Fulfilled>
+        </Async>
+      </CategoryCrossfilterContext.Provider>
+    );
+  }
+}
+
+export default Category;
+
+const StillLoading = ({ metadataField, checkboxID }) => {
+  /*
+  We are still loading this category, so render a "busy" signal.
+  */
+  return (
+    <div
+      style={{
+        maxWidth: globals.maxControlsWidth,
+      }}
+    >
       <div
         style={{
-          maxWidth: globals.maxControlsWidth,
+          display: "flex",
+          justifyContent: "space-between",
+          alignItems: "baseline",
         }}
       >
         <div
           style={{
             display: "flex",
-            justifyContent: "space-between",
-            alignItems: "baseline",
+            justifyContent: "flex-start",
+            alignItems: "flex-start",
           }}
         >
-          <div
-            style={{
-              display: "flex",
-              justifyContent: "flex-start",
-              alignItems: "flex-start",
-            }}
-          >
-            <label htmlFor={checkboxID} className="bp3-control bp3-checkbox">
-              <input disabled id={checkboxID} checked type="checkbox" />
-              <span className="bp3-control-indicator" />
-            </label>
-            <Truncate>
-              <span
-                style={{
-                  cursor: "pointer",
-                  display: "inline-block",
-                  width: LABEL_WIDTH,
-                }}
-              >
-                {metadataField}
-              </span>
-            </Truncate>
-          </div>
-          <div>
-            <Button minimal loading intent="primary" />
-          </div>
-        </div>
-      </div>
-    );
-  }
-
-  render() {
-    const { isChecked } = this.state;
-    const {
-      metadataField,
-      categoricalSelection,
-      isColorAccessor,
-      isExpanded,
-      schema,
-    } = this.props;
-
-    const isStillLoading = !(categoricalSelection?.[metadataField] ?? false);
-    if (isStillLoading) {
-      return this.renderIsStillLoading();
-    }
-
-    const checkboxID = `category-select-${metadataField}`;
-
-    const isUserAnno =
-      schema?.annotations?.obsByName[metadataField]?.writable ?? false;
-    const isTruncated = _.get(
-      categoricalSelection,
-      [metadataField, "isTruncated"],
-      false
-    );
-
-    if (
-      !isUserAnno &&
-      schema?.annotations?.obsByName[metadataField]?.categories?.length === 1
-    ) {
-      return (
-        <div style={{ marginBottom: 10, marginTop: 4 }}>
+          <label htmlFor={checkboxID} className="bp3-control bp3-checkbox">
+            <input disabled id={checkboxID} checked type="checkbox" />
+            <span className="bp3-control-indicator" />
+          </label>
           <Truncate>
-            <span style={{ maxWidth: 150, fontWeight: 700 }}>
+            <span
+              style={{
+                cursor: "pointer",
+                display: "inline-block",
+                width: LABEL_WIDTH,
+              }}
+            >
               {metadataField}
             </span>
           </Truncate>
-          <Truncate>
-            <span style={{ maxWidth: 150 }}>
-              {`: ${schema.annotations.obsByName[metadataField].categories[0]}`}
-            </span>
-          </Truncate>
         </div>
-      );
-    }
+        <div>
+          <Button minimal loading intent="primary" />
+        </div>
+      </div>
+    </div>
+  );
+};
+
+const ErrorLoading = ({ metadataField, error }) => {
+  console.error(error); // log error to console as it is unexpected.
+  return (
+    <div style={{ marginBottom: 10, marginTop: 4 }}>
+      <span
+        style={{
+          cursor: "pointer",
+          display: "inline-block",
+          width: LABEL_WIDTH,
+          fontStyle: "italic",
+        }}
+      >
+        {`Failure loading ${metadataField}`}
+      </span>
+    </div>
+  );
+};
+
+const CategoryHeader = React.memo(
+  ({
+    metadataField,
+    checkboxID,
+    isUserAnno,
+    isTruncated,
+    isColorAccessor,
+    isExpanded,
+    selectionState,
+    onColorChangeClick,
+    onCategoryMenuClick,
+    onCategoryMenuKeyPress,
+    onCategoryToggleAllClick,
+  }) => {
+    /*
+    Render category name and controls (eg, color-by button).
+    */
+    const checkboxRef = useRef(null);
+
+    useEffect(() => {
+      checkboxRef.current.indeterminate = selectionState === "some";
+    }, [checkboxRef.current, selectionState]);
 
     return (
-      <CategoryFlipperLayout
-        metadataField={metadataField}
-        isExpanded={isExpanded}
-        isUserAnno={isUserAnno}
-      >
+      <>
         <div
           style={{
             display: "flex",
@@ -228,12 +380,9 @@ class Category extends React.Component {
               id={checkboxID}
               data-testclass="category-select"
               data-testid={`${metadataField}:category-select`}
-              onChange={this.handleToggleAllClick.bind(this)}
-              ref={(el) => {
-                this.checkbox = el;
-                return el;
-              }}
-              checked={isChecked}
+              onChange={onCategoryToggleAllClick}
+              ref={checkboxRef}
+              checked={selectionState === "all"}
               type="checkbox"
             />
             <span className="bp3-control-indicator" />
@@ -241,16 +390,13 @@ class Category extends React.Component {
           <span
             role="menuitem"
             tabIndex="0"
+            data-testclass="category-expand"
             data-testid={`${metadataField}:category-expand`}
-            onKeyPress={(e) => {
-              if (e.key === "Enter") {
-                this.handleCategoryClick();
-              }
-            }}
+            onKeyPress={onCategoryMenuKeyPress}
             style={{
               cursor: "pointer",
             }}
-            onClick={this.handleCategoryClick}
+            onClick={onCategoryMenuClick}
           >
             <Truncate>
               <span
@@ -299,7 +445,7 @@ class Category extends React.Component {
             <AnchorButton
               data-testclass="colorby"
               data-testid={`colorby-${metadataField}`}
-              onClick={this.handleColorChange}
+              onClick={onColorChangeClick}
               active={isColorAccessor}
               intent={isColorAccessor ? "primary" : "none"}
               disabled={isTruncated}
@@ -307,9 +453,168 @@ class Category extends React.Component {
             />
           </Tooltip>
         </div>
-      </CategoryFlipperLayout>
+      </>
     );
   }
-}
+);
 
-export default Category;
+const CategoryRender = React.memo(
+  ({
+    metadataField,
+    checkboxID,
+    isUserAnno,
+    isTruncated,
+    isColorAccessor,
+    isExpanded,
+    selectionState,
+    categoryData,
+    categorySummary,
+    colorAccessor,
+    colorData,
+    colorTable,
+    onColorChangeClick,
+    onCategoryMenuClick,
+    onCategoryMenuKeyPress,
+    onCategoryToggleAllClick,
+  }) => {
+    /*
+    Render the core of the category, including checkboxes, controls, etc.
+    */
+    const { numCategoryValues } = categorySummary;
+    const isSingularValue = !isUserAnno && numCategoryValues === 1;
+
+    if (isSingularValue) {
+      /*
+      Entire category has a single value, special case.
+      */
+      const theOneValue = categorySummary.categoryValues[0];
+      return (
+        <div style={{ marginBottom: 10, marginTop: 4 }}>
+          <Truncate>
+            <span style={{ maxWidth: 150, fontWeight: 700 }}>
+              {metadataField}
+            </span>
+          </Truncate>
+          <Truncate>
+            <span style={{ maxWidth: 150 }}>{`: ${theOneValue}`}</span>
+          </Truncate>
+        </div>
+      );
+    }
+
+    /*
+    Otherwise, our normal multi-layout layout
+    */
+    return (
+      <div
+        style={{
+          maxWidth: globals.maxControlsWidth,
+        }}
+        data-testclass="category"
+        data-testid={`category-${metadataField}`}
+      >
+        <div
+          style={{
+            display: "flex",
+            justifyContent: "space-between",
+            alignItems: "baseline",
+          }}
+        >
+          <CategoryHeader
+            metadataField={metadataField}
+            checkboxID={checkboxID}
+            isUserAnno={isUserAnno}
+            isTruncated={isTruncated}
+            isExpanded={isExpanded}
+            isColorAccessor={isColorAccessor}
+            selectionState={selectionState}
+            onColorChangeClick={onColorChangeClick}
+            onCategoryToggleAllClick={onCategoryToggleAllClick}
+            onCategoryMenuClick={onCategoryMenuClick}
+            onCategoryMenuKeyPress={onCategoryMenuKeyPress}
+          />
+        </div>
+        <div style={{ marginLeft: 26 }}>
+          {
+            /* values*/
+            isExpanded ? (
+              <CategoryValueList
+                isUserAnno={isUserAnno}
+                metadataField={metadataField}
+                categoryData={categoryData}
+                categorySummary={categorySummary}
+                colorAccessor={colorAccessor}
+                colorData={colorData}
+                colorTable={colorTable}
+              />
+            ) : null
+          }
+        </div>
+        <div>
+          {isExpanded && isTruncated ? (
+            <p style={{ paddingLeft: 15 }}>... truncated list ...</p>
+          ) : null}
+        </div>
+      </div>
+    );
+  }
+);
+
+const CategoryValueList = React.memo(
+  ({
+    isUserAnno,
+    metadataField,
+    categoryData,
+    categorySummary,
+    colorAccessor,
+    colorData,
+    colorTable,
+  }) => {
+    const tuples = [...categorySummary.categoryValueIndices];
+
+    /*
+    Render the value list.  If this is a user annotation, we use a flipper
+    animation, if read-only, we don't bother and save a few bits of perf.
+    */
+    if (!isUserAnno) {
+      return (
+        <>
+          {tuples.map(([value, index]) => (
+            <Value
+              key={value}
+              isUserAnno={isUserAnno}
+              metadataField={metadataField}
+              categoryIndex={index}
+              categoryData={categoryData}
+              categorySummary={categorySummary}
+              colorAccessor={colorAccessor}
+              colorData={colorData}
+              colorTable={colorTable}
+            />
+          ))}
+        </>
+      );
+    }
+
+    /* User annotation */
+    const flipKey = tuples.map((t) => t[0]).join("");
+    return (
+      <Flipper flipKey={flipKey}>
+        {tuples.map(([value, index]) => (
+          <Flipped key={value} flipId={value}>
+            <Value
+              isUserAnno={isUserAnno}
+              metadataField={metadataField}
+              categoryIndex={index}
+              categoryData={categoryData}
+              categorySummary={categorySummary}
+              colorAccessor={colorAccessor}
+              colorData={colorData}
+              colorTable={colorTable}
+            />
+          </Flipped>
+        ))}
+      </Flipper>
+    );
+  }
+);

client/src/components/categorical/categoryContext.js

@@ -0,0 +1,7 @@
+import React from "react";
+
+/*
+CategoryCrossfilterContext is used to pass a snapshot of the crossfilter 
+matching the current category summary.
+*/
+export const CategoryCrossfilterContext = React.createContext(null);

client/src/components/categorical/index.js

@@ -9,11 +9,11 @@ import AnnoDialog from "./annoDialog";
 import AnnoSelect from "./annoSelect";
 import LabelInput from "./labelInput";
 import { labelPrompt } from "./labelUtil";
+import actions from "../../actions";
 
 @connect((state) => ({
   writableCategoriesEnabled: state.config?.parameters?.annotations ?? false,
-  schema: state.world?.schema,
-  config: state.config,
+  schema: state.annoMatrix?.schema,
   ontology: state.ontology,
 }))
 class Categories extends React.Component {
@@ -30,11 +30,12 @@ class Categories extends React.Component {
   handleCreateUserAnno = (e) => {
     const { dispatch } = this.props;
     const { newCategoryText, categoryToDuplicate } = this.state;
-    dispatch({
-      type: "annotation: create category",
-      data: newCategoryText,
-      categoryToDuplicate,
-    });
+    dispatch(
+      actions.annotationCreateCategoryAction(
+        newCategoryText,
+        categoryToDuplicate
+      )
+    );
     this.setState({
       createAnnoModeActive: false,
       categoryToDuplicate: null,
@@ -126,12 +127,11 @@ class Categories extends React.Component {
       newCategoryText,
       expandedCats,
     } = this.state;
-    const { writableCategoriesEnabled, schema, config, ontology } = this.props;
+    const { writableCategoriesEnabled, schema, ontology } = this.props;
     const ontologyEnabled = ontology?.enabled ?? false;
     /* all names, sorted in display order.  Will be rendered in this order */
     const allCategoryNames = ControlsHelpers.selectableCategoryNames(
-      schema,
-      ControlsHelpers.maxCategoryItems(config)
+      schema
     ).sort();
 
     return (

client/src/components/categorical/labelInput.js

@@ -142,7 +142,7 @@ export default class LabelInput extends React.PureComponent {
       return (
         <InputGroup
           autoFocus={autoFocus}
-          {...props.inputProps} // eslint-disable-line react/jsx-props-no-spreading
+          {...props.inputProps} // eslint-disable-line react/jsx-props-no-spreading --- Allows for modularity
           value={label}
           onChange={this.handleChange}
         />

client/src/components/categorical/value/index.js

@@ -1,5 +1,6 @@
 import { connect } from "react-redux";
 import React from "react";
+import * as d3 from "d3";
 
 import {
   Button,
@@ -10,7 +11,6 @@ import {
   Icon,
   PopoverInteractionKind,
 } from "@blueprintjs/core";
-import Occupancy from "./occupancy";
 import * as globals from "../../../globals";
 import styles from "../categorical.css";
 import AnnoDialog from "../annoDialog";
@@ -19,110 +19,106 @@ import Truncate from "../../util/truncate";
 
 import { AnnotationsHelpers } from "../../../util/stateManager";
 import { labelPrompt, isLabelErroneous } from "../labelUtil";
+import actions from "../../../actions";
+import MiniHistogram from "../../miniHistogram";
+import MiniStackedBar from "../../miniStackedBar";
+import { CategoryCrossfilterContext } from "../categoryContext";
+
+const VALUE_HEIGHT = 11;
+const CHART_WIDTH = 100;
 
 /* this is defined outside of the class so we can use it in connect() */
-function _currentLabel(ownProps, categoricalSelection) {
-  const { metadataField, categoryIndex } = ownProps;
-  return String(
-    categoricalSelection[metadataField].categoryValues[categoryIndex]
-  ).valueOf();
+function _currentLabelAsString(ownProps) {
+  const { label } = ownProps;
+  // when called as a function, the String() constructor performs type conversion,
+  // and returns a primitive string.
+  return String(label);
 }
 
 @connect((state, ownProps) => {
   const { pointDilation, categoricalSelection } = state;
-  const { metadataField } = ownProps;
+  const { metadataField, categorySummary, categoryIndex } = ownProps;
   const isDilated =
     pointDilation.metadataField === metadataField &&
-    pointDilation.categoryField ===
-      _currentLabel(ownProps, categoricalSelection);
+    pointDilation.categoryField === _currentLabelAsString(ownProps);
+
+  const category = categoricalSelection[metadataField];
+  const label = categorySummary.categoryValues[categoryIndex];
+  const isSelected = category.get(label) ?? true;
+
   return {
-    categoricalSelection,
     annotations: state.annotations,
-    colorScale: state.colors.scale,
-    colorAccessor: state.colors.colorAccessor,
-    schema: state.world?.schema,
-    world: state.world,
-    crossfilter: state.crossfilter,
+    schema: state.annoMatrix?.schema,
     ontology: state.ontology,
     isDilated,
+    isSelected,
+    label,
   };
 })
 class CategoryValue extends React.Component {
   constructor(props) {
     super(props);
     this.state = {
-      editedLabelText: this.currentLabel(),
+      editedLabelText: this.currentLabelAsString(),
     };
   }
 
   componentDidUpdate(prevProps) {
-    const { categoricalSelection, metadataField, categoryIndex } = this.props;
+    const { metadataField, categoryIndex, categorySummary } = this.props;
     if (
-      prevProps.categoricalSelection !== categoricalSelection ||
       prevProps.metadataField !== metadataField ||
-      prevProps.categoryIndex !== categoryIndex
+      prevProps.categoryIndex !== categoryIndex ||
+      prevProps.categorySummary !== categorySummary
     ) {
-      // adequately checked to prevent looping
-      // eslint-disable-next-line react/no-did-update-set-state
+      // eslint-disable-next-line react/no-did-update-set-state --- adequately checked to prevent looping
       this.setState({
-        editedLabelText: this.currentLabel(),
+        editedLabelText: this.currentLabelAsString(),
       });
     }
   }
 
-  handleDeleteValue = () => {
-    const { dispatch, metadataField } = this.props;
-    const label = this.getLabel();
+  // If coloring by and this isn't the colorAccessor and it isn't being edited
+  get shouldRenderStackedBarOrHistogram() {
+    const { colorAccessor, isColorBy, annotations } = this.props;
 
-    dispatch({
-      type: "annotation: delete label",
-      metadataField,
-      label,
-    });
+    return colorAccessor && !isColorBy && !annotations.isEditingLabelName;
+  }
+
+  handleDeleteValue = () => {
+    const { dispatch, metadataField, label } = this.props;
+    dispatch(actions.annotationDeleteLabelFromCategory(metadataField, label));
   };
 
   handleAddCurrentSelectionToThisLabel = () => {
-    const { dispatch, metadataField, categoryIndex } = this.props;
-    const label = this.getLabel();
-    dispatch({
-      type: "annotation: label current cell selection",
-      metadataField,
-      categoryIndex,
-      label,
-    });
+    const { dispatch, metadataField, label } = this.props;
+    dispatch(actions.annotationLabelCurrentSelection(metadataField, label));
   };
 
   handleEditValue = (e) => {
-    const { dispatch, metadataField, categoryIndex } = this.props;
+    const { dispatch, metadataField, label } = this.props;
     const { editedLabelText } = this.state;
-    const label = this.getLabel();
     this.cancelEditMode();
-    dispatch({
-      type: "annotation: label edited",
-      editedLabel: editedLabelText,
-      metadataField,
-      categoryIndex,
-      label,
-    });
+    dispatch(
+      actions.annotationRenameLabelInCategory(
+        metadataField,
+        label,
+        editedLabelText
+      )
+    );
     e.preventDefault();
   };
 
   handleCreateArbitraryLabel = (txt) => {
-    const { dispatch, metadataField, categoryIndex } = this.props;
-    const label = this.getLabel();
+    const { dispatch, metadataField, label } = this.props;
     this.cancelEditMode();
-    dispatch({
-      type: "annotation: label edited",
-      metadataField,
-      editedLabel: txt,
-      categoryIndex,
-      label,
-    });
+    dispatch(
+      actions.annotationRenameLabelInCategory(metadataField, label, txt)
+    );
   };
 
   labelNameError = (name) => {
     const { metadataField, ontology, schema } = this.props;
-    if (name === this.currentLabel()) return false;
+    if (name === this.currentLabelAsString()) return false;
     return isLabelErroneous(name, metadataField, ontology, schema);
   };
 
@@ -131,33 +127,45 @@ class CategoryValue extends React.Component {
   };
 
   activateEditLabelMode = () => {
-    const { dispatch, metadataField, categoryIndex } = this.props;
+    const { dispatch, metadataField, categoryIndex, label } = this.props;
     dispatch({
       type: "annotation: activate edit label mode",
       metadataField,
       categoryIndex,
+      label,
     });
   };
 
   cancelEditMode = () => {
-    const { dispatch, metadataField, categoryIndex } = this.props;
+    const { dispatch, metadataField, categoryIndex, label } = this.props;
     this.setState({
-      editedLabelText: this.currentLabel(),
+      editedLabelText: this.currentLabelAsString(),
     });
     dispatch({
       type: "annotation: cancel edit label mode",
       metadataField,
       categoryIndex,
+      label,
     });
   };
 
   toggleOff = () => {
-    const { dispatch, metadataField, categoryIndex } = this.props;
-    dispatch({
-      type: "categorical metadata filter deselect",
+    const {
+      dispatch,
       metadataField,
       categoryIndex,
-    });
+      categorySummary,
+    } = this.props;
+    const label = categorySummary.categoryValues[categoryIndex];
+    dispatch(
+      actions.selectCategoricalMetadataAction(
+        "categorical metadata filter deselect",
+        metadataField,
+        categorySummary.allCategoryValues,
+        label,
+        false
+      )
+    );
   };
 
   shouldComponentUpdate = (nextProps, nextState) => {
@@ -171,60 +179,74 @@ class CategoryValue extends React.Component {
     If and only if true, update the component
     */
     const { props, state } = this;
-    const { metadataField, categoryIndex, categoricalSelection } = props;
-    const { categoricalSelection: newCategoricalSelection } = nextProps;
+    const { categoryIndex, categorySummary, isSelected } = props;
+    const {
+      categoryIndex: newCategoryIndex,
+      categorySummary: newCategorySummary,
+      isSelected: newIsSelected,
+    } = nextProps;
 
-    const valueSelectionChange =
-      categoricalSelection[metadataField].categoryValueSelected[
-        categoryIndex
-      ] !==
-      newCategoricalSelection[metadataField].categoryValueSelected[
-        categoryIndex
-      ];
+    const label = categorySummary.categoryValues[categoryIndex];
+    const newLabel = newCategorySummary.categoryValues[newCategoryIndex];
+    const labelChanged = label !== newLabel;
+    const valueSelectionChange = isSelected !== newIsSelected;
 
-    const worldChange = props.world !== nextProps.world;
     const colorAccessorChange = props.colorAccessor !== nextProps.colorAccessor;
     const annotationsChange = props.annotations !== nextProps.annotations;
-    const crossfilterChange =
-      props.isUserAnno && props.crossfilter !== nextProps.crossfilter;
     const editingLabel = state.editedLabelText !== nextState.editedLabelText;
     const dilationChange = props.isDilated !== nextProps.isDilated;
 
+    const count = categorySummary.categoryValueCounts[categoryIndex];
+    const newCount = newCategorySummary.categoryValueCounts[newCategoryIndex];
+    const countChanged = count !== newCount;
+
     return (
+      labelChanged ||
       valueSelectionChange ||
-      worldChange ||
       colorAccessorChange ||
       annotationsChange ||
-      crossfilterChange ||
       editingLabel ||
-      dilationChange
+      dilationChange ||
+      countChanged
     );
   };
 
   toggleOn = () => {
-    const { dispatch, metadataField, categoryIndex } = this.props;
-    dispatch({
-      type: "categorical metadata filter select",
+    const {
+      dispatch,
       metadataField,
       categoryIndex,
-    });
+      categorySummary,
+    } = this.props;
+    const label = categorySummary.categoryValues[categoryIndex];
+    dispatch(
+      actions.selectCategoricalMetadataAction(
+        "categorical metadata filter select",
+        metadataField,
+        categorySummary.allCategoryValues,
+        label,
+        true
+      )
+    );
   };
 
   handleMouseEnter = () => {
-    const { dispatch, metadataField, categoryIndex } = this.props;
+    const { dispatch, metadataField, categoryIndex, label } = this.props;
     dispatch({
       type: "category value mouse hover start",
       metadataField,
       categoryIndex,
+      label,
     });
   };
 
   handleMouseExit = () => {
-    const { dispatch, metadataField, categoryIndex } = this.props;
+    const { dispatch, metadataField, categoryIndex, label } = this.props;
     dispatch({
       type: "category value mouse hover end",
       metadataField,
       categoryIndex,
+      label,
     });
   };
 
@@ -237,26 +259,103 @@ class CategoryValue extends React.Component {
     this.setState({ editedLabelText: e.target });
   };
 
-  getLabel = () => {
-    const { metadataField, categoryIndex, categoricalSelection } = this.props;
-    const category = categoricalSelection[metadataField];
-    const label = category.categoryValues[categoryIndex];
+  createHistogramBins = (
+    metadataField,
+    categoryData,
+    colorAccessor,
+    colorData,
+    categoryValue,
+    width,
+    height
+  ) => {
+    /*
+      Knowing that colorScale is based off continuous data,
+      createHistogramBins fetches the continuous data in relation to the cells relevant to the category value.
+      It then separates that data into 50 bins for drawing the mini-histogram
+    */
+    const groupBy = categoryData.col(metadataField);
+    const col = colorData.icol(0);
+    const range = col.summarize();
 
-    return label;
+    const histogramMap = col.histogram(
+      50,
+      [range.min, range.max],
+      groupBy
+    ); /* Because the signature changes we really need different names for histogram to differentiate signatures  */
+
+    const bins = histogramMap.has(categoryValue)
+      ? histogramMap.get(categoryValue)
+      : new Array(50).fill(0);
+
+    const xScale = d3.scaleLinear().domain([0, bins.length]).range([0, width]);
+
+    const largestBin = Math.max(...bins);
+
+    const yScale = d3.scaleLinear().domain([0, largestBin]).range([0, height]);
+
+    return {
+      xScale,
+      yScale,
+      bins,
+    };
   };
 
-  currentLabel() {
-    const { categoricalSelection } = this.props;
-    return _currentLabel(this.props, categoricalSelection);
+  createStackedGraphBins = (
+    metadataField,
+    categoryData,
+    colorAccessor,
+    colorData,
+    categoryValue,
+    colorTable,
+    schema,
+    width
+  ) => {
+    /*
+      Knowing that the color scale is based off of categorical data,
+      createOccupancyStack obtains a map showing the number if cells per colored value
+      Using the colorScale a stack of colored bars is drawn representing the map
+     */
+    const groupBy = categoryData.col(metadataField);
+    const occupancyMap = colorData
+      .col(colorAccessor)
+      .histogramCategorical(groupBy);
+
+    const occupancy = occupancyMap.get(categoryValue);
+
+    if (occupancy && occupancy.size > 0) {
+      // not all categories have occupancy, so occupancy may be undefined.
+      const scale = d3
+        .scaleLinear()
+        /* get all the keys d[1] as an array, then find the sum */
+        .domain([0, d3.sum(Array.from(occupancy.values()))])
+        .range([0, width]);
+      const categories =
+        schema.annotations.obsByName[colorAccessor]?.categories;
+
+      const dfColumn = colorData.col(colorAccessor);
+      const categoryValues = dfColumn.summarizeCategorical().categories;
+
+      return {
+        domainValues: categoryValues,
+        scale,
+        domain: categories,
+        occupancy,
+      };
+    }
+    return null;
+  };
+
+  currentLabelAsString() {
+    return _currentLabelAsString(this.props);
   }
 
-  isAddCurrentSelectionDisabled(category, value) {
+  isAddCurrentSelectionDisabled(crossfilter, category, value) {
     /*
     disable "add current selection to label", if one of the following is true:
     1. no cells are selected
     2. all currently selected cells already have this label, on this category
     */
-    const { crossfilter, world } = this.props;
+    const { categoryData } = this.props;
 
     // 1. no cells selected?
     if (crossfilter.countSelected() === 0) {
@@ -265,12 +364,7 @@ class CategoryValue extends React.Component {
     // 2. all selected cells already have the label
     const mask = crossfilter.allSelectedMask();
     if (
-      AnnotationsHelpers.allHaveLabelByMask(
-        world.obsAnnotations,
-        category,
-        value,
-        mask
-      )
+      AnnotationsHelpers.allHaveLabelByMask(categoryData, category, value, mask)
     ) {
       return true;
     }
@@ -278,43 +372,132 @@ class CategoryValue extends React.Component {
     return false;
   }
 
+  renderMiniStackedBar = () => {
+    const {
+      colorAccessor,
+      metadataField,
+      categoryData,
+      colorData,
+      colorTable,
+      schema,
+      label,
+    } = this.props;
+    const isColorBy = metadataField === colorAccessor;
+
+    if (
+      !this.shouldRenderStackedBarOrHistogram ||
+      !AnnotationsHelpers.isCategoricalAnnotation(schema, colorAccessor) ||
+      isColorBy
+    ) {
+      return null;
+    }
+
+    const { domainValues, scale, domain, occupancy } =
+      this.createStackedGraphBins(
+        metadataField,
+        categoryData,
+        colorAccessor,
+        colorData,
+        label,
+        colorTable,
+        schema,
+        CHART_WIDTH
+      ) ?? {};
+
+    if (!domainValues || !scale || !domain || !occupancy) {
+      return null;
+    }
+
+    return (
+      <MiniStackedBar
+        /* eslint-disable react/jsx-props-no-spreading -- Disable unneeded on next release of eslint-config-airbnb */
+        {...{
+          colorTable,
+          domainValues,
+          scale,
+          domain,
+          occupancy,
+        }}
+        /* eslint-enable react/jsx-props-no-spreading -- enable */
+        height={VALUE_HEIGHT}
+        width={CHART_WIDTH}
+      />
+    );
+  };
+
+  renderMiniHistogram = () => {
+    const {
+      colorAccessor,
+      metadataField,
+      colorData,
+      categoryData,
+      colorTable,
+      schema,
+      label,
+    } = this.props;
+    const colorScale = colorTable?.scale;
+
+    if (
+      !this.shouldRenderStackedBarOrHistogram ||
+      !AnnotationsHelpers.isContinuousAnnotation(schema, colorAccessor)
+    ) {
+      return null;
+    }
+
+    const { xScale, yScale, bins } =
+      this.createHistogramBins(
+        metadataField,
+        categoryData,
+        colorAccessor,
+        colorData,
+        label,
+        CHART_WIDTH,
+        VALUE_HEIGHT
+      ) ?? {};
+
+    return (
+      <MiniHistogram
+        /* eslint-disable react/jsx-props-no-spreading -- Disable unneeded on next release of eslint-config-airbnb */
+        {...{
+          colorScale,
+          xScale,
+          yScale,
+          bins,
+        }}
+        /* eslint-enable react/jsx-props-no-spreading -- enable */
+        obsOrVarContinuousFieldDisplayName={colorAccessor}
+        domainLabel={label}
+        height={VALUE_HEIGHT}
+        width={CHART_WIDTH}
+      />
+    );
+  };
+
   render() {
     const {
-      categoricalSelection,
       metadataField,
       categoryIndex,
       colorAccessor,
-      colorScale,
-      i,
-      schema,
+      colorTable,
       isUserAnno,
       annotations,
       ontology,
-      // flippedProps is potentially brittle, their docs want {...flippedProps} on our div,
-      // our lint doesn't like jsx spread, we are version pinned to prevent api change on their part
-      flippedProps,
       isDilated,
-      world,
+      isSelected,
+      categorySummary,
+      label,
     } = this.props;
+    const colorScale = colorTable?.scale;
     const ontologyEnabled = ontology?.enabled ?? false;
 
     const { editedLabelText } = this.state;
 
-    if (!categoricalSelection) return null;
-
-    const category = categoricalSelection[metadataField];
-    const selected = category.categoryValueSelected[categoryIndex];
-    const count = category.categoryValueCounts[categoryIndex];
-    const value = category.categoryValues[categoryIndex];
-    const displayString = this.currentLabel();
+    const count = categorySummary.categoryValueCounts[categoryIndex];
+    const displayString = this.currentLabelAsString();
 
     /* this is the color scale, so add swatches below */
     const isColorBy = metadataField === colorAccessor;
-    let categories = null;
-
-    if (isColorBy && schema) {
-      categories = schema.annotations.obsByName[colorAccessor]?.categories;
-    }
+    const { categoryValueIndices } = categorySummary;
 
     const editModeActive =
       isUserAnno &&
@@ -322,13 +505,14 @@ class CategoryValue extends React.Component {
       annotations.isEditingLabelName &&
       annotations.labelEditable.label === categoryIndex;
 
-    const valueToggleLabel = `value-toggle-checkbox-${displayString}`;
+    const valueToggleLabel = `value-toggle-checkbox-${metadataField}-${displayString}`;
 
-    const LEFT_MARGIN = 33;
+    const LEFT_MARGIN = 60;
     const CHECKBOX = 26;
-    const CELL_NUMBER = 61;
+    const CELL_NUMBER = 50;
     const ANNO_MENU = 26;
-    const LABEL_MARGIN = 24;
+    const LABEL_MARGIN = 16;
+    const CHART_MARGIN = 24;
 
     const otherElementsWidth =
       LEFT_MARGIN +
@@ -337,19 +521,16 @@ class CategoryValue extends React.Component {
       LABEL_MARGIN +
       (isUserAnno ? ANNO_MENU : 0);
 
-    const OCCUPANCY_WIDTH = 100;
-
     const labelWidth =
       colorAccessor && !isColorBy
-        ? globals.leftSidebarWidth - otherElementsWidth - OCCUPANCY_WIDTH
+        ? globals.leftSidebarWidth -
+          otherElementsWidth -
+          CHART_WIDTH -
+          CHART_MARGIN
         : globals.leftSidebarWidth - otherElementsWidth;
 
     return (
       <div
-        key={i}
-        data-flip-config={flippedProps["data-flip-config"]}
-        data-flip-id={flippedProps["data-flip-id"]}
-        data-portal-key={flippedProps["data-portal-key"]}
         className={
           /* This code is to change the styles on centroid label hover is causing over-rendering */
           `${styles.value}${isDilated ? ` ${styles.hover}` : ""}`
@@ -384,10 +565,10 @@ class CategoryValue extends React.Component {
             >
               <input
                 id={valueToggleLabel}
-                onChange={selected ? this.toggleOff : this.toggleOn}
+                onChange={isSelected ? this.toggleOff : this.toggleOn}
                 data-testclass="categorical-value-select"
                 data-testid={`categorical-value-select-${metadataField}-${displayString}`}
-                checked={selected}
+                checked={isSelected}
                 type="checkbox"
               />
               <span
@@ -460,16 +641,8 @@ class CategoryValue extends React.Component {
             ) : null}
           </div>
           <span style={{ flexShrink: 0 }}>
-            {colorAccessor && !isColorBy && !annotations.isEditingLabelName ? (
-              <Occupancy
-                categoryValue={value}
-                colorAccessor={colorAccessor}
-                metadataField={metadataField}
-                world={world}
-                colorScale={colorScale}
-                colorByIsCategorical={!!categoricalSelection[colorAccessor]}
-              />
-            ) : null}
+            {this.renderMiniStackedBar()}
+            {this.renderMiniHistogram()}
           </span>
         </div>
         <div>
@@ -492,14 +665,14 @@ class CategoryValue extends React.Component {
             </span>
 
             <svg
-              display={isColorBy && categories ? "auto" : "none"}
+              display={isColorBy && categoryValueIndices ? "auto" : "none"}
               style={{
                 marginLeft: 5,
-                width: 11,
-                height: 11,
+                width: VALUE_HEIGHT,
+                height: VALUE_HEIGHT,
                 backgroundColor:
-                  isColorBy && categories
-                    ? colorScale(categories.indexOf(value))
+                  isColorBy && categoryValueIndices
+                    ? colorScale(categoryValueIndices.get(label))
                     : "inherit",
               }}
             />
@@ -514,32 +687,37 @@ class CategoryValue extends React.Component {
                   position={Position.RIGHT_TOP}
                   content={
                     <Menu>
-                      <MenuItem
-                        icon="plus"
-                        data-testclass="handleAddCurrentSelectionToThisLabel"
-                        data-testid={`${metadataField}:${displayString}:add-current-selection-to-this-label`}
-                        onClick={this.handleAddCurrentSelectionToThisLabel}
-                        text={
-                          <span>
-                            Re-label currently selected cells as
-                            <span
-                              style={{
-                                fontStyle:
-                                  displayString ===
-                                  globals.unassignedCategoryLabel
-                                    ? "italic"
-                                    : "auto",
-                              }}
-                            >
-                              {` ${displayString}`}
-                            </span>
-                          </span>
-                        }
-                        disabled={this.isAddCurrentSelectionDisabled(
-                          metadataField,
-                          value
+                      <CategoryCrossfilterContext.Consumer>
+                        {(crossfilter) => (
+                          <MenuItem
+                            icon="plus"
+                            data-testclass="handleAddCurrentSelectionToThisLabel"
+                            data-testid={`${metadataField}:${displayString}:add-current-selection-to-this-label`}
+                            onClick={this.handleAddCurrentSelectionToThisLabel}
+                            text={
+                              <span>
+                                Re-label currently selected cells as
+                                <span
+                                  style={{
+                                    fontStyle:
+                                      displayString ===
+                                      globals.unassignedCategoryLabel
+                                        ? "italic"
+                                        : "auto",
+                                  }}
+                                >
+                                  {` ${displayString}`}
+                                </span>
+                              </span>
+                            }
+                            disabled={this.isAddCurrentSelectionDisabled(
+                              crossfilter,
+                              metadataField,
+                              label
+                            )}
+                          />
                         )}
-                      />
+                      </CategoryCrossfilterContext.Consumer>
                       {displayString !== globals.unassignedCategoryLabel ? (
                         <MenuItem
                           icon="edit"

client/src/components/categorical/value/occupancy.js

@@ -9,7 +9,9 @@ import {
   Classes,
 } from "@blueprintjs/core";
 
-@connect()
+@connect((state) => ({
+  schema: state.annoMatrix?.schema,
+}))
 class Occupancy extends React.PureComponent {
   _WIDTH = 100;
 
@@ -21,16 +23,17 @@ class Occupancy extends React.PureComponent {
       createHistogram fetches the continous data in relation to the cells releveant to the catagory value.
       It then seperates that data into 50 bins for drawing the mini-histogram
     */
-    const { world, metadataField, colorAccessor, categoryValue } = this.props;
+    const {
+      metadataField,
+      categoryData,
+      colorData,
+      categoryValue,
+    } = this.props;
 
     if (!this.canvas) return;
 
-    const groupBy = world.obsAnnotations.col(metadataField);
-
-    const col =
-      world.obsAnnotations.col(colorAccessor) ||
-      world.varData.col(colorAccessor);
-
+    const groupBy = categoryData.col(metadataField);
+    const col = colorData.icol(0);
     const range = col.summarize();
 
     const histogramMap = col.histogram(
@@ -39,7 +42,6 @@ class Occupancy extends React.PureComponent {
       groupBy
     ); /* Because the signature changes we really need different names for histogram to differentiate signatures  */
 
-    // const categoryValue = category.categoryValues[categoryIndex];
     const bins = histogramMap.has(categoryValue)
       ? histogramMap.get(categoryValue)
       : new Array(50).fill(0);
@@ -79,20 +81,22 @@ class Occupancy extends React.PureComponent {
       Using the colorScale a stack of colored bars is drawn representing the map
      */
     const {
-      world,
       metadataField,
+      categoryData,
       colorAccessor,
       categoryValue,
-      colorScale,
+      colorTable,
+      schema,
+      colorData,
     } = this.props;
-    const { schema } = world;
+    const { scale: colorScale } = colorTable;
 
     const ctx = this.canvas?.getContext("2d");
 
     if (!ctx) return;
 
-    const groupBy = world.obsAnnotations.col(metadataField);
-    const occupancyMap = world.obsAnnotations
+    const groupBy = categoryData.col(metadataField);
+    const occupancyMap = colorData
       .col(colorAccessor)
       .histogramCategorical(groupBy);
 
@@ -109,7 +113,7 @@ class Occupancy extends React.PureComponent {
         schema.annotations.obsByName[colorAccessor]?.categories;
 
       let currentOffset = 0;
-      const dfColumn = world.obsAnnotations.col(colorAccessor);
+      const dfColumn = colorData.col(colorAccessor);
       const categoryValues = dfColumn.summarizeCategorical().categories;
 
       let o;

client/src/components/continuous/continuous.js

@@ -1,97 +1,28 @@
-// jshint esversion: 6
 /* rc slider https://www.npmjs.com/package/rc-slider */
 
 import React from "react";
 import { connect } from "react-redux";
-import { Button } from "@blueprintjs/core";
-
-import * as globals from "../../globals";
 import HistogramBrush from "../brushableHistogram";
 
 @connect((state) => ({
-  obsAnnotations: state.world?.obsAnnotations,
-  colorAccessor: state.colors.colorAccessor,
-  colorScale: state.colors.scale,
-  schema: state.world?.schema,
+  schema: state.annoMatrix?.schema,
 }))
 class Continuous extends React.PureComponent {
-  static renderIsStillLoading(zebra, key) {
-    return (
-      <div
-        key={key}
-        style={{
-          padding: globals.leftSidebarSectionPadding,
-          backgroundColor: zebra % 2 === 0 ? globals.lightestGrey : "white",
-        }}
-      >
-        <div
-          style={{
-            display: "flex",
-            justifyContent: "space-between",
-            justifyItems: "center",
-            alignItems: "center",
-          }}
-        >
-          <div style={{ minWidth: 30 }} />
-          <div style={{ display: "flex", alignSelf: "center" }}>
-            <span style={{ fontStyle: "italic" }}>{key}</span>
-          </div>
-          <div
-            style={{
-              display: "flex",
-              justifyContent: "flex-end",
-            }}
-          >
-            <Button minimal loading intent="primary" />
-          </div>
-        </div>
-      </div>
-    );
-  }
-
   render() {
-    const { obsAnnotations, schema } = this.props;
-
+    /* initial value for iterator to simulate index, ranges is an object */
+    const { schema } = this.props;
+    if (!schema) return null;
     const obsIndex = schema.annotations.obs.index;
     const allContinuousNames = schema.annotations.obs.columns
       .filter((col) => col.type === "int32" || col.type === "float32")
       .filter((col) => col.name !== obsIndex)
       .map((col) => col.name);
 
-    /* initial value for iterator to simulate index, ranges is an object */
-    let zebra = 0;
-
     return (
       <div>
-        {allContinuousNames.map((key) => {
-          if (!obsAnnotations.hasCol(key)) {
-            // still loading!
-            zebra += 1;
-            return Continuous.renderIsStillLoading(zebra, key);
-          }
-
-          // data loaded and available
-          const summary = obsAnnotations.col(key).summarize();
-          const nonFiniteExtent =
-            summary.min === undefined ||
-            summary.max === undefined ||
-            Number.isNaN(summary.min) ||
-            Number.isNaN(summary.max);
-          if (!summary.categorical && !nonFiniteExtent) {
-            zebra += 1;
-            return (
-              <HistogramBrush
-                key={key}
-                field={key}
-                isObs
-                zebra={zebra % 2 === 0}
-                ranges={summary}
-              />
-            );
-          }
-
-          return null;
-        })}
+        {allContinuousNames.map((key, zebra) => (
+          <HistogramBrush key={key} field={key} isObs zebra={zebra % 2 === 0} />
+        ))}
       </div>
     );
   }

client/src/components/continuousLegend/index.js

@@ -4,6 +4,11 @@ import { connect } from "react-redux";
 import * as d3 from "d3";
 import { interpolateCool } from "d3-scale-chromatic";
 
+import {
+  createColorTable,
+  createColorQuery,
+} from "../../util/stateManager/colorHelpers";
+
 // create continuous color legend
 // http://bl.ocks.org/syntagmatic/e8ccca52559796be775553b467593a9f
 const continuous = (selectorId, colorscale, colorAccessor) => {
@@ -101,34 +106,78 @@ const continuous = (selectorId, colorscale, colorAccessor) => {
 };
 
 @connect((state) => ({
-  colorAccessor: state.colors.colorAccessor,
-  colorScale: state.colors.scale,
+  annoMatrix: state.annoMatrix,
+  colors: state.colors,
 }))
 class ContinuousLegend extends React.Component {
+  constructor(props) {
+    super(props);
+    this.ref = null;
+    this.state = {
+      colorAccessor: null,
+      colorScale: null,
+    };
+  }
+
+  componentDidMount() {
+    this.updateState(null);
+  }
+
   componentDidUpdate(prevProps) {
-    const { colorAccessor, colorScale } = this.props;
-    if (
-      prevProps.colorAccessor !== colorAccessor ||
-      prevProps.colorScale !== colorScale
-    ) {
+    this.updateState(prevProps);
+  }
+
+  async updateState(prevProps) {
+    const { annoMatrix, colors } = this.props;
+    if (!colors || !annoMatrix) return;
+
+    if (colors !== prevProps?.colors || annoMatrix !== prevProps?.annoMatrix) {
+      const { schema } = annoMatrix;
+      const { colorMode, colorAccessor, userColors } = colors;
+      const colorQuery = createColorQuery(colorMode, colorAccessor, schema);
+      const colorDf = colorQuery ? await annoMatrix.fetch(...colorQuery) : null;
+      const colorTable = createColorTable(
+        colorMode,
+        colorAccessor,
+        colorDf,
+        schema,
+        userColors
+      );
+
+      const colorScale = colorTable.scale;
+      const range = colorScale?.range;
+      const [domainMin, domainMax] = colorScale?.domain?.() ?? [0, 0];
+
       /* always remove it, if it's not continuous we don't put it back. */
       d3.select("#continuous_legend").selectAll("*").remove();
-    }
 
-    if (colorAccessor && colorScale && colorScale.range) {
-      /* fragile! continuous range is 0 to 1, not [#fa4b2c, ...], make this a flag? */
-      if (colorScale.range()[0][0] !== "#") {
-        continuous(
-          "#continuous_legend",
-          d3.scaleSequential(interpolateCool).domain(colorScale.domain()),
-          colorAccessor
-        );
+      if (colorAccessor && colorScale && range && domainMin < domainMax) {
+        /* fragile! continuous range is 0 to 1, not [#fa4b2c, ...], make this a flag? */
+        if (range()[0][0] !== "#") {
+          continuous(
+            "#continuous_legend",
+            d3.scaleSequential(interpolateCool).domain(colorScale.domain()),
+            colorAccessor
+          );
+        }
       }
+
+      this.setState({
+        colorAccessor,
+        colorScale: colorTable.scale,
+      });
     }
   }
 
   render() {
-    const { colorAccessor } = this.props;
+    const { colorAccessor, colorScale } = this.state;
+    if (
+      colorScale?.domain &&
+      colorScale.domain()[1] === colorScale.domain()[0]
+    ) {
+      /* it's a single value, not a distribution, min max are the same */
+      return null;
+    }
     return (
       <div
         id="continuous_legend"

client/src/components/framework/toasters.js

@@ -5,6 +5,7 @@ import { Position, Toaster, Intent } from "@blueprintjs/core";
 const ToastTopCenter = Toaster.create({
   className: "recipe-toaster",
   position: Position.TOP,
+  maxToasts: 4,
 });
 
 /*
@@ -23,12 +24,15 @@ export const keepAroundErrorToast = (message) =>
 /*
 a hard network error
 */
-export const postNetworkErrorToast = (message) =>
-  ToastTopCenter.show({
-    message,
-    timeout: 30000,
-    intent: Intent.DANGER,
-  });
+export const postNetworkErrorToast = (message, key = undefined) =>
+  ToastTopCenter.show(
+    {
+      message,
+      timeout: 30000,
+      intent: Intent.DANGER,
+    },
+    key
+  );
 
 /*
 Async message to user

client/src/components/geneExpression/addGenes.js

@@ -1,4 +1,3 @@
-// jshint esversion: 6
 /* rc slider https://www.npmjs.com/package/rc-slider */
 
 import React from "react";
@@ -34,9 +33,6 @@ const renderGene = (fuzzySortResult, { handleClick, modifiers }) => {
       active={modifiers.active}
       disabled={modifiers.disabled}
       data-testid={`suggest-menu-item-${geneName}`}
-      // Use of annotations in this way is incorrect and dataset specific.
-      // See https://github.com/chanzuckerberg/cellxgene/issues/483
-      // label={gene.n_counts}
       key={geneName}
       onClick={(g) =>
         /* this fires when user clicks a menu item */
@@ -56,11 +52,9 @@ const filterGenes = (query, genes) =>
 
 @connect((state) => {
   return {
-    obsAnnotations: state.world?.obsAnnotations,
+    annoMatrix: state.annoMatrix,
     userDefinedGenes: state.controls.userDefinedGenes,
     userDefinedGenesLoading: state.controls.userDefinedGenesLoading,
-    world: state.world,
-    colorAccessor: state.colors.colorAccessor,
     differential: state.differential,
   };
 })
@@ -71,9 +65,19 @@ class AddGenes extends React.Component {
       bulkAdd: "",
       tab: "autosuggest",
       activeItem: null,
+      geneNames: [],
+      status: "pending",
     };
   }
 
+  componentDidMount() {
+    this.updateState();
+  }
+
+  componentDidUpdate(prevProps) {
+    this.updateState(prevProps);
+  }
+
   _genesToUpper = (listGenes) => {
     // Has to be a Map to preserve index
     const upperGenes = new Map();
@@ -84,13 +88,12 @@ class AddGenes extends React.Component {
     return upperGenes;
   };
 
-  // eslint-disable-next-line react/sort-comp
+  // eslint-disable-next-line react/sort-comp -- memo requires a defined _genesToUpper
   _memoGenesToUpper = memoize(this._genesToUpper, (arr) => arr);
 
   handleBulkAddClick = () => {
-    const { world, dispatch, userDefinedGenes } = this.props;
-    const varIndexName = world.schema.annotations.var.index;
-    const { bulkAdd } = this.state;
+    const { dispatch, userDefinedGenes } = this.props;
+    const { bulkAdd, geneNames } = this.state;
 
     /*
       test:
@@ -98,18 +101,14 @@ class AddGenes extends React.Component {
     */
     if (bulkAdd !== "") {
       const genes = _.pull(_.uniq(bulkAdd.split(/[ ,]+/)), "");
-      console.log("geneExpression genes", genes);
       if (genes.length === 0) {
         return keepAroundErrorToast("Must enter a gene name.");
       }
-      const worldGenes =
-        world.varAnnotations?.col(varIndexName)?.asArray() || [];
-
       // These gene lists are unique enough where memoization is useless
       const upperGenes = this._genesToUpper(genes);
       const upperUserDefinedGenes = this._genesToUpper(userDefinedGenes);
 
-      const upperWorldGenes = this._memoGenesToUpper(worldGenes);
+      const upperGeneNames = this._memoGenesToUpper(geneNames);
 
       dispatch({ type: "bulk user defined gene start" });
 
@@ -119,7 +118,7 @@ class AddGenes extends React.Component {
             return keepAroundErrorToast("That gene already exists");
           }
 
-          const indexOfGene = upperWorldGenes.get(upperGene);
+          const indexOfGene = upperGeneNames.get(upperGene);
 
           if (indexOfGene === undefined) {
             return keepAroundErrorToast(
@@ -129,7 +128,7 @@ class AddGenes extends React.Component {
             );
           }
           return dispatch(
-            actions.requestUserDefinedGene(worldGenes[indexOfGene])
+            actions.requestUserDefinedGene(geneNames[indexOfGene])
           );
         })
       ).then(
@@ -142,6 +141,27 @@ class AddGenes extends React.Component {
     return undefined;
   };
 
+  async updateState(prevProps) {
+    const { annoMatrix } = this.props;
+    if (!annoMatrix) return;
+    if (annoMatrix !== prevProps?.annoMatrix) {
+      const { schema } = annoMatrix;
+      const varIndex = schema.annotations.var.index;
+
+      this.setState({ status: "pending" });
+      try {
+        const df = await annoMatrix.fetch("var", varIndex);
+        this.setState({
+          status: "success",
+          geneNames: df.col(varIndex).asArray(),
+        });
+      } catch (error) {
+        this.setState({ status: "error" });
+        throw error;
+      }
+    }
+  }
+
   placeholderGeneNames() {
     /*
     return a string containing gene name suggestions for use as a user hint.
@@ -151,10 +171,7 @@ class AddGenes extends React.Component {
 
     NOTE: the random selection means it will re-render constantly.
     */
-    const { world } = this.props;
-    const { varAnnotations } = world;
-    const varIndexName = world.schema.annotations.var.index;
-    const geneNames = varAnnotations.col(varIndexName).asArray();
+    const { geneNames } = this.state;
     if (geneNames.length > 0) {
       const placeholder = [];
       let len = geneNames.length;
@@ -175,8 +192,8 @@ class AddGenes extends React.Component {
   }
 
   handleClick(g) {
-    const { world, dispatch, userDefinedGenes } = this.props;
-    const varIndexName = world.schema.annotations.var.index;
+    const { dispatch, userDefinedGenes } = this.props;
+    const { geneNames } = this.state;
     if (!g) return;
     const gene = g.target;
     if (userDefinedGenes.indexOf(gene) !== -1) {
@@ -185,27 +202,21 @@ class AddGenes extends React.Component {
       postUserErrorToast(
         `That's too many genes, you can have at most ${globals.maxUserDefinedGenes} user defined genes`
       );
-    } else if (
-      world.varAnnotations.col(varIndexName).indexOf(gene) === undefined
-    ) {
+    } else if (geneNames.indexOf(gene) === undefined) {
       postUserErrorToast("That doesn't appear to be a valid gene name.");
     } else {
       dispatch({ type: "single user defined gene start" });
-      dispatch(actions.requestUserDefinedGene(gene)).then(
-        () => dispatch({ type: "single user defined gene complete" }),
-        () => dispatch({ type: "single user defined gene error" })
-      );
+      dispatch(actions.requestUserDefinedGene(gene));
+      dispatch({ type: "single user defined gene complete" });
     }
   }
 
   render() {
-    const { world, userDefinedGenesLoading } = this.props;
-    const varIndexName = world?.schema?.annotations?.var?.index;
-    const varIndex = world?.varAnnotations?.col(varIndexName)?.asArray();
-    const { tab, bulkAdd, activeItem } = this.state;
+    const { userDefinedGenesLoading } = this.props;
+    const { tab, bulkAdd, activeItem, status, geneNames } = this.state;
 
     // may still be loading!
-    if (!varIndex) return null;
+    if (status !== "success") return null;
 
     return (
       <div>
@@ -263,7 +274,7 @@ class AddGenes extends React.Component {
               itemListPredicate={filterGenes}
               onActiveItemChange={(item) => this.setState({ activeItem: item })}
               itemRenderer={renderGene}
-              items={varIndex || ["No genes"]}
+              items={geneNames || ["No genes"]}
               popoverProps={{ minimal: true }}
             />
             <Button

client/src/components/geneExpression/index.js

@@ -1,4 +1,3 @@
-// jshint esversion: 6
 /* rc slider https://www.npmjs.com/package/rc-slider */
 
 import React from "react";
@@ -13,11 +12,7 @@ import testGeneSets from "./test_data";
 
 @connect((state) => {
   return {
-    obsAnnotations: state.world?.obsAnnotations,
     userDefinedGenes: state.controls.userDefinedGenes,
-    userDefinedGenesLoading: state.controls.userDefinedGenesLoading,
-    world: state.world,
-    colorAccessor: state.colors.colorAccessor,
     differential: state.differential,
   };
 })
@@ -35,13 +30,7 @@ class GeneExpression extends React.Component {
   };
 
   render() {
-    const { world, userDefinedGenes, differential } = this.props;
-    const varIndexName = world?.schema?.annotations?.var?.index;
-    const varIndex = world?.varAnnotations?.col(varIndexName)?.asArray();
-
-    // may still be loading!
-    if (!varIndex) return null;
-
+    const { userDefinedGenes, differential } = this.props;
     return (
       <div
         style={{
@@ -50,19 +39,13 @@ class GeneExpression extends React.Component {
       >
         <div>
           <AddGenes />
-          {world && userDefinedGenes.length > 0
+          {userDefinedGenes.length > 0
             ? _.map(userDefinedGenes, (geneName, index) => {
-                const values = world.varData.col(geneName);
-                if (!values) {
-                  return null;
-                }
-                const summary = values.summarize();
                 return (
                   <HistogramBrush
                     key={geneName}
                     field={geneName}
                     zebra={index % 2 === 0}
-                    ranges={summary}
                     isUserDefined
                   />
                 );
@@ -72,18 +55,11 @@ class GeneExpression extends React.Component {
         <div>
           {differential.diffExp
             ? _.map(differential.diffExp, (value, index) => {
-                const name = world.varAnnotations.at(value[0], varIndexName);
-                const values = world.varData.col(name);
-                if (!values) {
-                  return null;
-                }
-                const summary = values.summarize();
                 return (
                   <HistogramBrush
-                    key={name}
-                    field={name}
+                    key={value[0]}
+                    field={value[0]}
                     zebra={index % 2 === 0}
-                    ranges={summary}
                     isDiffExp
                     logFoldChange={value[1]}
                     pval={value[2]}

client/src/components/graph/drawPointsRegl.js

@@ -1,6 +1,6 @@
 import { glPointFlags, glPointSize } from "../../util/glHelpers";
 
-export default function (regl) {
+export default function drawPointsRegl(regl) {
   return regl({
     vert: `
     precision mediump float;

client/src/components/graph/graph.js

@@ -1,18 +1,26 @@
 // jshint esversion: 6
 import React from "react";
 import * as d3 from "d3";
-import { connect } from "react-redux";
+import { connect, shallowEqual } from "react-redux";
 import { mat3, vec2 } from "gl-matrix";
 import _regl from "regl";
 import memoize from "memoize-one";
+import Async from "react-async";
+import { Button } from "@blueprintjs/core";
 
 import setupSVGandBrushElements from "./setupSVGandBrush";
 import _camera from "../../util/camera";
 import _drawPoints from "./drawPointsRegl";
-import { isTypedArray } from "../../util/typeHelpers";
+import {
+  createColorTable,
+  createColorQuery,
+} from "../../util/stateManager/colorHelpers";
+import * as globals from "../../globals";
 
 import GraphOverlayLayer from "./overlays/graphOverlayLayer";
 import CentroidLabels from "./overlays/centroidLabels";
+import actions from "../../actions";
+import renderThrottle from "../../util/renderThrottle";
 
 /*
 Simple 2D transforms control all point painting.  There are three:
@@ -21,7 +29,6 @@ Simple 2D transforms control all point painting.  There are three:
   * camera - apply a 2D camera transformation (pan, zoom)
   * projection - apply any transformation required for screen size and layout
 */
-
 function createProjectionTF(viewportWidth, viewportHeight) {
   /*
   the projection transform accounts for the screen size & other layout
@@ -53,38 +60,49 @@ function createModelTF() {
   return m;
 }
 
-function renderThrottle(callback) {
-  /*
-  This wraps a call to requestAnimationFrame(), enforcing a single
-  render callback at any given time (ie, you can call this any number
-  of times, and it will coallesce multiple inter-frame calls into a
-  single render).
-  */
-  let rafCurrentlyInProgress = null;
-  return function f() {
-    if (rafCurrentlyInProgress) return;
-    const context = this;
-    rafCurrentlyInProgress = window.requestAnimationFrame(() => {
-      callback.apply(context);
-      rafCurrentlyInProgress = null;
-    });
-  };
-}
+const flagSelected = 1;
+const flagNaN = 2;
+const flagHighlight = 4;
 
 @connect((state) => ({
-  universe: state.universe,
-  world: state.world,
-  crossfilter: state.crossfilter,
-  colorRGB: state.colors.rgb,
+  annoMatrix: state.annoMatrix,
+  crossfilter: state.obsCrossfilter,
   selectionTool: state.graphSelection.tool,
   currentSelection: state.graphSelection.selection,
   layoutChoice: state.layoutChoice,
-  centroidLabels: state.centroidLabels,
   graphInteractionMode: state.controls.graphInteractionMode,
-  colorAccessor: state.colors.colorAccessor,
+  colors: state.colors,
   pointDilation: state.pointDilation,
 }))
 class Graph extends React.Component {
+  static createReglState(canvas) {
+    /*
+    Must be created for each canvas
+    */
+    // setup canvas, webgl draw function and camera
+    const camera = _camera(canvas);
+    const regl = _regl(canvas);
+    const drawPoints = _drawPoints(regl);
+
+    // preallocate webgl buffers
+    const pointBuffer = regl.buffer();
+    const colorBuffer = regl.buffer();
+    const flagBuffer = regl.buffer();
+
+    return {
+      camera,
+      regl,
+      drawPoints,
+      pointBuffer,
+      colorBuffer,
+      flagBuffer,
+    };
+  }
+
+  static watchAsync(props, prevProps) {
+    return !shallowEqual(props.watchProps, prevProps.watchProps);
+  }
+
   computePointPositions = memoize((X, Y, modelTF) => {
     /*
     compute the model coordinate for each point
@@ -111,18 +129,44 @@ class Graph extends React.Component {
   });
 
   computeSelectedFlags = memoize(
-    (crossfilter, flagSelected, flagUnselected) => {
+    (crossfilter, _flagSelected, _flagUnselected) => {
       const x = crossfilter.fillByIsSelected(
         new Float32Array(crossfilter.size()),
-        flagSelected,
-        flagUnselected
+        _flagSelected,
+        _flagUnselected
       );
       return x;
     }
   );
 
+  computeHighlightFlags = memoize(
+    (nObs, pointDilationData, pointDilationLabel) => {
+      const flags = new Float32Array(nObs);
+      if (pointDilationData) {
+        for (let i = 0, len = flags.length; i < len; i += 1) {
+          if (pointDilationData[i] === pointDilationLabel) {
+            flags[i] = flagHighlight;
+          }
+        }
+      }
+      return flags;
+    }
+  );
+
+  computeColorByFlags = memoize((nObs, colorByData) => {
+    const flags = new Float32Array(nObs);
+    if (colorByData) {
+      for (let i = 0, len = flags.length; i < len; i += 1) {
+        if (!Number.isFinite(colorByData[i])) {
+          flags[i] = flagNaN;
+        }
+      }
+    }
+    return flags;
+  });
+
   computePointFlags = memoize(
-    (world, crossfilter, colorAccessor, pointDilation) => {
+    (crossfilter, colorByData, pointDilationData, pointDilationLabel) => {
       /*
       We communicate with the shader using three flags:
       - isNaN -- the value is a NaN. Only makes sense when we have a colorAccessor
@@ -136,38 +180,25 @@ class Graph extends React.Component {
       continuous metadata, as they rely on different tests, and some of the flags
       (eg, isNaN) are meaningless in the face of categorical metadata.
       */
+      const nObs = crossfilter.size();
+      const flags = new Float32Array(nObs);
 
-      const flagSelected = 1;
-      const flagNaN = 2;
-      const flagHighlight = 4;
-
-      const flags = this.computeSelectedFlags(
+      const selectedFlags = this.computeSelectedFlags(
         crossfilter,
         flagSelected,
         0
-      ).slice();
+      );
+      const highlightFlags = this.computeHighlightFlags(
+        nObs,
+        pointDilationData,
+        pointDilationLabel
+      );
+      const colorByFlags = this.computeColorByFlags(nObs, colorByData);
 
-      const { metadataField, categoryField } = pointDilation;
-      const highlightData = metadataField
-        ? world.obsAnnotations.col(metadataField)?.asArray()
-        : null;
-      const colorByColumn = colorAccessor
-        ? world.obsAnnotations.col(colorAccessor)?.asArray() ||
-          world.varData.col(colorAccessor)?.asArray()
-        : null;
-      const colorByData =
-        colorByColumn && isTypedArray(colorByColumn) ? colorByColumn : null;
-
-      if (colorByData || highlightData) {
-        for (let i = 0, len = flags.length; i < len; i += 1) {
-          if (highlightData) {
-            flags[i] += highlightData[i] === categoryField ? flagHighlight : 0;
-          }
-          if (colorByData) {
-            flags[i] += Number.isFinite(colorByData[i]) ? 0 : flagNaN;
-          }
-        }
+      for (let i = 0; i < nObs; i += 1) {
+        flags[i] = selectedFlags[i] + highlightFlags[i] + colorByFlags[i];
       }
+
       return flags;
     }
   );
@@ -175,167 +206,68 @@ class Graph extends React.Component {
   constructor(props) {
     super(props);
     const viewport = this.getViewportDimensions();
-    this.count = 0;
-    this.renderCache = {
-      X: null,
-      Y: null,
-      positions: null,
-      colors: null,
-      sizes: null,
-      flags: null,
-    };
+    this.reglCanvas = null;
+    this.cachedAsyncProps = null;
+    const modelTF = createModelTF();
     this.state = {
       toolSVG: null,
       tool: null,
       container: null,
-      cameraRender: 0,
       viewport,
+
+      // projection
+      camera: null,
+      modelTF,
+      modelInvTF: mat3.invert([], modelTF),
+      projectionTF: createProjectionTF(viewport.width, viewport.height),
+
+      // regl state
+      regl: null,
+      drawPoints: null,
+      pointBuffer: null,
+      colorBuffer: null,
+      flagBuffer: null,
+
+      // component rendering derived state - these must stay synchronized
+      // with the reducer state they were generated from.
+      layoutState: {
+        layoutDf: null,
+        layoutChoice: null,
+      },
+      colorState: {
+        colors: null,
+        colorDf: null,
+        colorTable: null,
+      },
+      pointDilationState: {
+        pointDilation: null,
+        pointDilationDf: null,
+      },
     };
   }
 
   componentDidMount() {
     window.addEventListener("resize", this.handleResize);
-
-    // setup canvas, webgl draw function and camera
-    const camera = _camera(this.reglCanvas);
-    const regl = _regl(this.reglCanvas);
-    const drawPoints = _drawPoints(regl);
-
-    // preallocate webgl buffers
-    const pointBuffer = regl.buffer();
-    const colorBuffer = regl.buffer();
-    const flagBuffer = regl.buffer();
-
-    // create all default rendering transformations
-    const modelTF = createModelTF();
-    const projectionTF = createProjectionTF(
-      this.reglCanvas.width,
-      this.reglCanvas.height
-    );
-
-    // initial draw to canvas
-    this.renderPoints(
-      regl,
-      drawPoints,
-      colorBuffer,
-      pointBuffer,
-      flagBuffer,
-      camera,
-      projectionTF
-    );
-
-    this.setState({
-      regl,
-      drawPoints,
-      pointBuffer,
-      colorBuffer,
-      flagBuffer,
-      camera,
-      modelTF,
-      modelInvTF: mat3.invert([], modelTF),
-      projectionTF,
-    });
   }
 
   componentDidUpdate(prevProps, prevState) {
-    const { renderCache } = this;
     const {
-      world,
-      crossfilter,
-      colorRGB,
       selectionTool,
       currentSelection,
-      layoutChoice,
       graphInteractionMode,
-      pointDilation,
-      colorAccessor,
     } = this.props;
-    const { regl, toolSVG, camera, modelTF, viewport } = this.state;
-    let { projectionTF } = this.state;
+    const { toolSVG, viewport } = this.state;
     const hasResized =
-      prevState.viewport.height !== this.reglCanvas.height ||
-      prevState.viewport.width !== this.reglCanvas.width;
+      prevState.viewport.height !== viewport.height ||
+      prevState.viewport.width !== viewport.width;
     let stateChanges = {};
-    let needsRepaint = hasResized;
 
-    if (regl && world && crossfilter) {
-      /* update the regl and point rendering state */
-      const { obsLayout, nObs } = world;
-      const { drawPoints, pointBuffer, colorBuffer, flagBuffer } = this.state;
-
-      if (hasResized) {
-        projectionTF = createProjectionTF(
-          this.reglCanvas.width,
-          this.reglCanvas.height
-        );
-        stateChanges = {
-          ...stateChanges,
-          projectionTF,
-        };
-      }
-
-      /* coordinates for each point */
-      const X = obsLayout.col(layoutChoice.currentDimNames[0]).asArray();
-      const Y = obsLayout.col(layoutChoice.currentDimNames[1]).asArray();
-      const newPositions = this.computePointPositions(X, Y, modelTF);
-      if (renderCache.positions !== newPositions) {
-        /* update our cache & GL if the buffer changes */
-        renderCache.positions = newPositions;
-        pointBuffer({ data: newPositions, dimension: 2 });
-        needsRepaint = true;
-      }
-
-      /* colors for each point */
-      const newColors = this.computePointColors(colorRGB);
-      if (renderCache.colors !== newColors) {
-        /* update our cache & GL if the buffer changes */
-        renderCache.colors = newColors;
-        colorBuffer({ data: newColors, dimension: 3 });
-        needsRepaint = true;
-      }
-
-      /* flags for each point */
-      const newFlags = this.computePointFlags(
-        world,
-        crossfilter,
-        colorAccessor,
-        pointDilation
-      );
-      if (renderCache.flags !== newFlags) {
-        renderCache.flags = newFlags;
-        needsRepaint = true;
-        flagBuffer({ data: newFlags, dimension: 1 });
-      }
-
-      this.count = nObs;
-
-      if (needsRepaint) {
-        this.renderPoints(
-          regl,
-          drawPoints,
-          colorBuffer,
-          pointBuffer,
-          flagBuffer,
-          camera,
-          projectionTF
-        );
-      }
-    }
-
-    if (hasResized) {
-      // If the window size has changed we want to recreate all SVGs
-      stateChanges = {
-        ...stateChanges,
-        ...this.createToolSVG(),
-      };
-    } else if (
-      (viewport.height && viewport.width && !toolSVG) ||
-      selectionTool !== prevProps.selectionTool
+    if (
+      (viewport.height && viewport.width && !toolSVG) || // first time init
+      hasResized || //  window size has changed we want to recreate all SVGs
+      selectionTool !== prevProps.selectionTool || // change of selection tool
+      prevProps.graphInteractionMode !== graphInteractionMode // lasso/zoom mode is switched
     ) {
-      // first time or change of selection tool
-      stateChanges = { ...stateChanges, ...this.createToolSVG() };
-    } else if (prevProps.graphInteractionMode !== graphInteractionMode) {
-      // If lasso/zoom is switched
       stateChanges = {
         ...stateChanges,
         ...this.createToolSVG(),
@@ -358,8 +290,7 @@ class Graph extends React.Component {
       );
     }
     if (Object.keys(stateChanges).length > 0) {
-      // Preventing update loop via stateChanges and diff checks
-      // eslint-disable-next-line react/no-did-update-set-state
+      // eslint-disable-next-line react/no-did-update-set-state --- Preventing update loop via stateChanges and diff checks
       this.setState(stateChanges);
     }
   }
@@ -368,12 +299,21 @@ class Graph extends React.Component {
     window.removeEventListener("resize", this.handleResize);
   }
 
+  setReglCanvas = (canvas) => {
+    this.reglCanvas = canvas;
+    this.setState({
+      ...Graph.createReglState(canvas),
+    });
+  };
+
   handleResize = () => {
     const { state } = this.state;
     const viewport = this.getViewportDimensions();
+    const projectionTF = createProjectionTF(viewport.width, viewport.height);
     this.setState({
       ...state,
       viewport,
+      projectionTF,
     });
   };
 
@@ -401,11 +341,13 @@ class Graph extends React.Component {
     Called from componentDidUpdate. Create the tool SVG, and return any
     state changes that should be passed to setState().
     */
-    const { viewport, selectionTool, graphInteractionMode } = this.props;
+    const { selectionTool, graphInteractionMode } = this.props;
+    const { viewport } = this.state;
 
     /* clear out whatever was on the div, even if nothing, but usually the brushes etc */
-
-    d3.select("#lasso-layer").selectAll(".lasso-group").remove();
+    const lasso = d3.select("#lasso-layer");
+    if (lasso.empty()) return {}; // still initializing
+    lasso.selectAll(".lasso-group").remove();
 
     // Don't render or recreate toolSVG if currently in zoom mode
     if (graphInteractionMode !== "select") {
@@ -441,6 +383,88 @@ class Graph extends React.Component {
     return { toolSVG: newToolSVG, tool, container };
   };
 
+  fetchAsyncProps = async (props) => {
+    const {
+      annoMatrix,
+      colors: colorsProp,
+      layoutChoice,
+      crossfilter,
+      pointDilation,
+      viewport,
+    } = props.watchProps;
+    const { modelTF } = this.state;
+
+    const [layoutDf, colorDf, pointDilationDf] = await this.fetchData(
+      annoMatrix,
+      layoutChoice,
+      colorsProp,
+      pointDilation
+    );
+    const { currentDimNames } = layoutChoice;
+    const X = layoutDf.col(currentDimNames[0]).asArray();
+    const Y = layoutDf.col(currentDimNames[1]).asArray();
+    const positions = this.computePointPositions(X, Y, modelTF);
+
+    const colorTable = this.updateColorTable(colorsProp, colorDf);
+    const colors = this.computePointColors(colorTable.rgb);
+
+    const { colorAccessor } = colorsProp;
+    const colorByData = colorDf?.col(colorAccessor)?.asArray();
+    const {
+      metadataField: pointDilationCategory,
+      categoryField: pointDilationLabel,
+    } = pointDilation;
+    const pointDilationData = pointDilationDf
+      ?.col(pointDilationCategory)
+      ?.asArray();
+    const flags = this.computePointFlags(
+      crossfilter,
+      colorByData,
+      pointDilationData,
+      pointDilationLabel
+    );
+
+    const { width, height } = viewport;
+    return {
+      positions,
+      colors,
+      flags,
+      width,
+      height,
+    };
+  };
+
+  async fetchData(annoMatrix, layoutChoice, colors, pointDilation) {
+    /*
+    fetch all data needed.  Includes:
+      - the color by dataframe
+      - the layout dataframe
+      - the point dilation dataframe
+    */
+    const { metadataField: pointDilationAccessor } = pointDilation;
+
+    const promises = [];
+    // layout
+    promises.push(annoMatrix.fetch("emb", layoutChoice.current));
+
+    // color
+    const query = this.createColorByQuery(colors);
+    if (query) {
+      promises.push(annoMatrix.fetch(...query));
+    } else {
+      promises.push(Promise.resolve(null));
+    }
+
+    // point highlighting
+    if (pointDilationAccessor) {
+      promises.push(annoMatrix.fetch("obs", pointDilationAccessor));
+    } else {
+      promises.push(Promise.resolve(null));
+    }
+
+    return Promise.all(promises);
+  }
+
   brushToolUpdate(tool, container) {
     /*
     this is called from componentDidUpdate(), so be very careful using
@@ -572,17 +596,22 @@ class Graph extends React.Component {
     // ignore programatically generated events
     if (d3.event.sourceEvent === null || !d3.event.selection) return;
 
-    const { dispatch } = this.props;
+    const { dispatch, layoutChoice } = this.props;
     const s = d3.event.selection;
-    const brushCoords = {
-      northwest: this.mapScreenToPoint([s[0][0], s[0][1]]),
-      southeast: this.mapScreenToPoint([s[1][0], s[1][1]]),
-    };
-
-    dispatch({
-      type: "graph brush change",
-      brushCoords,
-    });
+    const northwest = this.mapScreenToPoint(s[0]);
+    const southeast = this.mapScreenToPoint(s[1]);
+    const [minX, maxY] = northwest;
+    const [maxX, minY] = southeast;
+    dispatch(
+      actions.graphBrushChangeAction(layoutChoice.current, {
+        minX,
+        minY,
+        maxX,
+        maxY,
+        northwest,
+        southeast,
+      })
+    );
   }
 
   handleBrushStartAction() {
@@ -590,7 +619,7 @@ class Graph extends React.Component {
     if (!d3.event.sourceEvent) return;
 
     const { dispatch } = this.props;
-    dispatch({ type: "graph brush start" });
+    dispatch(actions.graphBrushStartAction());
   }
 
   handleBrushEndAction() {
@@ -601,65 +630,67 @@ class Graph extends React.Component {
     coordinates will be included if selection made, null
     if selection cleared.
     */
-    const { dispatch } = this.props;
+    const { dispatch, layoutChoice } = this.props;
     const s = d3.event.selection;
     if (s) {
-      const brushCoords = {
-        northwest: this.mapScreenToPoint(s[0]),
-        southeast: this.mapScreenToPoint(s[1]),
-      };
-      dispatch({
-        type: "graph brush end",
-        brushCoords,
-      });
+      const northwest = this.mapScreenToPoint(s[0]);
+      const southeast = this.mapScreenToPoint(s[1]);
+      const [minX, maxY] = northwest;
+      const [maxX, minY] = southeast;
+      dispatch(
+        actions.graphBrushEndAction(layoutChoice.current, {
+          minX,
+          minY,
+          maxX,
+          maxY,
+          northwest,
+          southeast,
+        })
+      );
     } else {
-      dispatch({
-        type: "graph brush deselect",
-      });
+      dispatch(actions.graphBrushDeselectAction(layoutChoice.current));
     }
   }
 
   handleBrushDeselectAction() {
-    const { dispatch } = this.props;
-    dispatch({
-      type: "graph brush deselect",
-    });
+    const { dispatch, layoutChoice } = this.props;
+    dispatch(actions.graphBrushDeselectAction(layoutChoice.current));
   }
 
   handleLassoStart() {
-    const { dispatch } = this.props;
-    dispatch({
-      type: "graph lasso start",
-    });
+    const { dispatch, layoutChoice } = this.props;
+    dispatch(actions.graphLassoStartAction(layoutChoice.current));
   }
 
   // when a lasso is completed, filter to the points within the lasso polygon
   handleLassoEnd(polygon) {
     const minimumPolygonArea = 10;
-    const { dispatch } = this.props;
+    const { dispatch, layoutChoice } = this.props;
 
     if (
       polygon.length < 3 ||
       Math.abs(d3.polygonArea(polygon)) < minimumPolygonArea
     ) {
       // if less than three points, or super small area, treat as a clear selection.
-      dispatch({ type: "graph lasso deselect" });
+      dispatch(actions.graphLassoDeselectAction(layoutChoice.current));
     } else {
-      dispatch({
-        type: "graph lasso end",
-        polygon: polygon.map((xy) => this.mapScreenToPoint(xy)), // transform the polygon
-      });
+      dispatch(
+        actions.graphLassoEndAction(
+          layoutChoice.current,
+          polygon.map((xy) => this.mapScreenToPoint(xy))
+        )
+      );
     }
   }
 
   handleLassoCancel() {
-    const { dispatch } = this.props;
-    dispatch({ type: "graph lasso cancel" });
+    const { dispatch, layoutChoice } = this.props;
+    dispatch(actions.graphLassoCancelAction(layoutChoice.current));
   }
 
   handleLassoDeselectAction() {
-    const { dispatch } = this.props;
-    dispatch({ type: "graph lasso deselect" });
+    const { dispatch, layoutChoice } = this.props;
+    dispatch(actions.graphLassoDeselectAction(layoutChoice.current));
   }
 
   handleDeselectAction() {
@@ -676,38 +707,6 @@ class Graph extends React.Component {
     });
   }
 
-  renderPoints(
-    regl,
-    drawPoints,
-    colorBuffer,
-    pointBuffer,
-    flagBuffer,
-    camera,
-    projectionTF
-  ) {
-    const { universe } = this.props;
-    if (!this.reglCanvas || !universe) return;
-    const cameraTF = camera.view();
-    const projView = mat3.multiply(mat3.create(), projectionTF, cameraTF);
-    const { width, height } = this.reglCanvas;
-    regl.poll();
-    regl.clear({
-      depth: 1,
-      color: [1, 1, 1, 1],
-    });
-    drawPoints({
-      distance: camera.distance(),
-      color: colorBuffer,
-      position: pointBuffer,
-      flag: flagBuffer,
-      count: this.count,
-      projView,
-      nPoints: universe.nObs,
-      minViewportDimension: Math.min(width, height),
-    });
-    regl._gl.flush();
-  }
-
   renderCanvas = renderThrottle(() => {
     const {
       regl,
@@ -729,9 +728,92 @@ class Graph extends React.Component {
     );
   });
 
+  updateReglAndRender(asyncProps) {
+    const { positions, colors, flags } = asyncProps;
+    this.cachedAsyncProps = asyncProps;
+    const { pointBuffer, colorBuffer, flagBuffer } = this.state;
+    pointBuffer({ data: positions, dimension: 2 });
+    colorBuffer({ data: colors, dimension: 3 });
+    flagBuffer({ data: flags, dimension: 1 });
+    this.renderCanvas();
+  }
+
+  updateColorTable(colors, colorDf) {
+    const { annoMatrix } = this.props;
+    const { schema } = annoMatrix;
+
+    /* update color table state */
+    if (!colors || !colorDf) {
+      return createColorTable(
+        null, // default mode
+        null,
+        null,
+        schema,
+        null
+      );
+    }
+
+    const { colorAccessor, userColors, colorMode } = colors;
+    return createColorTable(
+      colorMode,
+      colorAccessor,
+      colorDf,
+      schema,
+      userColors
+    );
+  }
+
+  createColorByQuery(colors) {
+    const { annoMatrix } = this.props;
+    const { schema } = annoMatrix;
+    const { colorMode, colorAccessor } = colors;
+    return createColorQuery(colorMode, colorAccessor, schema);
+  }
+
+  renderPoints(
+    regl,
+    drawPoints,
+    colorBuffer,
+    pointBuffer,
+    flagBuffer,
+    camera,
+    projectionTF
+  ) {
+    const { annoMatrix } = this.props;
+    if (!this.reglCanvas || !annoMatrix) return;
+
+    const { schema } = annoMatrix;
+    const cameraTF = camera.view();
+    const projView = mat3.multiply(mat3.create(), projectionTF, cameraTF);
+    const { width, height } = this.reglCanvas;
+    regl.poll();
+    regl.clear({
+      depth: 1,
+      color: [1, 1, 1, 1],
+    });
+    drawPoints({
+      distance: camera.distance(),
+      color: colorBuffer,
+      position: pointBuffer,
+      flag: flagBuffer,
+      count: annoMatrix.nObs,
+      projView,
+      nPoints: schema.dataframe.nObs,
+      minViewportDimension: Math.min(width, height),
+    });
+    regl._gl.flush();
+  }
+
   render() {
-    const { graphInteractionMode } = this.props;
-    const { modelTF, projectionTF, camera, viewport } = this.state;
+    const {
+      graphInteractionMode,
+      annoMatrix,
+      colors,
+      layoutChoice,
+      pointDilation,
+      crossfilter,
+    } = this.props;
+    const { modelTF, projectionTF, camera, viewport, regl } = this.state;
     const cameraTF = camera?.view()?.slice();
 
     return (
@@ -782,18 +864,99 @@ class Graph extends React.Component {
           }}
           className="graph-canvas"
           data-testid="layout-graph"
-          ref={(canvas) => {
-            this.reglCanvas = canvas;
-          }}
+          ref={this.setReglCanvas}
           onMouseDown={this.handleCanvasEvent}
           onMouseUp={this.handleCanvasEvent}
           onMouseMove={this.handleCanvasEvent}
           onDoubleClick={this.handleCanvasEvent}
           onWheel={this.handleCanvasEvent}
         />
+
+        <Async
+          watchFn={Graph.watchAsync}
+          promiseFn={this.fetchAsyncProps}
+          watchProps={{
+            annoMatrix,
+            colors,
+            layoutChoice,
+            pointDilation,
+            crossfilter,
+            viewport,
+          }}
+        >
+          <Async.Pending initial>
+            <StillLoading
+              displayName={layoutChoice.current}
+              width={viewport.width}
+              height={viewport.height}
+            />
+          </Async.Pending>
+          <Async.Rejected>
+            {(error) => (
+              <ErrorLoading
+                displayName={layoutChoice.current}
+                error={error}
+                width={viewport.width}
+                height={viewport.height}
+              />
+            )}
+          </Async.Rejected>
+          <Async.Fulfilled>
+            {(asyncProps) => {
+              if (regl && !shallowEqual(asyncProps, this.cachedAsyncProps)) {
+                this.updateReglAndRender(asyncProps);
+              }
+              return null;
+            }}
+          </Async.Fulfilled>
+        </Async>
       </div>
     );
   }
 }
 
+const ErrorLoading = ({ displayName, error, width, height }) => {
+  console.log(error); // log to console as this is an unepected error
+  return (
+    <div
+      style={{
+        position: "fixed",
+        fontWeight: 500,
+        top: height / 2,
+        left: globals.leftSidebarWidth + width / 2 - 50,
+      }}
+    >
+      <span>{`Failure loading ${displayName}`}</span>
+    </div>
+  );
+};
+
+const StillLoading = ({ displayName, width, height }) => {
+  /*
+  Render a busy/loading indicator
+  */
+  return (
+    <div
+      style={{
+        position: "fixed",
+        fontWeight: 500,
+        top: height / 2,
+        width,
+      }}
+    >
+      <div
+        style={{
+          display: "flex",
+          justifyContent: "center",
+          justifyItems: "center",
+          alignItems: "center",
+        }}
+      >
+        <Button minimal loading intent="primary" />
+        <span style={{ fontStyle: "italic" }}>Loading {displayName}</span>
+      </div>
+    </div>
+  );
+};
+
 export default Graph;

client/src/components/graph/overlays/centroidLabels.js

@@ -1,118 +1,217 @@
 import React, { PureComponent } from "react";
-import { connect } from "react-redux";
+import { connect, shallowEqual } from "react-redux";
+import Async from "react-async";
 
 import { categoryLabelDisplayStringLongLength } from "../../../globals";
+import calcCentroid from "../../../util/centroid";
+import { createColorQuery } from "../../../util/stateManager/colorHelpers";
 
 export default
 @connect((state) => ({
-  colorAccessor: state.colors.colorAccessor,
+  annoMatrix: state.annoMatrix,
+  colors: state.colors,
+  layoutChoice: state.layoutChoice,
   dilatedValue: state.pointDilation.categoryField,
-  labels: state.centroidLabels.labels,
   categoricalSelection: state.categoricalSelection,
+  showLabels: state.centroidLabels?.showLabels,
 }))
 class CentroidLabels extends PureComponent {
-  // Check to see if centroids have either just been displayed or removed from the overlay
+  static watchAsync(props, prevProps) {
+    return !shallowEqual(props.watchProps, prevProps.watchProps);
+  }
 
-  componentDidUpdate(prevProps) {
-    const { labels, overlayToggled } = this.props;
-    const prevSize = prevProps.labels.size;
-    const { size } = labels;
+  fetchAsyncProps = async (props) => {
+    const {
+      annoMatrix,
+      colors,
+      layoutChoice,
+      categoricalSelection,
+      showLabels,
+    } = props.watchProps;
+    const { schema } = annoMatrix;
+    const { colorAccessor } = colors;
 
-    const displayChangeOff = prevSize > 0 && size === undefined;
-    const displayChangeOn = prevSize === undefined && size > 0;
-
-    if (displayChangeOn || displayChangeOff) {
-      // Notify overlay layer of display change
-      overlayToggled("centroidLabels", displayChangeOn);
+    const [layoutDf, colorDf] = await this.fetchData();
+    let labels;
+    if (colorDf) {
+      labels = calcCentroid(
+        schema,
+        colorAccessor,
+        colorDf,
+        layoutChoice,
+        layoutDf
+      );
+    } else {
+      labels = new Map();
     }
+
+    const { overlaySetShowing } = this.props;
+    overlaySetShowing("centroidLabels", showLabels && labels.size > 0);
+
+    return {
+      labels,
+      colorAccessor,
+      category: categoricalSelection[colorAccessor],
+    };
+  };
+
+  handleMouseEnter = (e, colorAccessor, label) => {
+    const { dispatch } = this.props;
+    dispatch({
+      type: "category value mouse hover start",
+      metadataField: colorAccessor,
+      categoryField: label,
+    });
+  };
+
+  handleMouseOut = (e, colorAccessor, label) => {
+    const { dispatch } = this.props;
+    dispatch({
+      type: "category value mouse hover end",
+      metadataField: colorAccessor,
+      categoryField: label,
+    });
+  };
+
+  colorByQuery() {
+    const { annoMatrix, colors } = this.props;
+    const { schema } = annoMatrix;
+    const { colorMode, colorAccessor } = colors;
+    return createColorQuery(colorMode, colorAccessor, schema);
+  }
+
+  async fetchData() {
+    const { annoMatrix, layoutChoice } = this.props;
+    // fetch all data we need: layout, category
+    const promises = [];
+    // layout
+    promises.push(annoMatrix.fetch("emb", layoutChoice.current));
+    // category to label - we ONLY label on obs, never on X, etc.
+    const query = this.colorByQuery();
+    if (query && query[0] === "obs") {
+      promises.push(annoMatrix.fetch(...query));
+    } else {
+      promises.push(Promise.resolve(null));
+    }
+
+    return Promise.all(promises);
   }
 
   render() {
     const {
-      labels,
       inverseTransform,
       dilatedValue,
-      dispatch,
-      colorAccessor,
       categoricalSelection,
+      showLabels,
+      colors,
+      annoMatrix,
+      layoutChoice,
     } = this.props;
 
-    if (!colorAccessor || labels.size === undefined || labels.size === 0)
-      return null;
+    return (
+      <Async
+        watchFn={CentroidLabels.watchAsync}
+        promiseFn={this.fetchAsyncProps}
+        watchProps={{
+          annoMatrix,
+          colors,
+          layoutChoice,
+          categoricalSelection,
+          dilatedValue,
+          showLabels,
+        }}
+      >
+        <Async.Fulfilled>
+          {(asyncProps) => {
+            if (!showLabels) return null;
 
-    const {
-      categoryValueIndices,
-      categoryValueSelected,
-    } = categoricalSelection?.[colorAccessor];
+            const labelSVGS = [];
+            const deselectOpacity = 0.375;
+            const { category, colorAccessor, labels } = asyncProps;
 
-    const labelSVGS = [];
-    let fontSize = "15px";
-    let fontWeight = null;
-    const deselectOpacity = 0.375;
-    labels.forEach((coords, label) => {
-      fontSize = "15px";
-      fontWeight = null;
-      if (label === dilatedValue) {
-        fontSize = "18px";
-        fontWeight = "800";
-      }
+            labels.forEach((coords, label) => {
+              const selected = category.get(label) ?? true;
 
-      const selected = categoryValueSelected[categoryValueIndices.get(label)];
+              // Mirror LSB middle truncation
+              let displayLabel = label;
+              if (displayLabel.length > categoryLabelDisplayStringLongLength) {
+                displayLabel = `${label.slice(
+                  0,
+                  categoryLabelDisplayStringLongLength / 2
+                )}…${label.slice(-categoryLabelDisplayStringLongLength / 2)}`;
+              }
 
-      // Mirror LSB middle truncation
-      let displayLabel = label;
-      if (displayLabel.length > categoryLabelDisplayStringLongLength) {
-        displayLabel = `${label.slice(
-          0,
-          categoryLabelDisplayStringLongLength / 2
-        )}…${label.slice(-categoryLabelDisplayStringLongLength / 2)}`;
-      }
+              labelSVGS.push(
+                // eslint-disable-next-line jsx-a11y/mouse-events-have-key-events -- the mouse actions for centroid labels do not have a screen reader alternative
+                <Label
+                  key={label} // eslint-disable-line react/no-array-index-key --- label is not an index, eslint is confused
+                  label={label}
+                  dilatedValue={dilatedValue}
+                  coords={coords}
+                  inverseTransform={inverseTransform}
+                  opactity={selected ? 1 : deselectOpacity}
+                  colorAccessor={colorAccessor}
+                  displayLabel={displayLabel}
+                  onMouseEnter={this.handleMouseEnter}
+                  onMouseOut={this.handleMouseOut}
+                />
+              );
+            });
 
-      labelSVGS.push(
-        <g
-          // label is unique so disabling eslint rule
-          // eslint-disable-next-line react/no-array-index-key
-          key={label}
-          className="centroid-label"
-          transform={`translate(${coords[0]}, ${coords[1]})`}
-          data-testclass="centroid-label"
-          data-testid={`${label}-centroid-label`}
-        >
-          {/* The mouse actions for centroid labels do not have a screen reader alternative */}
-          {/* eslint-disable-next-line jsx-a11y/mouse-events-have-key-events */}
-          <text
-            transform={inverseTransform}
-            textAnchor="middle"
-            data-label={label}
-            style={{
-              fontSize,
-              fontWeight,
-              fill: "black",
-              userSelect: "none",
-              opacity: selected ? 1 : deselectOpacity,
-            }}
-            onMouseEnter={(e) =>
-              dispatch({
-                type: "category value mouse hover start",
-                metadataField: colorAccessor,
-                categoryField: e.target.getAttribute("data-label"),
-              })
-            }
-            onMouseOut={(e) =>
-              dispatch({
-                type: "category value mouse hover end",
-                metadataField: colorAccessor,
-                categoryField: e.target.getAttribute("data-label"),
-              })
-            }
-            pointerEvents="visiblePainted"
-          >
-            {displayLabel}
-          </text>
-        </g>
-      );
-    });
-
-    return <>{labelSVGS}</>;
+            return <>{labelSVGS}</>;
+          }}
+        </Async.Fulfilled>
+      </Async>
+    );
   }
 }
+
+const Label = ({
+  label,
+  dilatedValue,
+  coords,
+  inverseTransform,
+  opacity,
+  colorAccessor,
+  displayLabel,
+  onMouseEnter,
+  onMouseOut,
+}) => {
+  /*
+  Render a label at a given coordinate.
+  */
+  let fontSize = "15px";
+  let fontWeight = null;
+  if (label === dilatedValue) {
+    fontSize = "18px";
+    fontWeight = "800";
+  }
+
+  return (
+    <g
+      key={label}
+      className="centroid-label"
+      transform={`translate(${coords[0]}, ${coords[1]})`}
+      data-testclass="centroid-label"
+      data-testid={`${label}-centroid-label`}
+    >
+      {/* eslint-disable-next-line jsx-a11y/mouse-events-have-key-events --- the mouse actions for centroid labels do not have a screen reader alternative*/}
+      <text
+        transform={inverseTransform}
+        textAnchor="middle"
+        style={{
+          fontSize,
+          fontWeight,
+          fill: "black",
+          userSelect: "none",
+          opacity: { opacity },
+        }}
+        onMouseEnter={(e) => onMouseEnter(e, colorAccessor, label)}
+        onMouseOut={(e) => onMouseOut(e, colorAccessor, label)}
+        pointerEvents="visiblePainted"
+      >
+        {displayLabel}
+      </text>
+    </g>
+  );
+};

client/src/components/graph/overlays/graphOverlayLayer.js

@@ -33,7 +33,7 @@ export default class GraphOverlayLayer extends PureComponent {
   };
 
   // This is passed to all children, should be called when an overlay's display state is toggled along with the overlay name and its new display state in boolean form
-  overlayToggled = (overlay, displaying) => {
+  overlaySetShowing = (overlay, displaying) => {
     this.setState((state) => {
       return { ...state, display: { ...state.display, [overlay]: displaying } };
     });
@@ -67,7 +67,7 @@ export default class GraphOverlayLayer extends PureComponent {
     const newChildren = React.Children.map(children, (child) =>
       cloneElement(child, {
         inverseTransform,
-        overlayToggled: this.overlayToggled,
+        overlaySetShowing: this.overlaySetShowing,
       })
     );
 

client/src/components/graph/setupLasso.js

@@ -1,24 +1,26 @@
 // https://bl.ocks.org/pbeshai/8008075f9ce771ee8be39e8c38907570
 
 import * as d3 from "d3";
+import { Colors } from "@blueprintjs/core";
 
 const Lasso = () => {
   const dispatch = d3.dispatch("start", "end", "cancel");
 
-  const polygonToPath = (polygon) =>
-    `M${polygon.map((d) => d.join(",")).join("L")}`;
-
-  const distance = (pt1, pt2) =>
-    Math.sqrt((pt2[0] - pt1[0]) ** 2 + (pt2[1] - pt1[1]) ** 2);
-
-  // distance last point has to be to first point before it auto closes when mouse is released
-  const closeDistance = 75;
-
   const lasso = (svg) => {
     let lassoPolygon;
     let lassoPath;
     let closePath;
 
+    const polygonToPath = (polygon) =>
+      `M${polygon.map((d) => d.join(",")).join("L")}`;
+
+    const distance = (pt1, pt2) =>
+      Math.sqrt((pt2[0] - pt1[0]) ** 2 + (pt2[1] - pt1[1]) ** 2);
+
+    // distance last point has to be to first point before it auto closes when mouse is released
+    const closeDistance = 75;
+    const lassoPathColor = Colors.BLUE5;
+
     const handleDragStart = () => {
       lassoPolygon = [d3.mouse(svg.node())]; // current x y of mouse within element
 
@@ -29,18 +31,14 @@ const Lasso = () => {
       lassoPath = g
         .append("path")
         .attr("data-testid", "lasso-element")
-        .attr("fill", "#0bb")
         .attr("fill-opacity", 0.1)
-        .attr("stroke", "#0bb")
         .attr("stroke-dasharray", "3, 3");
 
       closePath = g
         .append("line")
         .attr("x2", lassoPolygon[0][0])
         .attr("y2", lassoPolygon[0][1])
-        .attr("stroke", "#0bb")
-        .attr("stroke-dasharray", "3, 3")
-        .attr("opacity", 0);
+        .attr("stroke-dasharray", "3, 3");
 
       dispatch.call("start", lasso, lassoPolygon);
     };
@@ -55,9 +53,17 @@ const Lasso = () => {
         distance(lassoPolygon[0], lassoPolygon[lassoPolygon.length - 1]) <
         closeDistance
       ) {
-        closePath.attr("x1", point[0]).attr("y1", point[1]).attr("opacity", 1);
+        const closePathColor = Colors.GREEN5;
+        closePath
+          .attr("x1", point[0])
+          .attr("y1", point[1])
+          .attr("opacity", 1)
+          .attr("stroke", closePathColor)
+          .attr("fill", closePathColor);
+        lassoPath.attr("stroke", closePathColor).attr("fill", closePathColor);
       } else {
         closePath.attr("opacity", 0);
+        lassoPath.attr("stroke", lassoPathColor).attr("fill", lassoPathColor);
       }
     };
 
@@ -121,9 +127,9 @@ const Lasso = () => {
         lassoPath = g
           .append("path")
           .attr("data-testid", "lasso-element")
-          .attr("fill", "#0bb")
+          .attr("fill", lassoPathColor)
           .attr("fill-opacity", 0.1)
-          .attr("stroke", "#0bb")
+          .attr("stroke", lassoPathColor)
           .attr("stroke-dasharray", "3, 3");
 
         lassoPath.attr("d", `${polygonToPath(lassoPolygon)}Z`);

client/src/components/graph/setupSVGandBrush.js

@@ -17,6 +17,7 @@ export default (
   viewport
 ) => {
   const svg = d3.select("#graph-wrapper").select("#lasso-layer");
+  if (svg.empty()) return {};
 
   if (selectionToolType === "brush") {
     const brush = d3

client/src/components/menubar/cellSetButtons.js

@@ -2,31 +2,19 @@
 import React from "react";
 import { AnchorButton, Tooltip } from "@blueprintjs/core";
 import { connect } from "react-redux";
-import { World } from "../../util/stateManager";
 import { tooltipHoverOpenDelay } from "../../globals";
+import actions from "../../actions";
 
-@connect()
+@connect((state) => ({
+  differential: state.differential,
+}))
 class CellSetButton extends React.PureComponent {
   set() {
-    const {
-      differential,
-      crossfilter,
-      dispatch,
-      eitherCellSetOneOrTwo,
-    } = this.props;
-
-    // Reducer and components assume that value will be null if
-    // no selection made.  World..getSelectedByIndex() returns a
-    // zero length TypedArray when nothing is selected.
-    let set = World.getSelectedByIndex(crossfilter);
-    if (set.length === 0) set = null;
+    const { differential, dispatch, eitherCellSetOneOrTwo } = this.props;
 
     if (!differential.diffExp) {
-      /* diffexp needs to be cleared before we store a new set */
-      dispatch({
-        type: `store current cell selection as differential set ${eitherCellSetOneOrTwo}`,
-        data: set,
-      });
+      // disallow this action if the user has active differential expression results
+      dispatch(actions.setCellSetFromSelection(eitherCellSetOneOrTwo));
     }
   }
 

client/src/components/menubar/clip.js

@@ -1,4 +1,3 @@
-// jshint esversion: 6
 import React from "react";
 import {
   Position,
@@ -11,7 +10,7 @@ import {
 import { tooltipHoverOpenDelay } from "../../globals";
 import styles from "./menubar.css";
 
-function Clip(props) {
+const Clip = React.memo((props) => {
   const {
     pendingClipPercentiles,
     clipPercentileMin,
@@ -129,6 +128,6 @@ function Clip(props) {
       />
     </div>
   );
-}
+});
 
 export default Clip;

client/src/components/menubar/diffexpButtons.js

@@ -1,4 +1,3 @@
-// jshint esversion: 6
 import React from "react";
 import { connect } from "react-redux";
 import { Button, ButtonGroup, AnchorButton, Tooltip } from "@blueprintjs/core";
@@ -8,15 +7,13 @@ import actions from "../../actions";
 import CellSetButton from "./cellSetButtons";
 
 @connect((state) => ({
-  config: state.config,
-  crossfilter: state.crossfilter,
   differential: state.differential,
   celllist1: state.differential?.celllist1,
   celllist2: state.differential?.celllist2,
   diffexpMayBeSlow: state.config?.parameters?.["diffexp-may-be-slow"] ?? false,
   diffexpCellcountMax: state.config?.limits?.["diffexp_cellcount_max"],
 }))
-class DiffexpButtons extends React.Component {
+class DiffexpButtons extends React.PureComponent {
   computeDiffExp = () => {
     const { dispatch, differential } = this.props;
     if (differential.celllist1 && differential.celllist2) {
@@ -66,14 +63,8 @@ class DiffexpButtons extends React.Component {
 
     return (
       <ButtonGroup className={styles.menubarButton}>
-        <CellSetButton
-          {...this.props} // eslint-disable-line react/jsx-props-no-spreading
-          eitherCellSetOneOrTwo={1}
-        />
-        <CellSetButton
-          {...this.props} // eslint-disable-line react/jsx-props-no-spreading
-          eitherCellSetOneOrTwo={2}
-        />
+        <CellSetButton eitherCellSetOneOrTwo={1} />
+        <CellSetButton eitherCellSetOneOrTwo={2} />
         {!differential.diffExp ? (
           <Tooltip
             content={warnMaxSizeExceeded ? tipMessageWarn : tipMessage}

client/src/components/menubar/embedding.js

@@ -1,6 +1,5 @@
 import React from "react";
 import {
-  AnchorButton,
   ButtonGroup,
   Popover,
   Button,
@@ -12,26 +11,25 @@ import {
 import { connect } from "react-redux";
 import * as globals from "../../globals";
 import styles from "./menubar.css";
-import { World } from "../../util/stateManager";
 import actions from "../../actions";
 
 @connect((state) => ({
-  universe: state.universe,
-  world: state.world,
   layoutChoice: state.layoutChoice,
-  reembedController: state.reembedController,
-  enableReembedding: state.config?.parameters?.["enable-reembedding"] ?? false,
+  // disabled temporarily. TODO - issue #1606
+  // reembedController: state.reembedController,
+  // enableReembedding: state.config?.parameters?.["enable-reembedding"] ?? false,
+  enableReembedding: false,
 }))
 class Embedding extends React.PureComponent {
   handleLayoutChoiceChange = (e) => {
     const { dispatch } = this.props;
-    dispatch({
-      type: "set layout choice",
-      layoutChoice: e.currentTarget.value,
-    });
+    dispatch(actions.layoutChoiceAction(e.currentTarget.value));
   };
 
+  // eslint-disable-next-line class-methods-use-this -- temporary disable
   renderReembedding() {
+    return null;
+    /* disabled pending rewrite. TODO - issue #1606
     const {
       enableReembedding,
       world,
@@ -63,6 +61,7 @@ class Embedding extends React.PureComponent {
         />
       </Tooltip>
     );
+*/
   }
 
   render() {

client/src/components/menubar/index.js

@@ -1,4 +1,3 @@
-// jshint esversion: 6
 import React from "react";
 import { connect } from "react-redux";
 import { ButtonGroup, AnchorButton, Tooltip } from "@blueprintjs/core";
@@ -13,33 +12,44 @@ import Subset from "./subset";
 import UndoRedoReset from "./undoRedo";
 import DiffexpButtons from "./diffexpButtons";
 
-@connect((state) => ({
-  universe: state.universe,
-  world: state.world,
-  crossfilter: state.crossfilter,
-  differential: state.differential,
-  graphInteractionMode: state.controls.graphInteractionMode,
-  clipPercentileMin: Math.round(100 * (state.world?.clipQuantiles?.min ?? 0)),
-  clipPercentileMax: Math.round(100 * (state.world?.clipQuantiles?.max ?? 1)),
-  userDefinedGenes: state.controls.userDefinedGenes,
-  diffexpGenes: state.controls.diffexpGenes,
-  colorAccessor: state.colors.colorAccessor,
-  scatterplotXXaccessor: state.controls.scatterplotXXaccessor,
-  scatterplotYYaccessor: state.controls.scatterplotYYaccessor,
-  celllist1: state.differential.celllist1,
-  celllist2: state.differential.celllist2,
-  libraryVersions: state.config?.["library_versions"],
-  undoDisabled: state["@@undoable/past"].length === 0,
-  redoDisabled: state["@@undoable/future"].length === 0,
-  aboutLink: state.config?.links?.["about-dataset"],
-  disableDiffexp: state.config?.parameters?.["disable-diffexp"] ?? false,
-  diffexpMayBeSlow: state.config?.parameters?.["diffexp-may-be-slow"] ?? false,
-  showCentroidLabels: state.centroidLabels.showLabels,
-  tosURL: state.config?.parameters?.["about_legal_tos"],
-  privacyURL: state.config?.parameters?.["about_legal_privacy"],
-  categoricalSelection: state.categoricalSelection,
-}))
-class MenuBar extends React.Component {
+@connect((state) => {
+  const { annoMatrix } = state;
+  const crossfilter = state.obsCrossfilter;
+  const selectedCount = crossfilter.countSelected();
+
+  const subsetPossible =
+    selectedCount !== 0 && selectedCount !== crossfilter.size(); // ie, not all are selected
+  const subsetResetPossible =
+    annoMatrix.nObs !== annoMatrix.schema.dataframe.nObs;
+
+  return {
+    subsetPossible,
+    subsetResetPossible,
+    differential: state.differential,
+    graphInteractionMode: state.controls.graphInteractionMode,
+    clipPercentileMin: Math.round(100 * (annoMatrix?.clipRange?.[0] ?? 0)),
+    clipPercentileMax: Math.round(100 * (annoMatrix?.clipRange?.[1] ?? 1)),
+    userDefinedGenes: state.controls.userDefinedGenes,
+    diffexpGenes: state.controls.diffexpGenes,
+    colorAccessor: state.colors.colorAccessor,
+    scatterplotXXaccessor: state.controls.scatterplotXXaccessor,
+    scatterplotYYaccessor: state.controls.scatterplotYYaccessor,
+    celllist1: state.differential.celllist1,
+    celllist2: state.differential.celllist2,
+    libraryVersions: state.config?.["library_versions"],
+    undoDisabled: state["@@undoable/past"].length === 0,
+    redoDisabled: state["@@undoable/future"].length === 0,
+    aboutLink: state.config?.links?.["about-dataset"],
+    disableDiffexp: state.config?.parameters?.["disable-diffexp"] ?? false,
+    diffexpMayBeSlow:
+      state.config?.parameters?.["diffexp-may-be-slow"] ?? false,
+    showCentroidLabels: state.centroidLabels.showLabels,
+    tosURL: state.config?.parameters?.["about_legal_tos"],
+    privacyURL: state.config?.parameters?.["about_legal_privacy"],
+    categoricalSelection: state.categoricalSelection,
+  };
+})
+class MenuBar extends React.PureComponent {
   static isValidDigitKeyEvent(e) {
     /*
     Return true if this event is necessary to enter a percent number input.
@@ -78,10 +88,10 @@ class MenuBar extends React.Component {
     const { pendingClipPercentiles } = this.state;
     const clipPercentileMin = pendingClipPercentiles?.clipPercentileMin;
     const clipPercentileMax = pendingClipPercentiles?.clipPercentileMax;
-
-    const { world } = this.props;
-    const currentClipMin = 100 * world?.clipQuantiles?.min;
-    const currentClipMax = 100 * world?.clipQuantiles?.max;
+    const {
+      clipPercentileMin: currentClipMin,
+      clipPercentileMax: currentClipMax,
+    } = this.props;
 
     // if you change this test, be careful with logic around
     // comparisons between undefined / NaN handling.
@@ -150,10 +160,7 @@ class MenuBar extends React.Component {
     const { clipPercentileMin, clipPercentileMax } = pendingClipPercentiles;
     const min = clipPercentileMin / 100;
     const max = clipPercentileMax / 100;
-    dispatch({
-      type: "set clip quantiles",
-      clipQuantiles: { min, max },
-    });
+    dispatch(actions.clipAction(min, max));
   };
 
   handleClipOpening = () => {
@@ -176,17 +183,14 @@ class MenuBar extends React.Component {
     });
   };
 
-  subsetPossible = () => {
-    const { crossfilter } = this.props;
-    return (
-      crossfilter.countSelected() !== 0 &&
-      crossfilter.countSelected() !== crossfilter.size()
-    );
+  handleSubset = () => {
+    const { dispatch } = this.props;
+    dispatch(actions.subsetAction());
   };
 
-  subsetResetPossible = () => {
-    const { world, universe } = this.props;
-    return world.nObs !== universe.nObs;
+  handleSubsetReset = () => {
+    const { dispatch } = this.props;
+    dispatch(actions.resetSubsetAction());
   };
 
   render() {
@@ -206,6 +210,8 @@ class MenuBar extends React.Component {
       tosURL,
       categoricalSelection,
       colorAccessor,
+      subsetPossible,
+      subsetResetPossible,
     } = this.props;
     const { pendingClipPercentiles } = this.state;
 
@@ -314,16 +320,10 @@ class MenuBar extends React.Component {
           </Tooltip>
         </ButtonGroup>
         <Subset
-          subsetPossible={this.subsetPossible()}
-          subsetResetPossible={this.subsetResetPossible()}
-          handleSubset={() => {
-            dispatch(actions.setWorldToSelection());
-            dispatch({ type: "increment graph render counter" });
-          }}
-          handleSubsetReset={() => {
-            dispatch(actions.resetWorldToUniverse());
-            dispatch({ type: "increment graph render counter" });
-          }}
+          subsetPossible={subsetPossible}
+          subsetResetPossible={subsetResetPossible}
+          handleSubset={this.handleSubset}
+          handleSubsetReset={this.handleSubsetReset}
         />
         {disableDiffexp ? null : <DiffexpButtons />}
       </div>

client/src/components/menubar/infoMenu.js

@@ -3,7 +3,7 @@ import React from "react";
 import { Button, Popover, Menu, MenuItem, Position } from "@blueprintjs/core";
 import styles from "./menubar.css";
 
-function InformationMenu(props) {
+const InformationMenu = React.memo((props) => {
   const { libraryVersions, aboutLink, tosURL, privacyURL } = props;
   return (
     <div className={`bp3-button-group ${styles.menubarButton}`}>
@@ -41,11 +41,11 @@ function InformationMenu(props) {
             />
             <MenuItem
               target="_blank"
-              text={`cellxgene v${
+              text={
                 libraryVersions && libraryVersions.cellxgene
                   ? libraryVersions.cellxgene
                   : null
-              }`}
+              }
             />
             <MenuItem text="MIT License" />
             {tosURL ? (
@@ -72,6 +72,6 @@ function InformationMenu(props) {
       </Popover>
     </div>
   );
-}
+});
 
 export default InformationMenu;

client/src/components/menubar/subset.js

@@ -3,7 +3,7 @@ import { AnchorButton, ButtonGroup, Tooltip } from "@blueprintjs/core";
 import styles from "./menubar.css";
 import * as globals from "../../globals";
 
-function Subset(props) {
+const Subset = React.memo((props) => {
   const {
     subsetPossible,
     subsetResetPossible,
@@ -41,6 +41,6 @@ function Subset(props) {
       </Tooltip>
     </ButtonGroup>
   );
-}
+});
 
 export default Subset;

client/src/components/menubar/undoRedo.js

@@ -1,10 +1,9 @@
-// jshint esversion: 6
 import React from "react";
 import { AnchorButton, Tooltip } from "@blueprintjs/core";
 import { tooltipHoverOpenDelay } from "../../globals";
 import styles from "./menubar.css";
 
-function InformationMenu(props) {
+const UndoRedo = React.memo((props) => {
   const { undoDisabled, redoDisabled, dispatch } = props;
   return (
     <div className={`bp3-button-group ${styles.menubarButton}`}>
@@ -46,6 +45,6 @@ function InformationMenu(props) {
       </Tooltip>
     </div>
   );
-}
+});
 
-export default InformationMenu;
+export default UndoRedo;

client/src/components/miniHistogram/index.js

@@ -0,0 +1,100 @@
+import React from "react";
+import {
+  Popover,
+  PopoverInteractionKind,
+  Position,
+  Classes,
+} from "@blueprintjs/core";
+
+export default class MiniHistogram extends React.PureComponent {
+  constructor(props) {
+    super(props);
+    this.canvasRef = React.createRef();
+  }
+
+  drawHistogram = () => {
+    const { xScale, yScale, bins, width, height } = this.props;
+
+    if (!bins) return;
+
+    const ctx = this.canvasRef.current.getContext("2d");
+
+    ctx.clearRect(0, 0, width, height);
+
+    ctx.fillStyle = "#000";
+
+    let x;
+    let y;
+
+    const rectWidth = width / bins.length;
+
+    for (let i = 0, { length } = bins; i < length; i += 1) {
+      x = xScale(i);
+      y = yScale(bins[i]);
+      ctx.fillRect(x, height - y, rectWidth, y);
+    }
+  };
+
+  componentDidMount = () => {
+    this.drawHistogram();
+  };
+
+  componentDidUpdate = (prevProps) => {
+    const { obsOrVarContinuousFieldDisplayName } = this.props;
+    if (
+      prevProps.obsOrVarContinuousFieldDisplayName !==
+      obsOrVarContinuousFieldDisplayName
+    )
+      this.drawHistogram();
+  };
+
+  render() {
+    const {
+      domainLabel,
+      obsOrVarContinuousFieldDisplayName,
+      width,
+      height,
+    } = this.props;
+
+    return (
+      <Popover
+        interactionKind={PopoverInteractionKind.HOVER_TARGET_ONLY}
+        hoverOpenDelay={1500}
+        hoverCloseDelay={200}
+        position={Position.LEFT}
+        modifiers={{
+          preventOverflow: { enabled: false },
+          hide: { enabled: false },
+        }}
+        lazy
+        usePortal
+        popoverClassName={Classes.POPOVER_CONTENT_SIZING}
+      >
+        <canvas
+          className="bp3-popover-targer"
+          style={{
+            marginRight: 5,
+            width,
+            height,
+            borderBottom: "solid rgb(230, 230, 230) 0.25px",
+          }}
+          width={width}
+          height={height}
+          ref={this.canvasRef}
+        />
+        <div key="text" style={{ fontSize: "14px" }}>
+          <p style={{ margin: "0" }}>
+            This histograms shows the distribution of{" "}
+            <strong>{obsOrVarContinuousFieldDisplayName}</strong> within{" "}
+            <strong>{domainLabel}</strong>.
+            <br />
+            <br />
+            The x axis is the same for each histogram, while the y axis is
+            scaled to the largest bin within this histogram instead of the
+            largest bin within the whole category.
+          </p>
+        </div>
+      </Popover>
+    );
+  }
+}

client/src/components/miniStackedBar/index.js

@@ -0,0 +1,74 @@
+// jshint esversion: 6
+import React from "react";
+
+export default class MiniStackedBar extends React.PureComponent {
+  constructor(props) {
+    super(props);
+    this.canvasRef = React.createRef();
+  }
+
+  drawStacks = () => {
+    const {
+      domainValues,
+      scale,
+      domain,
+      colorTable,
+      occupancy,
+      width,
+      height,
+    } = this.props;
+
+    if (!colorTable || !domainValues) return;
+
+    const { scale: colorScale } = colorTable;
+
+    const ctx = this.canvasRef?.current.getContext("2d");
+
+    ctx.clearRect(0, 0, width, height);
+    let currentOffset = 0;
+
+    let occupancyValue;
+    let scaledValue;
+    let value;
+
+    for (let i = 0, { length } = domainValues; i < length; i += 1) {
+      value = domainValues[i];
+      occupancyValue = occupancy.get(value);
+      scaledValue = scale(occupancyValue);
+      ctx.fillStyle = occupancyValue
+        ? colorScale(domain.indexOf(value))
+        : "rgb(255,255,255)";
+      ctx.fillRect(currentOffset, 0, occupancyValue ? scaledValue : 0, height);
+      currentOffset += occupancyValue ? scaledValue : 0;
+    }
+  };
+
+  componentDidUpdate = (prevProps) => {
+    const { occupancy } = this.props;
+    if (occupancy !== prevProps.occupancy) this.drawStacks();
+  };
+
+  componentDidMount = () => {
+    this.drawStacks();
+  };
+
+  render() {
+    const { width, height } = this.props;
+    const { canvas } = this;
+    if (canvas) canvas.getContext("2d").clearRect(0, 0, width, height);
+
+    return (
+      <canvas
+        className="bp3-popover-targer"
+        style={{
+          marginRight: 5,
+          width,
+          height,
+        }}
+        width={width}
+        height={height}
+        ref={this.canvasRef}
+      />
+    );
+  }
+}

client/src/components/scatterplot/drawPointsRegl.js

@@ -1,6 +1,6 @@
 import { glPointFlags, glPointSize } from "../../util/glHelpers";
 
-export default function (regl) {
+export default function drawPointsRegl(regl) {
   return regl({
     vert: `
     precision mediump float;

client/src/components/scatterplot/scatterplot.js

@@ -1,18 +1,25 @@
-import React from "react";
-import { connect } from "react-redux";
+import React, { useEffect, useRef } from "react";
+import { connect, shallowEqual } from "react-redux";
 import { Button, ButtonGroup } from "@blueprintjs/core";
 import _regl from "regl";
 import * as d3 from "d3";
 import { mat3 } from "gl-matrix";
 import memoize from "memoize-one";
-import { isTypedArray } from "../../util/typeHelpers";
+import Async from "react-async";
 
 import * as globals from "../../globals";
-import setupScatterplot from "./setupScatterplot";
 import styles from "./scatterplot.css";
 import _drawPoints from "./drawPointsRegl";
 import { margin, width, height } from "./util";
-import finiteExtent from "../../util/finiteExtent";
+import {
+  createColorTable,
+  createColorQuery,
+} from "../../util/stateManager/colorHelpers";
+import renderThrottle from "../../util/renderThrottle";
+
+const flagSelected = 1;
+const flagNaN = 2;
+const flagHighlight = 4;
 
 function createProjectionTF(viewportWidth, viewportHeight) {
   /*
@@ -22,40 +29,63 @@ function createProjectionTF(viewportWidth, viewportHeight) {
   return mat3.projection(m, viewportWidth, viewportHeight);
 }
 
+function getScale(col, rangeMin, rangeMax) {
+  if (!col) return null;
+  const { min, max } = col.summarize();
+  return d3.scaleLinear().domain([min, max]).range([rangeMin, rangeMax]);
+}
+const getXScale = memoize(getScale);
+const getYScale = memoize(getScale);
+
 @connect((state) => {
-  const { world, crossfilter, universe } = state;
+  const { obsCrossfilter: crossfilter } = state;
   const { scatterplotXXaccessor, scatterplotYYaccessor } = state.controls;
-  const expressionX = scatterplotXXaccessor
-    ? world.varData.col(scatterplotXXaccessor)?.asArray()
-    : null;
-  const expressionY = scatterplotYYaccessor
-    ? world.varData.col(scatterplotYYaccessor)?.asArray()
-    : null;
 
   return {
-    world,
-    universe,
-
-    colorRGB: state.colors.rgb,
-    colorScale: state.colors.scale,
-    colorAccessor: state.colors.colorAccessor,
-
+    annoMatrix: state.annoMatrix,
+    colors: state.colors,
     pointDilation: state.pointDilation,
 
     // Accessors are var/gene names (strings)
     scatterplotXXaccessor,
     scatterplotYYaccessor,
-    opacityForDeselectedCells: state.controls.opacityForDeselectedCells,
 
     differential: state.differential,
-
-    expressionX,
-    expressionY,
-
     crossfilter,
   };
 })
 class Scatterplot extends React.PureComponent {
+  static createReglState(canvas) {
+    /*
+    Must be created for each canvas
+    */
+
+    // regl will create a top-level, full-screen canvas if we pass it a null.
+    // canvas should never be null, so protect against that.
+    if (!canvas) return {};
+
+    // setup canvas, webgl draw function and camera
+    const regl = _regl(canvas);
+    const drawPoints = _drawPoints(regl);
+
+    // preallocate webgl buffers
+    const pointBuffer = regl.buffer();
+    const colorBuffer = regl.buffer();
+    const flagBuffer = regl.buffer();
+
+    return {
+      regl,
+      drawPoints,
+      pointBuffer,
+      colorBuffer,
+      flagBuffer,
+    };
+  }
+
+  static watchAsync(props, prevProps) {
+    return !shallowEqual(props.watchProps, prevProps.watchProps);
+  }
+
   computePointPositions = memoize((X, Y, xScale, yScale) => {
     const positions = new Float32Array(2 * X.length);
     for (let i = 0, len = X.length; i < len; i += 1) {
@@ -77,46 +107,23 @@ class Scatterplot extends React.PureComponent {
   });
 
   computeSelectedFlags = memoize(
-    (crossfilter, flagSelected, flagUnselected) => {
+    (crossfilter, _flagSelected, _flagUnselected) => {
       const x = crossfilter.fillByIsSelected(
         new Float32Array(crossfilter.size()),
-        flagSelected,
-        flagUnselected
+        _flagSelected,
+        _flagUnselected
       );
       return x;
     }
   );
 
-  computePointFlags = memoize(
-    (world, crossfilter, colorAccessor, pointDilation) => {
-      const flagSelected = 1;
-      const flagNaN = 2;
-      const flagHighlight = 4;
-
-      const flags = this.computeSelectedFlags(
-        crossfilter,
-        flagSelected,
-        0
-      ).slice();
-
-      const { metadataField, categoryField } = pointDilation;
-      const highlightData = metadataField
-        ? world.obsAnnotations.col(metadataField)?.asArray()
-        : null;
-      const colorByColumn = colorAccessor
-        ? world.obsAnnotations.col(colorAccessor)?.asArray() ||
-          world.varData.col(colorAccessor)?.asArray()
-        : null;
-      const colorByData =
-        colorByColumn && isTypedArray(colorByColumn) ? colorByColumn : null;
-
-      if (colorByData || highlightData) {
+  computeHighlightFlags = memoize(
+    (nObs, pointDilationData, pointDilationLabel) => {
+      const flags = new Float32Array(nObs);
+      if (pointDilationData) {
         for (let i = 0, len = flags.length; i < len; i += 1) {
-          if (highlightData) {
-            flags[i] += highlightData[i] === categoryField ? flagHighlight : 0;
-          }
-          if (colorByData) {
-            flags[i] += Number.isFinite(colorByData[i]) ? 0 : flagNaN;
+          if (pointDilationData[i] === pointDilationLabel) {
+            flags[i] = flagHighlight;
           }
         }
       }
@@ -124,189 +131,97 @@ class Scatterplot extends React.PureComponent {
     }
   );
 
+  computeColorByFlags = memoize((nObs, colorByData) => {
+    const flags = new Float32Array(nObs);
+    if (colorByData) {
+      for (let i = 0, len = flags.length; i < len; i += 1) {
+        if (!Number.isFinite(colorByData[i])) {
+          flags[i] = flagNaN;
+        }
+      }
+    }
+    return flags;
+  });
+
+  computePointFlags = memoize(
+    (crossfilter, colorByData, pointDilationData, pointDilationLabel) => {
+      /*
+      We communicate with the shader using three flags:
+      - isNaN -- the value is a NaN. Only makes sense when we have a colorAccessor
+      - isSelected -- the value is selected
+      - isHightlighted -- the value is highlighted in the UI (orthogonal from selection highlighting)
+
+      Due to constraints in webgl vertex shader attributes, these are encoded in a float, "kinda"
+      like bitmasks.
+
+      We also have separate code paths for generating flags for categorical and
+      continuous metadata, as they rely on different tests, and some of the flags
+      (eg, isNaN) are meaningless in the face of categorical metadata.
+      */
+      const nObs = crossfilter.size();
+
+      const selectedFlags = this.computeSelectedFlags(
+        crossfilter,
+        flagSelected,
+        0
+      );
+      const highlightFlags = this.computeHighlightFlags(
+        nObs,
+        pointDilationData,
+        pointDilationLabel
+      );
+      const colorByFlags = this.computeColorByFlags(nObs, colorByData);
+
+      const flags = new Float32Array(nObs);
+      for (let i = 0; i < nObs; i += 1) {
+        flags[i] = selectedFlags[i] + highlightFlags[i] + colorByFlags[i];
+      }
+
+      return flags;
+    }
+  );
+
   constructor(props) {
     super(props);
-    this.count = 0;
+    const viewport = this.getViewportDimensions();
     this.axes = false;
-    this.renderCache = {
-      positions: null,
-      colors: null,
-      flags: null,
-      xScale: null,
-      yScale: null,
-    };
+    this.reglCanvas = null;
+    this.renderCache = null;
     this.state = {
-      svg: null,
+      regl: null,
+      drawPoints: null,
       minimized: null,
-      viewport: {
-        height: null,
-        width: null,
-      },
+      viewport,
+      projectionTF: createProjectionTF(width, height),
     };
   }
 
   componentDidMount() {
-    const { svg } = setupScatterplot(width, height, margin);
-    let scales;
-    const { expressionX, expressionY } = this.props;
-
-    if (svg && expressionX && expressionY) {
-      scales = Scatterplot.setupScales(expressionX, expressionY);
-      this.drawAxesSVG(scales.xScale, scales.yScale, svg);
-      this.renderCache = { ...this.renderCache, ...scales };
-    }
-
-    const regl = _regl(this.reglCanvas);
-    const drawPoints = _drawPoints(regl);
-
-    // Create render transform
-    const projectionTF = createProjectionTF(
-      this.reglCanvas.width,
-      this.reglCanvas.height
-    );
-
-    // preallocate buffers
-    const pointBuffer = regl.buffer();
-    const colorBuffer = regl.buffer();
-    const flagBuffer = regl.buffer();
-
-    this.renderPoints(
-      regl,
-      drawPoints,
-      flagBuffer,
-      colorBuffer,
-      pointBuffer,
-      projectionTF
-    );
-
+    // this affect point render size for the scatterplot
     window.addEventListener("resize", this.handleResize);
-    const viewport = this.getViewportDimensions();
-
-    this.setState({
-      regl,
-      flagBuffer,
-      pointBuffer,
-      colorBuffer,
-      svg,
-      drawPoints,
-      projectionTF,
-      viewport,
-    });
-  }
-
-  componentDidUpdate(prevProps) {
-    const {
-      world,
-      crossfilter,
-      scatterplotXXaccessor,
-      scatterplotYYaccessor,
-      expressionX,
-      expressionY,
-      colorRGB,
-      colorAccessor,
-      pointDilation,
-    } = this.props;
-    const {
-      regl,
-      pointBuffer,
-      colorBuffer,
-      flagBuffer,
-      svg,
-      drawPoints,
-      projectionTF,
-    } = this.state;
-
-    if (
-      scatterplotXXaccessor !== prevProps.scatterplotXXaccessor ||
-      scatterplotYYaccessor !== prevProps.scatterplotYYaccessor ||
-      world !== prevProps.world // shape or clip of world changed
-    ) {
-      const scales = Scatterplot.setupScales(expressionX, expressionY);
-      this.drawAxesSVG(scales.xScale, scales.yScale, svg);
-      this.renderCache = { ...this.renderCache, ...scales };
-    }
-
-    if (world && regl) {
-      const { renderCache } = this;
-      const { xScale, yScale } = this.renderCache;
-      let needsRepaint = false;
-
-      const newPositions = this.computePointPositions(
-        expressionX,
-        expressionY,
-        xScale,
-        yScale
-      );
-      if (renderCache.positions !== newPositions) {
-        renderCache.positions = newPositions;
-        pointBuffer({ data: renderCache.positions, dimension: 2 });
-        needsRepaint = true;
-      }
-
-      /* colors for each point */
-      const newColors = this.computePointColors(colorRGB);
-      if (renderCache.colors !== newColors) {
-        renderCache.colors = newColors;
-        colorBuffer({ data: renderCache.colors, dimension: 3 });
-        needsRepaint = true;
-      }
-
-      const newFlags = this.computePointFlags(
-        world,
-        crossfilter,
-        colorAccessor,
-        pointDilation
-      );
-      if (renderCache.flags !== newFlags) {
-        renderCache.flags = newFlags;
-        flagBuffer({ data: renderCache.flags, dimension: 1 });
-        needsRepaint = true;
-      }
-
-      this.count = expressionX.length;
-
-      if (needsRepaint) {
-        this.renderPoints(
-          regl,
-          drawPoints,
-          flagBuffer,
-          colorBuffer,
-          pointBuffer,
-          projectionTF
-        );
-      }
-    }
   }
 
   componentWillUnmount() {
-    window.removeEventListener("resize", this.updateViewportDimensions);
+    window.removeEventListener("resize", this.handleResize);
   }
 
+  setReglCanvas = (canvas) => {
+    this.reglCanvas = canvas;
+    if (canvas) {
+      // no need to update this state if we are detaching.
+      this.setState({
+        ...Scatterplot.createReglState(canvas),
+      });
+    }
+  };
+
   getViewportDimensions = () => {
     return {
-      viewport: {
-        height: window.height,
-        width: window.width,
-      },
+      height: window.innerHeight,
+      width: window.innerWidth,
     };
   };
 
-  static setupScales(expressionX, expressionY) {
-    const xScale = d3
-      .scaleLinear()
-      .domain(finiteExtent(expressionX))
-      .range([0, width]);
-    const yScale = d3
-      .scaleLinear()
-      .domain(finiteExtent(expressionY))
-      .range([height, 0]);
-
-    return {
-      xScale,
-      yScale,
-    };
-  }
-
   handleResize = () => {
     const { state } = this.state;
     const viewport = this.getViewportDimensions();
@@ -316,52 +231,167 @@ class Scatterplot extends React.PureComponent {
     });
   };
 
-  updateViewportDimensions = () => {
-    this.setState(this.getViewportDimensions());
+  fetchAsyncProps = async (props) => {
+    const {
+      scatterplotXXaccessor,
+      scatterplotYYaccessor,
+      colors: colorsProp,
+      crossfilter,
+      pointDilation,
+    } = props.watchProps;
+
+    const [
+      expressionXDf,
+      expressionYDf,
+      colorDf,
+      pointDilationDf,
+    ] = await this.fetchData(
+      scatterplotXXaccessor,
+      scatterplotYYaccessor,
+      colorsProp,
+      pointDilation
+    );
+    const colorTable = this.updateColorTable(colorsProp, colorDf);
+
+    const xCol = expressionXDf.icol(0);
+    const yCol = expressionYDf.icol(0);
+    const xScale = getXScale(xCol, 0, width);
+    const yScale = getYScale(yCol, height, 0);
+    const positions = this.computePointPositions(
+      xCol.asArray(),
+      yCol.asArray(),
+      xScale,
+      yScale
+    );
+
+    const colors = this.computePointColors(colorTable.rgb);
+
+    const { colorAccessor } = colorsProp;
+    const colorByData = colorDf?.col(colorAccessor)?.asArray();
+    const {
+      metadataField: pointDilationCategory,
+      categoryField: pointDilationLabel,
+    } = pointDilation;
+    const pointDilationData = pointDilationDf
+      ?.col(pointDilationCategory)
+      ?.asArray();
+    const flags = this.computePointFlags(
+      crossfilter,
+      colorByData,
+      pointDilationData,
+      pointDilationLabel
+    );
+
+    return {
+      positions,
+      colors,
+      flags,
+      width,
+      height,
+      xScale,
+      yScale,
+    };
   };
 
-  drawAxesSVG(xScale, yScale, svg) {
-    const { scatterplotYYaccessor, scatterplotXXaccessor } = this.props;
-    svg.selectAll("*").remove();
+  createXQuery(geneName) {
+    const { annoMatrix } = this.props;
+    const { schema } = annoMatrix;
+    const varIndex = schema?.annotations?.var?.index;
+    if (!varIndex) return null;
+    return [
+      "X",
+      {
+        field: "var",
+        column: varIndex,
+        value: geneName,
+      },
+    ];
+  }
 
-    // the axes are much cleaner and easier now. No need to rotate and orient
-    // the axis, just call axisBottom, axisLeft etc.
-    const xAxis = d3.axisBottom().ticks(7).scale(xScale);
+  createColorByQuery(colors) {
+    const { annoMatrix } = this.props;
+    const { schema } = annoMatrix;
+    const { colorMode, colorAccessor } = colors;
+    return createColorQuery(colorMode, colorAccessor, schema);
+  }
 
-    const yAxis = d3.axisLeft().ticks(7).scale(yScale);
+  updateColorTable(colors, colorDf) {
+    /* update color table state */
+    const { annoMatrix } = this.props;
+    const { schema } = annoMatrix;
+    const { colorAccessor, userColors, colorMode } = colors;
+    return createColorTable(
+      colorMode,
+      colorAccessor,
+      colorDf,
+      schema,
+      userColors
+    );
+  }
 
-    // adding axes is also simpler now, just translate x-axis to (0,height)
-    // and it's alread defined to be a bottom axis.
-    svg
-      .append("g")
-      .attr("transform", `translate(0,${height})`)
-      .attr("class", "x axis")
-      .call(xAxis);
+  async fetchData(
+    scatterplotXXaccessor,
+    scatterplotYYaccessor,
+    colors,
+    pointDilation
+  ) {
+    const { annoMatrix } = this.props;
+    const { metadataField: pointDilationAccessor } = pointDilation;
 
-    // y-axis is translated to (0,0)
-    svg
-      .append("g")
-      .attr("transform", "translate(0,0)")
-      .attr("class", "y axis")
-      .call(yAxis);
+    const promises = [];
+    // X and Y dimensions
+    promises.push(
+      annoMatrix.fetch(...this.createXQuery(scatterplotXXaccessor))
+    );
+    promises.push(
+      annoMatrix.fetch(...this.createXQuery(scatterplotYYaccessor))
+    );
 
-    // adding label. For x-axis, it's at (10, 10), and for y-axis at (width, height-10).
-    svg
-      .append("text")
-      .attr("x", 10)
-      .attr("y", 10)
-      .attr("class", "label")
-      .style("font-style", "italic")
-      .text(scatterplotYYaccessor);
+    // color
+    const query = this.createColorByQuery(colors);
+    if (query) {
+      promises.push(annoMatrix.fetch(...query));
+    } else {
+      promises.push(Promise.resolve(null));
+    }
 
-    svg
-      .append("text")
-      .attr("x", width)
-      .attr("y", height - 10)
-      .attr("text-anchor", "end")
-      .attr("class", "label")
-      .style("font-style", "italic")
-      .text(scatterplotXXaccessor);
+    // point highlighting
+    if (pointDilationAccessor) {
+      promises.push(annoMatrix.fetch("obs", pointDilationAccessor));
+    } else {
+      promises.push(Promise.resolve(null));
+    }
+
+    return Promise.all(promises);
+  }
+
+  renderCanvas = renderThrottle(() => {
+    const {
+      regl,
+      drawPoints,
+      colorBuffer,
+      pointBuffer,
+      flagBuffer,
+      projectionTF,
+    } = this.state;
+    this.renderPoints(
+      regl,
+      drawPoints,
+      flagBuffer,
+      colorBuffer,
+      pointBuffer,
+      projectionTF
+    );
+  });
+
+  updateReglAndRender(newRenderCache) {
+    const { positions, colors, flags } = newRenderCache;
+    this.renderCache = newRenderCache;
+    const { pointBuffer, colorBuffer, flagBuffer } = this.state;
+    pointBuffer({ data: positions, dimension: 2 });
+    colorBuffer({ data: colors, dimension: 3 });
+    flagBuffer({ data: flags, dimension: 1 });
+    this.renderCanvas();
   }
 
   renderPoints(
@@ -372,8 +402,10 @@ class Scatterplot extends React.PureComponent {
     pointBuffer,
     projectionTF
   ) {
-    if (!this.reglCanvas) return;
-    const { universe } = this.props;
+    const { annoMatrix } = this.props;
+    if (!this.reglCanvas || !annoMatrix) return;
+
+    const { schema } = annoMatrix;
     const { viewport } = this.state;
     regl.poll();
     regl.clear({
@@ -385,8 +417,8 @@ class Scatterplot extends React.PureComponent {
       color: colorBuffer,
       position: pointBuffer,
       projection: projectionTF,
-      count: this.count,
-      nPoints: universe.nObs,
+      count: annoMatrix.nObs,
+      nPoints: schema.dataframe.nObs,
       minViewportDimension: Math.min(
         viewport.width - globals.leftSidebarWidth || width,
         viewport.height || height
@@ -396,8 +428,16 @@ class Scatterplot extends React.PureComponent {
   }
 
   render() {
-    const { dispatch } = this.props;
-    const { minimized } = this.state;
+    const {
+      dispatch,
+      annoMatrix,
+      scatterplotXXaccessor,
+      scatterplotYYaccessor,
+      colors,
+      crossfilter,
+      pointDilation,
+    } = this.props;
+    const { minimized, regl, viewport } = this.state;
 
     return (
       <div
@@ -459,10 +499,42 @@ class Scatterplot extends React.PureComponent {
               marginLeft: margin.left,
               marginTop: margin.top,
             }}
-            ref={(canvas) => {
-              this.reglCanvas = canvas;
-            }}
+            ref={this.setReglCanvas}
           />
+          <Async
+            watchFn={Scatterplot.watchAsync}
+            promiseFn={this.fetchAsyncProps}
+            watchProps={{
+              annoMatrix,
+              scatterplotXXaccessor,
+              scatterplotYYaccessor,
+              colors,
+              crossfilter,
+              pointDilation,
+              viewport,
+            }}
+          >
+            <Async.Pending initial>Loading...</Async.Pending>
+            <Async.Rejected>{(error) => error.message}</Async.Rejected>
+            <Async.Fulfilled>
+              {(asyncProps) => {
+                if (regl && !shallowEqual(asyncProps, this.renderCache)) {
+                  this.updateReglAndRender(asyncProps);
+                }
+                return (
+                  <ScatterplotAxis
+                    width={width}
+                    height={height}
+                    margin={margin}
+                    scatterplotYYaccessor={scatterplotXXaccessor}
+                    scatterplotXXaccessor={scatterplotYYaccessor}
+                    xScale={asyncProps.xScale}
+                    yScale={asyncProps.yScale}
+                  />
+                );
+              }}
+            </Async.Fulfilled>
+          </Async>
         </div>
       </div>
     );
@@ -470,3 +542,75 @@ class Scatterplot extends React.PureComponent {
 }
 
 export default Scatterplot;
+
+const ScatterplotAxis = React.memo(
+  ({ scatterplotYYaccessor, scatterplotXXaccessor, xScale, yScale }) => {
+    /*
+    Axis for the scatterplot, rendered with SVG/D3.  Props:
+      * scatterplotXXaccessor - name of X axis
+      * scatterplotXXaccessor - name of Y axis
+      * xScale - D3 scale for X axis (domain to range)
+      * yScale - D3 scale for Y axis (domain to range)
+
+    This also relies on the GLOBAL width/height/margin constants.  If those become
+    become variables, may need to add the params.
+    */
+
+    const svgRef = useRef(null);
+
+    useEffect(() => {
+      if (!svgRef.current) return;
+      const svg = d3.select(svgRef.current);
+
+      svg.selectAll("*").remove();
+
+      // the axes are much cleaner and easier now. No need to rotate and orient
+      // the axis, just call axisBottom, axisLeft etc.
+      const xAxis = d3.axisBottom().ticks(7).scale(xScale);
+      const yAxis = d3.axisLeft().ticks(7).scale(yScale);
+
+      // adding axes is also simpler now, just translate x-axis to (0,height)
+      // and it's alread defined to be a bottom axis.
+      svg
+        .append("g")
+        .attr("transform", `translate(0,${height})`)
+        .attr("class", "x axis")
+        .call(xAxis);
+
+      // y-axis is translated to (0,0)
+      svg
+        .append("g")
+        .attr("transform", "translate(0,0)")
+        .attr("class", "y axis")
+        .call(yAxis);
+
+      // adding label. For x-axis, it's at (10, 10), and for y-axis at (width, height-10).
+      svg
+        .append("text")
+        .attr("x", 10)
+        .attr("y", 10)
+        .attr("class", "label")
+        .style("font-style", "italic")
+        .text(scatterplotYYaccessor);
+
+      svg
+        .append("text")
+        .attr("x", width)
+        .attr("y", height - 10)
+        .attr("text-anchor", "end")
+        .attr("class", "label")
+        .style("font-style", "italic")
+        .text(scatterplotXXaccessor);
+    }, [scatterplotXXaccessor, scatterplotYYaccessor, xScale, yScale]);
+
+    return (
+      <svg
+        width={width + margin.left + margin.right}
+        height={height + margin.top + margin.bottom}
+        data-testid="scatterplot-svg"
+      >
+        <g ref={svgRef} transform={`translate(${margin.left},${margin.top})`} />
+      </svg>
+    );
+  }
+);

client/src/components/scatterplot/setupScatterplot.js

@@ -1,26 +0,0 @@
-// jshint esversion: 6
-/*****************************************
-******************************************
-      Setup SVG & Canvas elements
-******************************************
-******************************************/
-
-import * as d3 from "d3";
-
-const setupScatterplot = (width, height, margin) => {
-  const container = d3.select("#scatterplot");
-
-  const svg = container
-    .append("svg")
-    .attr("width", width + margin.left + margin.right)
-    .attr("height", height + margin.top + margin.bottom)
-    .attr("data-testid", "scatterplot-svg")
-    .append("g")
-    .attr("transform", `translate(${margin.left},${margin.top})`);
-
-  return {
-    svg,
-  };
-};
-
-export default setupScatterplot;

client/src/components/util/truncate.js

@@ -16,17 +16,19 @@ const FIRST_HALF_STYLE = {
   flexShrink: 1,
   minWidth: "5px",
 };
+const SECOND_HALF_STYLE = {
+  position: "relative",
+  overflow: "hidden",
+  whiteSpace: "nowrap",
+};
+const SECOND_HALF_SPACING_STYLE = {
+  color: "transparent",
+};
 
 const SECOND_HALF_INNER_STYLE = {
   position: "absolute",
   right: 0,
 };
-const SECOND_HALF_STYLE = {
-  color: "transparent",
-  position: "relative",
-  overflow: "hidden",
-  whiteSpace: "nowrap",
-};
 
 export default (props) => {
   const { children } = props;
@@ -40,23 +42,36 @@ export default (props) => {
   }
   const originalString = children.props.children;
 
-  const firstString = originalString.substr(0, originalString.length / 2);
-  const secondString = originalString.substr(originalString.length / 2);
+  let firstString;
+  let secondString;
+
+  if (originalString.length === 1) {
+    firstString = originalString;
+  } else {
+    firstString = originalString.substr(0, originalString.length / 2);
+    secondString = originalString.substr(originalString.length / 2);
+    if (firstString.charAt(firstString.length - 1) === " ") {
+      firstString = `${firstString.substr(0, firstString.length - 1)}\u00a0`;
+    }
+    if (secondString.charAt(0) === " ") {
+      secondString = `\u00a0${secondString.substr(1)}`;
+    }
+  }
 
   const inheritedColor = children.props.style.color;
 
   const splitStyle = { ...children.props.style, ...SPLIT_STYLE };
-  const secondHalfInnerStyle = {
+  const secondHalfContentStyle = {
     ...SECOND_HALF_INNER_STYLE,
-    color: inheritedColor || "initial",
+    color: inheritedColor || "inherit",
   };
 
   const truncatedJSX = (
     <span style={splitStyle}>
       <span style={FIRST_HALF_STYLE}>{firstString}</span>
       <span style={SECOND_HALF_STYLE}>
-        {secondString}
-        <span style={secondHalfInnerStyle}>{secondString}</span>
+        <span style={SECOND_HALF_SPACING_STYLE}>{secondString}</span>
+        <span style={secondHalfContentStyle}>{secondString}</span>
       </span>
     </span>
   );