Experimental - manual annotations (#837)

* icons, partway

* redux for values

* onChange

* cancel

* annotations lifecycle for category names

* copy categorical

* edit category

* add Dataframe.withColsFrom

* render user annotations; default add/delete annotation category

* add label name to actions

* category name edit

* error checking improvements

* change schema field isUserAnnotation to writable

* always have an unassigned label; implement delete label

* implement add new label and edit label name

* label current cell selection

* fix select exact bug in crossfilter

* clean up categorical reducer

* fix tests

* remove debugging printf

* implement subset/reset for user annotations

* undo redo support for user annotations

* remove duplicate button from categories

* add modal

* remove obsolete duplicate annotation reducers

* remove old debugging printf

* connect modal to annotation create and dup

* initial full-stack wiring

* finish up end-to-end wiring

* fix existing unit tests

* fix pytests to match new schema API

* remove debugging printfs

* add label file rotation

* remove obsolete comment

* add fbs encode/decode tests

* add tests for writable annotations

* simplify code

* fix hashing bug with FBS encoding

* lint

* fix smoke tests

* improve error checking in Dataframe.withColsFrom

* add unit test for Dataframe.withColsFrom

* add unit test for Dataframe.columns and Dataframe.renameCol

* fix bug in FBS encode, add better error checks, refactor

* add FBS encode/decode test

* add clarifying comment

* clean up action type names; fix state inconsistency in crossfilter update

* change autosave timer to 2.5sec

* sort categorical metadata render order so it remains consistent

* add temporary autogenerated label for add-new-label operation

* fix hover-over label menu interference with cell highlighting

* remove debugging code

* add missing reducer cases & fix typo

* make dataframe memoize more general purpose

* add dev mode for annos

* fix error on select duplicate

* handle zero occupancy categories

* correctly maintain unclipped AND clipped world

* correctly handle zero length FBS matrix and label files

* ensure all writable categorical schema contains an unassigned category

* handle case where building occupancy stack for category with no members

* dialog for creating label, disable button if duplicate or empty

* visually separate writeable

* edit category

* fix edit category name

* remove debugging code

* fix edit annotation label

* visually define unassigned, change options

* Pull in requirements.txt from `master`

* label currently selected cells

* duplicate label

* lint

* fix pytest merge issues

* rename --label-file to --experimental-label-file

* remove debugging console log

* spelling error fix; fix bug found in PR review.

* lint

client/src/actions/index.js

@@ -1,7 +1,7 @@
 // jshint esversion: 6
 import _ from "lodash";
 import * as globals from "../globals";
-import { Universe } from "../util/stateManager";
+import { Universe, MatrixFBS } from "../util/stateManager";
 import {
   catchErrorsWrap,
   doJsonRequest,
@@ -340,11 +340,57 @@ const resetInterface = () => (dispatch, getState) => {
   });
 };
 
+const saveObsAnnotations = () => async (dispatch, getState) => {
+  const { universe } = getState();
+  const { obsAnnotations, schema } = universe;
+
+  dispatch({
+    type: "writable obs annotations - save started"
+  });
+
+  const writableAnnotations = schema.annotations.obs.columns
+    .filter(s => s.writable)
+    .map(s => s.name);
+  const df = obsAnnotations.subset(null, writableAnnotations);
+  const matrix = MatrixFBS.encodeMatrixFBS(df);
+  try {
+    const res = await fetch(
+      `${globals.API.prefix}${globals.API.version}annotations/obs`,
+      {
+        method: "PUT",
+        body: matrix,
+        headers: new Headers({
+          "Content-Type": "application/octet-stream"
+        })
+      }
+    );
+    if (res.ok) {
+      dispatch({
+        type: "writable obs annotations - save complete",
+        obsAnnotations
+      });
+    } else {
+      dispatch({
+        type: "writable obs annotations - save error",
+        message: `HTTP error ${res.status} - ${res.statusText}`,
+        res
+      });
+    }
+  } catch (error) {
+    dispatch({
+      type: "writable obs annotations - save error",
+      message: error.toString(),
+      error
+    });
+  }
+};
+
 export default {
   regraph,
   resetInterface,
   requestSingleGeneExpressionCountsForColoringPOST,
   requestDifferentialExpression,
   requestUserDefinedGene,
-  doInitialDataLoad
+  doInitialDataLoad,
+  saveObsAnnotations
 };

client/src/components/app.js

@@ -8,6 +8,7 @@ import LeftSideBar from "./leftSidebar";
 import Legend from "./continuousLegend";
 import Graph from "./graph/graph";
 import MenuBar from "./menubar";
+import Autosave from "./autosave";
 
 import actions from "../actions";
 
@@ -89,6 +90,7 @@ class App extends React.Component {
           {loading ? null : <LeftSideBar />}
           {loading ? null : <MenuBar />}
           {loading ? null : <Graph key={graphRenderCounter} />}
+          {loading ? null : <Autosave />}
           <Legend />
         </div>
       </Container>

client/src/components/autosave/index.js

@@ -0,0 +1,81 @@
+import React from "react";
+import { connect } from "react-redux";
+
+import actions from "../../actions";
+
+@connect(state => ({
+	universe: state.universe,
+	obsAnnotations: state.universe.obsAnnotations,
+	saveInProgress: state.autosave?.saveInProgress ?? false,
+	lastSavedObsAnnotations: state.autosave?.lastSavedObsAnnotations,
+	error: state.autosave?.error,
+	writableCategoriesEnabled: state.config?.parameters?.["label_file"] ?? false
+}))
+class Autosave extends React.Component {
+	constructor(props) {
+		super(props);
+		this.state = {
+			timer: null
+		};
+	}
+
+	componentDidMount() {
+		const { writableCategoriesEnabled } = this.props;
+
+		let { timer } = this.state;
+		if (timer) clearInterval(timer);
+		if (writableCategoriesEnabled) {
+			timer = setInterval(this.tick, 2500);
+		} else {
+			timer = null;
+		}
+		this.setState({ timer });
+	}
+
+	componentWillUnmount() {
+		const { timer } = this.state;
+		if (timer) this.clearInterval(timer);
+	}
+
+	tick = () => {
+		const { dispatch, saveInProgress } = this.props;
+		if (this.needToSave() && !saveInProgress) {
+			dispatch(actions.saveObsAnnotations());
+		}
+	};
+
+	needToSave = () => {
+		/* return true if we need to save, false if we don't */
+		const { obsAnnotations, lastSavedObsAnnotations } = this.props;
+		return (
+			lastSavedObsAnnotations && obsAnnotations !== lastSavedObsAnnotations
+		);
+	};
+
+	statusMessage() {
+		const { error } = this.props;
+		if (error) {
+			return `Autosave error: ${error}`;
+		}
+		return this.needToSave() ? "Unsaved" : "All saved";
+	}
+
+	render() {
+		const { writableCategoriesEnabled } = this.props;
+		return writableCategoriesEnabled ? (
+			<div
+				id="autosave"
+				style={{
+					position: "fixed",
+					display: "inherit",
+					right: 0,
+					bottom: 0
+				}}
+			>
+				{this.statusMessage()}
+			</div>
+		) : null;
+	}
+}
+
+export default Autosave;

client/src/components/categorical/categorical.js

@@ -1,27 +1,161 @@
 // jshint esversion: 6
 import React from "react";
 import _ from "lodash";
+import {
+  Button,
+  Tooltip,
+  InputGroup,
+  Dialog,
+  Classes,
+  MenuItem
+} from "@blueprintjs/core";
+import { Select } from "@blueprintjs/select";
 import { connect } from "react-redux";
 import * as globals from "../../globals";
 import Category from "./category";
 
 @connect(state => ({
-  categoricalSelection: state.categoricalSelection
+  categoricalSelection: state.categoricalSelection,
+  writableCategoriesEnabled: state.config?.parameters?.["label_file"] ?? false,
+  schema: state.world?.schema
 }))
 class Categories extends React.Component {
+  constructor(props) {
+    super(props);
+    this.state = {
+      createAnnoModeActive: false,
+      newCategoryText: "",
+      categoryToDuplicate: null
+    };
+  }
+
+  handleCreateUserAnno = () => {
+    const { dispatch } = this.props;
+    const { newCategoryText, categoryToDuplicate } = this.state;
+    dispatch({
+      type: "annotation: create category",
+      data: newCategoryText,
+      categoryToDuplicate
+    });
+    this.setState({
+      createAnnoModeActive: false,
+      categoryToDuplicate: null,
+      newCategoryText: ""
+    });
+  };
+
+  handleEnableAnnoMode = () => {
+    this.setState({ createAnnoModeActive: true });
+  };
+
+  handleDisableAnnoMode = () => {
+    this.setState({ createAnnoModeActive: false });
+  };
+
+  handleModalDuplicateCategorySelection = d => {
+    this.setState({ categoryToDuplicate: d });
+  };
+
   render() {
-    const { categoricalSelection } = this.props;
+    const { createAnnoModeActive, categoryToDuplicate } = this.state;
+    const {
+      categoricalSelection,
+      writableCategoriesEnabled,
+      schema
+    } = this.props;
     if (!categoricalSelection) return null;
 
+    /* all names, sorted in display order.  Will be rendered in this order */
+    const allCategoryNames = Object.keys(categoricalSelection).sort();
+
     return (
       <div
         style={{
           padding: globals.leftSidebarSectionPadding
         }}
       >
-        {_.map(categoricalSelection, (catState, catName) => (
-          <Category key={catName} metadataField={catName} />
-        ))}
+        {/* READ ONLY CATEGORICAL FIELDS */}
+        {/* this is duplicative but flat, could be abstracted */}
+        {_.map(allCategoryNames, catName =>
+          !schema.annotations.obsByName[catName].writable ? (
+            <Category
+              key={catName}
+              metadataField={catName}
+              createAnnoModeActive={createAnnoModeActive}
+              isUserAnno={false}
+            />
+          ) : null
+        )}
+        {/* WRITEABLE FIELDS */}
+        {_.map(allCategoryNames, catName =>
+          schema.annotations.obsByName[catName].writable ? (
+            <Category
+              key={catName}
+              metadataField={catName}
+              createAnnoModeActive={createAnnoModeActive}
+              isUserAnno
+            />
+          ) : null
+        )}
+        {writableCategoriesEnabled ? (
+          <div>
+            <Dialog
+              icon="tag"
+              title="Create new category"
+              isOpen={createAnnoModeActive}
+              onClose={this.handleDisableAnnoMode}
+            >
+              <div className={Classes.DIALOG_BODY}>
+                <div style={{ marginBottom: 20 }}>
+                  <p>New, unique category name:</p>
+                  <InputGroup
+                    autoFocus
+                    onChange={e =>
+                      this.setState({ newCategoryText: e.target.value })
+                    }
+                    leftIcon="tag"
+                  />
+                </div>
+                <p>
+                  Optionally duplicate all labels & cell assignments from
+                  existing category into new category:
+                </p>
+                <Select
+                  items={allCategoryNames}
+                  filterable={false}
+                  itemRenderer={(d, { handleClick }) => {
+                    return <MenuItem onClick={handleClick} key={d} text={d} />;
+                  }}
+                  noResults={<MenuItem disabled text="No results." />}
+                  onItemSelect={d => {
+                    this.handleModalDuplicateCategorySelection(d);
+                  }}
+                >
+                  {/* children become the popover target; render value here */}
+                  <Button
+                    text={
+                      categoryToDuplicate || "None (all cells 'unassigned')"
+                    }
+                    rightIcon="double-caret-vertical"
+                  />
+                </Select>
+              </div>
+              <div className={Classes.DIALOG_FOOTER}>
+                <div className={Classes.DIALOG_FOOTER_ACTIONS}>
+                  <Tooltip content="Close this dialog without creating a category.">
+                    <Button onClick={this.handleDisableAnnoMode}>Cancel</Button>
+                  </Tooltip>
+                  <Button onClick={this.handleCreateUserAnno} intent="primary">
+                    Create new category
+                  </Button>
+                </div>
+              </div>
+            </Dialog>
+            <Button onClick={this.handleEnableAnnoMode} intent="primary">
+              Create new category
+            </Button>
+          </div>
+        ) : null}
       </div>
     );
   }

client/src/components/categorical/category.js

@@ -2,7 +2,19 @@ import React from "react";
 import _ from "lodash";
 import { connect } from "react-redux";
 import { FaChevronRight, FaChevronDown } from "react-icons/fa";
-import { Button, Tooltip } from "@blueprintjs/core";
+import {
+  Button,
+  Tooltip,
+  InputGroup,
+  Menu,
+  Dialog,
+  MenuItem,
+  Popover,
+  Classes,
+  Icon,
+  Position,
+  PopoverInteractionKind
+} from "@blueprintjs/core";
 
 import * as globals from "../../globals";
 import Value from "./value";
@@ -10,14 +22,18 @@ import sortedCategoryValues from "./util";
 
 @connect(state => ({
   colorAccessor: state.colors.colorAccessor,
-  categoricalSelection: state.categoricalSelection
+  categoricalSelection: state.categoricalSelection,
+  annotations: state.annotations,
+  universe: state.universe
 }))
 class Category extends React.Component {
   constructor(props) {
     super(props);
     this.state = {
       isChecked: true,
-      isExpanded: false
+      isExpanded: false,
+      newCategoryText: "",
+      newLabelText: ""
     };
   }
 
@@ -51,6 +67,74 @@ class Category extends React.Component {
     }
   }
 
+  activateAddNewLabelMode = () => {
+    const { dispatch, metadataField } = this.props;
+    dispatch({
+      type: "annotation: activate add new label mode",
+      data: metadataField
+    });
+  };
+
+  disableAddNewLabelMode = () => {
+    const { dispatch } = this.props;
+    dispatch({
+      type: "annotation: disable add new label mode"
+    });
+  };
+
+  handleAddNewLabelToCategory = () => {
+    const { dispatch, metadataField } = this.props;
+    const { newLabelText } = this.state;
+    /*
+    XXX TODO - temporary code generates random label string.   Remove
+    when the label creation UI is implemented.
+
+    const { newLabelText } = this.state;
+    */
+    // const newLabelText = `label${Math.random()}`;
+    dispatch({
+      type: "annotation: add new label to category",
+      metadataField,
+      newLabelText
+    });
+    this.setState({ newLabelText: "" });
+  };
+
+  activateEditCategoryMode = () => {
+    const { dispatch, metadataField } = this.props;
+    dispatch({
+      type: "annotation: activate category edit mode",
+      data: metadataField
+    });
+  };
+
+  disableEditCategoryMode = () => {
+    const { dispatch } = this.props;
+    dispatch({
+      type: "annotation: disable category edit mode"
+    });
+  };
+
+  handleEditCategory = () => {
+    const { dispatch, metadataField } = this.props;
+    const { newCategoryText } = this.state;
+
+    dispatch({
+      type: "annotation: category edited",
+      metadataField,
+      newCategoryText,
+      data: newCategoryText
+    });
+  };
+
+  handleDeleteCategory = () => {
+    const { dispatch, metadataField } = this.props;
+    dispatch({
+      type: "annotation: delete category",
+      metadataField
+    });
+  };
+
   handleColorChange = () => {
     const { dispatch, metadataField } = this.props;
     dispatch({
@@ -88,12 +172,13 @@ class Category extends React.Component {
   }
 
   renderCategoryItems() {
-    const { categoricalSelection, metadataField } = this.props;
+    const { categoricalSelection, metadataField, isUserAnno } = this.props;
 
     const cat = categoricalSelection[metadataField];
     const optTuples = sortedCategoryValues([...cat.categoryValueIndices]);
     return _.map(optTuples, (tuple, i) => (
       <Value
+        isUserAnno={isUserAnno}
         optTuples={optTuples}
         key={tuple[1]}
         metadataField={metadataField}
@@ -104,9 +189,17 @@ class Category extends React.Component {
   }
 
   render() {
-    const { isExpanded, isChecked } = this.state;
-    const { metadataField, colorAccessor, categoricalSelection } = this.props;
+    const { isExpanded, isChecked, newLabelText, newCategoryText } = this.state;
+    const {
+      metadataField,
+      colorAccessor,
+      categoricalSelection,
+      isUserAnno,
+      annotations,
+      universe
+    } = this.props;
     const { isTruncated } = categoricalSelection[metadataField];
+
     return (
       <div
         style={{
@@ -144,7 +237,45 @@ class Category extends React.Component {
               <span className="bp3-control-indicator" />
               {""}
             </label>
-
+            {/* Dialog uses portal, can be anywhere/factored out */}
+            <Dialog
+              icon="tag"
+              title="Edit category name"
+              isOpen={
+                annotations.isEditingCategoryName &&
+                annotations.categoryEditable === metadataField
+              }
+              onClose={this.disableEditCategoryMode}
+            >
+              <div className={Classes.DIALOG_BODY}>
+                <div style={{ marginBottom: 20 }}>
+                  <p>New, unique category name:</p>
+                  <InputGroup
+                    autoFocus
+                    onChange={e =>
+                      this.setState({ newCategoryText: e.target.value })
+                    }
+                    leftIcon="tag"
+                  />
+                </div>
+              </div>
+              <div className={Classes.DIALOG_FOOTER}>
+                <div className={Classes.DIALOG_FOOTER_ACTIONS}>
+                  <Tooltip content="Close this dialog without editing the category.">
+                    <Button onClick={this.disableEditCategoryMode}>
+                      Cancel
+                    </Button>
+                  </Tooltip>
+                  <Button
+                    disabled={newCategoryText.length === 0}
+                    onClick={this.handleEditCategory}
+                    intent="primary"
+                  >
+                    Edit category name
+                  </Button>
+                </div>
+              </div>
+            </Dialog>
             <span
               data-testid={`category-expand-${metadataField}`}
               style={{
@@ -155,6 +286,9 @@ class Category extends React.Component {
                 this.setState({ isExpanded: !isExpanded });
               }}
             >
+              {isUserAnno ? (
+                <Icon style={{ marginRight: 5 }} icon="tag" iconSize={16} />
+              ) : null}
               {metadataField}
               {isExpanded ? (
                 <FaChevronDown
@@ -169,20 +303,105 @@ class Category extends React.Component {
               )}
             </span>
           </div>
-          <Tooltip
-            content="Use as color scale"
-            position="bottom"
-            hoverOpenDelay={globals.tooltipHoverOpenDelay}
-          >
-            <Button
-              data-testclass="colorby"
-              data-testid={`colorby-${metadataField}`}
-              onClick={this.handleColorChange}
-              active={colorAccessor === metadataField}
-              intent={colorAccessor === metadataField ? "primary" : "none"}
-              icon="tint"
-            />
-          </Tooltip>
+          <div>
+            {isUserAnno ? (
+              <>
+                <Dialog
+                  icon="tag"
+                  title="Add new label"
+                  isOpen={annotations.isAddingNewLabel}
+                  onClose={this.disableAddNewLabelMode}
+                >
+                  <div className={Classes.DIALOG_BODY}>
+                    <div style={{ marginBottom: 20 }}>
+                      <p>New, unique label name:</p>
+                      <InputGroup
+                        autoFocus
+                        onChange={e =>
+                          this.setState({ newLabelText: e.target.value })
+                        }
+                        leftIcon="tag"
+                      />
+                    </div>
+                  </div>
+                  <div className={Classes.DIALOG_FOOTER}>
+                    <div className={Classes.DIALOG_FOOTER_ACTIONS}>
+                      <Tooltip content="Close this dialog without adding a label.">
+                        <Button onClick={this.disableAddNewLabelMode}>
+                          Cancel
+                        </Button>
+                      </Tooltip>
+                      <Button
+                        disabled={
+                          newLabelText.length === 0 ||
+                          universe.schema.annotations.obsByName[
+                            metadataField
+                          ].categories.indexOf(newLabelText) !== -1
+                        }
+                        onClick={this.handleAddNewLabelToCategory}
+                        intent="primary"
+                      >
+                        Add new label to category
+                      </Button>
+                    </div>
+                  </div>
+                </Dialog>
+                <Popover
+                  interactionKind={PopoverInteractionKind.HOVER}
+                  boundary="window"
+                  position={Position.RIGHT}
+                  content={
+                    <Menu>
+                      <MenuItem
+                        icon="tag"
+                        data-testclass="handleAddNewLabelToCategory"
+                        data-testid={`handleAddNewLabelToCategory-${metadataField}`}
+                        onClick={this.activateAddNewLabelMode}
+                        text="Add a new label to this category"
+                      />
+                      <MenuItem
+                        icon="edit"
+                        data-testclass="activateEditCategoryMode"
+                        data-testid={`activateEditCategoryMode-${metadataField}`}
+                        onClick={this.activateEditCategoryMode}
+                        text="Edit this category's name"
+                      />
+                      <MenuItem
+                        icon="delete"
+                        intent="danger"
+                        data-testclass="handleDeleteCategory"
+                        data-testid={`handleDeleteCategory-${metadataField}`}
+                        onClick={this.handleDeleteCategory}
+                        text="Delete this category, all associated labels, and remove all cell assignments"
+                      />
+                    </Menu>
+                  }
+                >
+                  <Button
+                    style={{ marginLeft: 0 }}
+                    data-testclass="seeActions"
+                    data-testid={`seeActions-${metadataField}`}
+                    icon="more"
+                    minimal
+                  />
+                </Popover>
+              </>
+            ) : null}
+            <Tooltip
+              content="Use as color scale"
+              position="bottom"
+              hoverOpenDelay={globals.tooltipHoverOpenDelay}
+            >
+              <Button
+                data-testclass="colorby"
+                data-testid={`colorby-${metadataField}`}
+                onClick={this.handleColorChange}
+                active={colorAccessor === metadataField}
+                intent={colorAccessor === metadataField ? "primary" : "none"}
+                icon="tint"
+              />
+            </Tooltip>
+          </div>
         </div>
         <div style={{ marginLeft: 26 }}>
           {isExpanded ? this.renderCategoryItems() : null}

client/src/components/categorical/occupancy.js

@@ -45,7 +45,10 @@ class Occupancy extends React.Component {
       groupBy
     ); /* Because the signature changes we really need different names for histogram to differentiate signatures  */
 
-    const bins = histogramMap.get(category.categoryValues[categoryIndex]);
+    const categoryValue = category.categoryValues[categoryIndex];
+    const bins = histogramMap.has(categoryValue)
+      ? histogramMap.get(categoryValue)
+      : new Array(50).fill(0);
 
     const xScale = d3
       .scaleLinear()
@@ -102,30 +105,34 @@ class Occupancy extends React.Component {
 
     const occupancy = occupancyMap.get(category.categoryValues[categoryIndex]);
 
-    const x = d3
-      .scaleLinear()
-      /* get all the keys d[1] as an array, then find the sum */
-      .domain([0, d3.sum(Array.from(occupancy.values()))])
-      .range([0, this._WIDTH]);
-    const categories = schema.annotations.obsByName[colorAccessor]?.categories;
+    if (occupancy && occupancy.size > 0) {
+      // not all categories have occupancy, so occupancy may be undefined.
+      const x = d3
+        .scaleLinear()
+        /* get all the keys d[1] as an array, then find the sum */
+        .domain([0, d3.sum(Array.from(occupancy.values()))])
+        .range([0, this._WIDTH]);
+      const categories =
+        schema.annotations.obsByName[colorAccessor]?.categories;
 
-    let currentOffset = 0;
-    const dfColumn = world.obsAnnotations.col(colorAccessor);
-    const categoryValues = dfColumn.summarize().categories;
+      let currentOffset = 0;
+      const dfColumn = world.obsAnnotations.col(colorAccessor);
+      const categoryValues = dfColumn.summarize().categories;
 
-    let o;
-    let scaledValue;
-    let value;
+      let o;
+      let scaledValue;
+      let value;
 
-    for (let i = 0, { length } = categoryValues; i < length; i += 1) {
-      value = categoryValues[i];
-      o = occupancy.get(value);
-      scaledValue = x(o);
-      ctx.fillStyle = o
-        ? colorScale(categories.indexOf(value))
-        : "rgb(255,255,255)";
-      ctx.fillRect(currentOffset, 0, o ? scaledValue : 0, this._HEIGHT);
-      currentOffset += o ? scaledValue : 0;
+      for (let i = 0, { length } = categoryValues; i < length; i += 1) {
+        value = categoryValues[i];
+        o = occupancy.get(value);
+        scaledValue = x(o);
+        ctx.fillStyle = o
+          ? colorScale(categories.indexOf(value))
+          : "rgb(255,255,255)";
+        ctx.fillRect(currentOffset, 0, o ? scaledValue : 0, this._HEIGHT);
+        currentOffset += o ? scaledValue : 0;
+      }
     }
   };
 

client/src/components/categorical/value.js

@@ -1,18 +1,108 @@
 // jshint esversion: 6
 import { connect } from "react-redux";
 import React from "react";
+
+import {
+  Button,
+  InputGroup,
+  Menu,
+  MenuItem,
+  Popover,
+  Position,
+  Icon,
+  PopoverInteractionKind
+} from "@blueprintjs/core";
 import Occupancy from "./occupancy";
 import * as globals from "../../globals";
 import styles from "./categorical.css";
 
 @connect(state => ({
   categoricalSelection: state.categoricalSelection,
+  annotations: state.annotations,
   colorScale: state.colors.scale,
   colorAccessor: state.colors.colorAccessor,
   schema: state.world?.schema,
   world: state.world
 }))
 class CategoryValue extends React.Component {
+  constructor(props) {
+    super(props);
+    this.state = {
+      editedLabelText: ""
+    };
+  }
+
+  handleDeleteValue = () => {
+    const {
+      dispatch,
+      metadataField,
+      categoryIndex,
+      categoricalSelection
+    } = this.props;
+    const category = categoricalSelection[metadataField];
+    const label = category.categoryValues[categoryIndex];
+    dispatch({
+      type: "annotation: delete label",
+      metadataField,
+      label
+    });
+  };
+
+  handleAddCurrentSelectionToThisLabel = () => {
+    const {
+      dispatch,
+      metadataField,
+      categoryIndex,
+      categoricalSelection
+    } = this.props;
+    const category = categoricalSelection[metadataField];
+    const label = category.categoryValues[categoryIndex];
+    dispatch({
+      type: "annotation: label current cell selection",
+      metadataField,
+      categoryIndex,
+      label
+    });
+  };
+
+  handleEditValue = () => {
+    const {
+      dispatch,
+      metadataField,
+      categoryIndex,
+      categoricalSelection
+    } = this.props;
+    const { editedLabelText } = this.state;
+    const category = categoricalSelection[metadataField];
+    const label = category.categoryValues[categoryIndex];
+    dispatch({
+      type: "annotation: label edited",
+      editedLabel: editedLabelText,
+      metadataField,
+      categoryIndex,
+      label
+    });
+    this.setState({ editedLabelText: "" });
+  };
+
+  activateEditLabelMode = () => {
+    const { dispatch, metadataField, categoryIndex } = this.props;
+    dispatch({
+      type: "annotation: activate edit label mode",
+      metadataField,
+      categoryIndex
+    });
+  };
+
+  cancelEdit = () => {
+    const { dispatch, metadataField, categoryIndex } = this.props;
+    dispatch({
+      type: "annotation: cancel edit label mode",
+      metadataField,
+      categoryIndex
+    });
+  };
+
   toggleOff = () => {
     const { dispatch, metadataField, categoryIndex } = this.props;
     dispatch({
@@ -23,12 +113,12 @@ class CategoryValue extends React.Component {
   };
 
   shouldComponentUpdate = nextProps => {
-    /* 
+    /*
     Checks to see if at least one of the following changed:
     * world state
     * the color accessor (what is currently being colored by)
     * if this catagorical value's selection status has changed
-    
+
     If and only if true, update the component
     */
     const { props } = this;
@@ -45,8 +135,14 @@ class CategoryValue extends React.Component {
 
     const worldChange = props.world !== nextProps.world;
     const colorAccessorChange = props.colorAccessor !== nextProps.colorAccessor;
+    const annotationsChange = props.annotations !== nextProps.annotations;
 
-    return valueSelectionChange || worldChange || colorAccessorChange;
+    return (
+      valueSelectionChange ||
+      worldChange ||
+      colorAccessorChange ||
+      annotationsChange
+    );
   };
 
   toggleOn = () => {
@@ -84,7 +180,9 @@ class CategoryValue extends React.Component {
       colorAccessor,
       colorScale,
       i,
-      schema
+      schema,
+      isUserAnno,
+      annotations
     } = this.props;
 
     if (!categoricalSelection) return null;
@@ -121,41 +219,103 @@ class CategoryValue extends React.Component {
         onMouseEnter={this.handleMouseEnter}
         onMouseLeave={this.handleMouseExit}
       >
-        <div
-          style={{
-            margin: 0,
-            padding: 0,
-            userSelect: "none",
-            width: globals.leftSidebarWidth - 130,
-            display: "flex",
-            justifyContent: "space-between"
-          }}
-        >
-          <div style={{ display: "flex" }}>
-            <label className="bp3-control bp3-checkbox" style={{ margin: 0 }}>
-              <input
-                onChange={selected ? this.toggleOff : this.toggleOn}
-                data-testclass="categorical-value-select"
-                data-testid={`categorical-value-select-${metadataField}-${displayString}`}
-                checked={selected}
-                type="checkbox"
-              />
+        <div style={{ display: "flex", justifyContent: "space-between" }}>
+          <div
+            style={{
+              margin: 0,
+              padding: 0,
+              userSelect: "none",
+              width: globals.leftSidebarWidth - 240,
+              display: "flex",
+              justifyContent: "flex-start"
+            }}
+          >
+            <div style={{ display: "flex" }}>
+              <label className="bp3-control bp3-checkbox" style={{ margin: 0 }}>
+                <input
+                  onChange={selected ? this.toggleOff : this.toggleOn}
+                  data-testclass="categorical-value-select"
+                  data-testid={`categorical-value-select-${metadataField}-${displayString}`}
+                  checked={selected}
+                  type="checkbox"
+                />
+                <span
+                  className="bp3-control-indicator"
+                  onMouseEnter={this.handleMouseExit}
+                  onMouseLeave={this.handleMouseEnter}
+                />
+              </label>
               <span
-                className="bp3-control-indicator"
-                onMouseEnter={this.handleMouseExit}
-                onMouseLeave={this.handleMouseEnter}
-              />
-            </label>
-            <span
-              data-testid={`categorical-value-${metadataField}-${displayString}`}
-              data-testclass="categorical-value"
-              style={{ wordBreak: "break-all" }}
-            >
-              {displayString}
-            </span>
+                data-testid={`categorical-value-${metadataField}-${displayString}`}
+                data-testclass="categorical-value"
+                style={{
+                  wordBreak: "break-all",
+                  color:
+                    displayString === globals.unassignedCategoryLabel
+                      ? "#ababab"
+                      : "black",
+                  fontStyle:
+                    displayString === globals.unassignedCategoryLabel
+                      ? "italic"
+                      : "auto"
+                }}
+              >
+                {annotations.isEditingLabelName &&
+                annotations.labelEditable.category === metadataField &&
+                annotations.labelEditable.label === categoryIndex
+                  ? null
+                  : displayString}
+              </span>
+            </div>
+            {isUserAnno &&
+            annotations.isEditingLabelName &&
+            annotations.labelEditable.category === metadataField &&
+            annotations.labelEditable.label === categoryIndex ? (
+              <form
+                onSubmit={e => {
+                  e.preventDefault();
+                  this.handleEditValue();
+                }}
+              >
+                <InputGroup
+                  style={{ position: "relative", top: -1 }}
+                  ref={input => {
+                    this.editableInput = input;
+                  }}
+                  small
+                  onChange={e => {
+                    this.setState({ editedLabelText: e.target.value });
+                  }}
+                  defaultValue={displayString}
+                  rightElement={
+                    <Button
+                      minimal
+                      style={{ position: "relative", top: -1 }}
+                      type="button"
+                      icon="small-tick"
+                      data-testclass="submitEdit"
+                      data-testid="submitEdit"
+                      onClick={this.handleEditValue}
+                    />
+                  }
+                />
+              </form>
+            ) : null}
+            {/*
+            CANCEL IT, WITH BUTTON, ESCAPE KEY, CLICK OUT, UNDO?
+
+            <Button
+            minimal
+            style={{ position: "relative", top: -1 }}
+            type="button"
+            icon="cross"
+            data-testclass="submitEdit"
+            data-testid="submitEdit"
+            onClick={this.cancelEdit}
+          /> */}
           </div>
           <span style={{ flexShrink: 0 }}>
-            {colorAccessor && !isColorBy ? (
+            {colorAccessor && !isColorBy && !annotations.isEditingLabelName ? (
               <Occupancy category={category} {...this.props} />
             ) : null}
           </span>
@@ -164,10 +324,22 @@ class CategoryValue extends React.Component {
           <span
             data-testclass="categorical-value-count"
             data-testid={`categorical-value-count-${metadataField}-${displayString}`}
+            style={{
+              color:
+                displayString === globals.unassignedCategoryLabel
+                  ? "#ababab"
+                  : "black",
+              fontStyle:
+                displayString === globals.unassignedCategoryLabel
+                  ? "italic"
+                  : "auto"
+            }}
           >
             {count}
           </span>
+
           <svg
+            display={isColorBy && categories ? "auto" : "none"}
             style={{
               marginLeft: 5,
               width: 11,
@@ -178,6 +350,57 @@ class CategoryValue extends React.Component {
                   : "inherit"
             }}
           />
+          {isUserAnno ? (
+            <span
+              onMouseEnter={this.handleMouseExit}
+              onMouseLeave={this.handleMouseEnter}
+            >
+              <Popover
+                interactionKind={PopoverInteractionKind.HOVER}
+                boundary="window"
+                position={Position.RIGHT_TOP}
+                content={
+                  <Menu>
+                    <MenuItem
+                      icon="plus"
+                      data-testclass="handleAddCurrentSelectionToThisLabel"
+                      data-testid={`handleAddCurrentSelectionToThisLabel-${metadataField}`}
+                      onClick={this.handleAddCurrentSelectionToThisLabel}
+                      text={`Label currently selected cells as ${displayString}`}
+                    />
+                    {displayString !== globals.unassignedCategoryLabel ? (
+                      <MenuItem
+                        icon="edit"
+                        text="Edit this label's name"
+                        data-testclass="handleEditValue"
+                        data-testid={`handleEditValue-${metadataField}`}
+                        onClick={this.activateEditLabelMode}
+                      />
+                    ) : null}
+                    {displayString !== globals.unassignedCategoryLabel ? (
+                      <MenuItem
+                        icon="delete"
+                        intent="danger"
+                        data-testclass="handleDeleteValue"
+                        data-testid={`handleDeleteValue-${metadataField}`}
+                        onClick={this.handleDeleteValue}
+                        text="Delete this value, and reassign all cells to type 'unknown'"
+                      />
+                    ) : null}
+                  </Menu>
+                }
+              >
+                <Button
+                  style={{ marginLeft: 0, position: "relative", top: -1 }}
+                  data-testclass="seeActions"
+                  data-testid={`seeActions-${metadataField}`}
+                  icon="more"
+                  small
+                  minimal
+                />
+              </Popover>
+            </span>
+          ) : null}
         </span>
       </div>
     );

client/src/components/framework/custom-icons.js

@@ -0,0 +1,16 @@
+/* https://github.com/palantir/blueprint/issues/2348 */
+
+<defs>
+<clipPath id="clip0">
+<rect width="16" height="16" fill="white"/>
+</clipPath>
+</defs>
+<g clip-path="url(#clip0)">
+<rect width="16" height="16" fill="white"/>
+<path d="M1.33415 8.75877C0.939491 8.36411 0.727699 7.82249 0.749957 7.2648L0.926361 2.84501C0.967947 1.80308 1.80308 0.967947 2.84501 0.926361L7.2648 0.749958C7.82249 0.727699 8.36411 0.939492 8.75877 1.33415L14.3595 6.93485C15.1405 7.7159 15.1405 8.98223 14.3595 9.76328L9.76328 14.3595C8.98223 15.1405 7.7159 15.1405 6.93485 14.3595L1.33415 8.75877Z" fill="black"/>
+<circle cx="4.5" cy="4.5" r="1.5" fill="white"/>
+<circle cx="4.5" cy="11.5" r="3.75" stroke="white" stroke-width="0.5"/>
+<circle cx="4.5" cy="11.5" r="3.5" fill="black"/>
+<line x1="4.5" y1="10" x2="4.5" y2="13" stroke="white"/>
+<line x1="3" y1="11.5" x2="6" y2="11.5" stroke="white"/>
+</g>

client/src/globals.js

@@ -3,6 +3,9 @@ import { Colors } from "@blueprintjs/core";
 /* if a categorical metadata field has more options than this, truncate */
 export const maxCategoricalOptionsToDisplay = 100;
 
+/* default "unassigned" value for user-created categorical metadata */
+export const unassignedCategoryLabel = "unassigned";
+
 /*
 these are default values for configuration  the CLI may supply.
 See the REST API and CLI specs for more info.

client/src/reducers/annotations.js

@@ -0,0 +1,88 @@
+/*
+Reducers for annotation UI-state.
+*/
+const Annotations = (
+  state = {
+    isEditingCategoryName: false,
+    isEditingLabelName: false,
+    categoryEditable: false,
+    labelEditable: { category: null, label: null }
+  },
+  action
+) => {
+  switch (action.type) {
+    /* CATEGORY */
+    case "annotation: activate add new label mode":
+      console.log(action.type, action);
+      return {
+        ...state,
+        isAddingNewLabel: true,
+        categoryAddingNewLabel: action.data
+      };
+    case "annotation: disable add new label mode":
+      return {
+        ...state,
+        isAddingNewLabel: false,
+        categoryAddingNewLabel: null
+      };
+    case "annotation: add new label to category":
+      console.log(action.type, action);
+      return {
+        ...state,
+        isAddingNewLabel: false,
+        categoryAddingNewLabel: null
+      };
+    case "annotation: activate category edit mode":
+      console.log(action.type, action);
+      return {
+        ...state,
+        isEditingCategoryName: true,
+        categoryEditable: action.data
+      };
+    case "annotation: disable category edit mode":
+      console.log(action.type, action);
+      return {
+        ...state,
+        isEditingCategoryName: false,
+        categoryEditable: null
+      };
+    case "annotation: category edited":
+      console.log(action.type, action);
+      return {
+        ...state,
+        isEditingCategoryName: true,
+        categoryEditable: null
+      };
+
+    /* LABEL */
+    case "annotation: activate edit label mode":
+      console.log(action.type, action);
+      return {
+        ...state,
+        isEditingLabelName: true,
+        labelEditable: {
+          category: action.metadataField,
+          label: action.categoryIndex
+        }
+      };
+    case "annotation: cancel edit label mode":
+      console.log(action.type, action);
+      return {
+        ...state,
+        isEditingLabelName: false,
+        labelEditable: { category: null, label: null }
+      };
+    case "annotation: label edited":
+      console.log(action.type, action);
+      return {
+        ...state,
+        isEditingLabelName: false,
+        labelEditable: { category: null, label: null }
+        /* Bruce to persist new label name */
+      };
+    default:
+      return state;
+  }
+};
+
+export default Annotations;

client/src/reducers/autosave.js

@@ -0,0 +1,53 @@
+const Autosave = (
+	state = {
+		saveInProgress: false,
+		error: false,
+		lastSavedObsAnnotations: null
+	},
+	action,
+	nextSharedState
+) => {
+	switch (action.type) {
+		case "initial data load complete (universe exists)": {
+			/* don't save on init */
+			const { universe } = nextSharedState;
+			return {
+				...state,
+				error: false,
+				saveInProgress: false,
+				lastSavedObsAnnotations: universe.obsAnnotations
+			};
+		}
+
+		case "writable obs annotations - save started": {
+			return {
+				...state,
+				saveInProgress: true
+			};
+		}
+
+		case "writable obs annotations - save error": {
+			const { message } = action;
+			return {
+				...state,
+				error: message,
+				saveInProgress: false
+			};
+		}
+
+		case "writable obs annotations - save complete": {
+			const lastSavedObsAnnotations = action.obsAnnotations;
+			return {
+				...state,
+				saveInProgress: false,
+				error: false,
+				lastSavedObsAnnotations
+			};
+		}
+
+		default:
+			return { ...state };
+	}
+};
+
+export default Autosave;

client/src/reducers/categoricalSelection.js

@@ -1,4 +1,4 @@
-import { ControlsHelpers } from "../util/stateManager";
+import { ControlsHelpers as CH } from "../util/stateManager";
 import * as globals from "../globals";
 
 function maxCategoryItems(state) {
@@ -20,10 +20,11 @@ const CategoricalSelection = (
     case "reset World to eq Universe":
     case "set clip quantiles": {
       const { world } = nextSharedState;
-      return ControlsHelpers.createCategoricalSelection(
-        maxCategoryItems(prevSharedState),
-        world
+      const newState = CH.createCategoricalSelection(
+        world,
+        CH.selectableCategoryNames(world, maxCategoryItems(prevSharedState))
       );
+      return newState;
     }
 
     case "categorical metadata filter select": {
@@ -96,6 +97,43 @@ const CategoricalSelection = (
       return newCategoricalSelection;
     }
 
+    case "annotation: create category": {
+      const { world } = nextSharedState;
+      const name = action.data;
+      return {
+        ...state,
+        ...CH.createCategoricalSelection(world, [name])
+      };
+    }
+
+    case "annotation: category edited": {
+      const name = action.metadataField;
+      const newName = action.newCategoryText;
+      const { [name]: catSeln, ...newState } = state;
+      newState[newName] = catSeln;
+      return newState;
+    }
+
+    case "annotation: delete category": {
+      const name = action.metadataField;
+      const { [name]: _, ...newState } = state;
+      return newState;
+    }
+
+    case "annotation: label current cell selection":
+    case "annotation: add new label to category":
+    case "annotation: label edited":
+    case "annotation: delete label": {
+      /* need to rebuild the state for this annotation */
+      const { world } = nextSharedState;
+      const name = action.metadataField;
+      const { [name]: _, ...partialState } = state;
+      return {
+        ...partialState,
+        ...CH.createCategoricalSelection(world, [name])
+      };
+    }
+
     default: {
       return state;
     }

client/src/reducers/colors.js

@@ -108,6 +108,27 @@ const ColorsReducer = (
       };
     }
 
+    case "annotation: add new label to category":
+    case "annotation: label current cell selection":
+    case "annotation: delete label": {
+      const { world } = nextSharedState;
+      const { colorMode, colorAccessor } = state;
+      const { metadataField } = action;
+      if (
+        colorMode !== "color by categorical metadata" ||
+        colorAccessor !== metadataField
+      )
+        return state;
+
+      /* else, we need to rebuild colors as labels have changed! */
+      const { rgb, scale } = ColorHelpers.createColors(
+        world,
+        colorMode,
+        colorAccessor
+      );
+      return { ...state, rgb, scale };
+    }
+
     case "clear differential expression": {
       const { world: prevWorld, controls: prevControls } = prevSharedState;
       const resetColorState = ColorHelpers.checkIfColorByDiffexpAndResetColors(

client/src/reducers/controls.js

@@ -7,7 +7,7 @@ import { WorldUtil } from "../util/stateManager";
 const Controls = (
   state = {
     // data loading flag
-    loading: false,
+    loading: true,
     error: null,
 
     // all of the data + selection state

client/src/reducers/crossfilter.js

@@ -1,7 +1,11 @@
 import _ from "lodash";
 
 import Crossfilter from "../util/typedCrossfilter";
-import { World, ControlsHelpers } from "../util/stateManager";
+import {
+  World,
+  ControlsHelpers as CH,
+  AnnotationsHelpers as AH
+} from "../util/stateManager";
 import {
   layoutDimensionName,
   obsAnnoDimensionName,
@@ -12,7 +16,7 @@ import {
 
 const XYDimName = layoutDimensionName("XY");
 
-const CrossfilterReducer = (
+const CrossfilterReducerBase = (
   state = null,
   action,
   nextSharedState,
@@ -32,12 +36,14 @@ const CrossfilterReducer = (
     case "reset World to eq Universe": {
       const { userDefinedGenes, diffexpGenes } = prevSharedState.controls;
       const { world } = nextSharedState;
-      const crossfilter = ControlsHelpers.createGeneDimensions(
+      let { crossfilter } = prevSharedState.resetCache;
+      crossfilter = CH.createGeneDimensions(
         userDefinedGenes,
         diffexpGenes,
         world,
-        prevSharedState.resetCache.crossfilter
+        crossfilter
       );
+      crossfilter = AH.createWritableAnnotationDimensions(world, crossfilter);
       return crossfilter;
     }
 
@@ -51,7 +57,7 @@ const CrossfilterReducer = (
         world,
         layoutChoice.currentDimNames
       );
-      crossfilter = ControlsHelpers.createGeneDimensions(
+      crossfilter = CH.createGeneDimensions(
         userDefinedGenes,
         diffexpGenes,
         world,
@@ -136,6 +142,37 @@ const CrossfilterReducer = (
       return crossfilter;
     }
 
+    case "annotation: create category": {
+      const name = action.data;
+      const { world } = nextSharedState;
+      const colData = world.obsAnnotations.col(name).asArray();
+      return state.addDimension(obsAnnoDimensionName(name), "enum", colData);
+    }
+
+    case "annotation: category edited": {
+      const name = action.metadataField;
+      const newName = action.newCategoryText;
+      return state.renameDimension(
+        obsAnnoDimensionName(name),
+        obsAnnoDimensionName(newName)
+      );
+    }
+
+    case "annotation: delete category": {
+      return state.delDimension(obsAnnoDimensionName(action.metadataField));
+    }
+
+    case "annotation: label current cell selection":
+    case "annotation: label edited":
+    case "annotation: delete label": {
+      /* we need to reindex the dimension.  For now, just drop it and add another */
+      const name = action.metadataField;
+      const dimName = obsAnnoDimensionName(name);
+      const { world } = nextSharedState;
+      const colData = world.obsAnnotations.col(name).asArray();
+      return state.delDimension(dimName).addDimension(dimName, "enum", colData);
+    }
+
     case "graph brush end":
     case "graph brush change": {
       const [minX, maxY] = action.brushCoords.northwest;
@@ -185,12 +222,12 @@ const CrossfilterReducer = (
     case "categorical metadata filter select":
     case "categorical metadata filter deselect": {
       const { categoricalSelection } = nextSharedState;
-      const { world } = prevSharedState;
       const cat = categoricalSelection[action.metadataField];
-      const col = world.obsAnnotations.col(action.metadataField);
+      const { categoryValues, categoryValueSelected } = cat;
+      const values = categoryValues.filter((v, i) => categoryValueSelected[i]);
       return state.select(obsAnnoDimensionName(action.metadataField), {
         mode: "exact",
-        values: ControlsHelpers.selectedValuesForCategory(cat, col)
+        values
       });
     }
 
@@ -212,4 +249,29 @@ const CrossfilterReducer = (
   }
 };
 
+/*
+  IMPORTANT: the system assumes that crossfilter.data() will point at the
+  same value as world.obsAnnotations.  For actions handled in this reducer,
+  make sure that this remains true.
+
+  This wrapper performs only this function.
+*/
+const CrossfilterReducer = (
+  state,
+  action,
+  nextSharedState,
+  prevSharedState
+) => {
+  const nextState = CrossfilterReducerBase(
+    state,
+    action,
+    nextSharedState,
+    prevSharedState
+  );
+  if (!nextState || nextState.all() === nextSharedState.world.obsAnnotations) {
+    return nextState;
+  }
+  return nextState.setData(nextSharedState.world.obsAnnotations);
+};
+
 export default CrossfilterReducer;

client/src/reducers/index.js

@@ -17,38 +17,44 @@ import responsive from "./responsive";
 import controls from "./controls";
 import resetCache from "./resetCache";
 import centroidLabel from "./centroidLabel";
+import annotations from "./annotations";
+import autosave from "./autosave";
 
 import undoableConfig from "./undoableConfig";
 
 const Reducer = undoable(
-  cascadeReducers([
-    ["config", config],
-    ["universe", universe],
-    ["world", world],
-    ["layoutChoice", layoutChoice],
-    ["categoricalSelection", categoricalSelection],
-    ["continuousSelection", continuousSelection],
-    ["graphSelection", graphSelection],
-    ["crossfilter", crossfilter],
-    ["colors", colors],
-    ["controls", controls],
-    ["differential", differential],
-    ["responsive", responsive],
-    ["centroidLabel", centroidLabel],
-    ["resetCache", resetCache]
-  ]),
-  [
-    "world",
-    "categoricalSelection",
-    "continuousSelection",
-    "graphSelection",
-    "crossfilter",
-    "colors",
-    "controls",
-    "differential",
-    "layoutChoice"
-  ],
-  undoableConfig
+    cascadeReducers([
+        ["config", config],
+        ["universe", universe],
+        ["world", world],
+        ["annotations", annotations],
+        ["layoutChoice", layoutChoice],
+        ["categoricalSelection", categoricalSelection],
+        ["continuousSelection", continuousSelection],
+        ["graphSelection", graphSelection],
+        ["crossfilter", crossfilter],
+        ["colors", colors],
+        ["controls", controls],
+        ["differential", differential],
+        ["responsive", responsive],
+        ["centroidLabel", centroidLabel],
+        ["autosave", autosave],
+        ["resetCache", resetCache]
+    ]),
+    [
+        "universe",
+        "world",
+        "categoricalSelection",
+        "continuousSelection",
+        "graphSelection",
+        "crossfilter",
+        "colors",
+        "controls",
+        "differential",
+        "layoutChoice",
+        "annotations"
+    ],
+    undoableConfig
 );
 
 const store = createStore(Reducer, applyMiddleware(thunk));

client/src/reducers/resetCache.js

@@ -1,13 +1,13 @@
 /*
 Reducer which caches derived state to be used in a reset or other 
-recomputation.
+recomputation.  Add stuff here you want stashed at init time (or whenever),
+for later use.
 
-Currently this only caches the baseline (full universe) world & crossfilter, 
-for use in a Reset.
+Currently this only caches the baseline (full universe) crossfilter, 
+which improves Reset UI performance.
 */
 const ResetCacheReducer = (
   state = {
-    world: null,
     crossfilter: null
   },
   action,
@@ -15,10 +15,9 @@ const ResetCacheReducer = (
 ) => {
   switch (action.type) {
     case "initial data load complete (universe exists)": {
-      const { world, crossfilter } = nextSharedState;
+      const { crossfilter } = nextSharedState;
       return {
         ...state,
-        world,
         crossfilter
       };
     }

client/src/reducers/undoableConfig.js

@@ -78,7 +78,15 @@ const saveOnActions = new Set([
   "set clip quantiles",
 
   "set layout choice",
-  "change graph interaction mode"
+  "change graph interaction mode",
+
+  // user editable annotations
+  "annotation: create category",
+  "annotation: add new label to category",
+  "annotation: delete category",
+  "annotation: label edited",
+  "annotation: label current cell selection",
+  "annotation: delete label"
 ]);
 
 /**

client/src/reducers/universe.js

@@ -1,4 +1,9 @@
-import { ControlsHelpers } from "../util/stateManager";
+import { unassignedCategoryLabel } from "../globals";
+import {
+  World,
+  ControlsHelpers as CH,
+  AnnotationsHelpers as AH
+} from "../util/stateManager";
 
 const Universe = (state = null, action, nextSharedState, prevSharedState) => {
   switch (action.type) {
@@ -30,12 +35,186 @@ const Universe = (state = null, action, nextSharedState, prevSharedState) => {
           Object.keys(action.expressionData)
         )
       ];
-      varData = ControlsHelpers.pruneVarDataCache(varData, allTheGenesWeNeed);
+      varData = CH.pruneVarDataCache(varData, allTheGenesWeNeed);
+      return { ...state, varData };
+    }
 
-      return {
-        ...state,
-        varData
+    case "annotation: create category": {
+      /* create a new annotation category, with all values set to 'unassigned' */
+      const name = action.data;
+      const { categoryToDuplicate } = action;
+
+      /* name must be a string,  non-zero length */
+      if (typeof name !== "string" || name.length === 0)
+        throw new Error("user annotations require string name");
+
+      /* ensure the name isn't already in use! */
+      if (state.obsAnnotations.hasCol(name))
+        throw new Error("name collision on annotation category create");
+
+      /* ensure the duplicate col exists */
+      if (
+        categoryToDuplicate &&
+        !state.obsAnnotations.hasCol(categoryToDuplicate)
+      )
+        throw new Error("categoryToDuplicate does not exist");
+
+      let schema;
+      let data;
+      if (categoryToDuplicate) {
+        /* duplicate the named annotation */
+        schema = AH.dupObsAnnoSchema(state.schema, categoryToDuplicate, name, {
+          writable: true
+        });
+        /* if we are duplicating a non-writable annotation, it may not have an unassigned category */
+        const s = schema.annotations.obsByName[categoryToDuplicate];
+        if (s.categories.indexOf(unassignedCategoryLabel) === -1) {
+          s.categories = s.categories.concat(unassignedCategoryLabel);
+        }
+        data = state.obsAnnotations.col(categoryToDuplicate).asArray();
+      } else {
+        /* else, all are unassined */
+        const categories = [unassignedCategoryLabel];
+        schema = AH.addObsAnnoSchema(state.schema, name, {
+          name,
+          categories,
+          type: "categorical",
+          writable: true
+        });
+        data = new Array(state.nObs).fill(unassignedCategoryLabel);
+      }
+
+      const obsAnnotations = state.obsAnnotations.withCol(name, data);
+      return { ...state, obsAnnotations, schema };
+    }
+
+    case "annotation: category edited": {
+      /* change the name of an obs annotation category */
+      const name = action.metadataField;
+      const newName = action.newCategoryText;
+      if (!AH.isUserAnnotation(state, name))
+        throw new Error("unable to edit read-only annotation");
+      if (typeof newName !== "string" || newName.length === 0)
+        throw new Error("user annotations require string name");
+
+      const colSchema = {
+        ...state.schema.annotations.obsByName[name],
+        name: newName
       };
+      const schema = AH.addObsAnnoSchema(
+        AH.removeObsAnnoSchema(state.schema, name),
+        newName,
+        colSchema
+      );
+      const obsAnnotations = state.obsAnnotations.renameCol(name, newName);
+      return { ...state, schema, obsAnnotations };
+    }
+
+    case "annotation: delete category": {
+      /* delete annotation category from schema and obsAnnotations */
+      const name = action.metadataField;
+      if (!AH.isUserAnnotation(state, name))
+        throw new Error("unable to delete read-only annotation");
+
+      const schema = AH.removeObsAnnoSchema(state.schema, name);
+      const obsAnnotations = state.obsAnnotations.dropCol(name);
+      return { ...state, schema, obsAnnotations };
+    }
+
+    case "annotation: add new label to category": {
+      const annotationName = action.metadataField;
+      const newLabelName = action.newLabelText;
+      if (!AH.isUserAnnotation(state, annotationName))
+        throw new Error("unable to modify read-only annotation");
+      if (typeof newLabelName !== "string" || newLabelName.length === 0)
+        throw new Error(
+          "user annotations require a non-zero length string name"
+        );
+
+      /* add the new label to the annotation */
+      const schema = AH.addObsAnnoCategory(
+        state.schema,
+        annotationName,
+        newLabelName
+      );
+      return { ...state, schema };
+    }
+
+    case "annotation: label edited": {
+      const annotationName = action.metadataField;
+      const oldLabelName = action.label;
+      const newLabelName = action.editedLabel;
+      if (!AH.isUserAnnotation(state, annotationName))
+        throw new Error("unable to modify read-only annotation");
+      if (typeof newLabelName !== "string" || newLabelName.length === 0)
+        throw new Error(
+          "user annotations require a non-zero length string name"
+        );
+
+      /* remove old label, add new label */
+      const schema = AH.addObsAnnoCategory(
+        AH.removeObsAnnoCategory(state.schema, annotationName, oldLabelName),
+        annotationName,
+        newLabelName
+      );
+
+      /* change all values in obsAnnotation */
+      const obsAnnotations = AH.setLabelByValue(
+        state.obsAnnotations,
+        annotationName,
+        oldLabelName,
+        newLabelName
+      );
+
+      return { ...state, schema, obsAnnotations };
+    }
+
+    case "annotation: delete label": {
+      /* delete the label from the annotation, and set all cells with this value to unassigned */
+      const annotationName = action.metadataField;
+      const labelName = action.label;
+      if (!AH.isUserAnnotation(state, annotationName))
+        throw new Error("unable to modify read-only annotation");
+      if (labelName === unassignedCategoryLabel)
+        throw new Error("may not remove the unassigned label");
+
+      /* remove the category from the schema */
+      const schema = AH.removeObsAnnoCategory(
+        state.schema,
+        annotationName,
+        labelName
+      );
+
+      /* set all values to unassigned in obsAnnotations */
+      const obsAnnotations = AH.setLabelByValue(
+        state.obsAnnotations,
+        annotationName,
+        labelName,
+        unassignedCategoryLabel
+      );
+
+      return { ...state, schema, obsAnnotations };
+    }
+
+    case "annotation: label current cell selection": {
+      const { metadataField, label } = action;
+      const { world, crossfilter } = prevSharedState;
+
+      /*
+      selection state is relative to world.  We need to convert it
+      to a mask for Universe before applying it.
+      */
+      const worldMask = crossfilter.allSelectedMask();
+      const mask = World.worldEqUniverse(world, state)
+        ? worldMask
+        : AH.worldToUniverseMask(worldMask, world.obsAnnotations, state.nObs);
+      const obsAnnotations = AH.setLabelByMask(
+        state.obsAnnotations,
+        metadataField,
+        mask,
+        label
+      );
+      return { ...state, obsAnnotations };
     }
 
     default: {

client/src/reducers/world.js

@@ -1,7 +1,22 @@
-import { World, ControlsHelpers } from "../util/stateManager";
+import { unassignedCategoryLabel } from "../globals";
+import {
+  World,
+  ControlsHelpers as CH,
+  AnnotationsHelpers as AH
+} from "../util/stateManager";
 import clip from "../util/clip";
 import quantile from "../util/quantile";
 
+/*
+important note: much of this code assumes that wriable (user) annotations
+will NOT contain scalar data (ie, will only contain categorical labelled
+data), and therefore will never need to be clipped.  Put another way, it
+assumes that for these annotations, the clipped & unclipped data is equal.
+
+If we ever start allowing user editable scalar data, this assumption will
+need to be revisited.
+*/
+
 const WorldReducer = (
   state = null,
   action,
@@ -9,16 +24,13 @@ const WorldReducer = (
   prevSharedState
 ) => {
   switch (action.type) {
-    case "initial data load complete (universe exists)": {
+    case "initial data load complete (universe exists)":
+    case "reset World to eq Universe": {
       const { universe } = nextSharedState;
       const world = World.createWorldFromEntireUniverse(universe);
       return world;
     }
 
-    case "reset World to eq Universe": {
-      return prevSharedState.resetCache.world;
-    }
-
     case "set World to current selection": {
       /* Set viewable world to be the currently selected data */
       const world = World.createWorldBySelection(
@@ -80,7 +92,7 @@ const WorldReducer = (
           Object.keys(action.expressionData)
         )
       ];
-      unclippedVarData = ControlsHelpers.pruneVarDataCache(
+      unclippedVarData = CH.pruneVarDataCache(
         unclippedVarData,
         allTheGenesWeNeed
       );
@@ -122,6 +134,135 @@ const WorldReducer = (
       };
     }
 
+    case "annotation: create category": {
+      const name = action.data;
+      const { universe } = nextSharedState;
+      const { schema } = universe;
+
+      /*
+      if world !== universe, we have to subset the newly created annotation,
+      else, just use it as is.
+      */
+      let newAnnotation = null;
+      if (!World.worldEqUniverse(state, universe)) {
+        newAnnotation = universe.obsAnnotations
+          .subset(state.obsAnnotations.rowIndex.keys(), [name], null)
+          .icol(0)
+          .asArray();
+      } else {
+        newAnnotation = universe.obsAnnotations.col(name).asArray();
+      }
+      const obsAnnotations = state.obsAnnotations.withCol(
+        name,
+        newAnnotation,
+        state.obsAnnotations.rowIndex
+      );
+      const unclipped = {
+        ...state.unclipped,
+        obsAnnotations: state.unclipped.obsAnnotations.withCol(
+          name,
+          newAnnotation,
+          state.unclipped.obsAnnotations.rowIndex
+        )
+      };
+      return { ...state, schema, obsAnnotations, unclipped };
+    }
+
+    case "annotation: category edited": {
+      /* change the name of an obs annotation */
+      const name = action.metadataField;
+      const newName = action.newCategoryText;
+      const { schema } = nextSharedState.universe;
+      const obsAnnotations = state.obsAnnotations.renameCol(name, newName);
+      const unclipped = {
+        ...state.unclipped,
+        obsAnnotations: state.unclipped.obsAnnotations.renameCol(name, newName)
+      };
+      return { ...state, schema, obsAnnotations, unclipped };
+    }
+
+    case "annotation: delete category": {
+      /* remove a category from obs annotation */
+      const { schema } = nextSharedState.universe;
+      const name = action.metadataField;
+      const obsAnnotations = state.obsAnnotations.dropCol(name);
+      const unclipped = {
+        ...state.unclipped,
+        obsAnnotations: state.unclipped.obsAnnotations.dropCol(name)
+      };
+      return { ...state, schema, obsAnnotations, unclipped };
+    }
+
+    case "annotation: add new label to category": {
+      /* add a new label to the schema - schema updated by universe reducer, we just need to note it */
+      const { schema } = nextSharedState.universe;
+      return { ...state, schema };
+    }
+
+    case "annotation: label edited": {
+      const { schema } = nextSharedState.universe;
+      const { metadataField } = action;
+      const oldLabelName = action.label;
+      const newLabelName = action.editedLabel;
+
+      /* set all values to to new label */
+      const unclipped = {
+        ...state.unclipped,
+        obsAnnotations: AH.setLabelByValue(
+          state.unclipped.obsAnnotations,
+          metadataField,
+          oldLabelName,
+          newLabelName
+        )
+      };
+      const obsAnnotations = state.obsAnnotations.replaceColData(
+        metadataField,
+        unclipped.obsAnnotations.col(metadataField).asArray()
+      );
+      return { ...state, schema, obsAnnotations, unclipped };
+    }
+
+    case "annotation: delete label": {
+      const { schema } = nextSharedState.universe;
+      const { label, metadataField } = action;
+
+      /* set all values to unassigned in obsAnnotations */
+      const unclipped = {
+        ...state.unclipped,
+        obsAnnotations: AH.setLabelByValue(
+          state.unclipped.obsAnnotations,
+          metadataField,
+          label,
+          unassignedCategoryLabel
+        )
+      };
+      const obsAnnotations = state.obsAnnotations.replaceColData(
+        metadataField,
+        unclipped.obsAnnotations.col(metadataField).asArray()
+      );
+      return { ...state, schema, obsAnnotations, unclipped };
+    }
+
+    case "annotation: label current cell selection": {
+      const { metadataField, label } = action;
+      const { crossfilter } = prevSharedState;
+      const mask = crossfilter.allSelectedMask();
+      const unclipped = {
+        ...state.unclipped,
+        obsAnnotations: AH.setLabelByMask(
+          state.unclipped.obsAnnotations,
+          metadataField,
+          mask,
+          label
+        )
+      };
+      const obsAnnotations = state.obsAnnotations.replaceColData(
+        metadataField,
+        unclipped.obsAnnotations.col(metadataField).asArray()
+      );
+      return { ...state, obsAnnotations, unclipped };
+    }
+
     default: {
       return state;
     }

client/src/util/dataframe/dataframe.js

@@ -345,12 +345,50 @@ class Dataframe {
     );
   }
 
+  withColsFrom(dataframe) {
+    /*
+    return a new dataframe containing all columns from both `this` and the
+    provided dataframe.
+
+    The row index from `this` will be used.  Both dataframes must have identical
+    dimensionality, and no overlapping columns labels.
+     */
+    const dims = [this.dims[0], this.dims[1] + dataframe.dims[1]];
+    const { rowIndex } = this;
+    const columns = [...this.__columns, ...dataframe.__columns];
+    const colIndex = this.colIndex.withLabels(dataframe.colIndex.keys());
+    const columnsAccessor = [
+      ...this.__columnsAccessor,
+      ...dataframe.__columnsAccessor
+    ];
+    return new this.constructor(
+      dims,
+      columns,
+      rowIndex,
+      colIndex,
+      columnsAccessor
+    );
+  }
+
   dropCol(label) {
     /*
     Create a new dataframe, omitting one columns.
 
     const newDf = df.dropCol("colors");
+
+    Corner case to manage: if dropping the last column, return an empty dataframe.
     */
+    if (!this.hasCol(label)) {
+      throw new RangeError(`unknown label: ${label}`);
+    }
+
+    /* 
+    Corner case to manage: if dropping the last column, return an empty dataframe. 
+    */
+    if (this.dims[1] === 1) {
+      return Dataframe.empty();
+    }
+
     const dims = [this.dims[0], this.dims[1] - 1];
     const coffset = this.colIndex.getOffset(label);
     const columns = [...this.__columns];
@@ -367,6 +405,50 @@ class Dataframe {
     );
   }
 
+  renameCol(oldLabel, newLabel) {
+    /*
+    Accelerator for dropping a column and then adding it again with a new label
+    */
+    const coffset = this.colIndex.getOffset(oldLabel);
+    const colIndex = this.colIndex.dropLabel(oldLabel).withLabel(newLabel);
+
+    const columns = [...this.__columns];
+    columns.push(columns[coffset]);
+    columns.splice(coffset, 1);
+
+    const columnsAccessor = [...this.__columnsAccessor];
+    columnsAccessor.push(columnsAccessor[coffset]);
+    columnsAccessor.splice(coffset, 1);
+
+    return new this.constructor(
+      this.dims,
+      columns,
+      this.rowIndex,
+      colIndex,
+      columnsAccessor
+    );
+  }
+
+  replaceColData(label, newColData) {
+    /*
+    Accelerator for dropping a column then adding it again with same
+    label and different values.
+    */
+    const coffset = this.colIndex.getOffset(label);
+    const columns = [...this.__columns];
+    columns[coffset] = newColData;
+    const columnsAccessor = [...this.__columnsAccessor];
+    columnsAccessor[coffset] = null;
+
+    return new this.constructor(
+      this.dims,
+      columns,
+      this.rowIndex,
+      this.colIndex,
+      columnsAccessor
+    );
+  }
+
   static empty(rowIndex = null, colIndex = null) {
     return new Dataframe([0, 0], [], rowIndex, colIndex);
   }
@@ -443,6 +525,8 @@ class Dataframe {
         return newCol;
       });
     }
+
+    if (dims[0] === 0 || dims[1] === 0) return Dataframe.empty();
     return new Dataframe(dims, columns, rowIndex, colIndex);
   }
 
@@ -526,6 +610,11 @@ class Dataframe {
   Data access with row/col.
   **/
 
+  columns() {
+    /* return all column accessors as an array, in offset order */
+    return [...this.__columnsAccessor];
+  }
+
   col(columnLabel) {
     /*
     Return accessor bound to a column.  Allows random row access

client/src/util/dataframe/labelIndex.js

@@ -80,6 +80,10 @@ class IdentityInt32Index {
     return this.__promote([...this.keys(), label]);
   }
 
+  withLabels(labels) {
+    return this.__promote([...this.keys(), ...labels]);
+  }
+
   dropLabel(label) {
     if (label === this.maxOffset - 1) {
       return new IdentityInt32Index(label);
@@ -163,6 +167,10 @@ class DenseInt32Index {
     return this.__promote([...this.keys(), label]);
   }
 
+  withLabels(labels) {
+    return this.__promote([...this.keys(), ...labels]);
+  }
+
   dropLabel(label) {
     const labelArray = [...this.keys()];
     labelArray.splice(labelArray.indexOf(label), 1);
@@ -187,6 +195,11 @@ class KeyIndex {
       index.set(v, i);
     });
 
+    if (index.size !== rindex.length) {
+      /* if true, there was a duplicate in the keys */
+      throw new Error("duplicate label provided to KeyIndex");
+    }
+
     this.index = index;
     this.rindex = rindex;
     this.__compile();
@@ -218,6 +231,10 @@ class KeyIndex {
     return new KeyIndex([...this.rindex, label]);
   }
 
+  withLabels(labels) {
+    return new KeyIndex([...this.rindex, ...labels]);
+  }
+
   dropLabel(label) {
     const idx = this.rindex.indexOf(label);
     const labelArray = [...this.rindex];

client/src/util/dataframe/util.js

@@ -21,7 +21,7 @@ export function callOnceLazy(f) {
   return result;
 }
 
-export function memoize(fn, hashFn) {
+export function memoize(fn, hashFn, maxResultsCached = -1) {
   /* 
   function memoization, with user-provided hash.  hashFn must return a
   key which will be unique as a Map key (ie, obeys "sameValueZero" algorithm
@@ -36,7 +36,19 @@ export function memoize(fn, hashFn) {
     }
     const result = fn(...args);
     cache.set(key, result);
+
+    if (maxResultsCached > -1 && cache.size > maxResultsCached) {
+      /* Least recent insertion deletion */
+      cache.delete(cache.keys().next().value);
+    }
+
     return result;
   };
+
+  wrap.clear = function clear() {
+    /* clear memoization cache */
+    cache.clear();
+  };
+
   return wrap;
 }

client/src/util/stateManager/annotationsHelpers.js

@@ -0,0 +1,168 @@
+/*
+Helper functions for user-editable nnotations state management.
+See also reducers/annotations.js
+*/
+import { unassignedCategoryLabel } from "../../globals";
+import * as SchemaHelpers from "./schemaHelpers";
+import { obsAnnoDimensionName } from "../nameCreators";
+
+/*
+There are a number of state constraints assumed throughout the
+application:
+	- all obs annotations are in {world|universe}.obsAnnotations,
+		regardless of whether or not they are user editable.
+	- the {world|universe}.schema is always up to date and matches
+		the data
+	- the schema flag `writable` correctly indicates whether
+	  the annotation is editable/mutable.
+
+In addition, the current state management only allows for
+categorical annotations to be writable.
+*/
+
+export function isCategoricalAnnotation(schema, name) {
+	/* we treat any string, categorical or boolean as a categorical */
+	const { type } = schema.annotations.obsByName[name];
+	return type === "string" || type === "boolean" || type === "categorical";
+}
+
+export function isContinuousAnnotation(schema, name) {
+	return !isCategoricalAnnotation(schema, name);
+}
+
+function _isUserAnnotation(schema, name) {
+	return schema.annotations.obsByName[name]?.writable;
+}
+
+export function isUserAnnotation(worldOrUniverse, name) {
+	return _isUserAnnotation(worldOrUniverse.schema, name);
+}
+
+export function removeObsAnnoSchema(schema, name) {
+	/*
+	remove named annotation from obs annotation schema
+	*/
+
+	/* only remove if it exists and is a user annotation */
+	if (!_isUserAnnotation(schema, name))
+		throw new Error("removing non-user-defined schema");
+	return SchemaHelpers.removeObsAnnoColumn(schema, name);
+}
+
+export function addObsAnnoSchema(schema, name, colSchema) {
+	/*
+	add a categorical type to the obs annotation schema 
+	*/
+
+	/* collision detection */
+	if (schema.annotations.obs.columns.some(v => v.name === name))
+		throw Error("annotations may not contain duplicate category names");
+	if (name !== colSchema.name) throw Error("column schema does not match");
+	return SchemaHelpers.addObsAnnoColumn(schema, name, colSchema);
+}
+
+export function dupObsAnnoSchema(schema, sourceName, dupName, defaultSchema) {
+	/*
+	duplicate the obs annotation `sourceName` schema, but with the name `dupName`
+	*/
+	const colSchema = {
+		...schema.annotations.obsByName[sourceName],
+		...defaultSchema,
+		name: dupName
+	};
+	/* existance check */
+	if (!colSchema) throw Error("source annotation does not exist");
+	/* collision detection */
+	if (schema.annotations.obs.columns.some(v => v.name === dupName))
+		throw Error("annotations may not contain duplicate category names");
+	return SchemaHelpers.addObsAnnoColumn(schema, dupName, colSchema);
+}
+
+export function removeObsAnnoCategory(schema, name, category) {
+	/* don't allow deletion of unassigned category on writable annotations */
+
+	if (!_isUserAnnotation(schema, name))
+		throw new Error("unable to modify read-only schema");
+	if (category === unassignedCategoryLabel)
+		throw new Error("may not remove unassigned category label");
+
+	return SchemaHelpers.removeObsAnnoCategory(schema, name, category);
+}
+
+export function addObsAnnoCategory(schema, name, category) {
+	if (!_isUserAnnotation(schema, name))
+		throw new Error("unable to modify read-only schema");
+
+	return SchemaHelpers.addObsAnnoCategory(schema, name, category);
+}
+
+export function setLabelByValue(df, colName, fromLabel, toLabel) {
+	/* 
+	in the dataframe column `colName`, set any value of `fromLabel` to `toLabel`
+	*/
+	const keys = df.colIndex.keys();
+	const ndf = df.mapColumns((col, colIdx) => {
+		if (colName !== keys[colIdx]) return col;
+
+		/* clone data and return it. */
+		const newCol = col.slice();
+		for (let i = 0, l = newCol.length; i < l; i += 1) {
+			if (newCol[i] === fromLabel) newCol[i] = toLabel;
+		}
+		return newCol;
+	});
+	return ndf;
+}
+
+export function setLabelByMask(df, colName, mask, label) {
+	/* 
+	in the dataframe column `colName`, set the masked rows to 'label'
+	*/
+	const keys = df.colIndex.keys();
+	const ndf = df.mapColumns((col, colIdx) => {
+		if (colName !== keys[colIdx]) return col;
+
+		/* clone data and return it. */
+		const newCol = col.slice();
+		for (let i = 0, l = newCol.length; i < l; i += 1) {
+			if (mask[i]) newCol[i] = label;
+		}
+		return newCol;
+	});
+	return ndf;
+}
+
+export function worldToUniverseMask(worldMask, worldObsAnnotations, nObs) {
+	/*
+	given world seleciton mask, return a selection mask for entire universe 
+	that has same selection state.
+	*/
+	const mask = new Uint8Array(nObs);
+	const { rowIndex } = worldObsAnnotations;
+
+	for (let i = 0, l = worldMask.length; i < l; i += 1) {
+		if (worldMask[i]) {
+			const label = rowIndex.getLabel(i);
+			mask[label] = 1;
+		}
+	}
+	return mask;
+}
+
+export function createWritableAnnotationDimensions(world, crossfilter) {
+	const { obsAnnotations, schema } = world;
+	const writableAnnotations = schema.annotations.obs.columns
+		.filter(s => s.writable)
+		.map(s => s.name);
+
+	crossfilter = writableAnnotations.reduce((xflt, anno) => {
+		const dimName = obsAnnoDimensionName(anno);
+		if (xflt.hasDimension(dimName)) xflt = xflt.delDimension(dimName);
+		return xflt.addDimension(
+			dimName,
+			"enum",
+			obsAnnotations.col(anno).asArray()
+		);
+	}, crossfilter);
+	return crossfilter;
+}

client/src/util/stateManager/colorHelpers.js

@@ -50,8 +50,9 @@ function createColorsByCategoricalMetadata(world, accessor) {
   }, {});
 
   const rgb = new Array(world.nObs);
-  const data = world.obsAnnotations.col(accessor).asArray();
-  for (let i = 0, len = world.obsAnnotations.length; i < len; i += 1) {
+  const df = world.obsAnnotations;
+  const data = df.col(accessor).asArray();
+  for (let i = 0, len = df.length; i < len; i += 1) {
     const cat = data[i];
     rgb[i] = colors[cat];
   }

client/src/util/stateManager/controlsHelpers.js

@@ -37,16 +37,14 @@ Remember that option values can be ANY js type, except undefined/null.
     }
   }
 */
-function topNCategories(summary) {
-  const counts = _.map(summary.categories, cat =>
-    summary.categoryCounts.get(cat)
-  );
-  const sortIndex = fillRange(new Array(summary.numCategories)).sort(
+function topNCategories(colSchema, summary, N) {
+  const { categories } = colSchema;
+  const counts = _.map(categories, cat => summary.categoryCounts.get(cat) ?? 0);
+  const sortIndex = fillRange(new Array(categories.length)).sort(
     (a, b) => counts[b] - counts[a]
   );
-  const sortedCategories = _.map(sortIndex, i => summary.categories[i]);
+  const sortedCategories = _.map(sortIndex, i => categories[i]);
   const sortedCounts = _.map(sortIndex, i => counts[i]);
-  const N = globals.maxCategoricalOptionsToDisplay;
 
   if (sortedCategories.length < N) {
     return [sortedCategories, sortedCounts];
@@ -54,64 +52,56 @@ function topNCategories(summary) {
   return [sortedCategories.slice(0, N), sortedCounts.slice(0, N)];
 }
 
-export function createCategoricalSelection(maxCategoryItems, world) {
-  const res = {};
-  const obsIndexName = world.schema.annotations.obs.index;
-  _.forEach(world.obsAnnotations.colIndex.keys(), key => {
-    const summary = world.obsAnnotations.col(key).summarize();
-    if (summary.categories) {
-      const isColorField = key.includes("color") || key.includes("Color");
-      const isSelectableCategory =
-        !isColorField &&
-        key !== obsIndexName &&
-        summary.categories.length < maxCategoryItems;
-      if (isSelectableCategory) {
-        const [categoryValues, categoryValueCounts] = topNCategories(summary);
-        const categoryValueIndices = new Map(
-          categoryValues.map((v, i) => [v, i])
-        );
-        const numCategoryValues = categoryValueIndices.size;
-        const categoryValueSelected = new Array(numCategoryValues).fill(true);
-        const isTruncated = categoryValues.length < summary.numCategories;
-        res[key] = {
-          categoryValues, // array: of natively typed category values
-          categoryValueIndices, // map: category value (native type) -> category index
-          categoryValueSelected, // array: t/f selection state
-          numCategoryValues, // number: of values in the category
-          isTruncated, // bool: true if list was truncated
-          categoryValueCounts, // array: cardinality of each category,
-          categorySelected: true // bool - default state for entire category
-        };
-      }
-    }
-  });
-  return res;
+export function selectableCategoryNames(world, maxCategoryItems) {
+  const { schema } = world;
+  const { index, columns } = schema.annotations.obs;
+  return columns
+    .filter(colSchema => {
+      const { name, categories } = colSchema;
+      return (
+        categories && categories.length < maxCategoryItems && name !== index
+      );
+    })
+    .map(v => v.name);
 }
 
-/*
-given a categoricalSelection, return the list of all category values
-where selection state is true (ie, they are selected).
-*/
-export function selectedValuesForCategory(categorySelectionState, dfColumn) {
-  const {
-    categorySelected,
-    categoryValueSelected,
-    categoryValueIndices
-  } = categorySelectionState;
-  let selectedValues;
-  if (categorySelected) {
-    selectedValues = new Set(dfColumn.summarize().categories);
-  } else {
-    selectedValues = new Set();
-  }
-  categoryValueIndices.forEach((catIndex, catValue) => {
-    if (!categoryValueSelected[catIndex]) {
-      selectedValues.delete(catValue);
-    } else {
-      selectedValues.add(catValue);
-    }
-  });
-  return [...selectedValues.values()];
+export function createCategoricalSelection(world, names) {
+  const N = globals.maxCategoricalOptionsToDisplay;
+  const { obsAnnotations, schema } = world;
+
+  const res = names.reduce((acc, name) => {
+    const colSchema = schema.annotations.obsByName[name];
+    const { writable: isUserAnno } = colSchema;
+
+    /*
+    Summarize the annotation data currently in world.  Must return categoryValues
+    in sorted order, and must include all category values even if they are not
+    actively used in the current world.
+    */
+    const summary = obsAnnotations.col(name).summarize();
+    const [categoryValues, categoryValueCounts] = topNCategories(
+      colSchema,
+      summary,
+      N
+    );
+    const categoryValueIndices = new Map(categoryValues.map((v, i) => [v, i]));
+    const numCategoryValues = categoryValueIndices.size;
+    const categoryValueSelected = new Array(numCategoryValues).fill(true);
+    const isTruncated = categoryValues.length < summary.numCategories;
+
+    acc[name] = {
+      categoryValues, // array: of natively typed category values
+      categoryValueIndices, // map: category value (native type) -> category index
+      categoryValueSelected, // array: t/f selection state
+      numCategoryValues, // number: of values in the category
+      isTruncated, // bool: true if list was truncated
+      categoryValueCounts, // array: cardinality of each category,
+      categorySelected: true, // bool - default state for entire category
+      isUserAnno // bool
+    };
+    return acc;
+  }, {});
+  return res;
 }
 
 /*

client/src/util/stateManager/index.js

@@ -19,3 +19,6 @@ export * as Universe from "./universe";
 export * as World from "./world";
 export * as WorldUtil from "./worldUtil";
 export * as ControlsHelpers from "./controlsHelpers";
+export * as AnnotationsHelpers from "./annotationsHelpers";
+export * as SchemaHelpers from "./schemaHelpers";
+export * as MatrixFBS from "./matrix";

client/src/util/stateManager/matrix.js

@@ -1,5 +1,7 @@
 import { flatbuffers } from "flatbuffers";
 import { NetEncoding } from "./matrix_generated";
+import { isTypedArray } from "../typeHelpers";
+import { IdentityInt32Index, DenseInt32Index, KeyIndex } from "../dataframe";
 
 const utf8Decoder = new TextDecoder("utf-8");
 
@@ -41,25 +43,25 @@ Returns: object containing decoded Matrix:
   colIdx: []|null
 }
 */
-function decodeMatrixFBS(arrayBuffer, inplace = false) {
+export function decodeMatrixFBS(arrayBuffer, inplace = false) {
   const bb = new flatbuffers.ByteBuffer(new Uint8Array(arrayBuffer));
-  const df = NetEncoding.Matrix.getRootAsMatrix(bb);
+  const matrix = NetEncoding.Matrix.getRootAsMatrix(bb);
 
-  const nRows = df.nRows();
-  const nCols = df.nCols();
+  const nRows = matrix.nRows();
+  const nCols = matrix.nCols();
 
   /* decode columns */
-  const columnsLength = df.columnsLength();
+  const columnsLength = matrix.columnsLength();
   const columns = Array(columnsLength).fill(null);
   for (let c = 0; c < columnsLength; c += 1) {
-    const col = df.columns(c);
+    const col = matrix.columns(c);
     columns[c] = decodeTypedArray(col.uType(), col.u.bind(col), inplace);
   }
 
   /* decode col_idx */
   const colIdx = decodeTypedArray(
-    df.colIndexType(),
-    df.colIndex.bind(df),
+    matrix.colIndexType(),
+    matrix.colIndex.bind(matrix),
     inplace
   );
 
@@ -72,4 +74,91 @@ function decodeMatrixFBS(arrayBuffer, inplace = false) {
   };
 }
 
-export default decodeMatrixFBS;
+function encodeTypedArray(builder, uType, uData) {
+  const uTypeName = NetEncoding.TypedArray[uType];
+  const ArrayType = NetEncoding[uTypeName];
+  const dv = ArrayType.createDataVector(builder, uData);
+  builder.startObject(1);
+  builder.addFieldOffset(0, dv, 0);
+  return builder.endObject();
+}
+
+export function encodeMatrixFBS(df) {
+  /*
+  encode the dataframe as an FBS Matrix
+  */
+
+  /* row indexing not supported currently */
+  if (df.rowIndex.constructor !== IdentityInt32Index) {
+    throw new Error("FBS does not support row index encoding at this time");
+  }
+
+  const shape = df.dims;
+  const utf8Encoder = new TextEncoder("utf-8");
+  const builder = new flatbuffers.Builder(1024);
+
+  let encColIndex;
+  let encColIndexUType;
+  let encColumns;
+
+  if (shape[0] > 0 && shape[1] > 0) {
+    const columns = df.columns().map(col => col.asArray());
+
+    const cols = columns.map(carr => {
+      let uType;
+      let tarr;
+      if (isTypedArray(carr)) {
+        uType = NetEncoding.TypedArray[carr.constructor.name];
+        tarr = encodeTypedArray(builder, uType, carr);
+      } else {
+        uType = NetEncoding.TypedArray.JSONEncodedArray;
+        const json = JSON.stringify(carr);
+        const jsonUTF8 = utf8Encoder.encode(json);
+        tarr = encodeTypedArray(builder, uType, jsonUTF8);
+      }
+      NetEncoding.Column.startColumn(builder);
+      NetEncoding.Column.addUType(builder, uType);
+      NetEncoding.Column.addU(builder, tarr);
+      return NetEncoding.Column.endColumn(builder);
+    });
+
+    encColumns = NetEncoding.Matrix.createColumnsVector(builder, cols);
+
+    if (df.colIndex && shape[1] > 0) {
+      const colIndexType = df.colIndex.constructor;
+      if (colIndexType === IdentityInt32Index) {
+        encColIndex = undefined;
+      } else if (colIndexType === DenseInt32Index) {
+        encColIndexUType = NetEncoding.TypedArray.Int32Array;
+        encColIndex = encodeTypedArray(
+          builder,
+          encColIndexUType,
+          df.colIndex.keys()
+        );
+      } else if (colIndexType === KeyIndex) {
+        encColIndexUType = NetEncoding.TypedArray.JSONEncodedArray;
+        encColIndex = encodeTypedArray(
+          builder,
+          encColIndexUType,
+          utf8Encoder.encode(JSON.stringify(df.colIndex.keys()))
+        );
+      } else {
+        throw new Error("Index type FBS encoding unsupported");
+      }
+    }
+  }
+
+  NetEncoding.Matrix.startMatrix(builder);
+  NetEncoding.Matrix.addNRows(builder, shape[0]);
+  NetEncoding.Matrix.addNCols(builder, shape[1]);
+  if (encColumns) {
+    NetEncoding.Matrix.addColumns(builder, encColumns);
+  }
+  if (encColIndexUType) {
+    NetEncoding.Matrix.addColIndexType(builder, encColIndexUType);
+    NetEncoding.Matrix.addColIndex(builder, encColIndex);
+  }
+  const root = NetEncoding.Matrix.endMatrix(builder);
+  builder.finish(root);
+  return builder.asUint8Array();
+}

client/src/util/stateManager/schemaHelpers.js

@@ -0,0 +1,96 @@
+/*
+Helpers for schema management
+*/
+import _ from "lodash";
+
+import fromEntries from "../fromEntries";
+
+/*
+System wide schema assumptions:
+	- schema and data wil be consistent (eg, for user-created annotations)
+	- schema will be internally self-consistent (eg, index matches columns)
+	- world & universe schema are same - only data is subset
+*/
+
+export function indexEntireSchema(schema) {
+	/* Index schema for ease of use */
+	schema.annotations.obsByName = fromEntries(
+		schema.annotations.obs.columns.map(v => [v.name, v])
+	);
+	schema.annotations.varByName = fromEntries(
+		schema.annotations.var.columns.map(v => [v.name, v])
+	);
+	schema.layout.obsByName = fromEntries(
+		schema.layout.obs.map(v => [v.name, v])
+	);
+	schema.layout.varByName = fromEntries(
+		schema.layout.var.map(v => [v.name, v])
+	);
+
+	return schema;
+}
+
+function _copy(schema) {
+	/* redux copy conventions - WARNING, only for modifyign obs annotations */
+	return {
+		...schema,
+		annotations: {
+			...schema.annotations,
+			obs: _.cloneDeep(schema.annotations.obs)
+		}
+	};
+}
+
+function _reindex(schema) {
+	/* reindex obs annotations ONLY */
+	schema.annotations.obsByName = fromEntries(
+		schema.annotations.obs.columns.map(v => [v.name, v])
+	);
+	return schema;
+}
+
+export function removeObsAnnoColumn(schema, name) {
+	const newSchema = _copy(schema);
+	newSchema.annotations.obs.columns = schema.annotations.obs.columns.filter(
+		v => v.name !== name
+	);
+	return _reindex(newSchema);
+}
+
+export function addObsAnnoColumn(schema, name, defn) {
+	const newSchema = _copy(schema);
+	newSchema.annotations.obs.columns.push(defn);
+	return _reindex(newSchema);
+}
+
+export function removeObsAnnoCategory(schema, name, category) {
+	/* remove a category from a categorical annotation */
+	const categories = schema.annotations.obsByName[name]?.categories;
+	if (!categories)
+		throw new Error("column does not exist or is not categorical");
+
+	const idx = categories.indexOf(category);
+	if (idx === -1) throw new Error("category does not exist");
+
+	const newSchema = _reindex(_copy(schema));
+
+	/* remove category */
+	newSchema.annotations.obsByName[name].categories.splice(idx, 1);
+	return newSchema;
+}
+
+export function addObsAnnoCategory(schema, name, category) {
+	/* add a category to a categorical annotation */
+	const categories = schema.annotations.obsByName[name]?.categories;
+	if (!categories)
+		throw new Error("column does not exist or is not categorical");
+
+	const idx = categories.indexOf(category);
+	if (idx !== -1) throw new Error("category already exists");
+
+	const newSchema = _reindex(_copy(schema));
+
+	/* remove category */
+	newSchema.annotations.obsByName[name].categories.push(category);
+	return newSchema;
+}

client/src/util/stateManager/universe.js

@@ -1,11 +1,11 @@
-// jshint esversion: 6
-
 import _ from "lodash";
 
-import decodeMatrixFBS from "./matrix";
+import { unassignedCategoryLabel } from "../../globals";
+import { decodeMatrixFBS } from "./matrix";
 import * as Dataframe from "../dataframe";
-import fromEntries from "../fromEntries";
 import { isFpTypedArray } from "../typeHelpers";
+import { indexEntireSchema } from "./schemaHelpers";
+import { isCategoricalAnnotation } from "./annotationsHelpers";
 
 /*
 Private helper function - create and return a template Universe
@@ -18,10 +18,13 @@ function templateUniverse() {
     schema: {},
 
     /*
-    Annotations
+    annotations
     */
     obsAnnotations: Dataframe.Dataframe.empty(),
     varAnnotations: Dataframe.Dataframe.empty(),
+    /*
+    layout
+    */
     obsLayout: Dataframe.Dataframe.empty(),
 
     /*
@@ -122,6 +125,10 @@ function reconcileSchemaCategoriesWithSummary(universe) {
   For example, boolean defined fields in the schema do not contain
   explicit declaration of categories (nor do string fields).  In these
   cases, add a 'categories' field to the schema so it is accessible.
+
+  In addition, we have a client-side convention (UI) that all writable
+  annotations must have an 'unassigned' category, even if it is not currently
+  in use.
   */
 
   universe.schema.annotations.obs.columns.forEach(s => {
@@ -136,6 +143,10 @@ function reconcileSchemaCategoriesWithSummary(universe) {
       );
       s.categories = categories;
     }
+
+    if (s.writable && s.categories.indexOf(unassignedCategoryLabel) === -1) {
+      s.categories = s.categories.concat(unassignedCategoryLabel);
+    }
   });
 }
 
@@ -166,7 +177,7 @@ export function createUniverseFromResponse(
   /* layout */
   universe.obsLayout = LayoutFBSToDataframe(layoutFBSResponse);
 
-  /* sanity check */
+  /* sanity checks */
   if (
     universe.nObs !== universe.obsLayout.length ||
     universe.nObs !== universe.obsAnnotations.length ||
@@ -176,20 +187,19 @@ export function createUniverseFromResponse(
   }
 
   reconcileSchemaCategoriesWithSummary(universe);
+  indexEntireSchema(universe.schema);
+
+  /* sanity checks */
+  if (
+    schema.annotations.obs.columns.some(
+      s => s.writable && !isCategoricalAnnotation(schema, s.name)
+    )
+  ) {
+    throw new Error(
+      "Writable continuous obs annotations are not supproted - failed to laod"
+    );
+  }
 
-  /* Index schema for ease of use */
-  universe.schema.annotations.obsByName = fromEntries(
-    universe.schema.annotations.obs.columns.map(v => [v.name, v])
-  );
-  universe.schema.annotations.varByName = fromEntries(
-    universe.schema.annotations.var.columns.map(v => [v.name, v])
-  );
-  universe.schema.layout.obsByName = fromEntries(
-    universe.schema.layout.obs.map(v => [v.name, v])
-  );
-  universe.schema.layout.varByName = fromEntries(
-    universe.schema.layout.var.map(v => [v.name, v])
-  );
   return universe;
 }
 

client/src/util/stateManager/world.js

@@ -1,14 +1,7 @@
-// jshint esversion: 6
-
 import clip from "../clip";
-import {
-  layoutDimensionName,
-  obsAnnoDimensionName,
-  diffexpDimensionName,
-  userDefinedDimensionName
-} from "../nameCreators";
+import { layoutDimensionName, obsAnnoDimensionName } from "../nameCreators";
 import * as Dataframe from "../dataframe";
-import ImmutableTypedCrossfilter from "../typedCrossfilter/crossfilter";
+import { isContinuousAnnotation } from "./annotationsHelpers";
 
 /*
 
@@ -158,7 +151,7 @@ and world.varData.
 function setClippedDataframes(world) {
   const { schema } = world;
   const isContinuousObsAnnotation = (df, idx, label) =>
-    deduceDimensionType(schema.annotations.obsByName[label], label) !== "enum";
+    isContinuousAnnotation(schema, label);
   const obsQuantile = (label, q) =>
     world.unclipped.obsAnnotations.col(label).summarize().percentiles[100 * q];
   world.obsAnnotations = clipDataframe(
@@ -183,7 +176,7 @@ function setClippedDataframes(world) {
 /*
 Subset the current world based upon the current selection, maintaining any existing
 clip.  Returns new world.  Parameters:
-  * unvierse
+  * universe
   * world - the current world
   * crossfilter - the selection state
 */

client/src/util/typedCrossfilter/crossfilter.js

@@ -5,6 +5,7 @@ import BitArray from "./bitArray";
 import {
   sortArray,
   lowerBound,
+  binarySearch,
   lowerBoundIndirect,
   upperBoundIndirect
 } from "./sort";
@@ -56,6 +57,14 @@ export default class ImmutableTypedCrossfilter {
     return this.data;
   }
 
+  setData(data) {
+    return new ImmutableTypedCrossfilter(
+      data,
+      this.dimensions,
+      this.selectionCache
+    );
+  }
+
   dimensionNames() {
     /* return array of all dimensions (by name) */
     return Object.keys(this.dimensions);
@@ -113,6 +122,23 @@ export default class ImmutableTypedCrossfilter {
     return new ImmutableTypedCrossfilter(data, dimensions, selectionCache);
   }
 
+  renameDimension(oldName, newName) {
+    /*
+    rename a dimension
+    */
+    const { [oldName]: dim, ...dimensions } = this.dimensions;
+    const { data, selectionCache } = this;
+    dim.dim.rename(newName);
+    return new ImmutableTypedCrossfilter(
+      data,
+      {
+        ...dimensions,
+        [newName]: dim
+      },
+      selectionCache
+    );
+  }
+
   select(name, spec) {
     /*
     select on named dimension, as indicated by `spec`.   Spec is an object
@@ -288,6 +314,10 @@ class _ImmutableBaseDimension {
     this.name = name;
   }
 
+  rename(name) {
+    this.name = name;
+  }
+
   select(spec) {
     const { mode } = spec;
     if (mode === undefined) {
@@ -436,7 +466,7 @@ class ImmutableEnumDimension extends ImmutableScalarDimension {
     const { values } = spec;
     return super.selectExact({
       mode: spec.mode,
-      values: values.map(v => lowerBound(enumIndex, v, 0, enumIndex.length))
+      values: values.map(v => binarySearch(enumIndex, v, 0, enumIndex.length))
     });
   }
 

client/src/util/typedCrossfilter/sort.js

@@ -413,3 +413,16 @@ export function upperBoundIndirect(valueArray, indexArray, value, first, last) {
   }
   return upperBoundNonFloatIndirect(valueArray, indexArray, value, first, last);
 }
+
+// Search for `value` in the sorted array `arr`, in the range [first, last).
+// Return the first index where arr[index] == value, OR if value not present,
+// return `last`
+//
+// The same semantics/behavior as:
+//    C++: binary_search()
+//
+export function binarySearch(valueArray, value, first, last) {
+  const index = lowerBound(valueArray, value, first, last);
+  if (index !== last && value === valueArray[index]) return index;
+  return last;
+}