experimental re-embedding (#1186)

* first cut at re-embedding route and back-end support

* update and expand config route tests

* add scanpy_umap

* add reembedding to config route parameters

* front-end support for reembedding fetch and UI

* remove unused imports

* add loading state

* save reembedding in reducer state

* improve withColsFrom

* transmit reembed schema to client; pick unique embedding names

* display embeddings

* format

* lint

* spaces, tab size 2

* lint

* test hack for smoke-test race

* back out hack sleep

* add check for backed mode

* add unit test for reembedding

* lint

* hide re-embedding CLI param from help

client/src/actions/index.js

@@ -7,6 +7,7 @@ import {
   doBinaryRequest,
   dispatchNetworkErrorMessageToUser
 } from "../util/actionHelpers";
+import { requestReembed, reembedResetWorldToUniverse } from "./reembed";
 
 /*
 return promise to fetch the OBS annotations we need to load.  Omit anything
@@ -387,6 +388,7 @@ const requestDifferentialExpression = (set1, set2, num_genes = 10) => async (
 
 const resetWorldToUniverse = () => (dispatch, getState) => {
   const { universe } = getState();
+  reembedResetWorldToUniverse(dispatch, getState);
   dispatch({
     type: "reset World to eq Universe",
     universe
@@ -451,7 +453,8 @@ export default {
   requestDifferentialExpression,
   requestSingleGeneExpressionCountsForColoringPOST,
   requestUserDefinedGene,
+  requestReembed,
   resetWorldToUniverse,
   saveObsAnnotations,
-  setWorldToSelection,
+  setWorldToSelection
 };

client/src/actions/reembed.js

@@ -0,0 +1,113 @@
+import { API } from "../globals";
+import { Universe } from "../util/stateManager";
+import {
+  postNetworkErrorToast,
+  postAsyncSuccessToast,
+  postAsyncFailureToast
+} from "../components/framework/toasters";
+
+function abortableFetch(request, opts, timeout = 0) {
+  const controller = new AbortController();
+  const { signal } = controller;
+
+  return {
+    abort: () => controller.abort(),
+    isAborted: () => signal.aborted,
+    ready: () => {
+      if (timeout) {
+        setTimeout(() => controller.abort(), timeout);
+      }
+      return fetch(request, { ...opts, signal });
+    }
+  };
+}
+
+async function doReembedFetch(dispatch, getState) {
+  const state = getState();
+  let cells = state.world.obsAnnotations.rowIndex.keys();
+
+  // These lines ensure that we convert any TypedArray to an Array.
+  // This is necessary because JSON.stringify() does some very strange
+  // things with TypedArrays (they are marshalled to JSON objects, rather
+  // than being marshalled as a JSON array).
+  cells = Array.isArray(cells) ? cells : Array.from(cells);
+
+  const af = abortableFetch(
+    `${API.prefix}${API.version}layout/obs`,
+    {
+      method: "PUT",
+      headers: new Headers({
+        Accept: "application/octet-stream",
+        "Content-Type": "application/json"
+      }),
+      body: JSON.stringify({
+        method: "umap",
+        filter: { obs: { index: cells } }
+      }),
+      credentials: "include"
+    },
+    60000 // 1 minute timeout
+  );
+  dispatch({
+    type: "reembed: request start",
+    abortableFetch: af
+  });
+  const res = await af.ready();
+
+  if (
+    res.ok &&
+    res.headers.get("Content-Type").includes("application/octet-stream")
+  ) {
+    return res;
+  }
+
+  // else an error
+  let msg = `Unexpected HTTP response ${res.status}, ${res.statusText}`;
+  const body = await res.text();
+  if (body && body.length > 0) {
+    msg = `${msg} -- ${body}`;
+  }
+  postNetworkErrorToast(msg);
+  throw new Error(msg);
+}
+
+/*
+functions below are dispatch-able
+*/
+export function requestReembed() {
+  return async (dispatch, getState) => {
+    try {
+      const res = await doReembedFetch(dispatch, getState);
+      const schema = JSON.parse(res.headers.get("CxG-Schema"));
+      const buffer = await res.arrayBuffer();
+      const df = Universe.matrixFBSToDataframe(buffer);
+      dispatch({
+        type: "reembed: request completed"
+      });
+      dispatch({
+        type: "reembed: add reembedding",
+        embedding: df,
+        schema
+      });
+      postAsyncSuccessToast("Re-embedding has completed.");
+    } catch (error) {
+      dispatch({
+        type: "reembed: request aborted"
+      });
+      if (error.name === "AbortError") {
+        postAsyncFailureToast("Re-embedding calculation was aborted.");
+      } else {
+        postNetworkErrorToast(`Re-embedding: ${error.message}`);
+      }
+      console.log("Reembed exception:", error, error.name, error.message);
+    }
+  };
+}
+
+export function reembedResetWorldToUniverse(dispatch, getState) {
+  const { reembedController } = getState();
+  if (reembedController.pendingFetch) reembedController.pendingFetch.abort();
+  dispatch({
+    type: "reembed: clear all reembeddings"
+  });
+}

client/src/components/framework/toasters.js

@@ -2,7 +2,7 @@ import { Position, Toaster, Intent } from "@blueprintjs/core";
 
 /** Singleton toaster instance. Create separate instances for different options. */
 
-const ErrorToastTopCenter = Toaster.create({
+const ToastTopCenter = Toaster.create({
   className: "recipe-toaster",
   position: Position.TOP
 });
@@ -11,21 +11,38 @@ const ErrorToastTopCenter = Toaster.create({
 A "user" error - eg, bad input
 */
 export const postUserErrorToast = message =>
-  ErrorToastTopCenter.show({ message, intent: Intent.WARNING });
+  ToastTopCenter.show({ message, intent: Intent.WARNING });
 
 /*
 A toast the user must dismiss manually, because they need to act on its information,
 ie., 8 bulk add genes out of 40 were bad. Manually see which ones and fix.
 */
 export const keepAroundErrorToast = message =>
-  ErrorToastTopCenter.show({ message, timeout: 0, intent: Intent.WARNING });
+  ToastTopCenter.show({ message, timeout: 0, intent: Intent.WARNING });
 
 /*
 a hard network error
 */
 export const postNetworkErrorToast = message =>
-  ErrorToastTopCenter.show({
+  ToastTopCenter.show({
     message,
     timeout: 30000,
     intent: Intent.DANGER
   });
+
+/*
+Async message to user
+*/
+export const postAsyncSuccessToast = message =>
+  ToastTopCenter.show({
+    message,
+    timeout: 10000,
+    intent: Intent.SUCCESS
+  });
+
+export const postAsyncFailureToast = message =>
+  ToastTopCenter.show({
+    message,
+    timeout: 10000,
+    intent: Intent.WARNING
+  });

client/src/components/menubar/embedding.js

@@ -0,0 +1,122 @@
+import React from "react";
+import {
+  AnchorButton,
+  ButtonGroup,
+  Popover,
+  Button,
+  Radio,
+  RadioGroup,
+  Tooltip,
+  Position
+} from "@blueprintjs/core";
+import { connect } from "react-redux";
+import * as globals from "../../globals";
+import { World } from "../../util/stateManager";
+import actions from "../../actions";
+
+@connect(state => ({
+  universe: state.universe,
+  world: state.world,
+  layoutChoice: state.layoutChoice,
+  reembedController: state.reembedController,
+  enableReembedding: state.config?.parameters?.["enable-reembedding"] ?? false
+}))
+class Embedding extends React.PureComponent {
+  handleLayoutChoiceChange = e => {
+    const { dispatch } = this.props;
+    dispatch({
+      type: "set layout choice",
+      layoutChoice: e.currentTarget.value
+    });
+  };
+
+  renderReembedding() {
+    const {
+      enableReembedding,
+      world,
+      universe,
+      dispatch,
+      reembedController
+    } = this.props;
+
+    if (!enableReembedding) return null;
+
+    const loading = !!reembedController?.pendingFetch;
+    const disabled = World.worldEqUniverse(world, universe);
+    const tipContent = disabled
+      ? "Subset cells first, then click to recompute UMAP embedding."
+      : "Click to recompute UMAP embedding on the current cell subset.";
+
+    return (
+      <Tooltip
+        content={tipContent}
+        position="bottom"
+        hoverOpenDelay={globals.tooltipHoverOpenDelay}
+      >
+        <AnchorButton
+          icon="new-object"
+          style={{ marginRight: 10 }}
+          disabled={disabled}
+          onClick={() => dispatch(actions.requestReembed())}
+          loading={loading}
+        />
+      </Tooltip>
+    );
+  }
+
+  render() {
+    const { layoutChoice } = this.props;
+
+    return (
+      <ButtonGroup
+        style={{
+          marginRight: 10
+        }}
+      >
+        <Popover
+          target={
+            <Tooltip
+              content="Select embedding for visualization"
+              position="bottom"
+              hoverOpenDelay={globals.tooltipHoverOpenDelay}
+            >
+              <Button
+                type="button"
+                data-testid="layout-choice"
+                icon="heatmap"
+                style={{
+                  cursor: "pointer"
+                }}
+              />
+            </Tooltip>
+          }
+          position={Position.BOTTOM_RIGHT}
+          content={
+            <div
+              style={{
+                display: "flex",
+                justifyContent: "flex-start",
+                alignItems: "flex-start",
+                flexDirection: "column",
+                padding: 10
+              }}
+            >
+              <RadioGroup
+                label="Embedding Choice"
+                onChange={this.handleLayoutChoiceChange}
+                selectedValue={layoutChoice.current}
+              >
+                {layoutChoice.available.map(name => (
+                  <Radio label={name} value={name} key={name} />
+                ))}
+              </RadioGroup>
+            </div>
+          }
+        />
+        {this.renderReembedding()}
+      </ButtonGroup>
+    );
+  }
+}
+
+export default Embedding;

client/src/components/menubar/index.js

@@ -1,20 +1,12 @@
 // jshint esversion: 6
 import React from "react";
 import { connect } from "react-redux";
-import {
-  Button,
-  ButtonGroup,
-  AnchorButton,
-  Tooltip,
-  Popover,
-  Position,
-  RadioGroup,
-  Radio,
-} from "@blueprintjs/core";
+import { Button, ButtonGroup, AnchorButton, Tooltip } from "@blueprintjs/core";
 import * as globals from "../../globals";
 import actions from "../../actions";
 import CellSetButton from "./cellSetButtons";
 import Clip from "./clip";
+import Embedding from "./embedding";
 import InformationMenu from "./infoMenu";
 import Subset from "./subset";
 import UndoRedoReset from "./undoRedo";
@@ -24,7 +16,6 @@ import UndoRedoReset from "./undoRedo";
   world: state.world,
   crossfilter: state.crossfilter,
   differential: state.differential,
-  layoutChoice: state.layoutChoice,
   graphInteractionMode: state.controls.graphInteractionMode,
   clipPercentileMin: Math.round(100 * (state.world?.clipQuantiles?.min ?? 0)),
   clipPercentileMax: Math.round(100 * (state.world?.clipQuantiles?.max ?? 1)),
@@ -171,14 +162,6 @@ class MenuBar extends React.Component {
     this.setState({ pendingClipPercentiles: null });
   };
 
-  handleLayoutChoiceChange = e => {
-    const { dispatch } = this.props;
-    dispatch({
-      type: "set layout choice",
-      layoutChoice: e.currentTarget.value
-    });
-  };
-
   computeDiffExp = () => {
     const { dispatch, differential } = this.props;
     if (differential.celllist1 && differential.celllist2) {
@@ -224,7 +207,6 @@ class MenuBar extends React.Component {
     return world.nObs !== universe.nObs;
   };
 
-
   renderDiffExp() {
     /* diffexp-related buttons may be disabled */
     const { disableDiffexp, differential, diffexpMayBeSlow } = this.props;
@@ -295,7 +277,6 @@ class MenuBar extends React.Component {
       selectionTool,
       clipPercentileMin,
       clipPercentileMax,
-      layoutChoice,
       graphInteractionMode,
       aboutLink,
       showCentroidLabels
@@ -335,7 +316,7 @@ class MenuBar extends React.Component {
             dispatch({ type: "increment graph render counter" });
           }}
         />
-        <ButtonGroup style={{marginRight: "10px"}}>
+        <ButtonGroup style={{ marginRight: "10px" }}>
           <Tooltip
             content={selectionTooltip}
             position="bottom"
@@ -394,52 +375,7 @@ class MenuBar extends React.Component {
             }}
           />
         </Tooltip>
-        <ButtonGroup
-          style={{
-            marginRight: 10
-          }}
-        >
-          <Popover
-            target={
-              <Tooltip
-                content="Select embedding for visualization"
-                position="bottom"
-                hoverOpenDelay={globals.tooltipHoverOpenDelay}
-              >
-                <Button
-                  type="button"
-                  data-testid="layout-choice"
-                  icon="heatmap"
-                  style={{
-                    cursor: "pointer"
-                  }}
-                />
-              </Tooltip>
-            }
-            position={Position.BOTTOM_RIGHT}
-            content={
-              <div
-                style={{
-                  display: "flex",
-                  justifyContent: "flex-start",
-                  alignItems: "flex-start",
-                  flexDirection: "column",
-                  padding: 10
-                }}
-              >
-                <RadioGroup
-                  label="Embedding Choice"
-                  onChange={this.handleLayoutChoiceChange}
-                  selectedValue={layoutChoice.current}
-                >
-                  {layoutChoice.available.map(name => (
-                    <Radio label={name} value={name} key={name} />
-                  ))}
-                </RadioGroup>
-              </div>
-            }
-          />
-        </ButtonGroup>
+        <Embedding />
         <Clip
           pendingClipPercentiles={pendingClipPercentiles}
           clipPercentileMin={clipPercentileMin}

client/src/components/menubar/subset.js

@@ -1,18 +1,17 @@
 import React from "react";
-import {AnchorButton, ButtonGroup, Tooltip} from "@blueprintjs/core";
+import { AnchorButton, ButtonGroup, Tooltip } from "@blueprintjs/core";
 import * as globals from "../../globals";
 
-
 function Subset(props) {
   const {
     subsetPossible,
     subsetResetPossible,
     handleSubset,
-    handleSubsetReset,
+    handleSubsetReset
   } = props;
 
   return (
-    <ButtonGroup style={{marginRight: "10px"}}>
+    <ButtonGroup style={{ marginRight: "10px" }}>
       <Tooltip
         content="Subset to currently selected cells and associated metadata"
         position="bottom"

client/src/reducers/colors.js

@@ -12,12 +12,12 @@ const ColorsReducer = (
   prevSharedState
 ) => {
   switch (action.type) {
-    case "universe exists, but loading is still in progress":
-    case "reset World to eq Universe": {
+    case "universe exists, but loading is still in progress": {
+      /* initialize everything with default colors, no mode, no color-by accessor */
       const { world } = nextSharedState;
       const colorMode = null;
       const colorAccessor = null;
-      const { rgb, scale } = ColorHelpers.createColors(world, colorMode);
+      const { rgb, scale } = ColorHelpers.createColors(world);
       return {
         ...state,
         colorAccessor,
@@ -27,6 +27,22 @@ const ColorsReducer = (
       };
     }
 
+    case "reset World to eq Universe": {
+      /* need to rebuild colors as world may have changed, but don't switch modes */
+      const { world } = nextSharedState;
+      const { colorMode, colorAccessor } = state;
+      const { rgb, scale } = ColorHelpers.createColors(
+        world,
+        colorMode,
+        colorAccessor
+      );
+      return {
+        ...state,
+        rgb,
+        scale
+      };
+    }
+
     case "set clip quantiles":
     case "set World to current selection": {
       const { world: prevWorld, controls: prevControls } = prevSharedState;

client/src/reducers/index.js

@@ -21,6 +21,7 @@ import autosave from "./autosave";
 import ontology from "./ontology";
 import centroidLabels from "./centroidLabels";
 import pointDialation from "./pointDilation";
+import { reembedController, reembedding } from "./reembed";
 
 import undoableConfig from "./undoableConfig";
 
@@ -31,6 +32,7 @@ const Reducer = undoable(
     ["world", world],
     ["ontology", ontology],
     ["annotations", annotations],
+    ["reembedding", reembedding],
     ["layoutChoice", layoutChoice],
     ["categoricalSelection", categoricalSelection],
     ["continuousSelection", continuousSelection],
@@ -42,6 +44,7 @@ const Reducer = undoable(
     ["responsive", responsive],
     ["centroidLabels", centroidLabels],
     ["pointDilation", pointDialation],
+    ["reembedController", reembedController],
     ["autosave", autosave],
     ["resetCache", resetCache]
   ]),
@@ -57,7 +60,8 @@ const Reducer = undoable(
     "differential",
     "layoutChoice",
     "centroidLabels",
-    "annotations"
+    "annotations",
+    "reembedding"
   ],
   undoableConfig
 );

client/src/reducers/layoutChoice.js

@@ -14,6 +14,14 @@ function bestDefaultLayout(layouts) {
   return layouts[0];
 }
 
+function setToDefaultLayout(world) {
+  const { schema } = world;
+  const available = schema.layout.obs.map(v => v.name).sort();
+  const current = bestDefaultLayout(available);
+  const currentDimNames = schema.layout.obsByName[current].dims;
+  return { available, current, currentDimNames };
+}
+
 const LayoutChoice = (
   state = {
     available: [], // all available choices
@@ -24,14 +32,13 @@ const LayoutChoice = (
   nextSharedState
 ) => {
   switch (action.type) {
-    case "universe exists, but loading is still in progress":
-    case "reset World to eq Universe": {
+    case "universe exists, but loading is still in progress": {
       // set default to default
-      const { schema } = nextSharedState.world;
-      const available = schema.layout.obs.map(v => v.name).sort();
-      const current = bestDefaultLayout(available);
-      const currentDimNames = schema.layout.obsByName[current].dims;
-      return { available, current, currentDimNames };
+      const { universe } = nextSharedState;
+      return {
+        ...state,
+        ...setToDefaultLayout(universe)
+      };
     }
 
     case "set layout choice": {
@@ -41,6 +48,31 @@ const LayoutChoice = (
       return { ...state, current, currentDimNames };
     }
 
+    case "reembed: add reembedding": {
+      const name = action.schema.name;
+      const available = Array.from(new Set(state.available).add(name));
+      return {
+        ...state,
+        available
+      };
+    }
+
+    case "reembed: clear all reembeddings": {
+      const { universe } = nextSharedState;
+      const { current } = state;
+      const dflt = setToDefaultLayout(universe);
+      if (dflt.available.includes(current)) {
+        return {
+          ...state,
+          available: dflt.available
+        };
+      }
+      return {
+        ...state,
+        ...dflt
+      };
+    }
+
     default: {
       return state;
     }

client/src/reducers/reembed.js

@@ -0,0 +1,64 @@
+/*
+controller state is not part of the undo/redo history
+*/
+export const reembedController = (
+  state = {
+    pendingFetch: null
+  },
+  action
+) => {
+  switch (action.type) {
+    case "reembed: request start": {
+      return {
+        ...state,
+        pendingFetch: action.abortableFetch
+      };
+    }
+    case "reembed: request aborted":
+    case "reembed: request cancel":
+    case "reembed: request completed": {
+      return {
+        ...state,
+        pendingFetch: null
+      };
+    }
+    default: {
+      return state;
+    }
+  }
+};
+
+/*
+actual reembedding data is part of the undo/redo history
+*/
+export const reembedding = (
+  state = {
+    reembeddings: new Map()
+  },
+  action
+) => {
+  switch (action.type) {
+    case "reembed: add reembedding": {
+      const { schema, embedding } = action;
+      const { name } = schema.name;
+      const { reembeddings } = state;
+      return {
+        ...state,
+        reembeddings: new Map(reembeddings).set(name, {
+          name,
+          schema,
+          embedding
+        })
+      };
+    }
+    case "reembed: clear all reembeddings": {
+      return {
+        ...state,
+        reembeddings: new Map()
+      };
+    }
+    default: {
+      return state;
+    }
+  }
+};

client/src/reducers/world.js

@@ -4,6 +4,10 @@ import {
   ControlsHelpers,
   AnnotationsHelpers
 } from "../util/stateManager";
+import {
+  addObsLayout,
+  removeObsLayout
+} from "../util/stateManager/schemaHelpers";
 import clip from "../util/clip";
 import quantile from "../util/quantile";
 
@@ -32,6 +36,7 @@ const WorldReducer = (
     }
 
     case "universe: column load success": {
+      /* incremental initial data load - always assumes world == universe */
       const { universe } = nextSharedState;
       const { dim } = action;
       return {
@@ -276,6 +281,52 @@ const WorldReducer = (
       };
     }
 
+    case "reembed: add reembedding": {
+      // new embedding loaded, which *only* affects world's layout.
+      // It may be new, or it may replace a previous re-embedding.
+      const { obsLayout: origObsLayout, schema: origSchema } = state;
+      const { embedding, schema: embeddingSchema } = action;
+
+      const { dims, name } = embeddingSchema;
+      let obsLayout = origObsLayout;
+      let schema = origSchema;
+
+      // alias the names the server sent us, in case they were not the same as the schema
+      const embedingLabels = embedding.colIndex.keys();
+      const labels = {
+        [embedingLabels[0]]: dims[0],
+        [embedingLabels[1]]: dims[1]
+      };
+      obsLayout = obsLayout.withColsFrom(embedding, labels);
+      schema = addObsLayout(schema, embeddingSchema);
+      return {
+        ...state,
+        obsLayout,
+        schema
+      };
+    }
+
+    case "reembed: clear all reembeddings": {
+      // reembedding was cleared -- remove from layout
+      const { obsLayout: origObsLayout, schema: origSchema } = state;
+      const { reembedding } = prevSharedState;
+
+      let schema = origSchema;
+      let obsLayout = origObsLayout;
+
+      reembedding.reembeddings.forEach((emb, name) => {
+        const { dims } = emb.schema;
+        obsLayout = obsLayout.dropCol(dims[0]);
+        obsLayout = obsLayout.dropCol(dims[1]);
+        schema = removeObsLayout(schema, name);
+      });
+      return {
+        ...state,
+        obsLayout,
+        schema
+      };
+    }
+
     default: {
       return state;
     }

client/src/util/dataframe/dataframe.js

@@ -348,37 +348,90 @@ class Dataframe {
     );
   }
 
-  withColsFrom(dataframe) {
+  withColsFrom(dataframe, labels) {
     /*
     return a new dataframe containing all columns from both `this` and the
-    provided of dataframe.
+    provided dataframe argument.
 
     The row index from `this` will be used.  All dataframes must have identical
     dimensionality, and no overlapping columns labels.
 
     Special case, if either dataframe is empty, the other is returned unchanged.
+
+    Arguments:
+    * dataframe: a dataframe to combine with `this`
+    * labels: columns to pull from `dataframe` and combine with `this`.  If falsey,
+      all columns are used.  If an array, must contain a list of labels.  If an
+      Object or Map, the key is the columns to pull, which will be stored into the
+      new dataframe as the value.
+
+    Example:
+
+    newDf = df.withColsFrom(otherDf);         // combines all columns from both
+    newDf = df.withColsFrom(otherDf, ['a']);  // combines df with otherDf['a']
+    newDf = df.withColsFrom(otherDf, {a: 'b'}); // combines df with otherDf['a'], but calls it 'b'
+
      */
+
+    // resolve the source and dest label names.
+    let srcLabels;
+    let dstLabels;
+    if (!labels) {
+      // combine all columns
+      dstLabels = dataframe.colIndex.keys();
+      srcLabels = dstLabels;
+    } else if (Array.isArray(labels)) {
+      // combine subset of keys with no aliasing
+      dstLabels = labels;
+      srcLabels = labels;
+    } else if (labels instanceof Map) {
+      // aliasing with a Map
+      srcLabels = Array.from(labels.keys());
+      dstLabels = Array.from(labels.values());
+    } else {
+      // aliasing with an Object
+      srcLabels = Object.keys(labels);
+      dstLabels = Object.values(labels);
+    }
+
+    // if datafame is empty, and no specific labels specified, noop.
+    if (dataframe.isEmpty()) {
+      if (!labels || srcLabels.length === 0) return this;
+      throw new Error("Empty dataframe, unable to pick columns");
+    }
+
     if (this.isEmpty()) {
+      // 1. subset dataframe from source keys
+      // 2. alias names
+      dataframe = dataframe.subset(null, srcLabels);
+      for (let i = 0; i < srcLabels.length; i += 1) {
+        dataframe = dataframe.renameCol(srcLabels[i], dstLabels[i]);
+      }
       return dataframe;
     }
-    if (dataframe.isEmpty()) {
-      return this;
+
+    // otherwise, bulid a new dataframe combining columns from both
+
+    const srcOffsets = srcLabels.map(l => dataframe.colIndex.getOffset(l));
+
+    // check for label collisions
+    if (dstLabels.some(this.hasCol, this)) {
+      throw new Error("duplicate key collision");
     }
 
-    this.colIndex.keys().forEach(key => {
-      if (dataframe.has(key)) {
-        throw new Error("duplicate key collision");
-      }
-    });
-
-    const dims = [this.dims[0], this.dims[1] + dataframe.dims[1]];
+    // const dims = [this.dims[0], this.dims[1] + dataframe.dims[1]];
+    const dims = [this.dims[0], this.dims[1] + srcOffsets.length];
     const { rowIndex } = this;
-    const columns = [...this.__columns, ...dataframe.__columns];
-    const colIndex = this.colIndex.withLabels(dataframe.colIndex.keys());
+    const columns = [
+      ...this.__columns,
+      ...srcOffsets.map(i => dataframe.__columns[i])
+    ];
+    const colIndex = this.colIndex.withLabels(dstLabels);
     const columnsAccessor = [
       ...this.__columnsAccessor,
-      ...dataframe.__columnsAccessor
+      ...srcOffsets.map(i => dataframe.__columnsAccessor[i])
     ];
+
     return new this.constructor(
       dims,
       columns,

client/src/util/stateManager/schemaHelpers.js

@@ -1,5 +1,8 @@
 /*
 Helpers for schema management
+
+TODO: all this would be much more natural if done with a framework
+like immutable.js
 */
 import _ from "lodash";
 
@@ -31,8 +34,8 @@ export function indexEntireSchema(schema) {
   return schema;
 }
 
-function _copy(schema) {
-  /* redux copy conventions - WARNING, only for modifyign obs annotations */
+function _copyObsAnno(schema) {
+  /* redux copy conventions - WARNING, only for modifying obs annotations */
   return {
     ...schema,
     annotations: {
@@ -42,7 +45,17 @@ function _copy(schema) {
   };
 }
 
-function _reindex(schema) {
+function _copyObsLayout(schema) {
+  return {
+    ...schema,
+    layout: {
+      ...schema.layout,
+      obs: _.cloneDeep(schema.layout.obs)
+    }
+  };
+}
+
+function _reindexObsAnno(schema) {
   /* reindex obs annotations ONLY */
   schema.annotations.obsByName = fromEntries(
     schema.annotations.obs.columns.map(v => [v.name, v])
@@ -50,18 +63,25 @@ function _reindex(schema) {
   return schema;
 }
 
+function _reindexObsLayout(schema) {
+  schema.layout.obsByName = fromEntries(
+    schema.layout.obs.map(v => [v.name, v])
+  );
+  return schema;
+}
+
 export function removeObsAnnoColumn(schema, name) {
-  const newSchema = _copy(schema);
+  const newSchema = _copyObsAnno(schema);
   newSchema.annotations.obs.columns = schema.annotations.obs.columns.filter(
     v => v.name !== name
   );
-  return _reindex(newSchema);
+  return _reindexObsAnno(newSchema);
 }
 
 export function addObsAnnoColumn(schema, name, defn) {
-  const newSchema = _copy(schema);
+  const newSchema = _copyObsAnno(schema);
   newSchema.annotations.obs.columns.push(defn);
-  return _reindex(newSchema);
+  return _reindexObsAnno(newSchema);
 }
 
 export function removeObsAnnoCategory(schema, name, category) {
@@ -73,7 +93,7 @@ export function removeObsAnnoCategory(schema, name, category) {
   const idx = categories.indexOf(category);
   if (idx === -1) throw new Error("category does not exist");
 
-  const newSchema = _reindex(_copy(schema));
+  const newSchema = _reindexObsAnno(_copyObsAnno(schema));
 
   /* remove category.  Do not need to resort as this can't change presentation order */
   newSchema.annotations.obsByName[name].categories.splice(idx, 1);
@@ -89,7 +109,7 @@ export function addObsAnnoCategory(schema, name, category) {
   const idx = categories.indexOf(category);
   if (idx !== -1) throw new Error("category already exists");
 
-  const newSchema = _reindex(_copy(schema));
+  const newSchema = _reindexObsAnno(_copyObsAnno(schema));
 
   /* add category, retaining presentation sort order */
   const catAnno = newSchema.annotations.obsByName[name];
@@ -99,3 +119,17 @@ export function addObsAnnoCategory(schema, name, category) {
   ]);
   return newSchema;
 }
+
+export function addObsLayout(schema, layout) {
+  /* add or replace a layout */
+  const newSchema = _copyObsLayout(schema);
+  newSchema.layout.obs.push(layout);
+  return _reindexObsLayout(newSchema);
+}
+
+export function removeObsLayout(schema, name) {
+  /* remove a layout */
+  const newSchema = _copyObsLayout(schema);
+  newSchema.layout.obs = schema.layout.obs.filter(v => v.name !== name);
+  return _reindexObsLayout(newSchema);
+}