experimental re-embedding (#1186)

* first cut at re-embedding route and back-end support

* update and expand config route tests

* add scanpy_umap

* add reembedding to config route parameters

* front-end support for reembedding fetch and UI

* remove unused imports

* add loading state

* save reembedding in reducer state

* improve withColsFrom

* transmit reembed schema to client; pick unique embedding names

* display embeddings

* format

* lint

* spaces, tab size 2

* lint

* test hack for smoke-test race

* back out hack sleep

* add check for backed mode

* add unit test for reembedding

* lint

* hide re-embedding CLI param from help

client/src/util/dataframe/dataframe.js

@@ -348,37 +348,90 @@ class Dataframe {
     );
   }
 
-  withColsFrom(dataframe) {
+  withColsFrom(dataframe, labels) {
     /*
     return a new dataframe containing all columns from both `this` and the
-    provided of dataframe.
+    provided dataframe argument.
 
     The row index from `this` will be used.  All dataframes must have identical
     dimensionality, and no overlapping columns labels.
 
     Special case, if either dataframe is empty, the other is returned unchanged.
+
+    Arguments:
+    * dataframe: a dataframe to combine with `this`
+    * labels: columns to pull from `dataframe` and combine with `this`.  If falsey,
+      all columns are used.  If an array, must contain a list of labels.  If an
+      Object or Map, the key is the columns to pull, which will be stored into the
+      new dataframe as the value.
+
+    Example:
+
+    newDf = df.withColsFrom(otherDf);         // combines all columns from both
+    newDf = df.withColsFrom(otherDf, ['a']);  // combines df with otherDf['a']
+    newDf = df.withColsFrom(otherDf, {a: 'b'}); // combines df with otherDf['a'], but calls it 'b'
+
      */
+
+    // resolve the source and dest label names.
+    let srcLabels;
+    let dstLabels;
+    if (!labels) {
+      // combine all columns
+      dstLabels = dataframe.colIndex.keys();
+      srcLabels = dstLabels;
+    } else if (Array.isArray(labels)) {
+      // combine subset of keys with no aliasing
+      dstLabels = labels;
+      srcLabels = labels;
+    } else if (labels instanceof Map) {
+      // aliasing with a Map
+      srcLabels = Array.from(labels.keys());
+      dstLabels = Array.from(labels.values());
+    } else {
+      // aliasing with an Object
+      srcLabels = Object.keys(labels);
+      dstLabels = Object.values(labels);
+    }
+
+    // if datafame is empty, and no specific labels specified, noop.
+    if (dataframe.isEmpty()) {
+      if (!labels || srcLabels.length === 0) return this;
+      throw new Error("Empty dataframe, unable to pick columns");
+    }
+
     if (this.isEmpty()) {
+      // 1. subset dataframe from source keys
+      // 2. alias names
+      dataframe = dataframe.subset(null, srcLabels);
+      for (let i = 0; i < srcLabels.length; i += 1) {
+        dataframe = dataframe.renameCol(srcLabels[i], dstLabels[i]);
+      }
       return dataframe;
     }
-    if (dataframe.isEmpty()) {
-      return this;
+
+    // otherwise, bulid a new dataframe combining columns from both
+
+    const srcOffsets = srcLabels.map(l => dataframe.colIndex.getOffset(l));
+
+    // check for label collisions
+    if (dstLabels.some(this.hasCol, this)) {
+      throw new Error("duplicate key collision");
     }
 
-    this.colIndex.keys().forEach(key => {
-      if (dataframe.has(key)) {
-        throw new Error("duplicate key collision");
-      }
-    });
-
-    const dims = [this.dims[0], this.dims[1] + dataframe.dims[1]];
+    // const dims = [this.dims[0], this.dims[1] + dataframe.dims[1]];
+    const dims = [this.dims[0], this.dims[1] + srcOffsets.length];
     const { rowIndex } = this;
-    const columns = [...this.__columns, ...dataframe.__columns];
-    const colIndex = this.colIndex.withLabels(dataframe.colIndex.keys());
+    const columns = [
+      ...this.__columns,
+      ...srcOffsets.map(i => dataframe.__columns[i])
+    ];
+    const colIndex = this.colIndex.withLabels(dstLabels);
     const columnsAccessor = [
       ...this.__columnsAccessor,
-      ...dataframe.__columnsAccessor
+      ...srcOffsets.map(i => dataframe.__columnsAccessor[i])
     ];
+
     return new this.constructor(
       dims,
       columns,

client/src/util/stateManager/schemaHelpers.js

@@ -1,5 +1,8 @@
 /*
 Helpers for schema management
+
+TODO: all this would be much more natural if done with a framework
+like immutable.js
 */
 import _ from "lodash";
 
@@ -31,8 +34,8 @@ export function indexEntireSchema(schema) {
   return schema;
 }
 
-function _copy(schema) {
-  /* redux copy conventions - WARNING, only for modifyign obs annotations */
+function _copyObsAnno(schema) {
+  /* redux copy conventions - WARNING, only for modifying obs annotations */
   return {
     ...schema,
     annotations: {
@@ -42,7 +45,17 @@ function _copy(schema) {
   };
 }
 
-function _reindex(schema) {
+function _copyObsLayout(schema) {
+  return {
+    ...schema,
+    layout: {
+      ...schema.layout,
+      obs: _.cloneDeep(schema.layout.obs)
+    }
+  };
+}
+
+function _reindexObsAnno(schema) {
   /* reindex obs annotations ONLY */
   schema.annotations.obsByName = fromEntries(
     schema.annotations.obs.columns.map(v => [v.name, v])
@@ -50,18 +63,25 @@ function _reindex(schema) {
   return schema;
 }
 
+function _reindexObsLayout(schema) {
+  schema.layout.obsByName = fromEntries(
+    schema.layout.obs.map(v => [v.name, v])
+  );
+  return schema;
+}
+
 export function removeObsAnnoColumn(schema, name) {
-  const newSchema = _copy(schema);
+  const newSchema = _copyObsAnno(schema);
   newSchema.annotations.obs.columns = schema.annotations.obs.columns.filter(
     v => v.name !== name
   );
-  return _reindex(newSchema);
+  return _reindexObsAnno(newSchema);
 }
 
 export function addObsAnnoColumn(schema, name, defn) {
-  const newSchema = _copy(schema);
+  const newSchema = _copyObsAnno(schema);
   newSchema.annotations.obs.columns.push(defn);
-  return _reindex(newSchema);
+  return _reindexObsAnno(newSchema);
 }
 
 export function removeObsAnnoCategory(schema, name, category) {
@@ -73,7 +93,7 @@ export function removeObsAnnoCategory(schema, name, category) {
   const idx = categories.indexOf(category);
   if (idx === -1) throw new Error("category does not exist");
 
-  const newSchema = _reindex(_copy(schema));
+  const newSchema = _reindexObsAnno(_copyObsAnno(schema));
 
   /* remove category.  Do not need to resort as this can't change presentation order */
   newSchema.annotations.obsByName[name].categories.splice(idx, 1);
@@ -89,7 +109,7 @@ export function addObsAnnoCategory(schema, name, category) {
   const idx = categories.indexOf(category);
   if (idx !== -1) throw new Error("category already exists");
 
-  const newSchema = _reindex(_copy(schema));
+  const newSchema = _reindexObsAnno(_copyObsAnno(schema));
 
   /* add category, retaining presentation sort order */
   const catAnno = newSchema.annotations.obsByName[name];
@@ -99,3 +119,17 @@ export function addObsAnnoCategory(schema, name, category) {
   ]);
   return newSchema;
 }
+
+export function addObsLayout(schema, layout) {
+  /* add or replace a layout */
+  const newSchema = _copyObsLayout(schema);
+  newSchema.layout.obs.push(layout);
+  return _reindexObsLayout(newSchema);
+}
+
+export function removeObsLayout(schema, name) {
+  /* remove a layout */
+  const newSchema = _copyObsLayout(schema);
+  newSchema.layout.obs = schema.layout.obs.filter(v => v.name !== name);
+  return _reindexObsLayout(newSchema);
+}