load embeddings in parallel (#1352)

* load embeddings in parallel

* correctly capture unclipped

* test

* another test

* add convenient copy assets target

* cleanup

client/src/actions/index.js

@@ -1,6 +1,7 @@
 import _ from "lodash";
 import * as globals from "../globals";
 import { Universe, MatrixFBS } from "../util/stateManager";
+import * as Dataframe from "../util/dataframe";
 import {
   catchErrorsWrap,
   doJsonRequest,
@@ -74,24 +75,29 @@ function varAnnotationFetchAndLoad(dispatch, schema) {
 /*
 return promise fetching layout we need
 */
-function layoutFetchAndLoad(dispatch) {
+function layoutFetchAndLoad(dispatch, schema) {
+  const embeddings = schema?.schema?.layout?.obs ?? [];
+  const embNames = embeddings.map(e => e.name);
+  const baseURL = `${globals.API.prefix}${globals.API.version}layout/obs`;
+
+  const plimit = new PromiseLimit(4);
   return Promise.all(
-    ["layout/obs"]
-      .map(path => {
-        const url = `${globals.API.prefix}${globals.API.version}${path}`;
-        return doBinaryRequest(url);
+    embNames.map(e =>
+      plimit.add(() => {
+        const url = `${baseURL}?layout-name=${encodeURIComponent(e)}`;
+        return doBinaryRequest(url).then(buffer =>
+          Universe.matrixFBSToDataframe(buffer)
+        );
       })
-      .map(rqst => rqst.then(buffer => Universe.matrixFBSToDataframe(buffer)))
-      .map(resp =>
-        resp.then(df =>
-          dispatch({
-            type: "universe: column load success",
-            dim: "obsLayout",
-            dataframe: df
-          })
-        )
-      )
-  );
+    )
+  ).then(dfs => {
+    const df = Dataframe.Dataframe.empty().withColsFromAll(dfs);
+    dispatch({
+      type: "universe: column load success",
+      dim: "obsLayout",
+      dataframe: df
+    });
+  });
 }
 
 /*
@@ -130,7 +136,7 @@ const doInitialDataLoad = () =>
       Step 2 - load the minimum stuff required to display.
       */
       await Promise.all([
-        layoutFetchAndLoad(dispatch),
+        layoutFetchAndLoad(dispatch, schema),
         varAnnotationFetchAndLoad(dispatch, schema)
       ]);
 

client/src/reducers/world.js

@@ -39,14 +39,20 @@ const WorldReducer = (
       /* incremental initial data load - always assumes world == universe */
       const { universe } = nextSharedState;
       const { dim } = action;
+
+      // we don't clip anything except for varData and obsAnnotations
+      let unclipped = state.unclipped;
+      if (dim == "varData" || dim == "obsAnnotations") {
+        unclipped = {
+          ...unclipped,
+          [dim]: universe[dim].clone()
+        };
+      }
       return {
         ...state,
         schema: universe.schema,
         [dim]: universe[dim].clone(),
-        unclipped: {
-          ...state.unclipped,
-          [dim]: universe[dim].clone()
-        }
+        unclipped
       };
     }
 
@@ -106,10 +112,15 @@ const WorldReducer = (
       const { userDefinedGenes, diffexpGenes } = prevSharedState;
       const allTheGenesWeNeed = [
         ...new Set(
-          [userDefinedGenes, diffexpGenes, Object.keys(action.expressionData)].filter(ele => ele).flat()
+          [userDefinedGenes, diffexpGenes, Object.keys(action.expressionData)]
+            .filter(ele => ele)
+            .flat()
         )
       ];
-      unclippedVarData = ControlsHelpers.pruneVarDataCache(unclippedVarData, allTheGenesWeNeed);
+      unclippedVarData = ControlsHelpers.pruneVarDataCache(
+        unclippedVarData,
+        allTheGenesWeNeed
+      );
 
       // at this point, we have the unclipped data in unclippedVarData.
       // Now create clipped.