Redux refactor (#1571)

* refactor categorical controls state

* lint

* fix race condition in tests

* fix typo

* add missing update on subset

* remove obsolete code

* update jest and puppeteer major version; update all minors

* update when label changes

* remove lint from tests; increase timeouts in e2e tests

* initial refactoring to new async annomatrix

* refine error handling

* fix bad merge

* add continuous legend

* lint

* fix memoization in color table creators

* partial implementation of user defined annotations

* add new annotations action creator file

* first pass at user annotations

* additional user annotation bug fixes

* user annotation auto-save

* unit test cleanup

* lint

* refactor into multiple files

* cleanup

* add column GC

* fix several bugs in user annotations

* remove debug code

* no anonymous functions

* undo redo cleanup

* file cleanup

* scatterplot

* performance

* cleanup

* remove old code

* render in parallel with load

* fix race condition

* simply graph rendering

* render throttle DRY

* fix category label order

* fix typo in e2e test setup

* re-fix the e2e test setup

* be more tolerant of races

* anno matrix unit tests

* temp disable reembedding

* pilot port continuous histo to react-async

* name change

* lint

* fix repaint bug

* typo fix

* update snap to match new ids

* world/universe name cleanup

* move annoMatrix to src dir

* use private underscore naming convention

* fix corner case in all selected

* name cleanup

* add layout control

* init edge case

* lint

* port scatterplot

* fix label indexing bug and improve tests

* port category to react-async

* fix user annotation labelling while subset

* select all of prev layout on layout switch

* fix race with crossfilter update

* prettier lint

* fix misleading comment

* fix url composition in loader

* first pass at crossfilter tests

* lint

* lint

* fix typo

* improved error handling for network errors

* fix memoization bug

* add memo

* refactor for performnce

* add missing single-value handling in select exact parser

* small bugs discovered by tests

* lint

* additional crossfilter unit tests

* remove extraneous comment

* add support for automatic category determination

* lint

* fix render bug in category

* take advantage of schema categories guarantee

* lint

* do not clear history when resetting

* enhanced annomatrix gc

* lint

* finish renaming to follow conventions; fix clone race bug

* lint

* add priority based loading to improve initial data load UX

* crossfilter cache perf

* perf tuning

* remove timers

* documentation

* PR review changes

* PR review changes

* more PR review edits

* improve clarity of comment

* more PR review fixes

* port centroidLabels to use react-async

* remove dead code

* pr review updates

* oops, remove logging

client/src/actions/index.js

@@ -1,93 +1,21 @@
 import * as globals from "../globals";
-import { Universe, MatrixFBS } from "../util/stateManager";
-import * as Dataframe from "../util/dataframe";
+import {
+  AnnoMatrixLoader,
+  AnnoMatrixObsCrossfilter,
+  clip,
+  isubsetMask,
+} from "../annoMatrix";
 import {
   catchErrorsWrap,
   doJsonRequest,
-  doBinaryRequest,
   dispatchNetworkErrorMessageToUser,
 } from "../util/actionHelpers";
-import PromiseLimit from "../util/promiseLimit";
-import { requestReembed, reembedResetWorldToUniverse } from "./reembed";
+import {
+  requestReembed /* , reembedResetWorldToUniverse -- disabled temporarily, TODO issue #1606 */,
+} from "./reembed";
 import { loadUserColorConfig } from "../util/stateManager/colorHelpers";
-
-/*
-return promise to fetch the OBS annotations we need to load.  Omit anything
-we don't need.
-*/
-async function obsAnnotationFetchAndLoad(dispatch, schema) {
-  const obsAnnotations = schema?.schema?.annotations?.obs ?? {};
-  const index = obsAnnotations.index ?? false;
-  const columns = (obsAnnotations.columns ?? []).filter(
-    (col) => col.name !== index
-  );
-
-  const plimit = new PromiseLimit(5);
-  return Promise.all(
-    columns.map((col) =>
-      plimit.add(() =>
-        fetchBinary(
-          `annotations/obs?annotation-name=${encodeURIComponent(col.name)}`
-        )
-          .then((buffer) => MatrixFBS.matrixFBSToDataframe(buffer))
-          .then((df) =>
-            dispatch({
-              type: "universe: column load success",
-              dim: "obsAnnotations",
-              dataframe: df,
-            })
-          )
-      )
-    )
-  );
-}
-
-/*
-return promise fetching VAR annotations we need to load.  Only index is currently used.
-*/
-async function varAnnotationFetchAndLoad(dispatch, schema) {
-  const varAnnotations = schema?.schema?.annotations?.var ?? {};
-  const index = varAnnotations.index ?? false;
-  const names = index ? [index] : [];
-  return Promise.all(
-    names.map((name) =>
-      fetchBinary(`annotations/var?annotation-name=${encodeURIComponent(name)}`)
-        .then((buffer) => MatrixFBS.matrixFBSToDataframe(buffer))
-        .then((df) =>
-          dispatch({
-            type: "universe: column load success",
-            dim: "varAnnotations",
-            dataframe: df,
-          })
-        )
-    )
-  );
-}
-
-/*
-return promise fetching layout we need
-*/
-function layoutFetchAndLoad(dispatch, schema) {
-  const embeddings = schema?.schema?.layout?.obs ?? [];
-  const embNames = embeddings.map((e) => e.name);
-
-  const plimit = new PromiseLimit(5);
-  return Promise.all(
-    embNames.map((e) =>
-      plimit.add(() =>
-        fetchBinary(
-          `layout/obs?layout-name=${encodeURIComponent(e)}`
-        ).then((buffer) => MatrixFBS.matrixFBSToDataframe(buffer))
-      )
-    )
-  ).then((dfs) =>
-    dispatch({
-      type: "universe: column load success",
-      dim: "obsLayout",
-      dataframe: Dataframe.Dataframe.empty().withColsFromAll(dfs),
-    })
-  );
-}
+import * as selnActions from "./selection";
+import * as annoActions from "./annotation";
 
 /*
 return promise fetching user-configured colors
@@ -101,178 +29,62 @@ async function userColorsFetchAndLoad(dispatch) {
   );
 }
 
+async function schemaFetch() {
+  return fetchJson("schema");
+}
+
+async function configFetch(dispatch) {
+  return fetchJson("config").then((response) => {
+    const config = { ...globals.configDefaults, ...response.config };
+    dispatch({
+      type: "configuration load complete",
+      config,
+    });
+    return config;
+  });
+}
+
 /*
-Bootstrap application with the initial data loading.
-  * /config - application configuration
-  * /schema - schema of dataframe
-  * /annotations - all metadata annotation
-  * /layout - all default layout
+Application bootstrap
 */
 const doInitialDataLoad = () =>
   catchErrorsWrap(async (dispatch) => {
     dispatch({ type: "initial data load start" });
 
     try {
-      /*
-      Step 1 - config & schema, all JSON
-      */
-      const requestJson = ["config", "schema"].map(fetchJson);
-      const [responseConfig, schema] = await Promise.all(requestJson);
-      /* set config defaults */
-      const config = { ...globals.configDefaults, ...responseConfig.config };
-      const universe = Universe.createUniverseFromResponse(config, schema);
-      dispatch({
-        type: "universe exists, but loading is still in progress",
-        universe,
-      });
-      dispatch({
-        type: "configuration load complete",
-        config,
-      });
-
-      /*
-      Step 2 - load the minimum stuff required to display.
-      */
-      await Promise.all([
+      const [, schema] = await Promise.all([
+        configFetch(dispatch),
+        schemaFetch(dispatch),
         userColorsFetchAndLoad(dispatch),
-        layoutFetchAndLoad(dispatch, schema),
-        varAnnotationFetchAndLoad(dispatch, schema),
       ]);
 
-      /*
-      Step 3 - load everything else
-      */
-      await obsAnnotationFetchAndLoad(dispatch, schema);
-
+      const baseDataUrl = `${globals.API.prefix}${globals.API.version}`;
+      const annoMatrix = new AnnoMatrixLoader(baseDataUrl, schema.schema);
+      const obsCrossfilter = new AnnoMatrixObsCrossfilter(annoMatrix);
       dispatch({
-        type: "initial data load complete (universe exists)",
-        universe,
+        type: "annoMatrix: init complete",
+        annoMatrix,
+        obsCrossfilter,
       });
+      dispatch({ type: "initial data load complete" });
     } catch (error) {
       dispatch({ type: "initial data load error", error });
     }
   }, true);
 
-/*
-Set the view (world) to current selection.   Placeholder for an async action
-which also does re-layout.
-*/
-const setWorldToSelection = () => (dispatch, getState) => {
-  const { universe, world, crossfilter } = getState();
-  dispatch({
-    type: "set World to current selection",
-    universe,
-    world,
-    crossfilter,
-  });
-};
-
-/* double URI encode - needed for query-param filters */
-function dubEncURIComponent(s) {
-  return encodeURIComponent(encodeURIComponent(s));
-}
-
-/*
-Fetch expression vectors for each gene in genes.  This is NOT an action
-function, but rather a helper to be called from an action helper that
-needs expression data.
-
-Transparently utilizes cached data if it is already present.
-*/
-async function _doRequestExpressionData(dispatch, getState, genes) {
-  const state = getState();
-  const { universe } = state;
-  const varIndexName = universe.schema.annotations.var.index;
-
-  /* helper for this function only */
-  const fetchData = async (geneNames) => {
-    const query = geneNames
-      .map(
-        (g) =>
-          `var:${dubEncURIComponent(varIndexName)}=${dubEncURIComponent(g)}`
-      )
-      .join("&");
-    // TODO: why convert to an Object and not a Dataframe?
-    return fetchBinary(`data/var?${query}`).then((buffer) =>
-      Universe.convertDataFBStoObject(universe, buffer)
-    );
-  };
-
-  /* preload data already in cache */
-  let expressionData = genes.reduce((acc, g) => {
-    const data = universe.varData.col(g);
-    if (data) {
-      acc[g] = data.asArray();
-    }
-    return acc;
-  }, {}); // --> { gene: data }
-
-  /* make a list of genes for which we do not have data */
-  const genesToFetch = genes.filter((g) => expressionData[g] === undefined);
-
-  dispatch({ type: "expression load start" });
-
-  /* Fetch data for any genes not in cache */
-  if (genesToFetch.length) {
-    try {
-      const newExpressionData = await fetchData(genesToFetch);
-      expressionData = {
-        ...expressionData,
-        ...newExpressionData,
-      };
-    } catch (error) {
-      dispatch({ type: "expression load error", error });
-      throw error; // rethrow
-    }
-  }
-
-  dispatch({ type: "expression load success", expressionData });
-  return expressionData;
-}
-
 function requestSingleGeneExpressionCountsForColoringPOST(gene) {
-  return async (dispatch, getState) => {
-    dispatch({ type: "get single gene expression for coloring started" });
-    try {
-      await _doRequestExpressionData(dispatch, getState, [gene]);
-      const { world } = getState();
-      dispatch({
-        type: "color by expression",
-        gene,
-        data: {
-          [gene]: world.varData.col(gene).asArray(),
-        },
-      });
-    } catch (error) {
-      dispatch({
-        type: "get single gene expression for coloring error",
-        error,
-      });
-    }
+  return {
+    type: "color by expression",
+    gene,
   };
 }
 
-const requestUserDefinedGene = (gene) => async (dispatch, getState) => {
-  dispatch({ type: "request user defined gene started" });
-  try {
-    await await _doRequestExpressionData(dispatch, getState, [gene]);
-    const { world } = getState();
-
-    /* then send the success case action through */
-    return dispatch({
-      type: "request user defined gene success",
-      data: {
-        genes: [gene],
-        expression: world.varData.col(gene).asArray(),
-      },
-    });
-  } catch (error) {
-    return dispatch({
-      type: "request user defined gene error",
-      error,
-    });
-  }
-};
+const requestUserDefinedGene = (gene) => ({
+  type: "request user defined gene success",
+  data: {
+    genes: [gene],
+  },
+});
 
 const dispatchDiffExpErrors = (dispatch, response) => {
   switch (response.status) {
@@ -308,9 +120,8 @@ const requestDifferentialExpression = (set1, set2, num_genes = 10) => async (
     1. get the most differentially expressed genes
     2. get expression data for each
     */
-    const state = getState();
-    const { universe } = state;
-    const varIndexName = universe.schema.annotations.var.index;
+    const { annoMatrix } = getState();
+    const varIndexName = annoMatrix.schema.annotations.var.index;
 
     // Legal values are null, Array or TypedArray.  Null is initial state.
     if (!set1) set1 = [];
@@ -345,22 +156,12 @@ const requestDifferentialExpression = (set1, set2, num_genes = 10) => async (
       return dispatchDiffExpErrors(dispatch, res);
     }
 
-    const data = await res.json();
-    // result is [ [varIdx, ...], ... ]
-    const topNGenes = data.map((r) =>
-      universe.varAnnotations.at(r[0], varIndexName)
-    );
-
-    /*
-    Kick off secondary action to fetch all of the expression data for the
-    topN expressed genes.
-    */
-    const plimit = new PromiseLimit(5);
-    await Promise.all(
-      topNGenes.map((gene) =>
-        plimit.add(() => _doRequestExpressionData(dispatch, getState, [gene]))
-      )
-    );
+    const response = await res.json();
+    const varIndex = await annoMatrix.fetch("var", varIndexName);
+    const data = response.map((v) => [
+      varIndex.at(v[0], varIndexName),
+      ...v.slice(1),
+    ]);
 
     /* then send the success case action through */
     return dispatch({
@@ -375,66 +176,73 @@ const requestDifferentialExpression = (set1, set2, num_genes = 10) => async (
   }
 };
 
-const resetWorldToUniverse = () => (dispatch, getState) => {
-  const { universe } = getState();
-  reembedResetWorldToUniverse(dispatch, getState);
+const clipAction = (min, max) => (dispatch, getState) => {
+  /*
+  apply a clip to the current annoMatrix.  By convention, the clip
+  view is ALWAYS the top view.
+  */
+  const { annoMatrix: prevAnnoMatrix } = getState();
+  const annoMatrix = prevAnnoMatrix.isClipped
+    ? clip(prevAnnoMatrix.viewOf, min, max)
+    : clip(prevAnnoMatrix, min, max);
+  const obsCrossfilter = new AnnoMatrixObsCrossfilter(annoMatrix);
   dispatch({
-    type: "reset World to eq Universe",
-    universe,
+    type: "set clip quantiles",
+    clipQuantiles: { min, max },
+    annoMatrix,
+    obsCrossfilter,
   });
 };
 
-const saveObsAnnotations = () => async (dispatch, getState) => {
-  const { universe, annotations } = getState();
-  const { obsAnnotations, schema } = universe;
-  const { dataCollectionNameIsReadOnly, dataCollectionName } = annotations;
+const subsetAction = () => (dispatch, getState) => {
+  /*
+  Subset the annoMatrix to the current crossfilter selection
+  */
+  const {
+    annoMatrix: prevAnnoMatrix,
+    obsCrossfilter: prevObsCrossfilter,
+  } = getState();
 
+  const annoMatrix = isubsetMask(
+    prevAnnoMatrix,
+    prevObsCrossfilter.allSelectedMask()
+  );
+  const obsCrossfilter = new AnnoMatrixObsCrossfilter(annoMatrix);
   dispatch({
-    type: "writable obs annotations - save started",
+    type: "subset to selection",
+    annoMatrix,
+    obsCrossfilter,
   });
+};
 
-  const writableAnnotations = schema.annotations.obs.columns
-    .filter((s) => s.writable)
-    .map((s) => s.name);
-  const df = obsAnnotations.subset(null, writableAnnotations);
-  const matrix = MatrixFBS.encodeMatrixFBS(df);
-  try {
-    const queryString =
-      !dataCollectionNameIsReadOnly && !!dataCollectionName
-        ? `?annotation-collection-name=${encodeURIComponent(
-            dataCollectionName
-          )}`
-        : "";
-    const res = await fetch(
-      `${globals.API.prefix}${globals.API.version}annotations/obs${queryString}`,
-      {
-        method: "PUT",
-        body: matrix,
-        headers: new Headers({
-          "Content-Type": "application/octet-stream",
-        }),
-        credentials: "include",
-      }
-    );
-    if (res.ok) {
-      dispatch({
-        type: "writable obs annotations - save complete",
-        obsAnnotations,
-      });
-    } else {
-      dispatch({
-        type: "writable obs annotations - save error",
-        message: `HTTP error ${res.status} - ${res.statusText}`,
-        res,
-      });
-    }
-  } catch (error) {
-    dispatch({
-      type: "writable obs annotations - save error",
-      message: error.toString(),
-      error,
-    });
+const resetSubsetAction = () => (dispatch, getState) => {
+  /*
+  Reset the annoMatrix to all data.  Because we may have multiple views
+  stacked, we pop them all.  By convention, any clip transformation will
+  be the top of the stack, and must be preserved.
+  */
+
+  const { annoMatrix: prevAnnoMatrix } = getState();
+
+  const clipRange = prevAnnoMatrix.isClipped ? prevAnnoMatrix.clipRange : null;
+
+  /* pop all views */
+  let annoMatrix = prevAnnoMatrix;
+  while (annoMatrix.isView) {
+    annoMatrix = annoMatrix.viewOf;
   }
+
+  /* re-apply the clip, if any */
+  if (clipRange !== null) {
+    annoMatrix = clip(annoMatrix, ...clipRange);
+  }
+
+  const obsCrossfilter = new AnnoMatrixObsCrossfilter(annoMatrix);
+  dispatch({
+    type: "reset subset",
+    annoMatrix,
+    obsCrossfilter,
+  });
 };
 
 function fetchJson(pathAndQuery) {
@@ -443,19 +251,37 @@ function fetchJson(pathAndQuery) {
   );
 }
 
-function fetchBinary(pathAndQuery) {
-  return doBinaryRequest(
-    `${globals.API.prefix}${globals.API.version}${pathAndQuery}`
-  );
-}
-
 export default {
   doInitialDataLoad,
   requestDifferentialExpression,
   requestSingleGeneExpressionCountsForColoringPOST,
   requestUserDefinedGene,
   requestReembed,
-  resetWorldToUniverse,
-  saveObsAnnotations,
-  setWorldToSelection,
+  selectContinuousMetadataAction: selnActions.selectContinuousMetadataAction,
+  selectCategoricalMetadataAction: selnActions.selectCategoricalMetadataAction,
+  selectCategoricalAllMetadataAction:
+    selnActions.selectCategoricalAllMetadataAction,
+  graphBrushStartAction: selnActions.graphBrushStartAction,
+  graphBrushChangeAction: selnActions.graphBrushChangeAction,
+  graphBrushDeselectAction: selnActions.graphBrushDeselectAction,
+  graphBrushCancelAction: selnActions.graphBrushCancelAction,
+  graphBrushEndAction: selnActions.graphBrushEndAction,
+  graphLassoStartAction: selnActions.graphLassoStartAction,
+  graphLassoEndAction: selnActions.graphLassoEndAction,
+  graphLassoCancelAction: selnActions.graphLassoCancelAction,
+  graphLassoDeselectAction: selnActions.graphLassoDeselectAction,
+  clipAction,
+  subsetAction,
+  resetSubsetAction,
+  annotationCreateCategoryAction: annoActions.annotationCreateCategoryAction,
+  annotationRenameCategoryAction: annoActions.annotationRenameCategoryAction,
+  annotationDeleteCategoryAction: annoActions.annotationDeleteCategoryAction,
+  annotationCreateLabelInCategory: annoActions.annotationCreateLabelInCategory,
+  annotationDeleteLabelFromCategory:
+    annoActions.annotationDeleteLabelFromCategory,
+  annotationRenameLabelInCategory: annoActions.annotationRenameLabelInCategory,
+  annotationLabelCurrentSelection: annoActions.annotationLabelCurrentSelection,
+  saveObsAnnotationsAction: annoActions.saveObsAnnotationsAction,
+  needToSaveObsAnnotations: annoActions.needToSaveObsAnnotations,
+  layoutChoiceAction: selnActions.layoutChoiceAction,
 };