Add user-defined category-label colors (#1402)

* Add user-defined category-label colors

Fixes https://github.com/chanzuckerberg/cellxgene/issues/1152

As described in https://github.com/chanzuckerberg/cellxgene/issues/1307

* Respond to feedback from @bkmartinjr in nodejs

* Respond to feedback from @bkmartinjr in python

* Add tests to the server module

* Autoformat python, run linter

* Make colors_get error handling specific

* Respond to feedback from @bkmartinjr

* Respond to feedback from @bkmartinjr

* Fix whitespace

* Fix python lint errrors

* Update documentation

* Add --disable-user-colors option to launch and cxgtool.py

* Fix python formatting

* Rename '--disable-user-colors' to '--disable-custom-colors'

client/src/actions/index.js

@@ -9,12 +9,13 @@ import {
 } from "../util/actionHelpers";
 import { PromiseLimit } from "../util/promiseLimit";
 import { requestReembed, reembedResetWorldToUniverse } from "./reembed";
+import { loadUserColorConfig } from "../util/stateManager/colorHelpers";
 
 /*
 return promise to fetch the OBS annotations we need to load.  Omit anything
 we don't need.
 */
-function obsAnnotationFetchAndLoad(dispatch, schema) {
+async function obsAnnotationFetchAndLoad(dispatch, schema) {
   const obsAnnotations = schema?.schema?.annotations?.obs ?? {};
   const index = obsAnnotations.index ?? false;
   const columns = (obsAnnotations.columns ?? []).filter(
@@ -23,21 +24,18 @@ function obsAnnotationFetchAndLoad(dispatch, schema) {
 
   const plimit = new PromiseLimit(5);
   return Promise.all(
-    columns.map((col) =>
-      plimit.add(() => {
-        const path = `annotations/obs?annotation-name=${encodeURIComponent(
-          col.name
-        )}`;
-        const url = `${globals.API.prefix}${globals.API.version}${path}`;
-        return doBinaryRequest(url).then((buffer) => {
-          const df = Universe.matrixFBSToDataframe(buffer);
-          dispatch({
-            type: "universe: column load success",
-            dim: "obsAnnotations",
-            dataframe: df,
-          });
-        });
-      })
+    columns.map(col =>
+      plimit.add(() =>
+        fetchBinary(`annotations/obs?annotation-name=${encodeURIComponent(col.name)}`)
+          .then(buffer => Universe.matrixFBSToDataframe(buffer))
+          .then(df =>
+            dispatch({
+              type: "universe: column load success",
+              dim: "obsAnnotations",
+              dataframe: df
+            })
+          )
+      )
     )
   );
 }
@@ -45,31 +43,22 @@ function obsAnnotationFetchAndLoad(dispatch, schema) {
 /*
 return promise fetching VAR annotations we need to load.  Only index is currently used.
 */
-function varAnnotationFetchAndLoad(dispatch, schema) {
+async function varAnnotationFetchAndLoad(dispatch, schema) {
   const varAnnotations = schema?.schema?.annotations?.var ?? {};
   const index = varAnnotations.index ?? false;
   const names = index ? [index] : [];
   return Promise.all(
-    names
-      .map((name) => {
-        const path = `annotations/var?annotation-name=${encodeURIComponent(
-          name
-        )}`;
-        const url = `${globals.API.prefix}${globals.API.version}${path}`;
-        return doBinaryRequest(url);
-      })
-      .map((rqst) =>
-        rqst.then((buffer) => Universe.matrixFBSToDataframe(buffer))
-      )
-      .map((resp) =>
-        resp.then((df) =>
+    names.map(name =>
+      fetchBinary(`annotations/var?annotation-name=${encodeURIComponent(name)}`)
+        .then(buffer => Universe.matrixFBSToDataframe(buffer))
+        .then(df =>
           dispatch({
             type: "universe: column load success",
             dim: "varAnnotations",
-            dataframe: df,
+            dataframe: df
           })
         )
-      )
+    )
   );
 }
 
@@ -79,26 +68,35 @@ return promise fetching layout we need
 function layoutFetchAndLoad(dispatch, schema) {
   const embeddings = schema?.schema?.layout?.obs ?? [];
   const embNames = embeddings.map((e) => e.name);
-  const baseURL = `${globals.API.prefix}${globals.API.version}layout/obs`;
 
   const plimit = new PromiseLimit(5);
   return Promise.all(
-    embNames.map((e) =>
-      plimit.add(() => {
-        const url = `${baseURL}?layout-name=${encodeURIComponent(e)}`;
-        return doBinaryRequest(url).then((buffer) =>
-          Universe.matrixFBSToDataframe(buffer)
-        );
-      })
+    embNames.map(e =>
+      plimit.add(() =>
+        fetchBinary(`layout/obs?layout-name=${encodeURIComponent(e)}`)
+          .then(buffer => Universe.matrixFBSToDataframe(buffer))
+      )
     )
-  ).then((dfs) => {
-    const df = Dataframe.Dataframe.empty().withColsFromAll(dfs);
+  ).then(dfs =>
     dispatch({
       type: "universe: column load success",
       dim: "obsLayout",
-      dataframe: df,
-    });
-  });
+      dataframe: Dataframe.Dataframe.empty().withColsFromAll(dfs)
+    })
+  );
+}
+
+/*
+return promise fetching user-configured colors
+*/
+async function userColorsFetchAndLoad(dispatch) {
+  return fetchJson("colors")
+    .then(response =>
+      dispatch({
+        type: "universe: user color load success",
+        userColors: loadUserColorConfig(response)
+      })
+    );
 }
 
 /*
@@ -116,13 +114,10 @@ const doInitialDataLoad = () =>
       /*
       Step 1 - config & schema, all JSON
       */
-      const requestJson = ["config", "schema"]
-        .map((r) => `${globals.API.prefix}${globals.API.version}${r}`)
-        .map((url) => doJsonRequest(url));
-      const stepOneResults = await Promise.all(requestJson);
+      const requestJson = ["config", "schema"].map(fetchJson);
+      const [responseConfig, schema] = await Promise.all(requestJson);
       /* set config defaults */
-      const config = { ...globals.configDefaults, ...stepOneResults[0].config };
-      const schema = stepOneResults[1];
+      const config = { ...globals.configDefaults, ...responseConfig.config };
       const universe = Universe.createUniverseFromResponse(config, schema);
       dispatch({
         type: "universe exists, but loading is still in progress",
@@ -137,6 +132,7 @@ const doInitialDataLoad = () =>
       Step 2 - load the minimum stuff required to display.
       */
       await Promise.all([
+        userColorsFetchAndLoad(dispatch),
         layoutFetchAndLoad(dispatch, schema),
         varAnnotationFetchAndLoad(dispatch, schema),
       ]);
@@ -169,13 +165,6 @@ const setWorldToSelection = () => (dispatch, getState) => {
   });
 };
 
-// Throws
-const dispatchExpressionErrors = (dispatch, res) => {
-  const msg = `Unexpected HTTP response while fetching expression data ${res.status}, ${res.statusText}`;
-  dispatchNetworkErrorMessageToUser(msg);
-  throw new Error(msg);
-};
-
 /* double URI encode - needed for query-param filters */
 function dubEncURIComponent(s) {
   return encodeURIComponent(encodeURIComponent(s));
@@ -196,16 +185,11 @@ async function _doRequestExpressionData(dispatch, getState, genes) {
   /* helper for this function only */
   const fetchData = async (geneNames) => {
     const query = geneNames
-      .map(
-        (g) =>
-          `var:${dubEncURIComponent(varIndexName)}=${dubEncURIComponent(g)}`
-      )
+      .map(g => `var:${dubEncURIComponent(varIndexName)}=${dubEncURIComponent(g)}`)
       .join("&");
-    const url = `${globals.API.prefix}${globals.API.version}data/var?${query}`;
-    return doBinaryRequest(url).then((buffer) =>
-      // TODO: why convert to an Object and not a Dataframe?
-      Universe.convertDataFBStoObject(universe, buffer)
-    );
+    // TODO: why convert to an Object and not a Dataframe?
+    return fetchBinary(`data/var?${query}`)
+      .then(buffer => Universe.convertDataFBStoObject(universe, buffer));
   };
 
   /* preload data already in cache */
@@ -367,7 +351,7 @@ const requestDifferentialExpression = (set1, set2, num_genes = 10) => async (
     */
     const plimit = new PromiseLimit(5);
     await Promise.all(
-      topNGenes.map((gene) =>
+      topNGenes.map(gene =>
         plimit.add(() => _doRequestExpressionData(dispatch, getState, [gene]))
       )
     );
@@ -447,6 +431,14 @@ const saveObsAnnotations = () => async (dispatch, getState) => {
   }
 };
 
+function fetchJson(pathAndQuery) {
+  return doJsonRequest(`${globals.API.prefix}${globals.API.version}${pathAndQuery}`);
+}
+
+function fetchBinary(pathAndQuery) {
+  return doBinaryRequest(`${globals.API.prefix}${globals.API.version}${pathAndQuery}`);
+}
+
 export default {
   doInitialDataLoad,
   requestDifferentialExpression,

client/src/components/categorical/category/index.js

@@ -205,7 +205,7 @@ class Category extends React.Component {
       return (
         <div style={{ marginBottom: 10, marginTop: 4 }}>
           <span style={{ fontWeight: 700 }}>
-            {truncatedString ? truncatedString : metadataField}
+            {truncatedString || metadataField}
           </span>
           : {schema.annotations.obsByName[metadataField].categories[0]}
         </div>

client/src/reducers/colors.js

@@ -27,6 +27,14 @@ const ColorsReducer = (
       };
     }
 
+    case "universe: user color load success": {
+      const { userColors } = action;
+      return {
+        ...state,
+        userColors
+      };
+    }
+
     case "reset World to eq Universe": {
       /* need to rebuild colors as world may have changed, but don't switch modes */
       const { world } = nextSharedState;
@@ -90,7 +98,7 @@ const ColorsReducer = (
 
     case "color by categorical metadata":
     case "color by continuous metadata": {
-      const { world } = prevSharedState;
+      const { world, colors } = prevSharedState;
 
       /* toggle between this mode and reset */
       const resetCurrent =
@@ -99,11 +107,7 @@ const ColorsReducer = (
       const colorMode = !resetCurrent ? action.type : null;
       const colorAccessor = !resetCurrent ? action.colorAccessor : null;
 
-      const { rgb, scale } = ColorHelpers.createColors(
-        world,
-        colorMode,
-        colorAccessor
-      );
+      const { rgb, scale } = ColorHelpers.createColors(world, colorMode, colorAccessor, colors.userColors);
       return {
         ...state,
         colorMode,
@@ -141,7 +145,7 @@ const ColorsReducer = (
     case "annotation: delete label": {
       const { world } = nextSharedState;
       const { colorMode, colorAccessor } = state;
-      const { metadataField } = action;
+      const { metadataField, colors } = action;
       if (
         colorMode !== "color by categorical metadata" ||
         colorAccessor !== metadataField
@@ -149,11 +153,7 @@ const ColorsReducer = (
         return state;
 
       /* else, we need to rebuild colors as labels have changed! */
-      const { rgb, scale } = ColorHelpers.createColors(
-        world,
-        colorMode,
-        colorAccessor
-      );
+      const { rgb, scale } = ColorHelpers.createColors(world, colorMode, colorAccessor);
       return { ...state, rgb, scale };
     }
 

client/src/reducers/undoableConfig.js

@@ -12,6 +12,7 @@ const skipOnActions = new Set([
   "interface reset started",
   "initial data load start",
   "universe: column load success",
+  "universe: user color load success",
   "universe exists, but loading is still in progress",
   "configuration load complete",
   "increment graph render counter",

client/src/util/stateManager/colorHelpers.js

@@ -11,13 +11,16 @@ import { range } from "../range";
 /*
 create new colors state object.   Paramters:
   - world - current world object
-  - mode - color-by mode.  One of: null, "color by expression",
-    "color by continuous metadata", "color by categorical metadata"
-  -
+  - colorMode - color-by mode. One of {null, "color by expression", "color by continuous metadata",
+    "color by categorical metadata"}
+  - colorAccessor - the obs annotations used for color-by
 */
-export function createColors(world, colorMode = null, colorAccessor = null) {
+export function createColors(world, colorMode = null, colorAccessor = null, userColors = null) {
   switch (colorMode) {
     case "color by categorical metadata": {
+      if (userColors && colorAccessor in userColors) {
+        return createUserColors(world, colorAccessor, userColors);
+      }
       return createColorsByCategoricalMetadata(world, colorAccessor);
     }
     case "color by continuous metadata": {
@@ -36,8 +39,29 @@ export function createColors(world, colorMode = null, colorAccessor = null) {
   }
 }
 
-function createColorsByCategoricalMetadata(world, accessor) {
-  const { categories } = world.schema.annotations.obsByName[accessor];
+export function loadUserColorConfig(userColors) {
+  const convertedUserColors = {};
+  Object.keys(userColors).forEach(category => {
+    const [colors, scaleMap] = Object.keys(userColors[category]).reduce((acc, label, i) => {
+      const color = parseRGB(userColors[category][label]);
+      acc[0][label] = color;
+      acc[1][i] = d3.rgb(255 * color[0], 255 * color[1], 255 * color[2]);
+      return acc;
+    }, [{}, {}]);
+    const scale = i => scaleMap[i];
+    convertedUserColors[category] = { colors, scale };
+  });
+  return convertedUserColors;
+}
+
+function createUserColors(world, colorAccessor, userColors) {
+  const { colors, scale } = userColors[colorAccessor];
+  const rgb = createRgbArray(world, colors, colorAccessor);
+  return { rgb, scale };
+}
+
+function createColorsByCategoricalMetadata(world, colorAccessor) {
+  const { categories } = world.schema.annotations.obsByName[colorAccessor];
 
   const scale = d3
     .scaleSequential(interpolateRainbow)
@@ -49,14 +73,19 @@ function createColorsByCategoricalMetadata(world, accessor) {
     return acc;
   }, {});
 
+  const rgb = createRgbArray(world, colors, colorAccessor);
+  return { rgb, scale };
+}
+
+export function createRgbArray(world, colors, colorAccessor) {
   const rgb = new Array(world.nObs);
   const df = world.obsAnnotations;
-  const data = df.col(accessor).asArray();
+  const data = df.col(colorAccessor).asArray();
   for (let i = 0, len = df.length; i < len; i += 1) {
-    const cat = data[i];
-    rgb[i] = colors[cat];
+    const label = data[i];
+    rgb[i] = colors[label];
   }
-  return { rgb, scale };
+  return rgb;
 }
 
 function createColorsByContinuousMetadata(world, accessor) {