Add user-defined category-label colors (#1402)

* Add user-defined category-label colors

Fixes https://github.com/chanzuckerberg/cellxgene/issues/1152

As described in https://github.com/chanzuckerberg/cellxgene/issues/1307

* Respond to feedback from @bkmartinjr in nodejs

* Respond to feedback from @bkmartinjr in python

* Add tests to the server module

* Autoformat python, run linter

* Make colors_get error handling specific

* Respond to feedback from @bkmartinjr

* Respond to feedback from @bkmartinjr

* Fix whitespace

* Fix python lint errrors

* Update documentation

* Add --disable-user-colors option to launch and cxgtool.py

* Fix python formatting

* Rename '--disable-user-colors' to '--disable-custom-colors'

client/src/actions/index.js

@@ -9,12 +9,13 @@ import {
 } from "../util/actionHelpers";
 import { PromiseLimit } from "../util/promiseLimit";
 import { requestReembed, reembedResetWorldToUniverse } from "./reembed";
+import { loadUserColorConfig } from "../util/stateManager/colorHelpers";
 
 /*
 return promise to fetch the OBS annotations we need to load.  Omit anything
 we don't need.
 */
-function obsAnnotationFetchAndLoad(dispatch, schema) {
+async function obsAnnotationFetchAndLoad(dispatch, schema) {
   const obsAnnotations = schema?.schema?.annotations?.obs ?? {};
   const index = obsAnnotations.index ?? false;
   const columns = (obsAnnotations.columns ?? []).filter(
@@ -23,21 +24,18 @@ function obsAnnotationFetchAndLoad(dispatch, schema) {
 
   const plimit = new PromiseLimit(5);
   return Promise.all(
-    columns.map((col) =>
+    columns.map(col =>
-      plimit.add(() => {
+      plimit.add(() =>
-        const path = `annotations/obs?annotation-name=${encodeURIComponent(
+        fetchBinary(`annotations/obs?annotation-name=${encodeURIComponent(col.name)}`)
-          col.name
+          .then(buffer => Universe.matrixFBSToDataframe(buffer))
-        )}`;
+          .then(df =>
-        const url = `${globals.API.prefix}${globals.API.version}${path}`;
+            dispatch({
-        return doBinaryRequest(url).then((buffer) => {
+              type: "universe: column load success",
-          const df = Universe.matrixFBSToDataframe(buffer);
+              dim: "obsAnnotations",
-          dispatch({
+              dataframe: df
-            type: "universe: column load success",
+            })
-            dim: "obsAnnotations",
+          )
-            dataframe: df,
+      )
-          });
-        });
-      })
     )
   );
 }
@@ -45,31 +43,22 @@ function obsAnnotationFetchAndLoad(dispatch, schema) {
 /*
 return promise fetching VAR annotations we need to load.  Only index is currently used.
 */
-function varAnnotationFetchAndLoad(dispatch, schema) {
+async function varAnnotationFetchAndLoad(dispatch, schema) {
   const varAnnotations = schema?.schema?.annotations?.var ?? {};
   const index = varAnnotations.index ?? false;
   const names = index ? [index] : [];
   return Promise.all(
-    names
+    names.map(name =>
-      .map((name) => {
+      fetchBinary(`annotations/var?annotation-name=${encodeURIComponent(name)}`)
-        const path = `annotations/var?annotation-name=${encodeURIComponent(
+        .then(buffer => Universe.matrixFBSToDataframe(buffer))
-          name
+        .then(df =>
-        )}`;
-        const url = `${globals.API.prefix}${globals.API.version}${path}`;
-        return doBinaryRequest(url);
-      })
-      .map((rqst) =>
-        rqst.then((buffer) => Universe.matrixFBSToDataframe(buffer))
-      )
-      .map((resp) =>
-        resp.then((df) =>
           dispatch({
             type: "universe: column load success",
             dim: "varAnnotations",
-            dataframe: df,
+            dataframe: df
           })
         )
-      )
+    )
   );
 }
 
@@ -79,26 +68,35 @@ return promise fetching layout we need
 function layoutFetchAndLoad(dispatch, schema) {
   const embeddings = schema?.schema?.layout?.obs ?? [];
   const embNames = embeddings.map((e) => e.name);
-  const baseURL = `${globals.API.prefix}${globals.API.version}layout/obs`;
 
   const plimit = new PromiseLimit(5);
   return Promise.all(
-    embNames.map((e) =>
+    embNames.map(e =>
-      plimit.add(() => {
+      plimit.add(() =>
-        const url = `${baseURL}?layout-name=${encodeURIComponent(e)}`;
+        fetchBinary(`layout/obs?layout-name=${encodeURIComponent(e)}`)
-        return doBinaryRequest(url).then((buffer) =>
+          .then(buffer => Universe.matrixFBSToDataframe(buffer))
-          Universe.matrixFBSToDataframe(buffer)
+      )
-        );
-      })
     )
-  ).then((dfs) => {
+  ).then(dfs =>
-    const df = Dataframe.Dataframe.empty().withColsFromAll(dfs);
     dispatch({
       type: "universe: column load success",
       dim: "obsLayout",
-      dataframe: df,
+      dataframe: Dataframe.Dataframe.empty().withColsFromAll(dfs)
-    });
+    })
-  });
+  );
+}
+
+/*
+return promise fetching user-configured colors
+*/
+async function userColorsFetchAndLoad(dispatch) {
+  return fetchJson("colors")
+    .then(response =>
+      dispatch({
+        type: "universe: user color load success",
+        userColors: loadUserColorConfig(response)
+      })
+    );
 }
 
 /*
@@ -116,13 +114,10 @@ const doInitialDataLoad = () =>
       /*
       Step 1 - config & schema, all JSON
       */
-      const requestJson = ["config", "schema"]
+      const requestJson = ["config", "schema"].map(fetchJson);
-        .map((r) => `${globals.API.prefix}${globals.API.version}${r}`)
+      const [responseConfig, schema] = await Promise.all(requestJson);
-        .map((url) => doJsonRequest(url));
-      const stepOneResults = await Promise.all(requestJson);
       /* set config defaults */
-      const config = { ...globals.configDefaults, ...stepOneResults[0].config };
+      const config = { ...globals.configDefaults, ...responseConfig.config };
-      const schema = stepOneResults[1];
       const universe = Universe.createUniverseFromResponse(config, schema);
       dispatch({
         type: "universe exists, but loading is still in progress",
@@ -137,6 +132,7 @@ const doInitialDataLoad = () =>
       Step 2 - load the minimum stuff required to display.
       */
       await Promise.all([
+        userColorsFetchAndLoad(dispatch),
         layoutFetchAndLoad(dispatch, schema),
         varAnnotationFetchAndLoad(dispatch, schema),
       ]);
@@ -169,13 +165,6 @@ const setWorldToSelection = () => (dispatch, getState) => {
   });
 };
 
-// Throws
-const dispatchExpressionErrors = (dispatch, res) => {
-  const msg = `Unexpected HTTP response while fetching expression data ${res.status}, ${res.statusText}`;
-  dispatchNetworkErrorMessageToUser(msg);
-  throw new Error(msg);
-};
-
 /* double URI encode - needed for query-param filters */
 function dubEncURIComponent(s) {
   return encodeURIComponent(encodeURIComponent(s));
@@ -196,16 +185,11 @@ async function _doRequestExpressionData(dispatch, getState, genes) {
   /* helper for this function only */
   const fetchData = async (geneNames) => {
     const query = geneNames
-      .map(
+      .map(g => `var:${dubEncURIComponent(varIndexName)}=${dubEncURIComponent(g)}`)
-        (g) =>
-          `var:${dubEncURIComponent(varIndexName)}=${dubEncURIComponent(g)}`
-      )
       .join("&");
-    const url = `${globals.API.prefix}${globals.API.version}data/var?${query}`;
+    // TODO: why convert to an Object and not a Dataframe?
-    return doBinaryRequest(url).then((buffer) =>
+    return fetchBinary(`data/var?${query}`)
-      // TODO: why convert to an Object and not a Dataframe?
+      .then(buffer => Universe.convertDataFBStoObject(universe, buffer));
-      Universe.convertDataFBStoObject(universe, buffer)
-    );
   };
 
   /* preload data already in cache */
@@ -367,7 +351,7 @@ const requestDifferentialExpression = (set1, set2, num_genes = 10) => async (
     */
     const plimit = new PromiseLimit(5);
     await Promise.all(
-      topNGenes.map((gene) =>
+      topNGenes.map(gene =>
         plimit.add(() => _doRequestExpressionData(dispatch, getState, [gene]))
       )
     );
@@ -447,6 +431,14 @@ const saveObsAnnotations = () => async (dispatch, getState) => {
   }
 };
 
+function fetchJson(pathAndQuery) {
+  return doJsonRequest(`${globals.API.prefix}${globals.API.version}${pathAndQuery}`);
+}
+
+function fetchBinary(pathAndQuery) {
+  return doBinaryRequest(`${globals.API.prefix}${globals.API.version}${pathAndQuery}`);
+}
+
 export default {
   doInitialDataLoad,
   requestDifferentialExpression,

client/src/components/categorical/category/index.js

@@ -205,7 +205,7 @@ class Category extends React.Component {
       return (
         <div style={{ marginBottom: 10, marginTop: 4 }}>
           <span style={{ fontWeight: 700 }}>
-            {truncatedString ? truncatedString : metadataField}
+            {truncatedString || metadataField}
           </span>
           : {schema.annotations.obsByName[metadataField].categories[0]}
         </div>

client/src/reducers/colors.js

@@ -27,6 +27,14 @@ const ColorsReducer = (
       };
     }
 
+    case "universe: user color load success": {
+      const { userColors } = action;
+      return {
+        ...state,
+        userColors
+      };
+    }
+
     case "reset World to eq Universe": {
       /* need to rebuild colors as world may have changed, but don't switch modes */
       const { world } = nextSharedState;
@@ -90,7 +98,7 @@ const ColorsReducer = (
 
     case "color by categorical metadata":
     case "color by continuous metadata": {
-      const { world } = prevSharedState;
+      const { world, colors } = prevSharedState;
 
       /* toggle between this mode and reset */
       const resetCurrent =
@@ -99,11 +107,7 @@ const ColorsReducer = (
       const colorMode = !resetCurrent ? action.type : null;
       const colorAccessor = !resetCurrent ? action.colorAccessor : null;
 
-      const { rgb, scale } = ColorHelpers.createColors(
+      const { rgb, scale } = ColorHelpers.createColors(world, colorMode, colorAccessor, colors.userColors);
-        world,
-        colorMode,
-        colorAccessor
-      );
       return {
         ...state,
         colorMode,
@@ -141,7 +145,7 @@ const ColorsReducer = (
     case "annotation: delete label": {
       const { world } = nextSharedState;
       const { colorMode, colorAccessor } = state;
-      const { metadataField } = action;
+      const { metadataField, colors } = action;
       if (
         colorMode !== "color by categorical metadata" ||
         colorAccessor !== metadataField
@@ -149,11 +153,7 @@ const ColorsReducer = (
         return state;
 
       /* else, we need to rebuild colors as labels have changed! */
-      const { rgb, scale } = ColorHelpers.createColors(
+      const { rgb, scale } = ColorHelpers.createColors(world, colorMode, colorAccessor);
-        world,
-        colorMode,
-        colorAccessor
-      );
       return { ...state, rgb, scale };
     }
 

client/src/reducers/undoableConfig.js

@@ -12,6 +12,7 @@ const skipOnActions = new Set([
   "interface reset started",
   "initial data load start",
   "universe: column load success",
+  "universe: user color load success",
   "universe exists, but loading is still in progress",
   "configuration load complete",
   "increment graph render counter",

client/src/util/stateManager/colorHelpers.js

@@ -11,13 +11,16 @@ import { range } from "../range";
 /*
 create new colors state object.   Paramters:
   - world - current world object
-  - mode - color-by mode.  One of: null, "color by expression",
+  - colorMode - color-by mode. One of {null, "color by expression", "color by continuous metadata",
-    "color by continuous metadata", "color by categorical metadata"
+    "color by categorical metadata"}
-  -
+  - colorAccessor - the obs annotations used for color-by
 */
-export function createColors(world, colorMode = null, colorAccessor = null) {
+export function createColors(world, colorMode = null, colorAccessor = null, userColors = null) {
   switch (colorMode) {
     case "color by categorical metadata": {
+      if (userColors && colorAccessor in userColors) {
+        return createUserColors(world, colorAccessor, userColors);
+      }
       return createColorsByCategoricalMetadata(world, colorAccessor);
     }
     case "color by continuous metadata": {
@@ -36,8 +39,29 @@ export function createColors(world, colorMode = null, colorAccessor = null) {
   }
 }
 
-function createColorsByCategoricalMetadata(world, accessor) {
+export function loadUserColorConfig(userColors) {
-  const { categories } = world.schema.annotations.obsByName[accessor];
+  const convertedUserColors = {};
+  Object.keys(userColors).forEach(category => {
+    const [colors, scaleMap] = Object.keys(userColors[category]).reduce((acc, label, i) => {
+      const color = parseRGB(userColors[category][label]);
+      acc[0][label] = color;
+      acc[1][i] = d3.rgb(255 * color[0], 255 * color[1], 255 * color[2]);
+      return acc;
+    }, [{}, {}]);
+    const scale = i => scaleMap[i];
+    convertedUserColors[category] = { colors, scale };
+  });
+  return convertedUserColors;
+}
+
+function createUserColors(world, colorAccessor, userColors) {
+  const { colors, scale } = userColors[colorAccessor];
+  const rgb = createRgbArray(world, colors, colorAccessor);
+  return { rgb, scale };
+}
+
+function createColorsByCategoricalMetadata(world, colorAccessor) {
+  const { categories } = world.schema.annotations.obsByName[colorAccessor];
 
   const scale = d3
     .scaleSequential(interpolateRainbow)
@@ -49,14 +73,19 @@ function createColorsByCategoricalMetadata(world, accessor) {
     return acc;
   }, {});
 
+  const rgb = createRgbArray(world, colors, colorAccessor);
+  return { rgb, scale };
+}
+
+export function createRgbArray(world, colors, colorAccessor) {
   const rgb = new Array(world.nObs);
   const df = world.obsAnnotations;
-  const data = df.col(accessor).asArray();
+  const data = df.col(colorAccessor).asArray();
   for (let i = 0, len = df.length; i < len; i += 1) {
-    const cat = data[i];
+    const label = data[i];
-    rgb[i] = colors[cat];
+    rgb[i] = colors[label];
   }
-  return { rgb, scale };
+  return rgb;
 }
 
 function createColorsByContinuousMetadata(world, accessor) {

docs/_config.yml

@@ -1,5 +1,6 @@
 theme: jekyll-theme-minimal
 show_downloads: false
+url: "https://chanzuckerberg.github.io"
 baseurl: "/cellxgene"
 
 logo: cellxgene-logo.png

docs/_site/index.html

@@ -5,21 +5,21 @@
     <meta http-equiv="X-UA-Compatible" content="IE=edge">
     <meta name="viewport" content="width=device-width, initial-scale=1">
 
-<!-- Begin Jekyll SEO tag v2.5.0 -->
+<!-- Begin Jekyll SEO tag v2.6.1 -->
 <title>Index | cellxgene</title>
 <meta name="generator" content="Jekyll v3.8.5" />
 <meta property="og:title" content="Index" />
 <meta property="og:locale" content="en_US" />
 <meta name="description" content="An interactive explorer for single-cell transcriptomics data" />
 <meta property="og:description" content="An interactive explorer for single-cell transcriptomics data" />
-<link rel="canonical" href="http://localhost:4000/cellxgene/" />
+<link rel="canonical" href="https://chanzuckerberg.github.io/cellxgene/" />
-<meta property="og:url" content="http://localhost:4000/cellxgene/" />
+<meta property="og:url" content="https://chanzuckerberg.github.io/cellxgene/" />
 <meta property="og:site_name" content="cellxgene" />
 <script type="application/ld+json">
-{"description":"An interactive explorer for single-cell transcriptomics data","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"http://localhost:4000/cellxgene/cellxgene-logo.png"}},"@type":"WebSite","headline":"Index","url":"http://localhost:4000/cellxgene/","name":"cellxgene","@context":"http://schema.org"}</script>
+{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"An interactive explorer for single-cell transcriptomics data","@type":"WebSite","url":"https://chanzuckerberg.github.io/cellxgene/","headline":"Index","name":"cellxgene","@context":"https://schema.org"}</script>
 <!-- End Jekyll SEO tag -->
 
-    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=565a3f148df7473603c355a32536cb0eea068885">
+    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=b45f000ecb36779dde84b689d463d17a077275d6">
     <!--[if lt IE 9]>
     <script src="https://cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv.min.js"></script>
     <![endif]-->

docs/_site/posts/annotations.html

@@ -5,21 +5,21 @@
     <meta http-equiv="X-UA-Compatible" content="IE=edge">
     <meta name="viewport" content="width=device-width, initial-scale=1">
 
-<!-- Begin Jekyll SEO tag v2.5.0 -->
+<!-- Begin Jekyll SEO tag v2.6.1 -->
 <title>annotations | cellxgene</title>
 <meta name="generator" content="Jekyll v3.8.5" />
 <meta property="og:title" content="annotations" />
 <meta property="og:locale" content="en_US" />
 <meta name="description" content="Creating annotations" />
 <meta property="og:description" content="Creating annotations" />
-<link rel="canonical" href="http://localhost:4000/cellxgene/posts/annotations.html" />
+<link rel="canonical" href="https://chanzuckerberg.github.io/cellxgene/posts/annotations.html" />
-<meta property="og:url" content="http://localhost:4000/cellxgene/posts/annotations.html" />
+<meta property="og:url" content="https://chanzuckerberg.github.io/cellxgene/posts/annotations.html" />
 <meta property="og:site_name" content="cellxgene" />
 <script type="application/ld+json">
-{"description":"Creating annotations","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"http://localhost:4000/cellxgene/cellxgene-logo.png"}},"@type":"WebPage","headline":"annotations","url":"http://localhost:4000/cellxgene/posts/annotations.html","@context":"http://schema.org"}</script>
+{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"Creating annotations","@type":"WebPage","url":"https://chanzuckerberg.github.io/cellxgene/posts/annotations.html","headline":"annotations","@context":"https://schema.org"}</script>
 <!-- End Jekyll SEO tag -->
 
-    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=565a3f148df7473603c355a32536cb0eea068885">
+    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=b45f000ecb36779dde84b689d463d17a077275d6">
     <!--[if lt IE 9]>
     <script src="https://cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv.min.js"></script>
     <![endif]-->

docs/_site/posts/contact.html

@@ -5,21 +5,21 @@
     <meta http-equiv="X-UA-Compatible" content="IE=edge">
     <meta name="viewport" content="width=device-width, initial-scale=1">
 
-<!-- Begin Jekyll SEO tag v2.5.0 -->
+<!-- Begin Jekyll SEO tag v2.6.1 -->
 <title>Contact | cellxgene</title>
 <meta name="generator" content="Jekyll v3.8.5" />
 <meta property="og:title" content="Contact" />
 <meta property="og:locale" content="en_US" />
 <meta name="description" content="Contact" />
 <meta property="og:description" content="Contact" />
-<link rel="canonical" href="http://localhost:4000/cellxgene/posts/contact.html" />
+<link rel="canonical" href="https://chanzuckerberg.github.io/cellxgene/posts/contact.html" />
-<meta property="og:url" content="http://localhost:4000/cellxgene/posts/contact.html" />
+<meta property="og:url" content="https://chanzuckerberg.github.io/cellxgene/posts/contact.html" />
 <meta property="og:site_name" content="cellxgene" />
 <script type="application/ld+json">
-{"description":"Contact","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"http://localhost:4000/cellxgene/cellxgene-logo.png"}},"@type":"WebPage","headline":"Contact","url":"http://localhost:4000/cellxgene/posts/contact.html","@context":"http://schema.org"}</script>
+{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"Contact","@type":"WebPage","url":"https://chanzuckerberg.github.io/cellxgene/posts/contact.html","headline":"Contact","@context":"https://schema.org"}</script>
 <!-- End Jekyll SEO tag -->
 
-    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=565a3f148df7473603c355a32536cb0eea068885">
+    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=b45f000ecb36779dde84b689d463d17a077275d6">
     <!--[if lt IE 9]>
     <script src="https://cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv.min.js"></script>
     <![endif]-->

docs/_site/posts/contribute.html

@@ -5,21 +5,21 @@
     <meta http-equiv="X-UA-Compatible" content="IE=edge">
     <meta name="viewport" content="width=device-width, initial-scale=1">
 
-<!-- Begin Jekyll SEO tag v2.5.0 -->
+<!-- Begin Jekyll SEO tag v2.6.1 -->
 <title>Code of conduct | cellxgene</title>
 <meta name="generator" content="Jekyll v3.8.5" />
 <meta property="og:title" content="Code of conduct" />
 <meta property="og:locale" content="en_US" />
 <meta name="description" content="An interactive explorer for single-cell transcriptomics data" />
 <meta property="og:description" content="An interactive explorer for single-cell transcriptomics data" />
-<link rel="canonical" href="http://localhost:4000/cellxgene/posts/contribute.html" />
+<link rel="canonical" href="https://chanzuckerberg.github.io/cellxgene/posts/contribute.html" />
-<meta property="og:url" content="http://localhost:4000/cellxgene/posts/contribute.html" />
+<meta property="og:url" content="https://chanzuckerberg.github.io/cellxgene/posts/contribute.html" />
 <meta property="og:site_name" content="cellxgene" />
 <script type="application/ld+json">
-{"description":"An interactive explorer for single-cell transcriptomics data","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"http://localhost:4000/cellxgene/cellxgene-logo.png"}},"@type":"WebPage","headline":"Code of conduct","url":"http://localhost:4000/cellxgene/posts/contribute.html","@context":"http://schema.org"}</script>
+{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"An interactive explorer for single-cell transcriptomics data","@type":"WebPage","url":"https://chanzuckerberg.github.io/cellxgene/posts/contribute.html","headline":"Code of conduct","@context":"https://schema.org"}</script>
 <!-- End Jekyll SEO tag -->
 
-    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=565a3f148df7473603c355a32536cb0eea068885">
+    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=b45f000ecb36779dde84b689d463d17a077275d6">
     <!--[if lt IE 9]>
     <script src="https://cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv.min.js"></script>
     <![endif]-->

docs/_site/posts/demo-data.html

@@ -5,21 +5,21 @@
     <meta http-equiv="X-UA-Compatible" content="IE=edge">
     <meta name="viewport" content="width=device-width, initial-scale=1">
 
-<!-- Begin Jekyll SEO tag v2.5.0 -->
+<!-- Begin Jekyll SEO tag v2.6.1 -->
 <title>demo-data | cellxgene</title>
 <meta name="generator" content="Jekyll v3.8.5" />
 <meta property="og:title" content="demo-data" />
 <meta property="og:locale" content="en_US" />
 <meta name="description" content="Demo datasets" />
 <meta property="og:description" content="Demo datasets" />
-<link rel="canonical" href="http://localhost:4000/cellxgene/posts/demo-data.html" />
+<link rel="canonical" href="https://chanzuckerberg.github.io/cellxgene/posts/demo-data.html" />
-<meta property="og:url" content="http://localhost:4000/cellxgene/posts/demo-data.html" />
+<meta property="og:url" content="https://chanzuckerberg.github.io/cellxgene/posts/demo-data.html" />
 <meta property="og:site_name" content="cellxgene" />
 <script type="application/ld+json">
-{"description":"Demo datasets","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"http://localhost:4000/cellxgene/cellxgene-logo.png"}},"@type":"WebPage","headline":"demo-data","url":"http://localhost:4000/cellxgene/posts/demo-data.html","@context":"http://schema.org"}</script>
+{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"Demo datasets","@type":"WebPage","url":"https://chanzuckerberg.github.io/cellxgene/posts/demo-data.html","headline":"demo-data","@context":"https://schema.org"}</script>
 <!-- End Jekyll SEO tag -->
 
-    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=565a3f148df7473603c355a32536cb0eea068885">
+    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=b45f000ecb36779dde84b689d463d17a077275d6">
     <!--[if lt IE 9]>
     <script src="https://cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv.min.js"></script>
     <![endif]-->

docs/_site/posts/gallery.html

@@ -5,21 +5,21 @@
     <meta http-equiv="X-UA-Compatible" content="IE=edge">
     <meta name="viewport" content="width=device-width, initial-scale=1">
 
-<!-- Begin Jekyll SEO tag v2.5.0 -->
+<!-- Begin Jekyll SEO tag v2.6.1 -->
 <title>Gallery | cellxgene</title>
 <meta name="generator" content="Jekyll v3.8.5" />
 <meta property="og:title" content="Gallery" />
 <meta property="og:locale" content="en_US" />
 <meta name="description" content="An interactive explorer for single-cell transcriptomics data" />
 <meta property="og:description" content="An interactive explorer for single-cell transcriptomics data" />
-<link rel="canonical" href="http://localhost:4000/cellxgene/posts/gallery.html" />
+<link rel="canonical" href="https://chanzuckerberg.github.io/cellxgene/posts/gallery.html" />
-<meta property="og:url" content="http://localhost:4000/cellxgene/posts/gallery.html" />
+<meta property="og:url" content="https://chanzuckerberg.github.io/cellxgene/posts/gallery.html" />
 <meta property="og:site_name" content="cellxgene" />
 <script type="application/ld+json">
-{"description":"An interactive explorer for single-cell transcriptomics data","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"http://localhost:4000/cellxgene/cellxgene-logo.png"}},"@type":"WebPage","headline":"Gallery","url":"http://localhost:4000/cellxgene/posts/gallery.html","@context":"http://schema.org"}</script>
+{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"An interactive explorer for single-cell transcriptomics data","@type":"WebPage","url":"https://chanzuckerberg.github.io/cellxgene/posts/gallery.html","headline":"Gallery","@context":"https://schema.org"}</script>
 <!-- End Jekyll SEO tag -->
 
-    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=565a3f148df7473603c355a32536cb0eea068885">
+    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=b45f000ecb36779dde84b689d463d17a077275d6">
     <!--[if lt IE 9]>
     <script src="https://cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv.min.js"></script>
     <![endif]-->

docs/_site/posts/hosted.html

@@ -5,21 +5,21 @@
     <meta http-equiv="X-UA-Compatible" content="IE=edge">
     <meta name="viewport" content="width=device-width, initial-scale=1">
 
-<!-- Begin Jekyll SEO tag v2.5.0 -->
+<!-- Begin Jekyll SEO tag v2.6.1 -->
 <title>Hosting cellxgene on the web | cellxgene</title>
 <meta name="generator" content="Jekyll v3.8.5" />
 <meta property="og:title" content="Hosting cellxgene on the web" />
 <meta property="og:locale" content="en_US" />
 <meta name="description" content="An interactive explorer for single-cell transcriptomics data" />
 <meta property="og:description" content="An interactive explorer for single-cell transcriptomics data" />
-<link rel="canonical" href="http://localhost:4000/cellxgene/posts/hosted.html" />
+<link rel="canonical" href="https://chanzuckerberg.github.io/cellxgene/posts/hosted.html" />
-<meta property="og:url" content="http://localhost:4000/cellxgene/posts/hosted.html" />
+<meta property="og:url" content="https://chanzuckerberg.github.io/cellxgene/posts/hosted.html" />
 <meta property="og:site_name" content="cellxgene" />
 <script type="application/ld+json">
-{"description":"An interactive explorer for single-cell transcriptomics data","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"http://localhost:4000/cellxgene/cellxgene-logo.png"}},"@type":"WebPage","headline":"Hosting cellxgene on the web","url":"http://localhost:4000/cellxgene/posts/hosted.html","@context":"http://schema.org"}</script>
+{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"An interactive explorer for single-cell transcriptomics data","@type":"WebPage","url":"https://chanzuckerberg.github.io/cellxgene/posts/hosted.html","headline":"Hosting cellxgene on the web","@context":"https://schema.org"}</script>
 <!-- End Jekyll SEO tag -->
 
-    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=565a3f148df7473603c355a32536cb0eea068885">
+    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=b45f000ecb36779dde84b689d463d17a077275d6">
     <!--[if lt IE 9]>
     <script src="https://cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv.min.js"></script>
     <![endif]-->

docs/_site/posts/install.html

@@ -5,21 +5,21 @@
     <meta http-equiv="X-UA-Compatible" content="IE=edge">
     <meta name="viewport" content="width=device-width, initial-scale=1">
 
-<!-- Begin Jekyll SEO tag v2.5.0 -->
+<!-- Begin Jekyll SEO tag v2.6.1 -->
 <title>Install | cellxgene</title>
 <meta name="generator" content="Jekyll v3.8.5" />
 <meta property="og:title" content="Install" />
 <meta property="og:locale" content="en_US" />
 <meta name="description" content="An interactive explorer for single-cell transcriptomics data" />
 <meta property="og:description" content="An interactive explorer for single-cell transcriptomics data" />
-<link rel="canonical" href="http://localhost:4000/cellxgene/posts/install.html" />
+<link rel="canonical" href="https://chanzuckerberg.github.io/cellxgene/posts/install.html" />
-<meta property="og:url" content="http://localhost:4000/cellxgene/posts/install.html" />
+<meta property="og:url" content="https://chanzuckerberg.github.io/cellxgene/posts/install.html" />
 <meta property="og:site_name" content="cellxgene" />
 <script type="application/ld+json">
-{"description":"An interactive explorer for single-cell transcriptomics data","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"http://localhost:4000/cellxgene/cellxgene-logo.png"}},"@type":"WebPage","headline":"Install","url":"http://localhost:4000/cellxgene/posts/install.html","@context":"http://schema.org"}</script>
+{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"An interactive explorer for single-cell transcriptomics data","@type":"WebPage","url":"https://chanzuckerberg.github.io/cellxgene/posts/install.html","headline":"Install","@context":"https://schema.org"}</script>
 <!-- End Jekyll SEO tag -->
 
-    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=565a3f148df7473603c355a32536cb0eea068885">
+    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=b45f000ecb36779dde84b689d463d17a077275d6">
     <!--[if lt IE 9]>
     <script src="https://cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv.min.js"></script>
     <![endif]-->

docs/_site/posts/launch.html

@@ -5,21 +5,21 @@
     <meta http-equiv="X-UA-Compatible" content="IE=edge">
     <meta name="viewport" content="width=device-width, initial-scale=1">
 
-<!-- Begin Jekyll SEO tag v2.5.0 -->
+<!-- Begin Jekyll SEO tag v2.6.1 -->
 <title>demo-data | cellxgene</title>
 <meta name="generator" content="Jekyll v3.8.5" />
 <meta property="og:title" content="demo-data" />
 <meta property="og:locale" content="en_US" />
 <meta name="description" content="Demo datasets" />
 <meta property="og:description" content="Demo datasets" />
-<link rel="canonical" href="http://localhost:4000/cellxgene/posts/launch.html" />
+<link rel="canonical" href="https://chanzuckerberg.github.io/cellxgene/posts/launch.html" />
-<meta property="og:url" content="http://localhost:4000/cellxgene/posts/launch.html" />
+<meta property="og:url" content="https://chanzuckerberg.github.io/cellxgene/posts/launch.html" />
 <meta property="og:site_name" content="cellxgene" />
 <script type="application/ld+json">
-{"description":"Demo datasets","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"http://localhost:4000/cellxgene/cellxgene-logo.png"}},"@type":"WebPage","headline":"demo-data","url":"http://localhost:4000/cellxgene/posts/launch.html","@context":"http://schema.org"}</script>
+{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"Demo datasets","@type":"WebPage","url":"https://chanzuckerberg.github.io/cellxgene/posts/launch.html","headline":"demo-data","@context":"https://schema.org"}</script>
 <!-- End Jekyll SEO tag -->
 
-    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=565a3f148df7473603c355a32536cb0eea068885">
+    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=b45f000ecb36779dde84b689d463d17a077275d6">
     <!--[if lt IE 9]>
     <script src="https://cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv.min.js"></script>
     <![endif]-->

docs/_site/posts/methods.html

@@ -5,21 +5,21 @@
     <meta http-equiv="X-UA-Compatible" content="IE=edge">
     <meta name="viewport" content="width=device-width, initial-scale=1">
 
-<!-- Begin Jekyll SEO tag v2.5.0 -->
+<!-- Begin Jekyll SEO tag v2.6.1 -->
 <title>Methods | cellxgene</title>
 <meta name="generator" content="Jekyll v3.8.5" />
 <meta property="og:title" content="Methods" />
 <meta property="og:locale" content="en_US" />
 <meta name="description" content="An interactive explorer for single-cell transcriptomics data" />
 <meta property="og:description" content="An interactive explorer for single-cell transcriptomics data" />
-<link rel="canonical" href="http://localhost:4000/cellxgene/posts/methods.html" />
+<link rel="canonical" href="https://chanzuckerberg.github.io/cellxgene/posts/methods.html" />
-<meta property="og:url" content="http://localhost:4000/cellxgene/posts/methods.html" />
+<meta property="og:url" content="https://chanzuckerberg.github.io/cellxgene/posts/methods.html" />
 <meta property="og:site_name" content="cellxgene" />
 <script type="application/ld+json">
-{"description":"An interactive explorer for single-cell transcriptomics data","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"http://localhost:4000/cellxgene/cellxgene-logo.png"}},"@type":"WebPage","headline":"Methods","url":"http://localhost:4000/cellxgene/posts/methods.html","@context":"http://schema.org"}</script>
+{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"An interactive explorer for single-cell transcriptomics data","@type":"WebPage","url":"https://chanzuckerberg.github.io/cellxgene/posts/methods.html","headline":"Methods","@context":"https://schema.org"}</script>
 <!-- End Jekyll SEO tag -->
 
-    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=565a3f148df7473603c355a32536cb0eea068885">
+    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=b45f000ecb36779dde84b689d463d17a077275d6">
     <!--[if lt IE 9]>
     <script src="https://cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv.min.js"></script>
     <![endif]-->

docs/_site/posts/prepare.html

@@ -5,21 +5,21 @@
     <meta http-equiv="X-UA-Compatible" content="IE=edge">
     <meta name="viewport" content="width=device-width, initial-scale=1">
 
-<!-- Begin Jekyll SEO tag v2.5.0 -->
+<!-- Begin Jekyll SEO tag v2.6.1 -->
 <title>prepare | cellxgene</title>
 <meta name="generator" content="Jekyll v3.8.5" />
 <meta property="og:title" content="prepare" />
 <meta property="og:locale" content="en_US" />
 <meta name="description" content="Preparing your data" />
 <meta property="og:description" content="Preparing your data" />
-<link rel="canonical" href="http://localhost:4000/cellxgene/posts/prepare.html" />
+<link rel="canonical" href="https://chanzuckerberg.github.io/cellxgene/posts/prepare.html" />
-<meta property="og:url" content="http://localhost:4000/cellxgene/posts/prepare.html" />
+<meta property="og:url" content="https://chanzuckerberg.github.io/cellxgene/posts/prepare.html" />
 <meta property="og:site_name" content="cellxgene" />
 <script type="application/ld+json">
-{"description":"Preparing your data","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"http://localhost:4000/cellxgene/cellxgene-logo.png"}},"@type":"WebPage","headline":"prepare","url":"http://localhost:4000/cellxgene/posts/prepare.html","@context":"http://schema.org"}</script>
+{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"Preparing your data","@type":"WebPage","url":"https://chanzuckerberg.github.io/cellxgene/posts/prepare.html","headline":"prepare","@context":"https://schema.org"}</script>
 <!-- End Jekyll SEO tag -->
 
-    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=565a3f148df7473603c355a32536cb0eea068885">
+    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=b45f000ecb36779dde84b689d463d17a077275d6">
     <!--[if lt IE 9]>
     <script src="https://cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv.min.js"></script>
     <![endif]-->
@@ -102,8 +102,8 @@
 <ul>
   <li>Expression values (raw or normalized) in <code class="language-plaintext highlighter-rouge">anndata.X</code></li>
   <li>At least one embedding (e.g., tSNE, UMAP) in <code class="language-plaintext highlighter-rouge">anndata.obsm</code>, specified with the prefix <code class="language-plaintext highlighter-rouge">X_</code> (e.g., by default scanpy stores UMAP coordinates in <code class="language-plaintext highlighter-rouge">anndata.obsm['X_umap']</code>)</li>
-  <li>A unique identifier for every cell is available in an <code class="language-plaintext highlighter-rouge">anndata.obs</code> field (you can specify this with the <code class="language-plaintext highlighter-rouge">--obs-names</code> option)</li>
+  <li>A unique identifier is required for each cell, which by default will be pulled from the <code class="language-plaintext highlighter-rouge">obs</code> DataFrame index. If the index is not unique or does not contain the cell ID, an alternative column can be specified with <code class="language-plaintext highlighter-rouge">--obs-names</code></li>
-  <li>A unique identifier for every gene is available in an <code class="language-plaintext highlighter-rouge">anndata.var</code> field (you can specify which field to use with the <code class="language-plaintext highlighter-rouge">--var-names</code> option)</li>
+  <li>A unique identifier is required for each gene, which by default will be pulled from the <code class="language-plaintext highlighter-rouge">var</code> DataFrame index. If the index is not unique or does not contain the cell ID, an alternative column can be specified with <code class="language-plaintext highlighter-rouge">--var-names</code></li>
 </ul>
 
 <h4 id="what-about-r-objects-from-seurat--bioconductor">What about R objects from seurat / bioconductor!?</h4>
@@ -112,6 +112,32 @@
 <h4 id="can-i-use-data-hosted-on-the-web-somewhere">Can I use data hosted on the web somewhere?</h4>
 <p>Yes! You can launch from a URL instead of a filepath. The same data format requirements apply. Please see <a href="launch">here</a> for more details.</p>
 
+<h1 id="data-format-options">Data format options</h1>
+
+<h4 id="category-colors">Category colors</h4>
+<p><code class="language-plaintext highlighter-rouge">cellxgene</code> will display <a href="https://github.com/chanzuckerberg/cellxgene/issues/1152#issue-564361541">scanpy-style color
+information</a>
+for category-label pairs. An example of this format is shown below:</p>
+
+<div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code>&gt;&gt;&gt; category = "louvain"
+&gt;&gt;&gt; # colors stored in adata.uns must be matplotlib-compatible color information
+&gt;&gt;&gt; adata.uns[f"{category}_colors"]
+array(['#1f77b4', '#ff7f0e', '#2ca02c', '#d62728', '#9467bd', '#8c564b', '#e377c2', '#bcbd22'], dtype='&lt;U7')
+&gt;&gt;&gt; # there must be a matching category in adata.obs
+&gt;&gt;&gt; category in adata.obs
+True
+</code></pre></div></div>
+
+<p>To test that you’ve done this properly, check that for your given <code class="language-plaintext highlighter-rouge">category</code> the number of colors match the number of category values and that the second command below results in a mapping from categories to colors.</p>
+
+<div class="language-plaintext highlighter-rouge"><div class="highlight"><pre class="highlight"><code>&gt;&gt;&gt; len(adata.obs[category].cat.categories) == len(adata.uns[f"{category}_colors"])
+True
+&gt;&gt;&gt; dict(zip(adata.obs[category].cat.categories, adata.uns[f"{category}_colors"]))
+{'CD4 T cells': '#1f77b4', 'CD14+ Monocytes': '#ff7f0e', 'B cells': '#2ca02c', 'CD8 T cells': '#d62728', 'NK cells': '#9467bd', 'FCGR3A+ Monocytes': '#8c564b', 'Dendritic cells': '#e377c2', 'Megakaryocytes': '#bcbd22'}
+</code></pre></div></div>
+
+<p>You can disable this feature using the <code class="language-plaintext highlighter-rouge">--disable-custom-colors</code> flag for <code class="language-plaintext highlighter-rouge">cellxgene launch</code>. cellxgene will then chose colors from its standard color palettes.</p>
+
 <h1 id="using-cellxgene-prepare">Using <code class="language-plaintext highlighter-rouge">cellxgene prepare</code></h1>
 
 <p>If your data is in a different format, and/or you still need to perform dimensionality reduction and/or clustering, <code class="language-plaintext highlighter-rouge">cellxgene</code> can do that for you with the <code class="language-plaintext highlighter-rouge">prepare</code> command.</p>
@@ -136,7 +162,7 @@
 <p><code class="language-plaintext highlighter-rouge">cellxgene prepare</code> is not meant as a way to formally process or analyze your data. It’s simply a utility for quickly wrangling your data into cellxgene-compatible format and computing a “vanilla” embedding so you can try out <code class="language-plaintext highlighter-rouge">cellxgene</code> and get a general sense of a dataset.</p>
 
 <h2 id="quickstart-for-cellxgene-prepare">Quickstart for <code class="language-plaintext highlighter-rouge">cellxgene prepare</code></h2>
-<p>To add <code class="language-plaintext highlighter-rouge">cellxgene prepare</code> to your <a href="install">cellxgene installation</a>, run<br />
+<p>To add <code class="language-plaintext highlighter-rouge">cellxgene prepare</code> to your <a href="install">cellxgene installation</a>, run
 <code class="language-plaintext highlighter-rouge">pip install cellxgene[prepare]</code></p>
 
 <p>Then run <code class="language-plaintext highlighter-rouge">prepare</code> on your data with:</p>
@@ -162,36 +188,36 @@
 
 <p>Let’s look at what <code class="language-plaintext highlighter-rouge">prepare</code> is doing to our data, and how each step relates to the command above. You can see a walkthrough of what’s going on under the hood for this example in <a href="https://github.com/chanzuckerberg/cellxgene-vignettes/blob/master/dataset-processing/pbmc3k-prepare-example.ipynb">this notebook</a>.</p>
 
-<p><strong>(A) - Compute quality control metrics and store this in our <code class="language-plaintext highlighter-rouge">AnnData</code> object for later inspection</strong> <br />
+<p><strong>(A) - Compute quality control metrics and store this in our <code class="language-plaintext highlighter-rouge">AnnData</code> object for later inspection</strong>
-<strong>(B) - Normalize the expression matrix using a basic preprocessing recipe</strong> <br />
+<strong>(B) - Normalize the expression matrix using a basic preprocessing recipe</strong>
-<strong>(auto) - Do some preprocessing to run PCA and compute the neighbor graph</strong><br />
+<strong>(auto) - Do some preprocessing to run PCA and compute the neighbor graph</strong>
-<strong>(auto) - Infer clusters with the Louvain algorithm and store these labels to visualize later</strong><br />
+<strong>(auto) - Infer clusters with the Louvain algorithm and store these labels to visualize later</strong>
-<strong>(C) - Compute and store UMAP and tSNE embeddings</strong><br />
+<strong>(C) - Compute and store UMAP and tSNE embeddings</strong>
 <strong>(D) - Write results to file</strong></p>
 
 <h2 id="options-for-cellxgene-prepare">Options for cellxgene <code class="language-plaintext highlighter-rouge">prepare</code></h2>
 
 <p><strong>For the most up-to-date and comprehensive list of options, run <code class="language-plaintext highlighter-rouge">cellxgene prepare --help</code></strong></p>
 
-<p><code class="language-plaintext highlighter-rouge">--embedding</code> controls which dimensionality reduction algorithm is applies to your data.<br />
+<p><code class="language-plaintext highlighter-rouge">--embedding</code> controls which dimensionality reduction algorithm is applies to your data.
 Options are <code class="language-plaintext highlighter-rouge">umap</code> and/or <code class="language-plaintext highlighter-rouge">tsne</code>. Defaults to both.</p>
 
 <p><code class="language-plaintext highlighter-rouge">--recipe</code> controls which normalization steps to apply to your data, based on one of the preprocessing <code class="language-plaintext highlighter-rouge">recipes</code> included with <code class="language-plaintext highlighter-rouge">scanpy</code>.
-These recipes include steps like cell filtering and gene selection; see the <code class="language-plaintext highlighter-rouge">scanpy</code> <a href="https://scanpy.readthedocs.io/en/latest/api/index.html#recipes">documentation</a> for more details. <br />
+These recipes include steps like cell filtering and gene selection; see the <code class="language-plaintext highlighter-rouge">scanpy</code> <a href="https://scanpy.readthedocs.io/en/latest/api/index.html#recipes">documentation</a> for more details.
 Options are <code class="language-plaintext highlighter-rouge">none</code>, <code class="language-plaintext highlighter-rouge">seurat</code>, or <code class="language-plaintext highlighter-rouge">zheng17</code>. Defaults to <code class="language-plaintext highlighter-rouge">none</code>.</p>
 
-<p><code class="language-plaintext highlighter-rouge">--sparse</code> is a flag determines whether to enforce a sparse matrix. For large datasets, <code class="language-plaintext highlighter-rouge">prepare</code> can take a long time to run (a few minutes for datasets with 10-100k cells, up to an hour or more for datasets with &gt;100k cells). If you want <code class="language-plaintext highlighter-rouge">prepare</code> to run faster we recommend using the <code class="language-plaintext highlighter-rouge">sparse</code> option.<br />
+<p><code class="language-plaintext highlighter-rouge">--sparse</code> is a flag determines whether to enforce a sparse matrix. For large datasets, <code class="language-plaintext highlighter-rouge">prepare</code> can take a long time to run (a few minutes for datasets with 10-100k cells, up to an hour or more for datasets with &gt;100k cells). If you want <code class="language-plaintext highlighter-rouge">prepare</code> to run faster we recommend using the <code class="language-plaintext highlighter-rouge">sparse</code> option.
 If this flag is not included, default is <code class="language-plaintext highlighter-rouge">False</code></p>
 
 <p><code class="language-plaintext highlighter-rouge">--skip-qc</code> by default, <code class="language-plaintext highlighter-rouge">cellxgene prepare</code> will compute quality control metrics (saved to <code class="language-plaintext highlighter-rouge">anndata.obs</code> and <code class="language-plaintext highlighter-rouge">anndata.var</code>) as described in the <code class="language-plaintext highlighter-rouge">scanpy</code> <a href="https://scanpy.readthedocs.io/en/stable/api/scanpy.pp.calculate_qc_metrics.html">documentation</a>. Pass this flag if you would like to skip this step.</p>
 
-<p><code class="language-plaintext highlighter-rouge">--make-obs-names-unique</code> / <code class="language-plaintext highlighter-rouge">--make-var-names-unique</code> determine whether to rename <code class="language-plaintext highlighter-rouge">obs</code> (cell) / <code class="language-plaintext highlighter-rouge">var</code> (gene) names, respectively, to be unique.<br />
+<p><code class="language-plaintext highlighter-rouge">--make-obs-names-unique</code> / <code class="language-plaintext highlighter-rouge">--make-var-names-unique</code> determine whether to rename <code class="language-plaintext highlighter-rouge">obs</code> (cell) / <code class="language-plaintext highlighter-rouge">var</code> (gene) names, respectively, to be unique.
 Default is <code class="language-plaintext highlighter-rouge">True</code>.</p>
 
-<p><code class="language-plaintext highlighter-rouge">--set-obs-names</code> controls which field in <code class="language-plaintext highlighter-rouge">anndata.obs</code> (cell metadata) is used as the <em>index</em> for cells (e.g., a cell ID column).<br />
+<p><code class="language-plaintext highlighter-rouge">--set-obs-names</code> controls which field in <code class="language-plaintext highlighter-rouge">anndata.obs</code> (cell metadata) is used as the <em>index</em> for cells (e.g., a cell ID column).
 Default is <code class="language-plaintext highlighter-rouge">anndata.obs.names</code></p>
 
-<p><code class="language-plaintext highlighter-rouge">--set-var-names</code> controls which field in <code class="language-plaintext highlighter-rouge">anndata.var</code> (gene metadata) is used as the <em>index</em> for genes.<br />
+<p><code class="language-plaintext highlighter-rouge">--set-var-names</code> controls which field in <code class="language-plaintext highlighter-rouge">anndata.var</code> (gene metadata) is used as the <em>index</em> for genes.
 Default is <code class="language-plaintext highlighter-rouge">anndata.var.names</code></p>
 
 <p><code class="language-plaintext highlighter-rouge">--output</code> and <code class="language-plaintext highlighter-rouge">--overwrite</code> control where the processed data is saved.</p>

docs/_site/posts/roadmap.html

@@ -5,21 +5,21 @@
     <meta http-equiv="X-UA-Compatible" content="IE=edge">
     <meta name="viewport" content="width=device-width, initial-scale=1">
 
-<!-- Begin Jekyll SEO tag v2.5.0 -->
+<!-- Begin Jekyll SEO tag v2.6.1 -->
 <title>roadmap | cellxgene</title>
 <meta name="generator" content="Jekyll v3.8.5" />
 <meta property="og:title" content="roadmap" />
 <meta property="og:locale" content="en_US" />
 <meta name="description" content="Roadmap" />
 <meta property="og:description" content="Roadmap" />
-<link rel="canonical" href="http://localhost:4000/cellxgene/posts/roadmap.html" />
+<link rel="canonical" href="https://chanzuckerberg.github.io/cellxgene/posts/roadmap.html" />
-<meta property="og:url" content="http://localhost:4000/cellxgene/posts/roadmap.html" />
+<meta property="og:url" content="https://chanzuckerberg.github.io/cellxgene/posts/roadmap.html" />
 <meta property="og:site_name" content="cellxgene" />
 <script type="application/ld+json">
-{"description":"Roadmap","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"http://localhost:4000/cellxgene/cellxgene-logo.png"}},"@type":"WebPage","headline":"roadmap","url":"http://localhost:4000/cellxgene/posts/roadmap.html","@context":"http://schema.org"}</script>
+{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"Roadmap","@type":"WebPage","url":"https://chanzuckerberg.github.io/cellxgene/posts/roadmap.html","headline":"roadmap","@context":"https://schema.org"}</script>
 <!-- End Jekyll SEO tag -->
 
-    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=565a3f148df7473603c355a32536cb0eea068885">
+    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=b45f000ecb36779dde84b689d463d17a077275d6">
     <!--[if lt IE 9]>
     <script src="https://cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv.min.js"></script>
     <![endif]-->

docs/_site/posts/troubleshooting.html

@@ -5,21 +5,21 @@
     <meta http-equiv="X-UA-Compatible" content="IE=edge">
     <meta name="viewport" content="width=device-width, initial-scale=1">
 
-<!-- Begin Jekyll SEO tag v2.5.0 -->
+<!-- Begin Jekyll SEO tag v2.6.1 -->
 <title>Troubleshooting | cellxgene</title>
 <meta name="generator" content="Jekyll v3.8.5" />
 <meta property="og:title" content="Troubleshooting" />
 <meta property="og:locale" content="en_US" />
 <meta name="description" content="Troubleshooting" />
 <meta property="og:description" content="Troubleshooting" />
-<link rel="canonical" href="http://localhost:4000/cellxgene/posts/troubleshooting.html" />
+<link rel="canonical" href="https://chanzuckerberg.github.io/cellxgene/posts/troubleshooting.html" />
-<meta property="og:url" content="http://localhost:4000/cellxgene/posts/troubleshooting.html" />
+<meta property="og:url" content="https://chanzuckerberg.github.io/cellxgene/posts/troubleshooting.html" />
 <meta property="og:site_name" content="cellxgene" />
 <script type="application/ld+json">
-{"description":"Troubleshooting","publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"http://localhost:4000/cellxgene/cellxgene-logo.png"}},"@type":"WebPage","headline":"Troubleshooting","url":"http://localhost:4000/cellxgene/posts/troubleshooting.html","@context":"http://schema.org"}</script>
+{"publisher":{"@type":"Organization","logo":{"@type":"ImageObject","url":"https://chanzuckerberg.github.io/cellxgene/cellxgene-logo.png"}},"description":"Troubleshooting","@type":"WebPage","url":"https://chanzuckerberg.github.io/cellxgene/posts/troubleshooting.html","headline":"Troubleshooting","@context":"https://schema.org"}</script>
 <!-- End Jekyll SEO tag -->
 
-    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=565a3f148df7473603c355a32536cb0eea068885">
+    <link rel="stylesheet" href="/cellxgene/assets/css/style.css?v=b45f000ecb36779dde84b689d463d17a077275d6">
     <!--[if lt IE 9]>
     <script src="https://cdnjs.cloudflare.com/ajax/libs/html5shiv/3.7.3/html5shiv.min.js"></script>
     <![endif]-->

docs/posts/prepare.md

@@ -9,8 +9,8 @@ If your data is in `h5ad` file (from the [`anndata`](https://anndata.readthedocs
 
 - Expression values (raw or normalized) in `anndata.X`
 - At least one embedding (e.g., tSNE, UMAP) in `anndata.obsm`, specified with the prefix `X_` (e.g., by default scanpy stores UMAP coordinates in `anndata.obsm['X_umap']`)
-- A unique identifier for every cell is available in an `anndata.obs` field (you can specify this with the `--obs-names` option)  
+- A unique identifier is required for each cell, which by default will be pulled from the `obs` DataFrame index. If the index is not unique or does not contain the cell ID, an alternative column can be specified with `--obs-names`
-- A unique identifier for every gene is available in an `anndata.var` field (you can specify which field to use with the `--var-names` option)  
+- A unique identifier is required for each gene, which by default will be pulled from the `var` DataFrame index. If the index is not unique or does not contain the cell ID, an alternative column can be specified with `--var-names`
 
 #### What about R objects from seurat / bioconductor!?
 We hear you! We'd also love to be able to ingest these files directly. This isn't currently possible, but in the meantime, you can use [sceasy](https://bioconda.github.io/recipes/r-sceasy/README.html) ([docs](https://cellgeni.readthedocs.io/en/latest/visualisations.html)) to convert to `h5ad`. Seurat also has some [handy conversion tools](https://satijalab.org/seurat/v3.0/conversion_vignette.html) that you can try out.
@@ -18,6 +18,34 @@ We hear you! We'd also love to be able to ingest these files directly. This isn'
 #### Can I use data hosted on the web somewhere?
 Yes! You can launch from a URL instead of a filepath. The same data format requirements apply. Please see [here](launch) for more details.
 
+# Data format options
+
+#### Category colors
+`cellxgene` will display [scanpy-style color
+information](https://github.com/chanzuckerberg/cellxgene/issues/1152#issue-564361541)
+for category-label pairs. An example of this format is shown below:
+
+```
+>>> category = "louvain"
+>>> # colors stored in adata.uns must be matplotlib-compatible color information
+>>> adata.uns[f"{category}_colors"]
+array(['#1f77b4', '#ff7f0e', '#2ca02c', '#d62728', '#9467bd', '#8c564b', '#e377c2', '#bcbd22'], dtype='<U7')
+>>> # there must be a matching category in adata.obs
+>>> category in adata.obs
+True
+```
+
+To test that you've done this properly, check that for your given `category` the number of colors match the number of category values and that the second command below results in a mapping from categories to colors.
+
+```
+>>> len(adata.obs[category].cat.categories) == len(adata.uns[f"{category}_colors"])
+True
+>>> dict(zip(adata.obs[category].cat.categories, adata.uns[f"{category}_colors"]))
+{'CD4 T cells': '#1f77b4', 'CD14+ Monocytes': '#ff7f0e', 'B cells': '#2ca02c', 'CD8 T cells': '#d62728', 'NK cells': '#9467bd', 'FCGR3A+ Monocytes': '#8c564b', 'Dendritic cells': '#e377c2', 'Megakaryocytes': '#bcbd22'}
+```
+
+You can disable this feature using the `--disable-custom-colors` flag for `cellxgene launch`. cellxgene will then chose colors from its standard color palettes.
+
 # Using `cellxgene prepare`
 
 If your data is in a different format, and/or you still need to perform dimensionality reduction and/or clustering, `cellxgene` can do that for you with the `prepare` command.

server/app/app.py

@@ -207,6 +207,13 @@ class DataVarAPI(Resource):
         return common_rest.data_var_get(request, data_adaptor)
 
 
+class ColorsAPI(Resource):
+    @cache_control(public=True, max_age=ONE_WEEK)
+    @rest_get_data_adaptor
+    def get(self, data_adaptor):
+        return common_rest.colors_get(data_adaptor)
+
+
 class DiffExpObsAPI(Resource):
     @cache_control(no_store=True)
     @rest_get_data_adaptor
@@ -235,6 +242,8 @@ def get_api_resources(bp_api):
     api.add_resource(AnnotationsObsAPI, "/annotations/obs")
     api.add_resource(AnnotationsVarAPI, "/annotations/var")
     api.add_resource(DataVarAPI, "/data/var")
+    # Display routes
+    api.add_resource(ColorsAPI, "/colors")
     # Computation routes
     api.add_resource(DiffExpObsAPI, "/diffexp/obs")
     api.add_resource(LayoutObsAPI, "/layout/obs")

server/cli/launch.py

@@ -73,6 +73,13 @@ def config_args(func):
         show_default=True,
         help="Will not display categories with more distinct values than specified.",
     )
+    @click.option(
+        "--disable-custom-colors",
+        is_flag=True,
+        default=False,
+        show_default=False,
+        help="Disable user-defined category-label colors drawn from source data file.",
+    )
     @click.option(
         "--diffexp-lfc-cutoff",
         "-de",
@@ -146,7 +153,7 @@ def dataset_args(func):
         "--about",
         default=DEFAULT_CONFIG.single_dataset__about,
         metavar="<URL>",
-        help="URL providing more information about the dataset " "(hint: must be a fully specified absolute URL).",
+        help="URL providing more information about the dataset (hint: must be a fully specified absolute URL).",
     )
     @functools.wraps(func)
     def wrapper(*args, **kwargs):
@@ -311,6 +318,7 @@ def launch(
     obs_names,
     var_names,
     max_category_items,
+    disable_custom_colors,
     diffexp_lfc_cutoff,
     title,
     scripts,
@@ -381,6 +389,7 @@ def launch(
             user_annotations__ontology__enable=experimental_annotations_ontology,
             user_annotations__ontology__obo_location=experimental_annotations_ontology_obo,
             presentation__max_categories=max_category_items,
+            presentation__custom_colors=not disable_custom_colors,
             embeddings__names=embedding,
             embeddings__enable_reembedding=experimental_enable_reembedding,
             diffexp__enable=not disable_diffexp,

server/cli/prepare.py

@@ -73,7 +73,8 @@ def prepare(
     (h5ad, loom, or a 10x directory), runs dimensionality reduction,
     computes nearest neighbors, computes an embedding, performs clustering,
     and saves the results. Includes additional options for naming annotations,
-    ensuring sparsity, and plotting results."""
+    ensuring sparsity, and plotting results.
+    """
 
     # collect slow imports here to make CLI startup more responsive
     click.echo("[cellxgene] Starting CLI...")

server/common/app_config.py

@@ -78,6 +78,7 @@ class AppConfig(object):
             self.user_annotations__ontology__obo_location = dc["user_annotations"]["ontology"]["obo_location"]
 
             self.presentation__max_categories = dc["presentation"]["max_categories"]
+            self.presentation__custom_colors = dc["presentation"]["custom_colors"]
 
             self.embeddings__names = dc["embeddings"]["names"]
             self.embeddings__enable_reembedding = dc["embeddings"]["enable_reembedding"]
@@ -275,6 +276,7 @@ class AppConfig(object):
 
     def handle_presentation(self, context):
         self.__check_attr("presentation__max_categories", int)
+        self.__check_attr("presentation__custom_colors", bool)
 
     def handle_single_dataset(self, context):
         self.__check_attr("single_dataset__datapath", (str, type(None)))
@@ -517,6 +519,7 @@ class AppConfig(object):
             "annotations_cell_ontology_enabled": False,
             "annotations_cell_ontology_obopath": None,
             "annotations_cell_ontology_terms": None,
+            "custom_colors": self.presentation__custom_colors,
             "diffexp-may-be-slow": False,
             "about_legal_tos": self.server__about_legal_tos,
             "about_legal_privacy": self.server__about_legal_privacy,

server/common/colors.py

@@ -0,0 +1,233 @@
+import re
+
+from server.common.errors import ColorFormatException
+
+HEX_COLOR_FORMAT = re.compile("^#[a-fA-F0-9]{6,6}$")
+
+# https://www.w3.org/TR/css-color-4/#named-colors
+CSS4_NAMED_COLORS = dict(
+    aliceblue="#f0f8ff",
+    antiquewhite="#faebd7",
+    aqua="#00ffff",
+    aquamarine="#7fffd4",
+    azure="#f0ffff",
+    beige="#f5f5dc",
+    bisque="#ffe4c4",
+    black="#000000",
+    blanchedalmond="#ffebcd",
+    blue="#0000ff",
+    blueviolet="#8a2be2",
+    brown="#a52a2a",
+    burlywood="#deb887",
+    cadetblue="#5f9ea0",
+    chartreuse="#7fff00",
+    chocolate="#d2691e",
+    coral="#ff7f50",
+    cornflowerblue="#6495ed",
+    cornsilk="#fff8dc",
+    crimson="#dc143c",
+    cyan="#00ffff",
+    darkblue="#00008b",
+    darkcyan="#008b8b",
+    darkgoldenrod="#b8860b",
+    darkgray="#a9a9a9",
+    darkgreen="#006400",
+    darkgrey="#a9a9a9",
+    darkkhaki="#bdb76b",
+    darkmagenta="#8b008b",
+    darkolivegreen="#556b2f",
+    darkorange="#ff8c00",
+    darkorchid="#9932cc",
+    darkred="#8b0000",
+    darksalmon="#e9967a",
+    darkseagreen="#8fbc8f",
+    darkslateblue="#483d8b",
+    darkslategray="#2f4f4f",
+    darkslategrey="#2f4f4f",
+    darkturquoise="#00ced1",
+    darkviolet="#9400d3",
+    deeppink="#ff1493",
+    deepskyblue="#00bfff",
+    dimgray="#696969",
+    dimgrey="#696969",
+    dodgerblue="#1e90ff",
+    firebrick="#b22222",
+    floralwhite="#fffaf0",
+    forestgreen="#228b22",
+    fuchsia="#ff00ff",
+    gainsboro="#dcdcdc",
+    ghostwhite="#f8f8ff",
+    gold="#ffd700",
+    goldenrod="#daa520",
+    gray="#808080",
+    green="#008000",
+    greenyellow="#adff2f",
+    grey="#808080",
+    honeydew="#f0fff0",
+    hotpink="#ff69b4",
+    indianred="#cd5c5c",
+    indigo="#4b0082",
+    ivory="#fffff0",
+    khaki="#f0e68c",
+    lavender="#e6e6fa",
+    lavenderblush="#fff0f5",
+    lawngreen="#7cfc00",
+    lemonchiffon="#fffacd",
+    lightblue="#add8e6",
+    lightcoral="#f08080",
+    lightcyan="#e0ffff",
+    lightgoldenrodyellow="#fafad2",
+    lightgray="#d3d3d3",
+    lightgreen="#90ee90",
+    lightgrey="#d3d3d3",
+    lightpink="#ffb6c1",
+    lightsalmon="#ffa07a",
+    lightseagreen="#20b2aa",
+    lightskyblue="#87cefa",
+    lightslategray="#778899",
+    lightslategrey="#778899",
+    lightsteelblue="#b0c4de",
+    lightyellow="#ffffe0",
+    lime="#00ff00",
+    limegreen="#32cd32",
+    linen="#faf0e6",
+    magenta="#ff00ff",
+    maroon="#800000",
+    mediumaquamarine="#66cdaa",
+    mediumblue="#0000cd",
+    mediumorchid="#ba55d3",
+    mediumpurple="#9370db",
+    mediumseagreen="#3cb371",
+    mediumslateblue="#7b68ee",
+    mediumspringgreen="#00fa9a",
+    mediumturquoise="#48d1cc",
+    mediumvioletred="#c71585",
+    midnightblue="#191970",
+    mintcream="#f5fffa",
+    mistyrose="#ffe4e1",
+    moccasin="#ffe4b5",
+    navajowhite="#ffdead",
+    navy="#000080",
+    oldlace="#fdf5e6",
+    olive="#808000",
+    olivedrab="#6b8e23",
+    orange="#ffa500",
+    orangered="#ff4500",
+    orchid="#da70d6",
+    palegoldenrod="#eee8aa",
+    palegreen="#98fb98",
+    paleturquoise="#afeeee",
+    palevioletred="#db7093",
+    papayawhip="#ffefd5",
+    peachpuff="#ffdab9",
+    peru="#cd853f",
+    pink="#ffc0cb",
+    plum="#dda0dd",
+    powderblue="#b0e0e6",
+    purple="#800080",
+    rebeccapurple="#663399",
+    red="#ff0000",
+    rosybrown="#bc8f8f",
+    royalblue="#4169e1",
+    saddlebrown="#8b4513",
+    salmon="#fa8072",
+    sandybrown="#f4a460",
+    seagreen="#2e8b57",
+    seashell="#fff5ee",
+    sienna="#a0522d",
+    silver="#c0c0c0",
+    skyblue="#87ceeb",
+    slateblue="#6a5acd",
+    slategray="#708090",
+    slategrey="#708090",
+    snow="#fffafa",
+    springgreen="#00ff7f",
+    steelblue="#4682b4",
+    tan="#d2b48c",
+    teal="#008080",
+    thistle="#d8bfd8",
+    tomato="#ff6347",
+    turquoise="#40e0d0",
+    violet="#ee82ee",
+    wheat="#f5deb3",
+    white="#ffffff",
+    whitesmoke="#f5f5f5",
+    yellow="#ffff00",
+    yellowgreen="#9acd32",
+)
+
+
+def convert_color_to_hex_format(unknown):
+    """
+    Try to convert color info to a hex triplet string https://en.wikipedia.org/wiki/Web_colors#Hex_triplet.
+
+    The function accepts for the following formats:
+    - A CSS4 color name, as supported by matplotlib https://matplotlib.org/3.1.0/gallery/color/named_colors.html
+    - RGB tuple/list with values ranging from 0.0 to 1.0, as in [0.5, 0.75, 1.0]
+    - RFB tuple/list with values ranging from 0 to 255, as in [128, 192, 255]
+    - Hex triplet string, as in "#08c0ff"
+
+    :param unknown: color info of unknown format
+    :return: a hex triplet representing that color
+    """
+    try:
+        if type(unknown) in (list, tuple) and len(unknown) == 3:
+            if all(0.0 <= ele <= 1.0 for ele in unknown):
+                tup = tuple(int(ele * 255) for ele in unknown)
+            elif all(0 <= ele <= 255 and isinstance(ele, int) for ele in unknown):
+                tup = tuple(unknown)
+            else:
+                raise ColorFormatException("Unknown color iterable format!")
+            return "#%02x%02x%02x" % tup
+        elif isinstance(unknown, str) and unknown.lower() in CSS4_NAMED_COLORS:
+            return CSS4_NAMED_COLORS[unknown.lower()]
+        elif isinstance(unknown, str) and HEX_COLOR_FORMAT.match(unknown):
+            return unknown.lower()
+        else:
+            raise ColorFormatException("Unknown color format type!")
+    except Exception as e:
+        raise ColorFormatException(e)
+
+
+def convert_anndata_category_colors_to_cxg_category_colors(data):
+    """
+    Convert color information from anndata files to the cellxgene color data format as described below:
+        {
+            "<category_name>": {
+                "<label_name>": "<color_hex_code>",
+                ...
+            },
+            ...
+        }
+
+    For more on the cxg color data structure, see https://github.com/chanzuckerberg/cellxgene/issues/1307.
+
+    For more on the anndata color data structure, see
+    https://github.com/chanzuckerberg/cellxgene/issues/1152#issuecomment-587276178.
+
+    Handling of malformed data:
+    - For any color info in a adata.uns[f"{category}_colors"] color array that convert_color_to_hex_format cannot
+      convert to a hex triplet string, a ColorFormatException is raised
+    - No category_name key group is returned for adata.uns[f"{category}_colors"] keys for which there is no
+      adata.obs[f"{category}"] key
+
+    :param data: the anndata file
+    :return: cellxgene color data structure as described above
+    """
+    cxg_colors = dict()
+    color_key_suffix = "_colors"
+    for uns_key in data.uns.keys():
+        # find uns array that describes colors for a category
+        if not uns_key.endswith(color_key_suffix):
+            continue
+
+        # check to see if we actually have observations for that category
+        category_name = uns_key[: -len(color_key_suffix)]
+        if category_name not in data.obs.keys():
+            continue
+
+        # create the cellxgene color entry for this category
+        cxg_colors[category_name] = dict(
+            zip(data.obs[category_name].cat.categories, [convert_color_to_hex_format(c) for c in data.uns[uns_key]])
+        )
+    return cxg_colors

server/common/default_config.py

@@ -19,6 +19,7 @@ server:
 
 presentation:
   max_categories: 1000
+  custom_colors: true
 
 multi_dataset:
   dataroot: null

server/common/errors.py

@@ -84,3 +84,9 @@ class ComputeError(Exception):
     """
 
     pass
+
+
+class ColorFormatException(Exception):
+    """Raised when color helper functions encounter an unknown color format"""
+
+    pass

server/common/rest.py

@@ -1,9 +1,11 @@
-import sys
-from http import HTTPStatus
 import copy
 import logging
+import sys
+from http import HTTPStatus
+
 from flask import make_response, jsonify, current_app, abort
 from werkzeug.urls import url_unquote
+
 from server.common.constants import Axis, DiffExpMode, JSON_NaN_to_num_warning_msg
 from server.common.errors import (
     FilterError,
@@ -12,6 +14,7 @@ from server.common.errors import (
     DisabledFeatureError,
     ExceedsLimitError,
     DatasetAccessError,
+    ColorFormatException,
 )
 
 import json
@@ -222,6 +225,15 @@ def data_var_get(request, data_adaptor):
         return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True)
 
 
+def colors_get(data_adaptor):
+    if not data_adaptor.config.presentation__custom_colors:
+        return make_response(jsonify({}), HTTPStatus.OK)
+    try:
+        return make_response(jsonify(data_adaptor.get_colors()), HTTPStatus.OK)
+    except ColorFormatException as e:
+        return abort_and_log(HTTPStatus.NOT_FOUND, str(e), include_exc_info=True)
+
+
 def diffexp_obs_post(request, data_adaptor):
     if not data_adaptor.config.diffexp__enable:
         return abort(HTTPStatus.NOT_IMPLEMENTED)

server/converters/cxgtool.py

@@ -4,28 +4,36 @@ into a cellxgene TileDB structure, aka a 'CXG'.
 
 The organization of the TileDB structure is:
 
-    the.cxg                 TileDB Group
+    the.cxg                         TileDB Group
-    |-- obs                 TileDB array containing cell (row) attributes, one attribute per
+    ├─ obs                          TileDB array containing cell (row) attributes, one attribute per
-    |                       dataframe columm, shape (n_obs,)
+    │                               dataframe column, shape (n_obs,)
-    |-- var                 TileDB array containing gene (column) attributes, with one attribute per
+    ├─ var                          TileDB array containing gene (column) attributes, with one attribute per
-    |                       dataframe column, shape (n_var,)
+    │                               dataframe column, shape (n_obs,)
-    |-- X                   Main count matrix as a 2D TileDB array, single unnanmed numeric attribute
+    ├─ X                            Main count matrix as a 2D TileDB array, single unnamed numeric attribute
-    |-- emb                 TileDB group, storing optional embeddings (group may be empty)
+    ├─ emb                          TileDB group, storing optional embeddings (group may be empty)
-    |   |-- <name1>         TileDB Array, single anon attribute, ND numeric array, shape (n_obs, N)
+    │  └─ <name1>                   TileDB Array, single anon attribute, ND numeric array, shape (n_obs, N)
-    |-- cxg_group_metadata  Empty array used only to stash metadata about the overall object.
+    └─ cxg_group_metadata           Empty array used only to stash metadata about the overall object.
+       └─ cxg_category_colors       CXG colors object as described below:
+                                        {
+                                             "<category_name>": {
+                                                 "<label_name>": "<color_hex_code>",
+                                                 ...
+                                             },
+                                             ...
+                                        }
     ...
 
-All arrays are defined to have a uint32 domain, zero based.  All X counds and embedding
+All arrays are defined to have a uint32 domain, zero based.  All X counts and embedding
 coordinates are coerced to float32, which is ample precision for visualization purposes.
 Dataframe (metadata) types are generally preserved, or where that is not possible,
-converted to somemthing with equal representative value in the cellxgene application
+converted to something with equal representative value in the cellxgene application
 (eg, categorical types are converted to string, bools to uint8, etc).
 
-The following objects are also decorated with auxilliary metadata using TileDB
+The following objects are also decorated with auxiliary metadata using TileDB
 array metadata:
 
 * cxg_group_metadata: minimally, will contain a 'cxg_version' field, which
-  is a semver string identifing the version number of the CXG layout.
+  is a semver string identifying the version number of the CXG layout.
   It may also contain 'cxg_parameters', a JSON-encoded parameter list
   describing CXG-wide dataset parameters.
 
@@ -40,6 +48,14 @@ including the global data layout, spatial tile size, and the like. The CXG is
 self-describing in these areas, and the actual values (eg, tile size) are empirically
 derived from benchmarking. They may change in the future.
 
+cxgtool.py will extract color information stored in arrays in the 'uns' anndata
+property with the key "{category_name}_colors". For this to work, the following
+command must result in a mapping from category names to matplotlib-compatible colors:
+
+```
+dict(zip(adata.obs[cat].cat.categories, adata.uns[f"{cat}_colors"]))
+```
+
 ---
 
 TODO/ISSUES:
@@ -55,10 +71,16 @@ import numpy as np
 from os.path import splitext, basename
 import json
 
+from server.common.colors import convert_anndata_category_colors_to_cxg_category_colors
+from server.common.errors import ColorFormatException
+
 
 # the CXG container version number.  Must be a semver string.
 CXG_VERSION = "0.1"
 
+# log_level must have a default
+log_level = 3
+
 
 def log(level, *args):
     global log_level
@@ -72,6 +94,12 @@ def main():
     parser.add_argument(
         "--backed", action="store_true", help="loaded in file backed mode. Will be slower, but use less memory."
     )
+    parser.add_argument(
+        "--disable-custom-colors",
+        action="store_true",
+        default=False,
+        help="Do not extract scanpy-compatible category colors from h5ad file.",
+    )
     parser.add_argument(
         "--obs-names", help="Name of annotation to use for observations. If not specified, will use the obs index."
     )
@@ -99,12 +127,20 @@ def main():
     container = out if splitext(out)[1] == ".cxg" else out + ".cxg"
     title = args.title if args.title is not None else basefname
 
-    write_cxg(adata, container, title, var_names=args.var_names, obs_names=args.obs_names, about=args.about)
+    write_cxg(
+        adata,
+        container,
+        title,
+        var_names=args.var_names,
+        obs_names=args.obs_names,
+        about=args.about,
+        extract_colors=not args.disable_custom_colors,
+    )
 
     log(1, "done")
 
 
-def write_cxg(adata, container, title, var_names=None, obs_names=None, about=None):
+def write_cxg(adata, container, title, var_names=None, obs_names=None, about=None, extract_colors=False):
     if not adata.var.index.is_unique:
         raise ValueError("Variable index is not unique - unable to convert.")
     if not adata.obs.index.is_unique:
@@ -129,7 +165,19 @@ def write_cxg(adata, container, title, var_names=None, obs_names=None, about=Non
     log(1, f"\t...group created, with name {container}")
 
     # dataset metadata
-    save_metadata(container, {"title": title, "about": about})
+    metadata_dict = dict(cxg_version=CXG_VERSION, cxg_properties=json.dumps({"title": title, "about": about}))
+    if extract_colors:
+        try:
+            metadata_dict["cxg_category_colors"] = json.dumps(
+                convert_anndata_category_colors_to_cxg_category_colors(adata)
+            )
+        except ColorFormatException:
+            log(
+                0,
+                "Warning: failed to extract colors from h5ad file! "
+                "Fix the h5ad file or rerun with --disable-custom-colors. See help for details.",
+            )
+    save_metadata(container, metadata_dict)
     log(1, "\t...dataset metadata saved")
 
     # var/gene dataframe
@@ -392,7 +440,7 @@ def save_X(container, adata, ctx):
     tiledb.consolidate(X_name, ctx=ctx)
 
 
-def save_metadata(container, metadata):
+def save_metadata(container, metadata_dict):
     """
     Save all dataset-wide metadata.   This includes:
     * CXG version
@@ -407,8 +455,8 @@ def save_metadata(container, metadata):
     with tiledb.from_numpy(a_name, np.zeros((1,))) as A:
         pass
     with tiledb.DenseArray(a_name, mode="w") as A:
-        A.meta["cxg_version"] = CXG_VERSION
+        for k, v in metadata_dict.items():
-        A.meta["cxg_properties"] = json.dumps(metadata)
+            A.meta[k] = v
 
 
 def sanitize_keys(keys):

server/data_anndata/anndata_adaptor.py

@@ -12,6 +12,7 @@ from server_timing import Timing as ServerTiming
 from server.data_common.data_adaptor import DataAdaptor
 from server.data_common.fbs.matrix import encode_matrix_fbs
 from server.common.utils import series_to_schema
+from server.common.colors import convert_anndata_category_colors_to_cxg_category_colors
 from server.common.constants import Axis, MAX_LAYOUTS
 from server.common.errors import PrepareError, DatasetAccessError, FilterError
 from server.compute.scanpy import scanpy_umap
@@ -333,6 +334,9 @@ class AnndataAdaptor(DataAdaptor):
             lfc_cutoff = self.config.diffexp__lfc_cutoff
         return diffexp_generic.diffexp_ttest(self, maskA, maskB, top_n, lfc_cutoff)
 
+    def get_colors(self):
+        return convert_anndata_category_colors_to_cxg_category_colors(self.data)
+
     def get_X_array(self, obs_mask=None, var_mask=None):
         if obs_mask is None:
             obs_mask = slice(None)

server/data_common/data_adaptor.py

@@ -83,6 +83,10 @@ class DataAdaptor(metaclass=ABCMeta):
     def query_obs_array(self, term_var):
         pass
 
+    @abstractmethod
+    def get_colors(self):
+        pass
+
     @abstractmethod
     def get_obs_index(self):
         pass

server/data_cxg/cxg_adaptor.py

@@ -194,6 +194,10 @@ class CxgAdaptor(DataAdaptor):
             lfc_cutoff = self.config.diffexp__lfc_cutoff
         return diffexp_cxg.diffexp_ttest(self, maskA, maskB, top_n, lfc_cutoff)
 
+    def get_colors(self):
+        meta = self.open_array("cxg_group_metadata").meta
+        return json.loads(meta["cxg_category_colors"]) if "cxg_category_colors" in meta else dict()
+
     def get_X_array(self, obs_mask=None, var_mask=None):
         obs_items = pack_selector_from_mask(obs_mask)
         var_items = pack_selector_from_mask(var_mask)

server/eb/app.py

@@ -68,7 +68,7 @@ class WSGIServer(Server):
         if len(style_hashes) > 0:
             csp["style-src"] = style_hashes
 
-        Talisman(app, force_https=app_config.server__force_https, frame_options='DENY', content_security_policy=csp)
+        Talisman(app, force_https=app_config.server__force_https, frame_options="DENY", content_security_policy=csp)
 
     @staticmethod
     def load_csp_hashes(app):

server/test/__init__.py

@@ -1,6 +1,6 @@
 import shutil
 import tempfile
-from os import path
+from os import path, popen
 
 import pandas as pd
 
@@ -11,11 +11,14 @@ from server.data_common.fbs.matrix import encode_matrix_fbs
 from server.data_common.matrix_loader import MatrixDataLoader, MatrixDataType
 
 
+PROJECT_ROOT = popen("git rev-parse --show-toplevel").read().strip()
+
+
 def data_with_tmp_annotations(ext: MatrixDataType, annotations_fixture=False):
     tmp_dir = tempfile.mkdtemp()
     annotations_file = path.join(tmp_dir, "test_annotations.csv")
     if annotations_fixture:
-        shutil.copyfile(f"test/test_datasets/pbmc3k-annotations.csv", annotations_file)
+        shutil.copyfile(f"{PROJECT_ROOT}/server/test/test_datasets/pbmc3k-annotations.csv", annotations_file)
     args = {
         "embeddings__names": ["umap"],
         "presentation__max_categories": 100,
@@ -24,7 +27,7 @@ def data_with_tmp_annotations(ext: MatrixDataType, annotations_fixture=False):
         "diffexp__lfc_cutoff": 0.01,
     }
     fname = {
-        MatrixDataType.H5AD: "../example-dataset/pbmc3k.h5ad",
+        MatrixDataType.H5AD: f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad",
         MatrixDataType.CXG: "test/test_datasets/pbmc3k.cxg",
     }[ext]
     data_locator = DataLocator(fname)
@@ -53,3 +56,21 @@ def skip_if(condition, reason: str):
         return wraps
 
     return decorator
+
+
+def app_config(data_locator, backed=False):
+    args = {
+        "embeddings__names": ["umap", "tsne", "pca"],
+        "presentation__max_categories": 100,
+        "single_dataset__obs_names": None,
+        "single_dataset__var_names": None,
+        "diffexp__lfc_cutoff": 0.01,
+        "adaptor__anndata_adaptor__backed": backed,
+        "single_dataset__datapath": data_locator,
+        "limits__diffexp_cellcount_max": None,
+        "limits__column_request_max": None,
+    }
+    config = AppConfig()
+    config.update(**args)
+    config.complete_config()
+    return config

server/test/test_anndata_adaptor.py

@@ -10,10 +10,11 @@ from parameterized import parameterized_class
 import numpy as np
 import pandas as pd
 
-from server.data_anndata.anndata_adaptor import AnndataAdaptor
-from server.common.errors import FilterError
 from server.common.data_locator import DataLocator
-from server.common.app_config import AppConfig
+from server.common.errors import FilterError
+from server.data_anndata.anndata_adaptor import AnndataAdaptor
+from server.test import PROJECT_ROOT, app_config
+from server.test.test_datasets.fixtures import pbmc3k_colors
 
 """
 Test the anndata adaptor using the pbmc3k data set.
@@ -23,30 +24,17 @@ Test the anndata adaptor using the pbmc3k data set.
 @parameterized_class(
     ("data_locator", "backed"),
     [
-        ("../example-dataset/pbmc3k.h5ad", False),
+        (f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad", False),
-        ("test/test_datasets/pbmc3k-CSC-gz.h5ad", False),
+        (f"{PROJECT_ROOT}/server/test/test_datasets/pbmc3k-CSC-gz.h5ad", False),
-        ("test/test_datasets/pbmc3k-CSR-gz.h5ad", False),
+        (f"{PROJECT_ROOT}/server/test/test_datasets/pbmc3k-CSR-gz.h5ad", False),
-        ("../example-dataset/pbmc3k.h5ad", True),
+        (f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad", True),
-        ("test/test_datasets/pbmc3k-CSC-gz.h5ad", True),
+        (f"{PROJECT_ROOT}/server/test/test_datasets/pbmc3k-CSC-gz.h5ad", True),
-        ("test/test_datasets/pbmc3k-CSR-gz.h5ad", True),
+        (f"{PROJECT_ROOT}/server/test/test_datasets/pbmc3k-CSR-gz.h5ad", True),
     ],
 )
 class AdaptorTest(unittest.TestCase):
     def setUp(self):
-        args = {
+        config = app_config(self.data_locator, self.backed)
-            "embeddings__names": ["umap", "tsne", "pca"],
-            "presentation__max_categories": 100,
-            "single_dataset__obs_names": None,
-            "single_dataset__var_names": None,
-            "diffexp__lfc_cutoff": 0.01,
-            "adaptor__anndata_adaptor__backed": self.backed,
-            "single_dataset__datapath": self.data_locator,
-            "limits__diffexp_cellcount_max": None,
-            "limits__column_request_max": None,
-        }
-        config = AppConfig()
-        config.update(**args)
-        config.complete_config()
         self.data = AnndataAdaptor(DataLocator(self.data_locator), config)
 
     def test_init(self):
@@ -92,6 +80,9 @@ class AdaptorTest(unittest.TestCase):
         self.assertEqual(np.sum(self.data.data.var[self.data.get_schema()["annotations"]["var"]["index"]].isna()), 0)
         self.assertEqual(np.sum(self.data.data.obs[self.data.get_schema()["annotations"]["obs"]["index"]].isna()), 0)
 
+    def test_get_colors(self):
+        self.assertEqual(self.data.get_colors(), pbmc3k_colors)
+
     def test_get_schema(self):
         with open(path.join(path.dirname(__file__), "schema.json")) as fh:
             schema = json.load(fh)

server/test/test_anndata_adaptor_data_load.py

@@ -4,6 +4,7 @@ import json
 from server.data_anndata.anndata_adaptor import AnndataAdaptor
 from server.common.data_locator import DataLocator
 from server.common.app_config import AppConfig
+from server.test import PROJECT_ROOT
 
 
 class DataLoadAdaptorTest(unittest.TestCase):
@@ -12,7 +13,7 @@ class DataLoadAdaptorTest(unittest.TestCase):
     """
 
     def setUp(self):
-        self.data_file = DataLocator("../example-dataset/pbmc3k.h5ad")
+        self.data_file = DataLocator(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad")
         config = AppConfig()
         config.update(single_dataset__datapath=self.data_file.path)
         config.complete_config()

server/test/test_api.py

@@ -8,8 +8,10 @@ import pandas as pd
 import requests
 
 import server.test.decode_fbs as decode_fbs
-from server.test import data_with_tmp_annotations, make_fbs
 from server.data_common.matrix_loader import MatrixDataType
+from server.test import data_with_tmp_annotations, make_fbs, PROJECT_ROOT
+from server.test.test_datasets.fixtures import pbmc3k_colors
+
 
 BAD_FILTER = {"filter": {"obs": {"annotation_value": [{"name": "xyz"}]}}}
 
@@ -255,6 +257,15 @@ class EndPoints(object):
         self.assertEqual(df["n_rows"], 2638)
         self.assertEqual(df["n_cols"], 1)
 
+    def test_colors(self):
+        endpoint = "colors"
+        url = f"{self.URL_BASE}{endpoint}"
+        result = self.session.get(url)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/json")
+        result_data = result.json()
+        self.assertEqual(result_data, pbmc3k_colors)
+
     def test_static(self):
         endpoint = "static"
         file = "assets/favicon.ico"
@@ -349,7 +360,7 @@ class EndPointsAnndata(unittest.TestCase, EndPoints):
                 "cellxgene",
                 "--no-upgrade-check",
                 "launch",
-                "../example-dataset/pbmc3k.h5ad",
+                f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad",
                 "--disable-annotations",
                 "--verbose",
                 "--port",
@@ -383,7 +394,7 @@ class EndPointsCxg(unittest.TestCase, EndPoints):
                 "cellxgene",
                 "--no-upgrade-check",
                 "launch",
-                "test/test_datasets/pbmc3k.cxg",
+                f"{PROJECT_ROOT}/server/test/test_datasets/pbmc3k.cxg",
                 "--disable-annotations",
                 "--verbose",
                 "--port",

server/test/test_colors.py

@@ -0,0 +1,40 @@
+import unittest
+
+import anndata
+from server.common.colors import convert_color_to_hex_format, convert_anndata_category_colors_to_cxg_category_colors
+from server.common.errors import ColorFormatException
+from server.test import PROJECT_ROOT
+from server.test.test_datasets.fixtures import pbmc3k_colors
+
+
+class ColorsTest(unittest.TestCase):
+    """ Test color helper functions """
+
+    def test_convert_color_to_hex_format(self):
+        self.assertEqual(convert_color_to_hex_format("wheat"), "#f5deb3")
+        self.assertEqual(convert_color_to_hex_format("WHEAT"), "#f5deb3")
+        self.assertEqual(convert_color_to_hex_format((245, 222, 179)), "#f5deb3")
+        self.assertEqual(convert_color_to_hex_format([245, 222, 179]), "#f5deb3")
+        self.assertEqual(convert_color_to_hex_format("#f5deb3"), "#f5deb3")
+        self.assertEqual(
+            convert_color_to_hex_format([0.9607843137254902, 0.8705882352941177, 0.7019607843137254]), "#f5deb3"
+        )
+        for bad_input in ["foo", "BAR", "#AABB", "#AABBCCDD", "#AABBGG", (1, 2), [1, 2], (1, 2, 3, 4), [1, 2, 3, 4]]:
+            with self.assertRaises(ColorFormatException):
+                convert_color_to_hex_format(bad_input)
+
+    def test_anndata_colors_to_cxg_colors(self):
+        # test standard behavior
+        adata = self._get_h5ad()
+        self.assertEqual(convert_anndata_category_colors_to_cxg_category_colors(adata), pbmc3k_colors)
+        # test that invalid color formats raise an exception
+        adata.uns["louvain_colors"][0] = "#NOTCOOL"
+        with self.assertRaises(ColorFormatException):
+            convert_anndata_category_colors_to_cxg_category_colors(adata)
+        # test that colors without a matching obs category are skipped
+        adata = self._get_h5ad()
+        del adata.obs["louvain"]
+        self.assertEqual(convert_anndata_category_colors_to_cxg_category_colors(adata), {})
+
+    def _get_h5ad(self):
+        return anndata.read_h5ad(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad")

server/test/test_cxg_adaptor.py

@@ -0,0 +1,16 @@
+import unittest
+
+from server.common.data_locator import DataLocator
+from server.data_cxg.cxg_adaptor import CxgAdaptor
+from server.test import PROJECT_ROOT, app_config
+from server.test.test_datasets.fixtures import pbmc3k_colors
+
+
+class TestCxgAdaptor(unittest.TestCase):
+    def setUp(self):
+        data_locator = f"{PROJECT_ROOT}/server/test/test_datasets/pbmc3k.cxg"
+        config = app_config(data_locator)
+        self.data = CxgAdaptor(DataLocator(data_locator), config)
+
+    def test_get_colors(self):
+        self.assertEqual(self.data.get_colors(), pbmc3k_colors)

server/test/test_cxgtool.py

@@ -0,0 +1,40 @@
+import random
+import shutil
+import string
+import unittest
+
+import anndata
+
+from server.common.data_locator import DataLocator
+from server.converters.cxgtool import write_cxg
+from server.data_cxg.cxg_adaptor import CxgAdaptor
+from server.test import PROJECT_ROOT, app_config
+from server.test.test_datasets.fixtures import pbmc3k_colors
+
+
+class TestCxgAdaptor(unittest.TestCase):
+    def setUp(self) -> None:
+        self.fixtures = []
+
+    def tearDown(self) -> None:
+        try:
+            for data_locator in self.fixtures:
+                print("REMOVING ", data_locator)
+                shutil.rmtree(data_locator)
+        except FileNotFoundError:
+            pass
+
+    def test_cxg_category_colors(self):
+        data = self.convert_pbmc3k(extract_colors=True)
+        self.assertEqual(data.get_colors(), pbmc3k_colors)
+        data = self.convert_pbmc3k(extract_colors=False)
+        self.assertEqual(data.get_colors(), {})
+
+    def convert_pbmc3k(self, **kwargs):
+        random_string = "".join(random.choice(string.ascii_letters) for _ in range(8))
+        data_locator = f"/tmp/test_{random_string}.cxg"
+        self.fixtures.append(data_locator)
+        source_h5ad = anndata.read_h5ad(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad")
+        write_cxg(adata=source_h5ad, container=data_locator, title="pbmc3k", **kwargs)
+        config = app_config(data_locator)
+        return CxgAdaptor(DataLocator(data_locator), config)

server/test/test_datasets/fixtures.py

@@ -0,0 +1,12 @@
+pbmc3k_colors = {
+    "louvain": {
+        "B cells": "#2ca02c",
+        "CD14+ Monocytes": "#ff7f0e",
+        "CD4 T cells": "#1f77b4",
+        "CD8 T cells": "#d62728",
+        "Dendritic cells": "#e377c2",
+        "FCGR3A+ Monocytes": "#8c564b",
+        "Megakaryocytes": "#bcbd22",
+        "NK cells": "#9467bd",
+    }
+}

server/test/test_diffexp.py

@@ -5,6 +5,8 @@ import server.compute.diffexp_cxg as diffexp_cxg
 import server.compute.diffexp_generic as diffexp_generic
 import numpy as np
 
+from server.test import PROJECT_ROOT
+
 
 class DiffExpTest(unittest.TestCase):
     """Tests the diffexp returns the expected results for one test case, using different
@@ -50,7 +52,7 @@ class DiffExpTest(unittest.TestCase):
 
     def test_anndata_default(self):
         """Test an anndata adaptor with its default diffexp algorithm (diffexp_generic)"""
-        adaptor = self.load_dataset("../example-dataset/pbmc3k.h5ad")
+        adaptor = self.load_dataset(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad")
         maskA = self.get_mask(adaptor, 1, 10)
         maskB = self.get_mask(adaptor, 2, 10)
         results = adaptor.compute_diffexp_ttest(maskA, maskB, 10)
@@ -58,7 +60,7 @@ class DiffExpTest(unittest.TestCase):
 
     def test_cxg_default(self):
         """Test a cxg adaptor with its default diffexp algorithm (diffexp_cxg)"""
-        adaptor = self.load_dataset("test/test_datasets/pbmc3k.cxg")
+        adaptor = self.load_dataset(f"{PROJECT_ROOT}/server/test/test_datasets/pbmc3k.cxg")
         maskA = self.get_mask(adaptor, 1, 10)
         maskB = self.get_mask(adaptor, 2, 10)
 
@@ -72,7 +74,7 @@ class DiffExpTest(unittest.TestCase):
 
     def test_cxg_generic(self):
         """Test a cxg adaptor with the generic adaptor"""
-        adaptor = self.load_dataset("test/test_datasets/pbmc3k.cxg")
+        adaptor = self.load_dataset(f"{PROJECT_ROOT}/server/test/test_datasets/pbmc3k.cxg")
         maskA = self.get_mask(adaptor, 1, 10)
         maskB = self.get_mask(adaptor, 2, 10)
         # run it directly

server/test/test_matrixcache.py

@@ -7,13 +7,15 @@ import shutil
 import os
 import time
 
+from server.test import PROJECT_ROOT
+
 
 class MatrixCacheTest(unittest.TestCase):
     def setup(self):
         pass
 
     def make_temporay_datasets(self, dirname, num):
-        source = "test/test_datasets/pbmc3k.cxg"
+        source = f"{PROJECT_ROOT}/server/test/test_datasets/pbmc3k.cxg"
         for i in range(num):
             target = os.path.join(dirname, str(i) + ".cxg")
             shutil.copytree(source, target)

server/test/test_nan_anndata_adaptor.py

@@ -8,39 +8,22 @@ import server.test.decode_fbs as decode_fbs
 from server.data_anndata.anndata_adaptor import AnndataAdaptor
 from server.common.errors import FilterError
 from server.common.data_locator import DataLocator
-from server.common.app_config import AppConfig
+from server.test import PROJECT_ROOT, app_config
 
 
 class NaNTest(unittest.TestCase):
     def setUp(self):
-        self.args = {
+        self.data_locator = DataLocator(f"{PROJECT_ROOT}/server/test/test_datasets/nan.h5ad")
-            "embeddings__names": ["umap"],
+        self.config = app_config(self.data_locator.path)
-            "presentation__max_categories": 100,
-            "single_dataset__obs_names": None,
-            "single_dataset__var_names": None,
-            "diffexp__lfc_cutoff": 0.01,
-            "limits__diffexp_cellcount_max": None,
-            "limits__column_request_max": None,
-        }
-        config = AppConfig()
-        config.update(**self.args)
-        locator = DataLocator("test/test_datasets/nan.h5ad")
-        config.update(single_dataset__datapath=locator.path)
-        config.complete_config()
 
         with warnings.catch_warnings():
             warnings.simplefilter("ignore", category=UserWarning)
-            self.data = AnndataAdaptor(locator, config)
+            self.data = AnndataAdaptor(self.data_locator, self.config)
             self.data._create_schema()
 
     def test_load(self):
         with self.assertWarns(UserWarning):
-            config = AppConfig()
+            self.data = AnndataAdaptor(self.data_locator, self.config)
-            config.update(**self.args)
-            locator = DataLocator("test/test_datasets/nan.h5ad")
-            config.update(single_dataset__datapath=locator.path)
-            config.complete_config()
-            self.data = AnndataAdaptor(locator, config)
 
     def test_init(self):
         self.assertEqual(self.data.cell_count, 100)