Add user-defined category-label colors (#1402)

* Add user-defined category-label colors

Fixes https://github.com/chanzuckerberg/cellxgene/issues/1152

As described in https://github.com/chanzuckerberg/cellxgene/issues/1307

* Respond to feedback from @bkmartinjr in nodejs

* Respond to feedback from @bkmartinjr in python

* Add tests to the server module

* Autoformat python, run linter

* Make colors_get error handling specific

* Respond to feedback from @bkmartinjr

* Respond to feedback from @bkmartinjr

* Fix whitespace

* Fix python lint errrors

* Update documentation

* Add --disable-user-colors option to launch and cxgtool.py

* Fix python formatting

* Rename '--disable-user-colors' to '--disable-custom-colors'

server/app/app.py

@@ -207,6 +207,13 @@ class DataVarAPI(Resource):
         return common_rest.data_var_get(request, data_adaptor)
 
 
+class ColorsAPI(Resource):
+    @cache_control(public=True, max_age=ONE_WEEK)
+    @rest_get_data_adaptor
+    def get(self, data_adaptor):
+        return common_rest.colors_get(data_adaptor)
+
+
 class DiffExpObsAPI(Resource):
     @cache_control(no_store=True)
     @rest_get_data_adaptor
@@ -235,6 +242,8 @@ def get_api_resources(bp_api):
     api.add_resource(AnnotationsObsAPI, "/annotations/obs")
     api.add_resource(AnnotationsVarAPI, "/annotations/var")
     api.add_resource(DataVarAPI, "/data/var")
+    # Display routes
+    api.add_resource(ColorsAPI, "/colors")
     # Computation routes
     api.add_resource(DiffExpObsAPI, "/diffexp/obs")
     api.add_resource(LayoutObsAPI, "/layout/obs")

server/cli/launch.py

@@ -73,6 +73,13 @@ def config_args(func):
         show_default=True,
         help="Will not display categories with more distinct values than specified.",
     )
+    @click.option(
+        "--disable-custom-colors",
+        is_flag=True,
+        default=False,
+        show_default=False,
+        help="Disable user-defined category-label colors drawn from source data file.",
+    )
     @click.option(
         "--diffexp-lfc-cutoff",
         "-de",
@@ -146,7 +153,7 @@ def dataset_args(func):
         "--about",
         default=DEFAULT_CONFIG.single_dataset__about,
         metavar="<URL>",
-        help="URL providing more information about the dataset " "(hint: must be a fully specified absolute URL).",
+        help="URL providing more information about the dataset (hint: must be a fully specified absolute URL).",
     )
     @functools.wraps(func)
     def wrapper(*args, **kwargs):
@@ -311,6 +318,7 @@ def launch(
     obs_names,
     var_names,
     max_category_items,
+    disable_custom_colors,
     diffexp_lfc_cutoff,
     title,
     scripts,
@@ -381,6 +389,7 @@ def launch(
             user_annotations__ontology__enable=experimental_annotations_ontology,
             user_annotations__ontology__obo_location=experimental_annotations_ontology_obo,
             presentation__max_categories=max_category_items,
+            presentation__custom_colors=not disable_custom_colors,
             embeddings__names=embedding,
             embeddings__enable_reembedding=experimental_enable_reembedding,
             diffexp__enable=not disable_diffexp,

server/cli/prepare.py

@@ -73,7 +73,8 @@ def prepare(
     (h5ad, loom, or a 10x directory), runs dimensionality reduction,
     computes nearest neighbors, computes an embedding, performs clustering,
     and saves the results. Includes additional options for naming annotations,
-    ensuring sparsity, and plotting results."""
+    ensuring sparsity, and plotting results.
+    """
 
     # collect slow imports here to make CLI startup more responsive
     click.echo("[cellxgene] Starting CLI...")

server/common/app_config.py

@@ -78,6 +78,7 @@ class AppConfig(object):
             self.user_annotations__ontology__obo_location = dc["user_annotations"]["ontology"]["obo_location"]
 
             self.presentation__max_categories = dc["presentation"]["max_categories"]
+            self.presentation__custom_colors = dc["presentation"]["custom_colors"]
 
             self.embeddings__names = dc["embeddings"]["names"]
             self.embeddings__enable_reembedding = dc["embeddings"]["enable_reembedding"]
@@ -275,6 +276,7 @@ class AppConfig(object):
 
     def handle_presentation(self, context):
         self.__check_attr("presentation__max_categories", int)
+        self.__check_attr("presentation__custom_colors", bool)
 
     def handle_single_dataset(self, context):
         self.__check_attr("single_dataset__datapath", (str, type(None)))
@@ -517,6 +519,7 @@ class AppConfig(object):
             "annotations_cell_ontology_enabled": False,
             "annotations_cell_ontology_obopath": None,
             "annotations_cell_ontology_terms": None,
+            "custom_colors": self.presentation__custom_colors,
             "diffexp-may-be-slow": False,
             "about_legal_tos": self.server__about_legal_tos,
             "about_legal_privacy": self.server__about_legal_privacy,

server/common/colors.py

@@ -0,0 +1,233 @@
+import re
+
+from server.common.errors import ColorFormatException
+
+HEX_COLOR_FORMAT = re.compile("^#[a-fA-F0-9]{6,6}$")
+
+# https://www.w3.org/TR/css-color-4/#named-colors
+CSS4_NAMED_COLORS = dict(
+    aliceblue="#f0f8ff",
+    antiquewhite="#faebd7",
+    aqua="#00ffff",
+    aquamarine="#7fffd4",
+    azure="#f0ffff",
+    beige="#f5f5dc",
+    bisque="#ffe4c4",
+    black="#000000",
+    blanchedalmond="#ffebcd",
+    blue="#0000ff",
+    blueviolet="#8a2be2",
+    brown="#a52a2a",
+    burlywood="#deb887",
+    cadetblue="#5f9ea0",
+    chartreuse="#7fff00",
+    chocolate="#d2691e",
+    coral="#ff7f50",
+    cornflowerblue="#6495ed",
+    cornsilk="#fff8dc",
+    crimson="#dc143c",
+    cyan="#00ffff",
+    darkblue="#00008b",
+    darkcyan="#008b8b",
+    darkgoldenrod="#b8860b",
+    darkgray="#a9a9a9",
+    darkgreen="#006400",
+    darkgrey="#a9a9a9",
+    darkkhaki="#bdb76b",
+    darkmagenta="#8b008b",
+    darkolivegreen="#556b2f",
+    darkorange="#ff8c00",
+    darkorchid="#9932cc",
+    darkred="#8b0000",
+    darksalmon="#e9967a",
+    darkseagreen="#8fbc8f",
+    darkslateblue="#483d8b",
+    darkslategray="#2f4f4f",
+    darkslategrey="#2f4f4f",
+    darkturquoise="#00ced1",
+    darkviolet="#9400d3",
+    deeppink="#ff1493",
+    deepskyblue="#00bfff",
+    dimgray="#696969",
+    dimgrey="#696969",
+    dodgerblue="#1e90ff",
+    firebrick="#b22222",
+    floralwhite="#fffaf0",
+    forestgreen="#228b22",
+    fuchsia="#ff00ff",
+    gainsboro="#dcdcdc",
+    ghostwhite="#f8f8ff",
+    gold="#ffd700",
+    goldenrod="#daa520",
+    gray="#808080",
+    green="#008000",
+    greenyellow="#adff2f",
+    grey="#808080",
+    honeydew="#f0fff0",
+    hotpink="#ff69b4",
+    indianred="#cd5c5c",
+    indigo="#4b0082",
+    ivory="#fffff0",
+    khaki="#f0e68c",
+    lavender="#e6e6fa",
+    lavenderblush="#fff0f5",
+    lawngreen="#7cfc00",
+    lemonchiffon="#fffacd",
+    lightblue="#add8e6",
+    lightcoral="#f08080",
+    lightcyan="#e0ffff",
+    lightgoldenrodyellow="#fafad2",
+    lightgray="#d3d3d3",
+    lightgreen="#90ee90",
+    lightgrey="#d3d3d3",
+    lightpink="#ffb6c1",
+    lightsalmon="#ffa07a",
+    lightseagreen="#20b2aa",
+    lightskyblue="#87cefa",
+    lightslategray="#778899",
+    lightslategrey="#778899",
+    lightsteelblue="#b0c4de",
+    lightyellow="#ffffe0",
+    lime="#00ff00",
+    limegreen="#32cd32",
+    linen="#faf0e6",
+    magenta="#ff00ff",
+    maroon="#800000",
+    mediumaquamarine="#66cdaa",
+    mediumblue="#0000cd",
+    mediumorchid="#ba55d3",
+    mediumpurple="#9370db",
+    mediumseagreen="#3cb371",
+    mediumslateblue="#7b68ee",
+    mediumspringgreen="#00fa9a",
+    mediumturquoise="#48d1cc",
+    mediumvioletred="#c71585",
+    midnightblue="#191970",
+    mintcream="#f5fffa",
+    mistyrose="#ffe4e1",
+    moccasin="#ffe4b5",
+    navajowhite="#ffdead",
+    navy="#000080",
+    oldlace="#fdf5e6",
+    olive="#808000",
+    olivedrab="#6b8e23",
+    orange="#ffa500",
+    orangered="#ff4500",
+    orchid="#da70d6",
+    palegoldenrod="#eee8aa",
+    palegreen="#98fb98",
+    paleturquoise="#afeeee",
+    palevioletred="#db7093",
+    papayawhip="#ffefd5",
+    peachpuff="#ffdab9",
+    peru="#cd853f",
+    pink="#ffc0cb",
+    plum="#dda0dd",
+    powderblue="#b0e0e6",
+    purple="#800080",
+    rebeccapurple="#663399",
+    red="#ff0000",
+    rosybrown="#bc8f8f",
+    royalblue="#4169e1",
+    saddlebrown="#8b4513",
+    salmon="#fa8072",
+    sandybrown="#f4a460",
+    seagreen="#2e8b57",
+    seashell="#fff5ee",
+    sienna="#a0522d",
+    silver="#c0c0c0",
+    skyblue="#87ceeb",
+    slateblue="#6a5acd",
+    slategray="#708090",
+    slategrey="#708090",
+    snow="#fffafa",
+    springgreen="#00ff7f",
+    steelblue="#4682b4",
+    tan="#d2b48c",
+    teal="#008080",
+    thistle="#d8bfd8",
+    tomato="#ff6347",
+    turquoise="#40e0d0",
+    violet="#ee82ee",
+    wheat="#f5deb3",
+    white="#ffffff",
+    whitesmoke="#f5f5f5",
+    yellow="#ffff00",
+    yellowgreen="#9acd32",
+)
+
+
+def convert_color_to_hex_format(unknown):
+    """
+    Try to convert color info to a hex triplet string https://en.wikipedia.org/wiki/Web_colors#Hex_triplet.
+
+    The function accepts for the following formats:
+    - A CSS4 color name, as supported by matplotlib https://matplotlib.org/3.1.0/gallery/color/named_colors.html
+    - RGB tuple/list with values ranging from 0.0 to 1.0, as in [0.5, 0.75, 1.0]
+    - RFB tuple/list with values ranging from 0 to 255, as in [128, 192, 255]
+    - Hex triplet string, as in "#08c0ff"
+
+    :param unknown: color info of unknown format
+    :return: a hex triplet representing that color
+    """
+    try:
+        if type(unknown) in (list, tuple) and len(unknown) == 3:
+            if all(0.0 <= ele <= 1.0 for ele in unknown):
+                tup = tuple(int(ele * 255) for ele in unknown)
+            elif all(0 <= ele <= 255 and isinstance(ele, int) for ele in unknown):
+                tup = tuple(unknown)
+            else:
+                raise ColorFormatException("Unknown color iterable format!")
+            return "#%02x%02x%02x" % tup
+        elif isinstance(unknown, str) and unknown.lower() in CSS4_NAMED_COLORS:
+            return CSS4_NAMED_COLORS[unknown.lower()]
+        elif isinstance(unknown, str) and HEX_COLOR_FORMAT.match(unknown):
+            return unknown.lower()
+        else:
+            raise ColorFormatException("Unknown color format type!")
+    except Exception as e:
+        raise ColorFormatException(e)
+
+
+def convert_anndata_category_colors_to_cxg_category_colors(data):
+    """
+    Convert color information from anndata files to the cellxgene color data format as described below:
+        {
+            "<category_name>": {
+                "<label_name>": "<color_hex_code>",
+                ...
+            },
+            ...
+        }
+
+    For more on the cxg color data structure, see https://github.com/chanzuckerberg/cellxgene/issues/1307.
+
+    For more on the anndata color data structure, see
+    https://github.com/chanzuckerberg/cellxgene/issues/1152#issuecomment-587276178.
+
+    Handling of malformed data:
+    - For any color info in a adata.uns[f"{category}_colors"] color array that convert_color_to_hex_format cannot
+      convert to a hex triplet string, a ColorFormatException is raised
+    - No category_name key group is returned for adata.uns[f"{category}_colors"] keys for which there is no
+      adata.obs[f"{category}"] key
+
+    :param data: the anndata file
+    :return: cellxgene color data structure as described above
+    """
+    cxg_colors = dict()
+    color_key_suffix = "_colors"
+    for uns_key in data.uns.keys():
+        # find uns array that describes colors for a category
+        if not uns_key.endswith(color_key_suffix):
+            continue
+
+        # check to see if we actually have observations for that category
+        category_name = uns_key[: -len(color_key_suffix)]
+        if category_name not in data.obs.keys():
+            continue
+
+        # create the cellxgene color entry for this category
+        cxg_colors[category_name] = dict(
+            zip(data.obs[category_name].cat.categories, [convert_color_to_hex_format(c) for c in data.uns[uns_key]])
+        )
+    return cxg_colors

server/common/default_config.py

@@ -19,6 +19,7 @@ server:
 
 presentation:
   max_categories: 1000
+  custom_colors: true
 
 multi_dataset:
   dataroot: null

server/common/errors.py

@@ -84,3 +84,9 @@ class ComputeError(Exception):
     """
 
     pass
+
+
+class ColorFormatException(Exception):
+    """Raised when color helper functions encounter an unknown color format"""
+
+    pass

server/common/rest.py

@@ -1,9 +1,11 @@
-import sys
-from http import HTTPStatus
 import copy
 import logging
+import sys
+from http import HTTPStatus
+
 from flask import make_response, jsonify, current_app, abort
 from werkzeug.urls import url_unquote
+
 from server.common.constants import Axis, DiffExpMode, JSON_NaN_to_num_warning_msg
 from server.common.errors import (
     FilterError,
@@ -12,6 +14,7 @@ from server.common.errors import (
     DisabledFeatureError,
     ExceedsLimitError,
     DatasetAccessError,
+    ColorFormatException,
 )
 
 import json
@@ -222,6 +225,15 @@ def data_var_get(request, data_adaptor):
         return abort_and_log(HTTPStatus.BAD_REQUEST, str(e), include_exc_info=True)
 
 
+def colors_get(data_adaptor):
+    if not data_adaptor.config.presentation__custom_colors:
+        return make_response(jsonify({}), HTTPStatus.OK)
+    try:
+        return make_response(jsonify(data_adaptor.get_colors()), HTTPStatus.OK)
+    except ColorFormatException as e:
+        return abort_and_log(HTTPStatus.NOT_FOUND, str(e), include_exc_info=True)
+
+
 def diffexp_obs_post(request, data_adaptor):
     if not data_adaptor.config.diffexp__enable:
         return abort(HTTPStatus.NOT_IMPLEMENTED)

server/converters/cxgtool.py

@@ -4,28 +4,36 @@ into a cellxgene TileDB structure, aka a 'CXG'.
 
 The organization of the TileDB structure is:
 
-    the.cxg                 TileDB Group
-    |-- obs                 TileDB array containing cell (row) attributes, one attribute per
-    |                       dataframe columm, shape (n_obs,)
-    |-- var                 TileDB array containing gene (column) attributes, with one attribute per
-    |                       dataframe column, shape (n_var,)
-    |-- X                   Main count matrix as a 2D TileDB array, single unnanmed numeric attribute
-    |-- emb                 TileDB group, storing optional embeddings (group may be empty)
-    |   |-- <name1>         TileDB Array, single anon attribute, ND numeric array, shape (n_obs, N)
-    |-- cxg_group_metadata  Empty array used only to stash metadata about the overall object.
+    the.cxg                         TileDB Group
+    ├─ obs                          TileDB array containing cell (row) attributes, one attribute per
+    │                               dataframe column, shape (n_obs,)
+    ├─ var                          TileDB array containing gene (column) attributes, with one attribute per
+    │                               dataframe column, shape (n_obs,)
+    ├─ X                            Main count matrix as a 2D TileDB array, single unnamed numeric attribute
+    ├─ emb                          TileDB group, storing optional embeddings (group may be empty)
+    │  └─ <name1>                   TileDB Array, single anon attribute, ND numeric array, shape (n_obs, N)
+    └─ cxg_group_metadata           Empty array used only to stash metadata about the overall object.
+       └─ cxg_category_colors       CXG colors object as described below:
+                                        {
+                                             "<category_name>": {
+                                                 "<label_name>": "<color_hex_code>",
+                                                 ...
+                                             },
+                                             ...
+                                        }
     ...
 
-All arrays are defined to have a uint32 domain, zero based.  All X counds and embedding
+All arrays are defined to have a uint32 domain, zero based.  All X counts and embedding
 coordinates are coerced to float32, which is ample precision for visualization purposes.
 Dataframe (metadata) types are generally preserved, or where that is not possible,
-converted to somemthing with equal representative value in the cellxgene application
+converted to something with equal representative value in the cellxgene application
 (eg, categorical types are converted to string, bools to uint8, etc).
 
-The following objects are also decorated with auxilliary metadata using TileDB
+The following objects are also decorated with auxiliary metadata using TileDB
 array metadata:
 
 * cxg_group_metadata: minimally, will contain a 'cxg_version' field, which
-  is a semver string identifing the version number of the CXG layout.
+  is a semver string identifying the version number of the CXG layout.
   It may also contain 'cxg_parameters', a JSON-encoded parameter list
   describing CXG-wide dataset parameters.
 
@@ -40,6 +48,14 @@ including the global data layout, spatial tile size, and the like. The CXG is
 self-describing in these areas, and the actual values (eg, tile size) are empirically
 derived from benchmarking. They may change in the future.
 
+cxgtool.py will extract color information stored in arrays in the 'uns' anndata
+property with the key "{category_name}_colors". For this to work, the following
+command must result in a mapping from category names to matplotlib-compatible colors:
+
+```
+dict(zip(adata.obs[cat].cat.categories, adata.uns[f"{cat}_colors"]))
+```
+
 ---
 
 TODO/ISSUES:
@@ -55,10 +71,16 @@ import numpy as np
 from os.path import splitext, basename
 import json
 
+from server.common.colors import convert_anndata_category_colors_to_cxg_category_colors
+from server.common.errors import ColorFormatException
+
 
 # the CXG container version number.  Must be a semver string.
 CXG_VERSION = "0.1"
 
+# log_level must have a default
+log_level = 3
+
 
 def log(level, *args):
     global log_level
@@ -72,6 +94,12 @@ def main():
     parser.add_argument(
         "--backed", action="store_true", help="loaded in file backed mode. Will be slower, but use less memory."
     )
+    parser.add_argument(
+        "--disable-custom-colors",
+        action="store_true",
+        default=False,
+        help="Do not extract scanpy-compatible category colors from h5ad file.",
+    )
     parser.add_argument(
         "--obs-names", help="Name of annotation to use for observations. If not specified, will use the obs index."
     )
@@ -99,12 +127,20 @@ def main():
     container = out if splitext(out)[1] == ".cxg" else out + ".cxg"
     title = args.title if args.title is not None else basefname
 
-    write_cxg(adata, container, title, var_names=args.var_names, obs_names=args.obs_names, about=args.about)
+    write_cxg(
+        adata,
+        container,
+        title,
+        var_names=args.var_names,
+        obs_names=args.obs_names,
+        about=args.about,
+        extract_colors=not args.disable_custom_colors,
+    )
 
     log(1, "done")
 
 
-def write_cxg(adata, container, title, var_names=None, obs_names=None, about=None):
+def write_cxg(adata, container, title, var_names=None, obs_names=None, about=None, extract_colors=False):
     if not adata.var.index.is_unique:
         raise ValueError("Variable index is not unique - unable to convert.")
     if not adata.obs.index.is_unique:
@@ -129,7 +165,19 @@ def write_cxg(adata, container, title, var_names=None, obs_names=None, about=Non
     log(1, f"\t...group created, with name {container}")
 
     # dataset metadata
-    save_metadata(container, {"title": title, "about": about})
+    metadata_dict = dict(cxg_version=CXG_VERSION, cxg_properties=json.dumps({"title": title, "about": about}))
+    if extract_colors:
+        try:
+            metadata_dict["cxg_category_colors"] = json.dumps(
+                convert_anndata_category_colors_to_cxg_category_colors(adata)
+            )
+        except ColorFormatException:
+            log(
+                0,
+                "Warning: failed to extract colors from h5ad file! "
+                "Fix the h5ad file or rerun with --disable-custom-colors. See help for details.",
+            )
+    save_metadata(container, metadata_dict)
     log(1, "\t...dataset metadata saved")
 
     # var/gene dataframe
@@ -392,7 +440,7 @@ def save_X(container, adata, ctx):
     tiledb.consolidate(X_name, ctx=ctx)
 
 
-def save_metadata(container, metadata):
+def save_metadata(container, metadata_dict):
     """
     Save all dataset-wide metadata.   This includes:
     * CXG version
@@ -407,8 +455,8 @@ def save_metadata(container, metadata):
     with tiledb.from_numpy(a_name, np.zeros((1,))) as A:
         pass
     with tiledb.DenseArray(a_name, mode="w") as A:
-        A.meta["cxg_version"] = CXG_VERSION
-        A.meta["cxg_properties"] = json.dumps(metadata)
+        for k, v in metadata_dict.items():
+            A.meta[k] = v
 
 
 def sanitize_keys(keys):

server/data_anndata/anndata_adaptor.py

@@ -12,6 +12,7 @@ from server_timing import Timing as ServerTiming
 from server.data_common.data_adaptor import DataAdaptor
 from server.data_common.fbs.matrix import encode_matrix_fbs
 from server.common.utils import series_to_schema
+from server.common.colors import convert_anndata_category_colors_to_cxg_category_colors
 from server.common.constants import Axis, MAX_LAYOUTS
 from server.common.errors import PrepareError, DatasetAccessError, FilterError
 from server.compute.scanpy import scanpy_umap
@@ -333,6 +334,9 @@ class AnndataAdaptor(DataAdaptor):
             lfc_cutoff = self.config.diffexp__lfc_cutoff
         return diffexp_generic.diffexp_ttest(self, maskA, maskB, top_n, lfc_cutoff)
 
+    def get_colors(self):
+        return convert_anndata_category_colors_to_cxg_category_colors(self.data)
+
     def get_X_array(self, obs_mask=None, var_mask=None):
         if obs_mask is None:
             obs_mask = slice(None)

server/data_common/data_adaptor.py

@@ -83,6 +83,10 @@ class DataAdaptor(metaclass=ABCMeta):
     def query_obs_array(self, term_var):
         pass
 
+    @abstractmethod
+    def get_colors(self):
+        pass
+
     @abstractmethod
     def get_obs_index(self):
         pass

server/data_cxg/cxg_adaptor.py

@@ -194,6 +194,10 @@ class CxgAdaptor(DataAdaptor):
             lfc_cutoff = self.config.diffexp__lfc_cutoff
         return diffexp_cxg.diffexp_ttest(self, maskA, maskB, top_n, lfc_cutoff)
 
+    def get_colors(self):
+        meta = self.open_array("cxg_group_metadata").meta
+        return json.loads(meta["cxg_category_colors"]) if "cxg_category_colors" in meta else dict()
+
     def get_X_array(self, obs_mask=None, var_mask=None):
         obs_items = pack_selector_from_mask(obs_mask)
         var_items = pack_selector_from_mask(var_mask)

server/eb/app.py

@@ -68,7 +68,7 @@ class WSGIServer(Server):
         if len(style_hashes) > 0:
             csp["style-src"] = style_hashes
 
-        Talisman(app, force_https=app_config.server__force_https, frame_options='DENY', content_security_policy=csp)
+        Talisman(app, force_https=app_config.server__force_https, frame_options="DENY", content_security_policy=csp)
 
     @staticmethod
     def load_csp_hashes(app):

server/test/__init__.py

@@ -1,6 +1,6 @@
 import shutil
 import tempfile
-from os import path
+from os import path, popen
 
 import pandas as pd
 
@@ -11,11 +11,14 @@ from server.data_common.fbs.matrix import encode_matrix_fbs
 from server.data_common.matrix_loader import MatrixDataLoader, MatrixDataType
 
 
+PROJECT_ROOT = popen("git rev-parse --show-toplevel").read().strip()
+
+
 def data_with_tmp_annotations(ext: MatrixDataType, annotations_fixture=False):
     tmp_dir = tempfile.mkdtemp()
     annotations_file = path.join(tmp_dir, "test_annotations.csv")
     if annotations_fixture:
-        shutil.copyfile(f"test/test_datasets/pbmc3k-annotations.csv", annotations_file)
+        shutil.copyfile(f"{PROJECT_ROOT}/server/test/test_datasets/pbmc3k-annotations.csv", annotations_file)
     args = {
         "embeddings__names": ["umap"],
         "presentation__max_categories": 100,
@@ -24,7 +27,7 @@ def data_with_tmp_annotations(ext: MatrixDataType, annotations_fixture=False):
         "diffexp__lfc_cutoff": 0.01,
     }
     fname = {
-        MatrixDataType.H5AD: "../example-dataset/pbmc3k.h5ad",
+        MatrixDataType.H5AD: f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad",
         MatrixDataType.CXG: "test/test_datasets/pbmc3k.cxg",
     }[ext]
     data_locator = DataLocator(fname)
@@ -53,3 +56,21 @@ def skip_if(condition, reason: str):
         return wraps
 
     return decorator
+
+
+def app_config(data_locator, backed=False):
+    args = {
+        "embeddings__names": ["umap", "tsne", "pca"],
+        "presentation__max_categories": 100,
+        "single_dataset__obs_names": None,
+        "single_dataset__var_names": None,
+        "diffexp__lfc_cutoff": 0.01,
+        "adaptor__anndata_adaptor__backed": backed,
+        "single_dataset__datapath": data_locator,
+        "limits__diffexp_cellcount_max": None,
+        "limits__column_request_max": None,
+    }
+    config = AppConfig()
+    config.update(**args)
+    config.complete_config()
+    return config

server/test/test_anndata_adaptor.py

@@ -10,10 +10,11 @@ from parameterized import parameterized_class
 import numpy as np
 import pandas as pd
 
-from server.data_anndata.anndata_adaptor import AnndataAdaptor
-from server.common.errors import FilterError
 from server.common.data_locator import DataLocator
-from server.common.app_config import AppConfig
+from server.common.errors import FilterError
+from server.data_anndata.anndata_adaptor import AnndataAdaptor
+from server.test import PROJECT_ROOT, app_config
+from server.test.test_datasets.fixtures import pbmc3k_colors
 
 """
 Test the anndata adaptor using the pbmc3k data set.
@@ -23,30 +24,17 @@ Test the anndata adaptor using the pbmc3k data set.
 @parameterized_class(
     ("data_locator", "backed"),
     [
-        ("../example-dataset/pbmc3k.h5ad", False),
-        ("test/test_datasets/pbmc3k-CSC-gz.h5ad", False),
-        ("test/test_datasets/pbmc3k-CSR-gz.h5ad", False),
-        ("../example-dataset/pbmc3k.h5ad", True),
-        ("test/test_datasets/pbmc3k-CSC-gz.h5ad", True),
-        ("test/test_datasets/pbmc3k-CSR-gz.h5ad", True),
+        (f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad", False),
+        (f"{PROJECT_ROOT}/server/test/test_datasets/pbmc3k-CSC-gz.h5ad", False),
+        (f"{PROJECT_ROOT}/server/test/test_datasets/pbmc3k-CSR-gz.h5ad", False),
+        (f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad", True),
+        (f"{PROJECT_ROOT}/server/test/test_datasets/pbmc3k-CSC-gz.h5ad", True),
+        (f"{PROJECT_ROOT}/server/test/test_datasets/pbmc3k-CSR-gz.h5ad", True),
     ],
 )
 class AdaptorTest(unittest.TestCase):
     def setUp(self):
-        args = {
-            "embeddings__names": ["umap", "tsne", "pca"],
-            "presentation__max_categories": 100,
-            "single_dataset__obs_names": None,
-            "single_dataset__var_names": None,
-            "diffexp__lfc_cutoff": 0.01,
-            "adaptor__anndata_adaptor__backed": self.backed,
-            "single_dataset__datapath": self.data_locator,
-            "limits__diffexp_cellcount_max": None,
-            "limits__column_request_max": None,
-        }
-        config = AppConfig()
-        config.update(**args)
-        config.complete_config()
+        config = app_config(self.data_locator, self.backed)
         self.data = AnndataAdaptor(DataLocator(self.data_locator), config)
 
     def test_init(self):
@@ -92,6 +80,9 @@ class AdaptorTest(unittest.TestCase):
         self.assertEqual(np.sum(self.data.data.var[self.data.get_schema()["annotations"]["var"]["index"]].isna()), 0)
         self.assertEqual(np.sum(self.data.data.obs[self.data.get_schema()["annotations"]["obs"]["index"]].isna()), 0)
 
+    def test_get_colors(self):
+        self.assertEqual(self.data.get_colors(), pbmc3k_colors)
+
     def test_get_schema(self):
         with open(path.join(path.dirname(__file__), "schema.json")) as fh:
             schema = json.load(fh)

server/test/test_anndata_adaptor_data_load.py

@@ -4,6 +4,7 @@ import json
 from server.data_anndata.anndata_adaptor import AnndataAdaptor
 from server.common.data_locator import DataLocator
 from server.common.app_config import AppConfig
+from server.test import PROJECT_ROOT
 
 
 class DataLoadAdaptorTest(unittest.TestCase):
@@ -12,7 +13,7 @@ class DataLoadAdaptorTest(unittest.TestCase):
     """
 
     def setUp(self):
-        self.data_file = DataLocator("../example-dataset/pbmc3k.h5ad")
+        self.data_file = DataLocator(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad")
         config = AppConfig()
         config.update(single_dataset__datapath=self.data_file.path)
         config.complete_config()

server/test/test_api.py

@@ -8,8 +8,10 @@ import pandas as pd
 import requests
 
 import server.test.decode_fbs as decode_fbs
-from server.test import data_with_tmp_annotations, make_fbs
 from server.data_common.matrix_loader import MatrixDataType
+from server.test import data_with_tmp_annotations, make_fbs, PROJECT_ROOT
+from server.test.test_datasets.fixtures import pbmc3k_colors
+
 
 BAD_FILTER = {"filter": {"obs": {"annotation_value": [{"name": "xyz"}]}}}
 
@@ -255,6 +257,15 @@ class EndPoints(object):
         self.assertEqual(df["n_rows"], 2638)
         self.assertEqual(df["n_cols"], 1)
 
+    def test_colors(self):
+        endpoint = "colors"
+        url = f"{self.URL_BASE}{endpoint}"
+        result = self.session.get(url)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/json")
+        result_data = result.json()
+        self.assertEqual(result_data, pbmc3k_colors)
+
     def test_static(self):
         endpoint = "static"
         file = "assets/favicon.ico"
@@ -349,7 +360,7 @@ class EndPointsAnndata(unittest.TestCase, EndPoints):
                 "cellxgene",
                 "--no-upgrade-check",
                 "launch",
-                "../example-dataset/pbmc3k.h5ad",
+                f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad",
                 "--disable-annotations",
                 "--verbose",
                 "--port",
@@ -383,7 +394,7 @@ class EndPointsCxg(unittest.TestCase, EndPoints):
                 "cellxgene",
                 "--no-upgrade-check",
                 "launch",
-                "test/test_datasets/pbmc3k.cxg",
+                f"{PROJECT_ROOT}/server/test/test_datasets/pbmc3k.cxg",
                 "--disable-annotations",
                 "--verbose",
                 "--port",

server/test/test_colors.py

@@ -0,0 +1,40 @@
+import unittest
+
+import anndata
+from server.common.colors import convert_color_to_hex_format, convert_anndata_category_colors_to_cxg_category_colors
+from server.common.errors import ColorFormatException
+from server.test import PROJECT_ROOT
+from server.test.test_datasets.fixtures import pbmc3k_colors
+
+
+class ColorsTest(unittest.TestCase):
+    """ Test color helper functions """
+
+    def test_convert_color_to_hex_format(self):
+        self.assertEqual(convert_color_to_hex_format("wheat"), "#f5deb3")
+        self.assertEqual(convert_color_to_hex_format("WHEAT"), "#f5deb3")
+        self.assertEqual(convert_color_to_hex_format((245, 222, 179)), "#f5deb3")
+        self.assertEqual(convert_color_to_hex_format([245, 222, 179]), "#f5deb3")
+        self.assertEqual(convert_color_to_hex_format("#f5deb3"), "#f5deb3")
+        self.assertEqual(
+            convert_color_to_hex_format([0.9607843137254902, 0.8705882352941177, 0.7019607843137254]), "#f5deb3"
+        )
+        for bad_input in ["foo", "BAR", "#AABB", "#AABBCCDD", "#AABBGG", (1, 2), [1, 2], (1, 2, 3, 4), [1, 2, 3, 4]]:
+            with self.assertRaises(ColorFormatException):
+                convert_color_to_hex_format(bad_input)
+
+    def test_anndata_colors_to_cxg_colors(self):
+        # test standard behavior
+        adata = self._get_h5ad()
+        self.assertEqual(convert_anndata_category_colors_to_cxg_category_colors(adata), pbmc3k_colors)
+        # test that invalid color formats raise an exception
+        adata.uns["louvain_colors"][0] = "#NOTCOOL"
+        with self.assertRaises(ColorFormatException):
+            convert_anndata_category_colors_to_cxg_category_colors(adata)
+        # test that colors without a matching obs category are skipped
+        adata = self._get_h5ad()
+        del adata.obs["louvain"]
+        self.assertEqual(convert_anndata_category_colors_to_cxg_category_colors(adata), {})
+
+    def _get_h5ad(self):
+        return anndata.read_h5ad(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad")

server/test/test_cxg_adaptor.py

@@ -0,0 +1,16 @@
+import unittest
+
+from server.common.data_locator import DataLocator
+from server.data_cxg.cxg_adaptor import CxgAdaptor
+from server.test import PROJECT_ROOT, app_config
+from server.test.test_datasets.fixtures import pbmc3k_colors
+
+
+class TestCxgAdaptor(unittest.TestCase):
+    def setUp(self):
+        data_locator = f"{PROJECT_ROOT}/server/test/test_datasets/pbmc3k.cxg"
+        config = app_config(data_locator)
+        self.data = CxgAdaptor(DataLocator(data_locator), config)
+
+    def test_get_colors(self):
+        self.assertEqual(self.data.get_colors(), pbmc3k_colors)

server/test/test_cxgtool.py

@@ -0,0 +1,40 @@
+import random
+import shutil
+import string
+import unittest
+
+import anndata
+
+from server.common.data_locator import DataLocator
+from server.converters.cxgtool import write_cxg
+from server.data_cxg.cxg_adaptor import CxgAdaptor
+from server.test import PROJECT_ROOT, app_config
+from server.test.test_datasets.fixtures import pbmc3k_colors
+
+
+class TestCxgAdaptor(unittest.TestCase):
+    def setUp(self) -> None:
+        self.fixtures = []
+
+    def tearDown(self) -> None:
+        try:
+            for data_locator in self.fixtures:
+                print("REMOVING ", data_locator)
+                shutil.rmtree(data_locator)
+        except FileNotFoundError:
+            pass
+
+    def test_cxg_category_colors(self):
+        data = self.convert_pbmc3k(extract_colors=True)
+        self.assertEqual(data.get_colors(), pbmc3k_colors)
+        data = self.convert_pbmc3k(extract_colors=False)
+        self.assertEqual(data.get_colors(), {})
+
+    def convert_pbmc3k(self, **kwargs):
+        random_string = "".join(random.choice(string.ascii_letters) for _ in range(8))
+        data_locator = f"/tmp/test_{random_string}.cxg"
+        self.fixtures.append(data_locator)
+        source_h5ad = anndata.read_h5ad(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad")
+        write_cxg(adata=source_h5ad, container=data_locator, title="pbmc3k", **kwargs)
+        config = app_config(data_locator)
+        return CxgAdaptor(DataLocator(data_locator), config)

server/test/test_datasets/fixtures.py

@@ -0,0 +1,12 @@
+pbmc3k_colors = {
+    "louvain": {
+        "B cells": "#2ca02c",
+        "CD14+ Monocytes": "#ff7f0e",
+        "CD4 T cells": "#1f77b4",
+        "CD8 T cells": "#d62728",
+        "Dendritic cells": "#e377c2",
+        "FCGR3A+ Monocytes": "#8c564b",
+        "Megakaryocytes": "#bcbd22",
+        "NK cells": "#9467bd",
+    }
+}

server/test/test_diffexp.py

@@ -5,6 +5,8 @@ import server.compute.diffexp_cxg as diffexp_cxg
 import server.compute.diffexp_generic as diffexp_generic
 import numpy as np
 
+from server.test import PROJECT_ROOT
+
 
 class DiffExpTest(unittest.TestCase):
     """Tests the diffexp returns the expected results for one test case, using different
@@ -50,7 +52,7 @@ class DiffExpTest(unittest.TestCase):
 
     def test_anndata_default(self):
         """Test an anndata adaptor with its default diffexp algorithm (diffexp_generic)"""
-        adaptor = self.load_dataset("../example-dataset/pbmc3k.h5ad")
+        adaptor = self.load_dataset(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad")
         maskA = self.get_mask(adaptor, 1, 10)
         maskB = self.get_mask(adaptor, 2, 10)
         results = adaptor.compute_diffexp_ttest(maskA, maskB, 10)
@@ -58,7 +60,7 @@ class DiffExpTest(unittest.TestCase):
 
     def test_cxg_default(self):
         """Test a cxg adaptor with its default diffexp algorithm (diffexp_cxg)"""
-        adaptor = self.load_dataset("test/test_datasets/pbmc3k.cxg")
+        adaptor = self.load_dataset(f"{PROJECT_ROOT}/server/test/test_datasets/pbmc3k.cxg")
         maskA = self.get_mask(adaptor, 1, 10)
         maskB = self.get_mask(adaptor, 2, 10)
 
@@ -72,7 +74,7 @@ class DiffExpTest(unittest.TestCase):
 
     def test_cxg_generic(self):
         """Test a cxg adaptor with the generic adaptor"""
-        adaptor = self.load_dataset("test/test_datasets/pbmc3k.cxg")
+        adaptor = self.load_dataset(f"{PROJECT_ROOT}/server/test/test_datasets/pbmc3k.cxg")
         maskA = self.get_mask(adaptor, 1, 10)
         maskB = self.get_mask(adaptor, 2, 10)
         # run it directly

server/test/test_matrixcache.py

@@ -7,13 +7,15 @@ import shutil
 import os
 import time
 
+from server.test import PROJECT_ROOT
+
 
 class MatrixCacheTest(unittest.TestCase):
     def setup(self):
         pass
 
     def make_temporay_datasets(self, dirname, num):
-        source = "test/test_datasets/pbmc3k.cxg"
+        source = f"{PROJECT_ROOT}/server/test/test_datasets/pbmc3k.cxg"
         for i in range(num):
             target = os.path.join(dirname, str(i) + ".cxg")
             shutil.copytree(source, target)

server/test/test_nan_anndata_adaptor.py

@@ -8,39 +8,22 @@ import server.test.decode_fbs as decode_fbs
 from server.data_anndata.anndata_adaptor import AnndataAdaptor
 from server.common.errors import FilterError
 from server.common.data_locator import DataLocator
-from server.common.app_config import AppConfig
+from server.test import PROJECT_ROOT, app_config
 
 
 class NaNTest(unittest.TestCase):
     def setUp(self):
-        self.args = {
-            "embeddings__names": ["umap"],
-            "presentation__max_categories": 100,
-            "single_dataset__obs_names": None,
-            "single_dataset__var_names": None,
-            "diffexp__lfc_cutoff": 0.01,
-            "limits__diffexp_cellcount_max": None,
-            "limits__column_request_max": None,
-        }
-        config = AppConfig()
-        config.update(**self.args)
-        locator = DataLocator("test/test_datasets/nan.h5ad")
-        config.update(single_dataset__datapath=locator.path)
-        config.complete_config()
+        self.data_locator = DataLocator(f"{PROJECT_ROOT}/server/test/test_datasets/nan.h5ad")
+        self.config = app_config(self.data_locator.path)
 
         with warnings.catch_warnings():
             warnings.simplefilter("ignore", category=UserWarning)
-            self.data = AnndataAdaptor(locator, config)
+            self.data = AnndataAdaptor(self.data_locator, self.config)
             self.data._create_schema()
 
     def test_load(self):
         with self.assertWarns(UserWarning):
-            config = AppConfig()
-            config.update(**self.args)
-            locator = DataLocator("test/test_datasets/nan.h5ad")
-            config.update(single_dataset__datapath=locator.path)
-            config.complete_config()
-            self.data = AnndataAdaptor(locator, config)
+            self.data = AnndataAdaptor(self.data_locator, self.config)
 
     def test_init(self):
         self.assertEqual(self.data.cell_count, 100)