Add user-defined category-label colors (#1402)

* Add user-defined category-label colors

Fixes https://github.com/chanzuckerberg/cellxgene/issues/1152

As described in https://github.com/chanzuckerberg/cellxgene/issues/1307

* Respond to feedback from @bkmartinjr in nodejs

* Respond to feedback from @bkmartinjr in python

* Add tests to the server module

* Autoformat python, run linter

* Make colors_get error handling specific

* Respond to feedback from @bkmartinjr

* Respond to feedback from @bkmartinjr

* Fix whitespace

* Fix python lint errrors

* Update documentation

* Add --disable-user-colors option to launch and cxgtool.py

* Fix python formatting

* Rename '--disable-user-colors' to '--disable-custom-colors'

server/test/__init__.py

@@ -1,6 +1,6 @@
 import shutil
 import tempfile
-from os import path
+from os import path, popen
 
 import pandas as pd
 
@@ -11,11 +11,14 @@ from server.data_common.fbs.matrix import encode_matrix_fbs
 from server.data_common.matrix_loader import MatrixDataLoader, MatrixDataType
 
 
+PROJECT_ROOT = popen("git rev-parse --show-toplevel").read().strip()
+
+
 def data_with_tmp_annotations(ext: MatrixDataType, annotations_fixture=False):
     tmp_dir = tempfile.mkdtemp()
     annotations_file = path.join(tmp_dir, "test_annotations.csv")
     if annotations_fixture:
-        shutil.copyfile(f"test/test_datasets/pbmc3k-annotations.csv", annotations_file)
+        shutil.copyfile(f"{PROJECT_ROOT}/server/test/test_datasets/pbmc3k-annotations.csv", annotations_file)
     args = {
         "embeddings__names": ["umap"],
         "presentation__max_categories": 100,
@@ -24,7 +27,7 @@ def data_with_tmp_annotations(ext: MatrixDataType, annotations_fixture=False):
         "diffexp__lfc_cutoff": 0.01,
     }
     fname = {
-        MatrixDataType.H5AD: "../example-dataset/pbmc3k.h5ad",
+        MatrixDataType.H5AD: f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad",
         MatrixDataType.CXG: "test/test_datasets/pbmc3k.cxg",
     }[ext]
     data_locator = DataLocator(fname)
@@ -53,3 +56,21 @@ def skip_if(condition, reason: str):
         return wraps
 
     return decorator
+
+
+def app_config(data_locator, backed=False):
+    args = {
+        "embeddings__names": ["umap", "tsne", "pca"],
+        "presentation__max_categories": 100,
+        "single_dataset__obs_names": None,
+        "single_dataset__var_names": None,
+        "diffexp__lfc_cutoff": 0.01,
+        "adaptor__anndata_adaptor__backed": backed,
+        "single_dataset__datapath": data_locator,
+        "limits__diffexp_cellcount_max": None,
+        "limits__column_request_max": None,
+    }
+    config = AppConfig()
+    config.update(**args)
+    config.complete_config()
+    return config

server/test/test_anndata_adaptor.py

@@ -10,10 +10,11 @@ from parameterized import parameterized_class
 import numpy as np
 import pandas as pd
 
-from server.data_anndata.anndata_adaptor import AnndataAdaptor
-from server.common.errors import FilterError
 from server.common.data_locator import DataLocator
-from server.common.app_config import AppConfig
+from server.common.errors import FilterError
+from server.data_anndata.anndata_adaptor import AnndataAdaptor
+from server.test import PROJECT_ROOT, app_config
+from server.test.test_datasets.fixtures import pbmc3k_colors
 
 """
 Test the anndata adaptor using the pbmc3k data set.
@@ -23,30 +24,17 @@ Test the anndata adaptor using the pbmc3k data set.
 @parameterized_class(
     ("data_locator", "backed"),
     [
-        ("../example-dataset/pbmc3k.h5ad", False),
-        ("test/test_datasets/pbmc3k-CSC-gz.h5ad", False),
-        ("test/test_datasets/pbmc3k-CSR-gz.h5ad", False),
-        ("../example-dataset/pbmc3k.h5ad", True),
-        ("test/test_datasets/pbmc3k-CSC-gz.h5ad", True),
-        ("test/test_datasets/pbmc3k-CSR-gz.h5ad", True),
+        (f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad", False),
+        (f"{PROJECT_ROOT}/server/test/test_datasets/pbmc3k-CSC-gz.h5ad", False),
+        (f"{PROJECT_ROOT}/server/test/test_datasets/pbmc3k-CSR-gz.h5ad", False),
+        (f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad", True),
+        (f"{PROJECT_ROOT}/server/test/test_datasets/pbmc3k-CSC-gz.h5ad", True),
+        (f"{PROJECT_ROOT}/server/test/test_datasets/pbmc3k-CSR-gz.h5ad", True),
     ],
 )
 class AdaptorTest(unittest.TestCase):
     def setUp(self):
-        args = {
-            "embeddings__names": ["umap", "tsne", "pca"],
-            "presentation__max_categories": 100,
-            "single_dataset__obs_names": None,
-            "single_dataset__var_names": None,
-            "diffexp__lfc_cutoff": 0.01,
-            "adaptor__anndata_adaptor__backed": self.backed,
-            "single_dataset__datapath": self.data_locator,
-            "limits__diffexp_cellcount_max": None,
-            "limits__column_request_max": None,
-        }
-        config = AppConfig()
-        config.update(**args)
-        config.complete_config()
+        config = app_config(self.data_locator, self.backed)
         self.data = AnndataAdaptor(DataLocator(self.data_locator), config)
 
     def test_init(self):
@@ -92,6 +80,9 @@ class AdaptorTest(unittest.TestCase):
         self.assertEqual(np.sum(self.data.data.var[self.data.get_schema()["annotations"]["var"]["index"]].isna()), 0)
         self.assertEqual(np.sum(self.data.data.obs[self.data.get_schema()["annotations"]["obs"]["index"]].isna()), 0)
 
+    def test_get_colors(self):
+        self.assertEqual(self.data.get_colors(), pbmc3k_colors)
+
     def test_get_schema(self):
         with open(path.join(path.dirname(__file__), "schema.json")) as fh:
             schema = json.load(fh)

server/test/test_anndata_adaptor_data_load.py

@@ -4,6 +4,7 @@ import json
 from server.data_anndata.anndata_adaptor import AnndataAdaptor
 from server.common.data_locator import DataLocator
 from server.common.app_config import AppConfig
+from server.test import PROJECT_ROOT
 
 
 class DataLoadAdaptorTest(unittest.TestCase):
@@ -12,7 +13,7 @@ class DataLoadAdaptorTest(unittest.TestCase):
     """
 
     def setUp(self):
-        self.data_file = DataLocator("../example-dataset/pbmc3k.h5ad")
+        self.data_file = DataLocator(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad")
         config = AppConfig()
         config.update(single_dataset__datapath=self.data_file.path)
         config.complete_config()

server/test/test_api.py

@@ -8,8 +8,10 @@ import pandas as pd
 import requests
 
 import server.test.decode_fbs as decode_fbs
-from server.test import data_with_tmp_annotations, make_fbs
 from server.data_common.matrix_loader import MatrixDataType
+from server.test import data_with_tmp_annotations, make_fbs, PROJECT_ROOT
+from server.test.test_datasets.fixtures import pbmc3k_colors
+
 
 BAD_FILTER = {"filter": {"obs": {"annotation_value": [{"name": "xyz"}]}}}
 
@@ -255,6 +257,15 @@ class EndPoints(object):
         self.assertEqual(df["n_rows"], 2638)
         self.assertEqual(df["n_cols"], 1)
 
+    def test_colors(self):
+        endpoint = "colors"
+        url = f"{self.URL_BASE}{endpoint}"
+        result = self.session.get(url)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/json")
+        result_data = result.json()
+        self.assertEqual(result_data, pbmc3k_colors)
+
     def test_static(self):
         endpoint = "static"
         file = "assets/favicon.ico"
@@ -349,7 +360,7 @@ class EndPointsAnndata(unittest.TestCase, EndPoints):
                 "cellxgene",
                 "--no-upgrade-check",
                 "launch",
-                "../example-dataset/pbmc3k.h5ad",
+                f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad",
                 "--disable-annotations",
                 "--verbose",
                 "--port",
@@ -383,7 +394,7 @@ class EndPointsCxg(unittest.TestCase, EndPoints):
                 "cellxgene",
                 "--no-upgrade-check",
                 "launch",
-                "test/test_datasets/pbmc3k.cxg",
+                f"{PROJECT_ROOT}/server/test/test_datasets/pbmc3k.cxg",
                 "--disable-annotations",
                 "--verbose",
                 "--port",

server/test/test_colors.py

@@ -0,0 +1,40 @@
+import unittest
+
+import anndata
+from server.common.colors import convert_color_to_hex_format, convert_anndata_category_colors_to_cxg_category_colors
+from server.common.errors import ColorFormatException
+from server.test import PROJECT_ROOT
+from server.test.test_datasets.fixtures import pbmc3k_colors
+
+
+class ColorsTest(unittest.TestCase):
+    """ Test color helper functions """
+
+    def test_convert_color_to_hex_format(self):
+        self.assertEqual(convert_color_to_hex_format("wheat"), "#f5deb3")
+        self.assertEqual(convert_color_to_hex_format("WHEAT"), "#f5deb3")
+        self.assertEqual(convert_color_to_hex_format((245, 222, 179)), "#f5deb3")
+        self.assertEqual(convert_color_to_hex_format([245, 222, 179]), "#f5deb3")
+        self.assertEqual(convert_color_to_hex_format("#f5deb3"), "#f5deb3")
+        self.assertEqual(
+            convert_color_to_hex_format([0.9607843137254902, 0.8705882352941177, 0.7019607843137254]), "#f5deb3"
+        )
+        for bad_input in ["foo", "BAR", "#AABB", "#AABBCCDD", "#AABBGG", (1, 2), [1, 2], (1, 2, 3, 4), [1, 2, 3, 4]]:
+            with self.assertRaises(ColorFormatException):
+                convert_color_to_hex_format(bad_input)
+
+    def test_anndata_colors_to_cxg_colors(self):
+        # test standard behavior
+        adata = self._get_h5ad()
+        self.assertEqual(convert_anndata_category_colors_to_cxg_category_colors(adata), pbmc3k_colors)
+        # test that invalid color formats raise an exception
+        adata.uns["louvain_colors"][0] = "#NOTCOOL"
+        with self.assertRaises(ColorFormatException):
+            convert_anndata_category_colors_to_cxg_category_colors(adata)
+        # test that colors without a matching obs category are skipped
+        adata = self._get_h5ad()
+        del adata.obs["louvain"]
+        self.assertEqual(convert_anndata_category_colors_to_cxg_category_colors(adata), {})
+
+    def _get_h5ad(self):
+        return anndata.read_h5ad(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad")

server/test/test_cxg_adaptor.py

@@ -0,0 +1,16 @@
+import unittest
+
+from server.common.data_locator import DataLocator
+from server.data_cxg.cxg_adaptor import CxgAdaptor
+from server.test import PROJECT_ROOT, app_config
+from server.test.test_datasets.fixtures import pbmc3k_colors
+
+
+class TestCxgAdaptor(unittest.TestCase):
+    def setUp(self):
+        data_locator = f"{PROJECT_ROOT}/server/test/test_datasets/pbmc3k.cxg"
+        config = app_config(data_locator)
+        self.data = CxgAdaptor(DataLocator(data_locator), config)
+
+    def test_get_colors(self):
+        self.assertEqual(self.data.get_colors(), pbmc3k_colors)

server/test/test_cxgtool.py

@@ -0,0 +1,40 @@
+import random
+import shutil
+import string
+import unittest
+
+import anndata
+
+from server.common.data_locator import DataLocator
+from server.converters.cxgtool import write_cxg
+from server.data_cxg.cxg_adaptor import CxgAdaptor
+from server.test import PROJECT_ROOT, app_config
+from server.test.test_datasets.fixtures import pbmc3k_colors
+
+
+class TestCxgAdaptor(unittest.TestCase):
+    def setUp(self) -> None:
+        self.fixtures = []
+
+    def tearDown(self) -> None:
+        try:
+            for data_locator in self.fixtures:
+                print("REMOVING ", data_locator)
+                shutil.rmtree(data_locator)
+        except FileNotFoundError:
+            pass
+
+    def test_cxg_category_colors(self):
+        data = self.convert_pbmc3k(extract_colors=True)
+        self.assertEqual(data.get_colors(), pbmc3k_colors)
+        data = self.convert_pbmc3k(extract_colors=False)
+        self.assertEqual(data.get_colors(), {})
+
+    def convert_pbmc3k(self, **kwargs):
+        random_string = "".join(random.choice(string.ascii_letters) for _ in range(8))
+        data_locator = f"/tmp/test_{random_string}.cxg"
+        self.fixtures.append(data_locator)
+        source_h5ad = anndata.read_h5ad(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad")
+        write_cxg(adata=source_h5ad, container=data_locator, title="pbmc3k", **kwargs)
+        config = app_config(data_locator)
+        return CxgAdaptor(DataLocator(data_locator), config)

server/test/test_datasets/fixtures.py

@@ -0,0 +1,12 @@
+pbmc3k_colors = {
+    "louvain": {
+        "B cells": "#2ca02c",
+        "CD14+ Monocytes": "#ff7f0e",
+        "CD4 T cells": "#1f77b4",
+        "CD8 T cells": "#d62728",
+        "Dendritic cells": "#e377c2",
+        "FCGR3A+ Monocytes": "#8c564b",
+        "Megakaryocytes": "#bcbd22",
+        "NK cells": "#9467bd",
+    }
+}

server/test/test_diffexp.py

@@ -5,6 +5,8 @@ import server.compute.diffexp_cxg as diffexp_cxg
 import server.compute.diffexp_generic as diffexp_generic
 import numpy as np
 
+from server.test import PROJECT_ROOT
+
 
 class DiffExpTest(unittest.TestCase):
     """Tests the diffexp returns the expected results for one test case, using different
@@ -50,7 +52,7 @@ class DiffExpTest(unittest.TestCase):
 
     def test_anndata_default(self):
         """Test an anndata adaptor with its default diffexp algorithm (diffexp_generic)"""
-        adaptor = self.load_dataset("../example-dataset/pbmc3k.h5ad")
+        adaptor = self.load_dataset(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad")
         maskA = self.get_mask(adaptor, 1, 10)
         maskB = self.get_mask(adaptor, 2, 10)
         results = adaptor.compute_diffexp_ttest(maskA, maskB, 10)
@@ -58,7 +60,7 @@ class DiffExpTest(unittest.TestCase):
 
     def test_cxg_default(self):
         """Test a cxg adaptor with its default diffexp algorithm (diffexp_cxg)"""
-        adaptor = self.load_dataset("test/test_datasets/pbmc3k.cxg")
+        adaptor = self.load_dataset(f"{PROJECT_ROOT}/server/test/test_datasets/pbmc3k.cxg")
         maskA = self.get_mask(adaptor, 1, 10)
         maskB = self.get_mask(adaptor, 2, 10)
 
@@ -72,7 +74,7 @@ class DiffExpTest(unittest.TestCase):
 
     def test_cxg_generic(self):
         """Test a cxg adaptor with the generic adaptor"""
-        adaptor = self.load_dataset("test/test_datasets/pbmc3k.cxg")
+        adaptor = self.load_dataset(f"{PROJECT_ROOT}/server/test/test_datasets/pbmc3k.cxg")
         maskA = self.get_mask(adaptor, 1, 10)
         maskB = self.get_mask(adaptor, 2, 10)
         # run it directly

server/test/test_matrixcache.py

@@ -7,13 +7,15 @@ import shutil
 import os
 import time
 
+from server.test import PROJECT_ROOT
+
 
 class MatrixCacheTest(unittest.TestCase):
     def setup(self):
         pass
 
     def make_temporay_datasets(self, dirname, num):
-        source = "test/test_datasets/pbmc3k.cxg"
+        source = f"{PROJECT_ROOT}/server/test/test_datasets/pbmc3k.cxg"
         for i in range(num):
             target = os.path.join(dirname, str(i) + ".cxg")
             shutil.copytree(source, target)

server/test/test_nan_anndata_adaptor.py

@@ -8,39 +8,22 @@ import server.test.decode_fbs as decode_fbs
 from server.data_anndata.anndata_adaptor import AnndataAdaptor
 from server.common.errors import FilterError
 from server.common.data_locator import DataLocator
-from server.common.app_config import AppConfig
+from server.test import PROJECT_ROOT, app_config
 
 
 class NaNTest(unittest.TestCase):
     def setUp(self):
-        self.args = {
-            "embeddings__names": ["umap"],
-            "presentation__max_categories": 100,
-            "single_dataset__obs_names": None,
-            "single_dataset__var_names": None,
-            "diffexp__lfc_cutoff": 0.01,
-            "limits__diffexp_cellcount_max": None,
-            "limits__column_request_max": None,
-        }
-        config = AppConfig()
-        config.update(**self.args)
-        locator = DataLocator("test/test_datasets/nan.h5ad")
-        config.update(single_dataset__datapath=locator.path)
-        config.complete_config()
+        self.data_locator = DataLocator(f"{PROJECT_ROOT}/server/test/test_datasets/nan.h5ad")
+        self.config = app_config(self.data_locator.path)
 
         with warnings.catch_warnings():
             warnings.simplefilter("ignore", category=UserWarning)
-            self.data = AnndataAdaptor(locator, config)
+            self.data = AnndataAdaptor(self.data_locator, self.config)
             self.data._create_schema()
 
     def test_load(self):
         with self.assertWarns(UserWarning):
-            config = AppConfig()
-            config.update(**self.args)
-            locator = DataLocator("test/test_datasets/nan.h5ad")
-            config.update(single_dataset__datapath=locator.path)
-            config.complete_config()
-            self.data = AnndataAdaptor(locator, config)
+            self.data = AnndataAdaptor(self.data_locator, self.config)
 
     def test_init(self):
         self.assertEqual(self.data.cell_count, 100)