mirror of
https://github.com/chanzuckerberg/cellxgene.git
synced 2026-10-05 18:18:12 +08:00
Add user-defined category-label colors (#1402)
* Add user-defined category-label colors Fixes https://github.com/chanzuckerberg/cellxgene/issues/1152 As described in https://github.com/chanzuckerberg/cellxgene/issues/1307 * Respond to feedback from @bkmartinjr in nodejs * Respond to feedback from @bkmartinjr in python * Add tests to the server module * Autoformat python, run linter * Make colors_get error handling specific * Respond to feedback from @bkmartinjr * Respond to feedback from @bkmartinjr * Fix whitespace * Fix python lint errrors * Update documentation * Add --disable-user-colors option to launch and cxgtool.py * Fix python formatting * Rename '--disable-user-colors' to '--disable-custom-colors'
This commit is contained in:
Matt Weiden
@@ -4,28 +4,36 @@ into a cellxgene TileDB structure, aka a 'CXG'.
|
||||
|
||||
The organization of the TileDB structure is:
|
||||
|
||||
the.cxg TileDB Group
|
||||
|-- obs TileDB array containing cell (row) attributes, one attribute per
|
||||
| dataframe columm, shape (n_obs,)
|
||||
|-- var TileDB array containing gene (column) attributes, with one attribute per
|
||||
| dataframe column, shape (n_var,)
|
||||
|-- X Main count matrix as a 2D TileDB array, single unnanmed numeric attribute
|
||||
|-- emb TileDB group, storing optional embeddings (group may be empty)
|
||||
| |-- <name1> TileDB Array, single anon attribute, ND numeric array, shape (n_obs, N)
|
||||
|-- cxg_group_metadata Empty array used only to stash metadata about the overall object.
|
||||
the.cxg TileDB Group
|
||||
├─ obs TileDB array containing cell (row) attributes, one attribute per
|
||||
│ dataframe column, shape (n_obs,)
|
||||
├─ var TileDB array containing gene (column) attributes, with one attribute per
|
||||
│ dataframe column, shape (n_obs,)
|
||||
├─ X Main count matrix as a 2D TileDB array, single unnamed numeric attribute
|
||||
├─ emb TileDB group, storing optional embeddings (group may be empty)
|
||||
│ └─ <name1> TileDB Array, single anon attribute, ND numeric array, shape (n_obs, N)
|
||||
└─ cxg_group_metadata Empty array used only to stash metadata about the overall object.
|
||||
└─ cxg_category_colors CXG colors object as described below:
|
||||
{
|
||||
"<category_name>": {
|
||||
"<label_name>": "<color_hex_code>",
|
||||
...
|
||||
},
|
||||
...
|
||||
}
|
||||
...
|
||||
|
||||
All arrays are defined to have a uint32 domain, zero based. All X counds and embedding
|
||||
All arrays are defined to have a uint32 domain, zero based. All X counts and embedding
|
||||
coordinates are coerced to float32, which is ample precision for visualization purposes.
|
||||
Dataframe (metadata) types are generally preserved, or where that is not possible,
|
||||
converted to somemthing with equal representative value in the cellxgene application
|
||||
converted to something with equal representative value in the cellxgene application
|
||||
(eg, categorical types are converted to string, bools to uint8, etc).
|
||||
|
||||
The following objects are also decorated with auxilliary metadata using TileDB
|
||||
The following objects are also decorated with auxiliary metadata using TileDB
|
||||
array metadata:
|
||||
|
||||
* cxg_group_metadata: minimally, will contain a 'cxg_version' field, which
|
||||
is a semver string identifing the version number of the CXG layout.
|
||||
is a semver string identifying the version number of the CXG layout.
|
||||
It may also contain 'cxg_parameters', a JSON-encoded parameter list
|
||||
describing CXG-wide dataset parameters.
|
||||
|
||||
@@ -40,6 +48,14 @@ including the global data layout, spatial tile size, and the like. The CXG is
|
||||
self-describing in these areas, and the actual values (eg, tile size) are empirically
|
||||
derived from benchmarking. They may change in the future.
|
||||
|
||||
cxgtool.py will extract color information stored in arrays in the 'uns' anndata
|
||||
property with the key "{category_name}_colors". For this to work, the following
|
||||
command must result in a mapping from category names to matplotlib-compatible colors:
|
||||
|
||||
```
|
||||
dict(zip(adata.obs[cat].cat.categories, adata.uns[f"{cat}_colors"]))
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
TODO/ISSUES:
|
||||
@@ -55,10 +71,16 @@ import numpy as np
|
||||
from os.path import splitext, basename
|
||||
import json
|
||||
|
||||
from server.common.colors import convert_anndata_category_colors_to_cxg_category_colors
|
||||
from server.common.errors import ColorFormatException
|
||||
|
||||
|
||||
# the CXG container version number. Must be a semver string.
|
||||
CXG_VERSION = "0.1"
|
||||
|
||||
# log_level must have a default
|
||||
log_level = 3
|
||||
|
||||
|
||||
def log(level, *args):
|
||||
global log_level
|
||||
@@ -72,6 +94,12 @@ def main():
|
||||
parser.add_argument(
|
||||
"--backed", action="store_true", help="loaded in file backed mode. Will be slower, but use less memory."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--disable-custom-colors",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Do not extract scanpy-compatible category colors from h5ad file.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--obs-names", help="Name of annotation to use for observations. If not specified, will use the obs index."
|
||||
)
|
||||
@@ -99,12 +127,20 @@ def main():
|
||||
container = out if splitext(out)[1] == ".cxg" else out + ".cxg"
|
||||
title = args.title if args.title is not None else basefname
|
||||
|
||||
write_cxg(adata, container, title, var_names=args.var_names, obs_names=args.obs_names, about=args.about)
|
||||
write_cxg(
|
||||
adata,
|
||||
container,
|
||||
title,
|
||||
var_names=args.var_names,
|
||||
obs_names=args.obs_names,
|
||||
about=args.about,
|
||||
extract_colors=not args.disable_custom_colors,
|
||||
)
|
||||
|
||||
log(1, "done")
|
||||
|
||||
|
||||
def write_cxg(adata, container, title, var_names=None, obs_names=None, about=None):
|
||||
def write_cxg(adata, container, title, var_names=None, obs_names=None, about=None, extract_colors=False):
|
||||
if not adata.var.index.is_unique:
|
||||
raise ValueError("Variable index is not unique - unable to convert.")
|
||||
if not adata.obs.index.is_unique:
|
||||
@@ -129,7 +165,19 @@ def write_cxg(adata, container, title, var_names=None, obs_names=None, about=Non
|
||||
log(1, f"\t...group created, with name {container}")
|
||||
|
||||
# dataset metadata
|
||||
save_metadata(container, {"title": title, "about": about})
|
||||
metadata_dict = dict(cxg_version=CXG_VERSION, cxg_properties=json.dumps({"title": title, "about": about}))
|
||||
if extract_colors:
|
||||
try:
|
||||
metadata_dict["cxg_category_colors"] = json.dumps(
|
||||
convert_anndata_category_colors_to_cxg_category_colors(adata)
|
||||
)
|
||||
except ColorFormatException:
|
||||
log(
|
||||
0,
|
||||
"Warning: failed to extract colors from h5ad file! "
|
||||
"Fix the h5ad file or rerun with --disable-custom-colors. See help for details.",
|
||||
)
|
||||
save_metadata(container, metadata_dict)
|
||||
log(1, "\t...dataset metadata saved")
|
||||
|
||||
# var/gene dataframe
|
||||
@@ -392,7 +440,7 @@ def save_X(container, adata, ctx):
|
||||
tiledb.consolidate(X_name, ctx=ctx)
|
||||
|
||||
|
||||
def save_metadata(container, metadata):
|
||||
def save_metadata(container, metadata_dict):
|
||||
"""
|
||||
Save all dataset-wide metadata. This includes:
|
||||
* CXG version
|
||||
@@ -407,8 +455,8 @@ def save_metadata(container, metadata):
|
||||
with tiledb.from_numpy(a_name, np.zeros((1,))) as A:
|
||||
pass
|
||||
with tiledb.DenseArray(a_name, mode="w") as A:
|
||||
A.meta["cxg_version"] = CXG_VERSION
|
||||
A.meta["cxg_properties"] = json.dumps(metadata)
|
||||
for k, v in metadata_dict.items():
|
||||
A.meta[k] = v
|
||||
|
||||
|
||||
def sanitize_keys(keys):
|
||||
|
||||
Reference in New Issue
Block a user