diff --git a/server/converters/cxgtool.py b/server/converters/cxgtool.py index 143f71c3..099d5535 100644 --- a/server/converters/cxgtool.py +++ b/server/converters/cxgtool.py @@ -47,6 +47,7 @@ TODO/ISSUES: * Possible future work: accept Loom files """ +import re import anndata import tiledb import argparse @@ -85,7 +86,7 @@ def main(): metavar="", help="URL providing more information about the dataset (hint: must be a fully specified absolute URL).", ) - parser.add_argument("--out", "-o", help="output CXG file name") + parser.add_argument("--out", "--output", "-o", help="output CXG file name") args = parser.parse_args() global log_level @@ -110,6 +111,13 @@ def write_cxg(adata, container, title, var_names=None, obs_names=None, about=Non if not adata.obs.index.is_unique: raise ValueError("Observation index is not unique - unable to convert.") + """ + TileDB bug TileDB-Inc/TileDB#1575 requires that we sanitize all column names + prior to saving. This can be reverted when the bug is fixed. + """ + log(0, "Warning: sanitizing all dataframe column names.") + clean_all_column_names(adata) + ctx = tiledb.Ctx( { "sm.num_reader_threads": 32, @@ -385,5 +393,65 @@ def save_metadata(container, metadata): A.meta["cxg_properties"] = json.dumps(metadata) +def sanitize_keys(keys): + """ + We need names to be safe to use as attribute names in tiledb. See: + TileDB-Inc/TileDB#1575 + TileDB-Inc/TileDB-Py#294 + This can be entirely removed once they add proper escaping. + + Args: list of keys + Returns: dict of {old_key: new_key, ...} + + Returned new keys will be both safe and unique. + + Masking out [~/.] and anything outside the ASCII range. + """ + # p = re.compile(r"[^a-zA-Z0-9!\-_\.\*'\(\)&$@=;:\+ ,\?]") + p = re.compile(r"[^ -\.0-\[\]-\}]") + clean_keys = {k: p.sub('_', k) for k in keys} + + used_keys = set() + clean_unique_keys = {} + for k, v in clean_keys.items(): + if v not in used_keys: + used_keys.add(v) + clean_unique_keys[k] = v + continue + + # else, needs deduping. + counter = 1 + while True: + candidate_name = v + '-' + str(counter) + if candidate_name not in used_keys: + used_keys.add(candidate_name) + clean_unique_keys[k] = candidate_name + break + counter += 1 + + for k, v, in clean_unique_keys.items(): + if k != v: + log(1, f"Renaming {k} to {v}") + return clean_unique_keys + + +def sanitize_df(df): + df.rename(columns=sanitize_keys(df.keys().tolist()), inplace=True) + + +def sanitize_mapping(mapping): + clean_keys = sanitize_keys([k for k in mapping.keys()]) + for old_key, new_key in clean_keys.items(): + if old_key != new_key: + mapping[new_key] = mapping[old_key] + del mapping[old_key] + + +def clean_all_column_names(adata): + sanitize_df(adata.obs) + sanitize_df(adata.var) + sanitize_mapping(adata.obsm) + + if __name__ == "__main__": main()