work-around for column name encoding bug (#1324)

* add attribute cleanup

* fix typo

* handle mappings

* logging

* fix regex

* remove debugging printfs

* update masked characters

* fix typo

* add missing incr

server/converters/cxgtool.py

@@ -47,6 +47,7 @@ TODO/ISSUES:
 * Possible future work: accept Loom files
 
 """
+import re
 import anndata
 import tiledb
 import argparse
@@ -85,7 +86,7 @@ def main():
         metavar="<URL>",
         help="URL providing more information about the dataset (hint: must be a fully specified absolute URL).",
     )
-    parser.add_argument("--out", "-o", help="output CXG file name")
+    parser.add_argument("--out", "--output", "-o", help="output CXG file name")
     args = parser.parse_args()
 
     global log_level
@@ -110,6 +111,13 @@ def write_cxg(adata, container, title, var_names=None, obs_names=None, about=Non
     if not adata.obs.index.is_unique:
         raise ValueError("Observation index is not unique - unable to convert.")
 
+    """
+    TileDB bug TileDB-Inc/TileDB#1575 requires that we sanitize all column names
+    prior to saving.  This can be reverted when the bug is fixed.
+    """
+    log(0, "Warning: sanitizing all dataframe column names.")
+    clean_all_column_names(adata)
+
     ctx = tiledb.Ctx(
         {
             "sm.num_reader_threads": 32,
@@ -385,5 +393,65 @@ def save_metadata(container, metadata):
         A.meta["cxg_properties"] = json.dumps(metadata)
 
 
+def sanitize_keys(keys):
+    """
+    We need names to be safe to use as attribute names in tiledb.  See:
+        TileDB-Inc/TileDB#1575
+        TileDB-Inc/TileDB-Py#294
+    This can be entirely removed once they add proper escaping.
+
+    Args: list of keys
+    Returns: dict of {old_key: new_key, ...}
+
+    Returned new keys will be both safe and unique.
+
+    Masking out [~/.] and anything outside the ASCII range.
+    """
+    # p = re.compile(r"[^a-zA-Z0-9!\-_\.\*'\(\)&$@=;:\+ ,\?]")
+    p = re.compile(r"[^ -\.0-\[\]-\}]")
+    clean_keys = {k: p.sub('_', k) for k in keys}
+
+    used_keys = set()
+    clean_unique_keys = {}
+    for k, v in clean_keys.items():
+        if v not in used_keys:
+            used_keys.add(v)
+            clean_unique_keys[k] = v
+            continue
+
+        # else, needs deduping.
+        counter = 1
+        while True:
+            candidate_name = v + '-' + str(counter)
+            if candidate_name not in used_keys:
+                used_keys.add(candidate_name)
+                clean_unique_keys[k] = candidate_name
+                break
+            counter += 1
+
+    for k, v, in clean_unique_keys.items():
+        if k != v:
+            log(1, f"Renaming {k} to {v}")
+    return clean_unique_keys
+
+
+def sanitize_df(df):
+    df.rename(columns=sanitize_keys(df.keys().tolist()), inplace=True)
+
+
+def sanitize_mapping(mapping):
+    clean_keys = sanitize_keys([k for k in mapping.keys()])
+    for old_key, new_key in clean_keys.items():
+        if old_key != new_key:
+            mapping[new_key] = mapping[old_key]
+            del mapping[old_key]
+
+
+def clean_all_column_names(adata):
+    sanitize_df(adata.obs)
+    sanitize_df(adata.var)
+    sanitize_mapping(adata.obsm)
+
+
 if __name__ == "__main__":
     main()