categorical type handling fix (#1342)

* fix numeric category conversion bug

* correctly compute categorical summaries

* lint

* remove debugging print

server/converters/cxgtool.py

@@ -52,7 +52,6 @@ import anndata
 import tiledb
 import argparse
 import numpy as np
-import pandas as pd
 from os.path import splitext, basename
 import json
 
@@ -152,45 +151,64 @@ def write_cxg(adata, container, title, var_names=None, obs_names=None, about=Non
     log(1, "\t...X created")
 
 
-def _can_cast_to_float32(col):
-    if col.dtype.kind == "f":
+"""
+TODO: the code used to handle type inferencing should not be duplicated between
+this tool and the server/common/utils code. When this tool is merged into
+the cellxgene CLI, consolidate.
+"""
+
+
+def dtype_to_schema(dtype):
+    if dtype == np.float32:
+        return (np.float32, {})
+    elif dtype == np.int32:
+        return (np.int32, {})
+    elif dtype == np.bool_:
+        return (np.uint8, {type: "boolean"})
+    elif dtype == np.str:
+        return (np.unicode, {"type": "string"})
+    elif dtype == "category":
+        typ, hint = cxg_type(dtype.categories)
+        return (typ, {"type": "categorical", "categories": dtype.categories.tolist()})
+    else:
+        raise TypeError(f"Annotations of type {dtype} are unsupported.")
+
+
+def _can_cast_to_float32(array):
+    if array.dtype.kind == "f":
         # force downcast for all floats
         return True
     return False
 
 
-def _can_cast_to_int32(col):
-    if col.dtype.kind in ["i", "u"]:
-        if np.can_cast(col.dtype, np.int32):
+def _can_cast_to_int32(array):
+    if array.dtype.kind in ["i", "u"]:
+        if np.can_cast(array.dtype, np.int32):
             return True
         ii32 = np.iinfo(np.int32)
-        if col.min() >= ii32.min and col.max() <= ii32.max:
+        if array.min() >= ii32.min and array.max() <= ii32.max:
             return True
     return False
 
 
-def cxg_type(col):
-    """ given a dtype, return an encoding dtype and any schema hints """
-    dtype = col.dtype
-    kind = dtype.kind
-    if _can_cast_to_float32(col):
-        return (np.float32, {})
-    elif _can_cast_to_int32(col):
-        return (np.int32, {})
-    elif dtype == np.bool_ or dtype == np.bool:
-        return (np.uint8, {type: "boolean"})
-    elif kind == "O" and isinstance(dtype, pd.CategoricalDtype):
-        typ, hint = cxg_type(dtype.categories)
-        hint["categories"] = dtype.categories.tolist()
-        return (typ, hint)
-    elif kind == "O":
-        return (np.unicode, {"type": "string"})
-    else:
-        raise TypeError(f"Annotations of type {dtype} are unsupported by cellxgene.")
+def cxg_type(array):
+    try:
+        return dtype_to_schema(array.dtype)
+    except TypeError:
+        dtype = array.dtype
+        data_kind = dtype.kind
+        if _can_cast_to_float32(array):
+            return (np.float32, {})
+        elif _can_cast_to_int32(array):
+            return (np.int32, {})
+        elif data_kind == "O" and dtype == "object":
+            return (np.unicode, {"type": "string"})
+        else:
+            raise TypeError(f"Annotations of type {dtype} are unsupported.")
 
 
-def cxg_dtype(col):
-    return cxg_type(col)[0]
+def cxg_dtype(array):
+    return cxg_type(array)[0]
 
 
 def create_dataframe(name, df, ctx):
@@ -407,9 +425,8 @@ def sanitize_keys(keys):
 
     Masking out [~/.] and anything outside the ASCII range.
     """
-    # p = re.compile(r"[^a-zA-Z0-9!\-_\.\*'\(\)&$@=;:\+ ,\?]")
     p = re.compile(r"[^ -\.0-\[\]-\}]")
-    clean_keys = {k: p.sub('_', k) for k in keys}
+    clean_keys = {k: p.sub("_", k) for k in keys}
 
     used_keys = set()
     clean_unique_keys = {}
@@ -422,7 +439,7 @@ def sanitize_keys(keys):
         # else, needs deduping.
         counter = 1
         while True:
-            candidate_name = v + '-' + str(counter)
+            candidate_name = v + "-" + str(counter)
             if candidate_name not in used_keys:
                 used_keys.add(candidate_name)
                 clean_unique_keys[k] = candidate_name