Fixing bugs in cxg conversion tool (#1782)

server/common/utils/cxg_generation_utils.py

@@ -67,7 +67,6 @@ def convert_dataframe_to_cxg_array(cxg_container, dataframe_name, dataframe, ind
         schema_hints = {}
         for column_name, column_values in dataframe.items():
             dtype, hints = get_dtype_and_schema_of_array(column_values)
-
             value[column_name] = column_values.to_numpy(dtype=dtype)
             if hints:
                 schema_hints.update({column_name: hints})

server/common/utils/type_conversion_utils.py

@@ -37,19 +37,19 @@ def get_dtype_from_dtype(dtype, array_values=None):
     dtype_name = dtype.name
     dtype_kind = dtype.kind
 
-    if dtype == np.float32 or dtype == np.int32:
-        return dtype
     if dtype_name == "bool":
         return np.uint8
     if dtype_name == "object" and dtype_kind == "O":
         return np.unicode
     if dtype_name == "category":
-        return get_dtype_from_dtype(dtype.categories.dtype, dtype.categories)
+        return get_dtype_from_dtype(dtype.categories.dtype, array_values)
 
-    if can_cast_to_float32(dtype):
-        return np.float32
     if can_cast_to_int32(dtype, array_values):
         return np.int32
+    if can_cast_to_float32(dtype, array_values):
+        return np.float32
+    if not can_cast_to_float32(dtype, array_values):
+        return np.float64
 
     raise TypeError(f"Annotations of type {dtype} are unsupported.")
 
@@ -72,19 +72,43 @@ def get_schema_type_hint_from_dtype(dtype, array_values=None):
     if dtype_name == "category":
         return {"type": "categorical", "categories": dtype.categories.tolist()}
 
-    if can_cast_to_float32(dtype):
-        return {"type": "float32"}
     if can_cast_to_int32(dtype, array_values):
         return {"type": "int32"}
+    if can_cast_to_float32(dtype, array_values):
+        return {"type": "float32"}
+    if dtype_kind == "f" and not can_cast_to_float32(dtype, array_values):
+        return {"type": "float64"}
 
     raise TypeError(f"Annotations of type {dtype} are unsupported.")
 
 
-def can_cast_to_float32(dtype):
+def can_cast_to_float32(dtype, array_values):
+    """
+    A dtype can be cast to float32 if it is a float type and converting it to float32 presents the same output as the
+    original values. Note that NaNs fail equality (i.e. np.NaN != np.NaN) so we use np.testing.assert_equal to ensure
+    that the arrays are equal minus NaNs.
+
+    We also handle a special case here where the array is a Series object with integer categorical values AND NaNs.
+    Since NaNs are floating points in numpy, we upcast the integer array to float32.
+    """
+
     if dtype.kind == "f":
-        if not np.can_cast(dtype, np.float32):
+        # Try to convert the array to float32
+        converted_float32_values = array_values.to_numpy(np.float32)
+        original_values = array_values.to_numpy()
+
+        # Verify that the two arrays are equal except for NaNs (which will equate to be unequal).
+        if not ((converted_float32_values != original_values) == np.isnan(original_values)).all():
+            return False
+
+        if dtype != np.float32:
             logging.warning(f"Type {dtype.name} will be converted to 32 bit float and may lose precision.")
+
         return True
+
+    if dtype.kind == "O" and array_values.hasnans:
+        return True
+
     return False
 
 
@@ -94,11 +118,30 @@ def can_cast_to_int32(dtype, array_values=None):
     the higher precision type has values that are entirely within the range of the downcast type.
     """
 
+    # Since a NaN is technically a float, any array that contains NaNs cannot be cast to an integer so immediately
+    # return False.
+    if array_values.hasnans:
+        return False
+
+    # If the array is categorical, then we need to order the array values so that functions min and max that occur
+    # later, can function. They do not function on unordered categories.
+    ordered_array_values = array_values
+    if array_values.dtype.name == "category" and not array_values.cat.ordered:
+        ordered_array_values = array_values.cat.as_ordered()
+
     if dtype.kind in ["i", "u"]:
         if np.can_cast(dtype, np.int32):
             return True
         ii32 = np.iinfo(np.int32)
-        if not array_values.empty and (
-                array_values.min() >= ii32.min and array_values.max() <= ii32.max) or array_values.empty:
+        if not ordered_array_values.empty and (
+                ordered_array_values.min() >= ii32.min and ordered_array_values.max() <= ii32.max) or \
+                ordered_array_values.empty:
             return True
     return False
+
+
+def convert_pandas_series_to_numpy(series_to_convert: pd.Series, dtype):
+    if series_to_convert.hasnans and dtype == np.int32:
+        logging.error("Cannot convert a pandas Series object to an integer dtype if it contains NaNs.")
+
+    return series_to_convert.to_numpy(dtype)