fork/cellxgene
1
0
Fork 0
mirror of https://github.com/chanzuckerberg/cellxgene.git synced 2026-10-03 01:18:12 +08:00
Code Issues Packages Projects Releases Wiki Activity

clean up type inferencing (#2332)

Browse Source 
* unit tests for 64 bit conversion

* clean up type handling

* type inference tests

* more type inference fixes

* use schema to determine user intent for data typing

* stop using deprecated API

* fbs type encoding test

* add missing test

* add more tests

* correctly infer X type for CXG adaptor

* lint

* fix typo

* ts migration

* cleanup from PR review

* lint

* PR review changes
This commit is contained in:
Bruce Martin
2021-07-28 15:10:12 -07:00
committed by GitHub
parent 1140676106
commit 32f60a1547
15 changed files with 730 additions and 346 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/common/utils/type_conversion_utils.py
+145 -112
View File
@@ -1,8 +1,42 @@
from typing import Union, Tuple
import logging
import numpy as np
import pandas as pd
"""
These routines drive all type inference for the schema generation and the
FBS (REST OTA) encoding. They are also used for CXG generation.
H5AD Type REST REST
(ndarray, Series, Index) FBS encoding schema type ERROR/exceptions
---------------------------- -------------- --------------- ----------------------
bool_/bool uint8 boolean
(u)int8, (u)int16, int32 int32 int32
uint32, (u)int64 int32 int32 CHECKS value bounds
float16, float32, float64 float32 float32[0]
categorical[T is numeric[4]]:
hasna = False T categorical[1]
hasna = True float32 categorical[1] CHECKS value bounds
categorical[T not numeric] JSON/str categorical[1,2]
(other object) JSON/str string
(all other) Always an ERROR[3]
Notes:
[0] IEEE format, includes non-finite numbers (NaN, Inf, ...)
[1] with NO categories enumerated (client side does it to handle rounding)
[2] NA (undefined) categories are assigned a JSON null value
[3] Includes all other numpy types: datetime, complex, etc.
[4] means float, int, uint (dtype.kind in ['i','u','f'])
"""
def get_dtypes_and_schemas_of_dataframe(dataframe: pd.DataFrame):
dtypes_by_column_name = {}
@@ -17,133 +51,132 @@ def get_dtypes_and_schemas_of_dataframe(dataframe: pd.DataFrame):
return dtypes_by_column_name, schema_type_hints_by_column_name
def get_dtype_of_array(array: pd.Series):
return get_dtype_and_schema_of_array(array)[0]
def get_encoding_dtype_of_array(array: Union[np.ndarray, pd.Series, pd.Index]) -> np.dtype:
return _get_type_info(array)[0]
def get_schema_type_hint_of_array(array: pd.Series):
return get_dtype_and_schema_of_array(array)[1]
def get_schema_type_hint_of_array(array: Union[np.ndarray, pd.Series, pd.Index]) -> dict:
return _get_type_info(array)[1]
def get_dtype_and_schema_of_array(array: pd.Series):
return (
get_dtype_from_dtype(array.dtype, array_values=array),
get_schema_type_hint_from_dtype(array.dtype, array_values=array),
)
def get_dtype_and_schema_of_array(array: Union[np.ndarray, pd.Series, pd.Index]) -> Tuple[np.dtype, dict]:
"""Return tuple (encoding_dtype, schema_type_hint)"""
return _get_type_info(array)
def get_dtype_from_dtype(dtype, array_values=None):
def get_schema_type_hint_from_dtype(dtype) -> dict:
res = _get_type_info_from_dtype(dtype)
if res is None:
raise TypeError(f"Annotations of type {dtype} are unsupported.")
else:
return res[1]
def _get_type_info_from_dtype(dtype) -> Union[Tuple[np.dtype, dict], None]:
"""
Given a data type, finds the equivalent data type that the array should be encoded as. Notably, this is relevant
for 64 bit values which will get downcast to 32 bit.
Best-effort to determine encoding type and schema hint from a dtype.
If this is not possible, or the type is unsupported, return None.
This should be a subset of the cases which are supported by
_get_type_info(). The latter should be preferred if the array (values)
are available for typing.
"""
if dtype.kind == "b":
return (np.uint8, {"type": "boolean"})
dtype_name = dtype.name
dtype_kind = dtype.kind
if dtype_name == "bool":
return np.uint8
if dtype_name == "object" and dtype_kind == "O":
return str
if dtype_name == "category":
return get_dtype_from_dtype(dtype.categories.dtype, array_values)
if can_cast_to_int32(dtype, array_values):
return np.int32
if can_cast_to_float32(dtype, array_values):
return np.float32
if not can_cast_to_float32(dtype, array_values):
return np.float64
raise TypeError(f"Annotations of type {dtype} are unsupported.")
def get_schema_type_hint_from_dtype(dtype, array_values=None):
"""
Returns a dictionary that contains type hints about the data type given, especially if the data type is 64 bit
and will be downcast to 32 bit.
"""
dtype_name = dtype.name
dtype_kind = dtype.kind
if dtype == np.float32 or dtype == np.int32:
return {"type": dtype_name}
if dtype_name == "bool":
return {"type": "boolean"}
if dtype_name == "object" and dtype_kind == "O":
return {"type": "string"}
if dtype_name == "category":
return {"type": "categorical", "categories": dtype.categories.tolist()}
if can_cast_to_int32(dtype, array_values):
return {"type": "int32"}
if can_cast_to_float32(dtype, array_values):
return {"type": "float32"}
if dtype_kind == "f" and not can_cast_to_float32(dtype, array_values):
return {"type": "float64"}
raise TypeError(f"Annotations of type {dtype} are unsupported.")
def can_cast_to_float32(dtype, array_values):
"""
Optimistically returns True signifying that a type downcast to float32 is possible whenever the incoming type is
a float.
We also handle a special case here where the array is a Series object with integer categorical values AND NaNs.
Since NaNs are floating points in numpy, we upcast the integer array to float32 and return True.
"""
if dtype.kind == "f":
if not np.can_cast(dtype, np.float32):
logging.warning(f"Type {dtype.name} will be converted to 32 bit float and may lose precision.")
return True
if dtype.kind == "O" and array_values.hasnans:
return True
return False
def can_cast_to_int32(dtype, array_values=None):
"""
A type can be cast to 32 bit, overriding the numpy `cast_cast` function if the values in the array that are of
the higher precision type has values that are entirely within the range of the downcast type.
"""
# Since a NaN is technically a float, any array that contains NaNs cannot be cast to an integer so immediately
# return False.
if array_values.hasnans:
return False
# If the array is categorical, then we need to order the array values so that functions min and max that occur
# later, can function. They do not function on unordered categories.
ordered_array_values = array_values
if array_values.dtype.name == "category" and not array_values.cat.ordered:
ordered_array_values = array_values.cat.as_ordered()
if dtype.kind == "U":
return (np.dtype(str), {"type": "string"})
if dtype.kind in ["i", "u"]:
if np.can_cast(dtype, np.int32):
return True
ii32 = np.iinfo(np.int32)
if (
not ordered_array_values.empty
and (ordered_array_values.min() >= ii32.min and ordered_array_values.max() <= ii32.max)
or ordered_array_values.empty
):
return True
return (np.int32, {"type": "int32"})
if dtype.kind == "f":
_float64_warning(dtype)
return (np.float32, {"type": "float32"})
if dtype.kind == "O" and not dtype.name == "category":
return (np.dtype(str), {"type": "string"})
return None
def _get_type_info(array: Union[np.ndarray, pd.Series, pd.Index]) -> Tuple[np.dtype, dict]:
"""
Determine encoding type and schema hint from an array. This allows more
flexible casting than may be possible by using just the dtype, as it can
account for category types and array values.
"""
if (
not isinstance(array, np.ndarray)
and not isinstance(array, pd.Series)
and not isinstance(array, pd.Index)
and not hasattr(array, "dtype")
):
raise TypeError("Unsupported data type.")
dtype = array.dtype
res = _get_type_info_from_dtype(dtype)
if res is not None:
return res
if dtype.kind == "O":
if dtype.name == "category":
# Sometimes CategoricalDType can be encoded as int or float without further fuss.
# Do not specify the categories in the schema - let the client-side figure it out
# on its own. Utilize Series.to_numpy() to do casting that handles categorical
# NA/NaN (missing or undefined) categories.
if dtype.categories.dtype.kind in ["f", "i", "u"]:
return (
_get_type_info(array.to_numpy())[0],
{"type": "categorical"},
)
else:
return (np.dtype(str), {"type": "categorical", "categories": dtype.categories.to_list()})
# all other extension types are str-encoded
return (np.dtype(str), {"type": "string"})
if dtype.kind in ["i", "u"] and _can_cast_array_values_to_int32(array):
return (np.int32, {"type": "int32"})
if dtype.kind == "f":
_float64_warning(array.dtype)
return (np.float32, {"type": "float32"})
raise TypeError(f"Annotations of type {dtype} are unsupported.")
def _float64_warning(dtype):
"""
Warn the user if we are down-casting a float64 to float32, and may potentially lose information.
"""
if dtype.kind == "f" and not np.can_cast(dtype, np.float32):
logging.warning(f"Type {dtype.name} will be converted to 32 bit float and may lose precision.")
def _can_cast_array_values_to_int32(array: Union[np.ndarray, pd.Series, pd.Index]) -> bool:
"""
Return true if the (U)INT array values can be safely cast to int32. We allow size reducing
casts (ie, int64 to int32) if no actual values require the larger size (ie, actual values
can be represented by the smaller type).
"""
assert array.dtype.kind in ["u", "i"]
if np.can_cast(array.dtype, np.int32):
return True
if array.size == 0:
return True
int32_machine_limits = np.iinfo(np.int32)
if array.min() >= int32_machine_limits.min and array.max() <= int32_machine_limits.max:
return True
return False
def convert_pandas_series_to_numpy(series_to_convert: pd.Series, dtype):
if series_to_convert.hasnans and dtype == np.int32:
logging.error("Cannot convert a pandas Series object to an integer dtype if it contains NaNs.")
return series_to_convert.to_numpy(dtype)
def convert_string_to_value(value: str):
"""convert a string to value with the most appropriate type"""
if value.lower() == "true":