diff --git a/backend/common/fbs/matrix.py b/backend/common/fbs/matrix.py index 0cb696d7..fd8f1cf6 100644 --- a/backend/common/fbs/matrix.py +++ b/backend/common/fbs/matrix.py @@ -5,6 +5,8 @@ import pandas as pd from flatbuffers import Builder from scipy import sparse +from backend.common.utils.type_conversion_utils import get_encoding_dtype_of_array + import backend.common.fbs.NetEncoding.Column as Column import backend.common.fbs.NetEncoding.Float32Array as Float32Array import backend.common.fbs.NetEncoding.Float64Array as Float64Array @@ -14,6 +16,7 @@ import backend.common.fbs.NetEncoding.Matrix as Matrix import backend.common.fbs.NetEncoding.TypedArray as TypedArray import backend.common.fbs.NetEncoding.Uint32Array as Uint32Array + # Serialization helper def serialize_column(builder, typed_arr): """ Serialize NetEncoding.Column """ @@ -84,7 +87,7 @@ def serialize_typed_array(builder, source_array, encoding_info): def column_encoding(arr): column_encoding_type_map = { # array protocol string: ( array_type, as_type ) - np.dtype(np.float64).str: (TypedArray.TypedArray.Float64Array, np.float64), + np.dtype(np.float64).str: (TypedArray.TypedArray.Float32Array, np.float32), np.dtype(np.float32).str: (TypedArray.TypedArray.Float32Array, np.float32), np.dtype(np.float16).str: (TypedArray.TypedArray.Float32Array, np.float32), np.dtype(np.int8).str: (TypedArray.TypedArray.Int32Array, np.int32), @@ -98,7 +101,8 @@ def column_encoding(arr): } column_encoding_default = (TypedArray.TypedArray.JSONEncodedArray, "json") - return column_encoding_type_map.get(arr.dtype.str, column_encoding_default) + encoding_dtype = np.dtype(get_encoding_dtype_of_array(arr)) + return column_encoding_type_map.get(encoding_dtype.str, column_encoding_default) def index_encoding(arr): @@ -198,7 +202,7 @@ def deserialize_typed_array(tarr): arr.Init(u.Bytes, u.Pos) narr = arr.DataAsNumpy() if u_type == TypedArray.TypedArray.JSONEncodedArray: - narr = json.loads(narr.tostring().decode("utf-8")) + narr = json.loads(narr.tobytes().decode("utf-8")) return narr diff --git a/backend/common/utils/type_conversion_utils.py b/backend/common/utils/type_conversion_utils.py index 971bf641..f054cfdb 100644 --- a/backend/common/utils/type_conversion_utils.py +++ b/backend/common/utils/type_conversion_utils.py @@ -1,8 +1,42 @@ +from typing import Union, Tuple import logging import numpy as np import pandas as pd +""" +These routines drive all type inference for the schema generation and the +FBS (REST OTA) encoding. They are also used for CXG generation. + + +H5AD Type REST REST +(ndarray, Series, Index) FBS encoding schema type ERROR/exceptions +---------------------------- -------------- --------------- ---------------------- +bool_/bool uint8 boolean +(u)int8, (u)int16, int32 int32 int32 +uint32, (u)int64 int32 int32 CHECKS value bounds +float16, float32, float64 float32 float32[0] + +categorical[T is numeric[4]]: + hasna = False T categorical[1] + hasna = True float32 categorical[1] CHECKS value bounds + +categorical[T not numeric] JSON/str categorical[1,2] + +(other object) JSON/str string + +(all other) Always an ERROR[3] + + +Notes: +[0] IEEE format, includes non-finite numbers (NaN, Inf, ...) +[1] with NO categories enumerated (client side does it to handle rounding) +[2] NA (undefined) categories are assigned a JSON null value +[3] Includes all other numpy types: datetime, complex, etc. +[4] means float, int, uint (dtype.kind in ['i','u','f']) + +""" + def get_dtypes_and_schemas_of_dataframe(dataframe: pd.DataFrame): dtypes_by_column_name = {} @@ -17,133 +51,132 @@ def get_dtypes_and_schemas_of_dataframe(dataframe: pd.DataFrame): return dtypes_by_column_name, schema_type_hints_by_column_name -def get_dtype_of_array(array: pd.Series): - return get_dtype_and_schema_of_array(array)[0] +def get_encoding_dtype_of_array(array: Union[np.ndarray, pd.Series, pd.Index]) -> np.dtype: + return _get_type_info(array)[0] -def get_schema_type_hint_of_array(array: pd.Series): - return get_dtype_and_schema_of_array(array)[1] +def get_schema_type_hint_of_array(array: Union[np.ndarray, pd.Series, pd.Index]) -> dict: + return _get_type_info(array)[1] -def get_dtype_and_schema_of_array(array: pd.Series): - return ( - get_dtype_from_dtype(array.dtype, array_values=array), - get_schema_type_hint_from_dtype(array.dtype, array_values=array), - ) +def get_dtype_and_schema_of_array(array: Union[np.ndarray, pd.Series, pd.Index]) -> Tuple[np.dtype, dict]: + """Return tuple (encoding_dtype, schema_type_hint)""" + return _get_type_info(array) -def get_dtype_from_dtype(dtype, array_values=None): +def get_schema_type_hint_from_dtype(dtype) -> dict: + res = _get_type_info_from_dtype(dtype) + if res is None: + raise TypeError(f"Annotations of type {dtype} are unsupported.") + else: + return res[1] + + +def _get_type_info_from_dtype(dtype) -> Union[Tuple[np.dtype, dict], None]: """ - Given a data type, finds the equivalent data type that the array should be encoded as. Notably, this is relevant - for 64 bit values which will get downcast to 32 bit. + Best-effort to determine encoding type and schema hint from a dtype. + If this is not possible, or the type is unsupported, return None. + + This should be a subset of the cases which are supported by + _get_type_info(). The latter should be preferred if the array (values) + are available for typing. """ + if dtype.kind == "b": + return (np.uint8, {"type": "boolean"}) - dtype_name = dtype.name - dtype_kind = dtype.kind - - if dtype_name == "bool": - return np.uint8 - if dtype_name == "object" and dtype_kind == "O": - return str - if dtype_name == "category": - return get_dtype_from_dtype(dtype.categories.dtype, array_values) - - if can_cast_to_int32(dtype, array_values): - return np.int32 - if can_cast_to_float32(dtype, array_values): - return np.float32 - if not can_cast_to_float32(dtype, array_values): - return np.float64 - - raise TypeError(f"Annotations of type {dtype} are unsupported.") - - -def get_schema_type_hint_from_dtype(dtype, array_values=None): - """ - Returns a dictionary that contains type hints about the data type given, especially if the data type is 64 bit - and will be downcast to 32 bit. - """ - - dtype_name = dtype.name - dtype_kind = dtype.kind - - if dtype == np.float32 or dtype == np.int32: - return {"type": dtype_name} - if dtype_name == "bool": - return {"type": "boolean"} - if dtype_name == "object" and dtype_kind == "O": - return {"type": "string"} - if dtype_name == "category": - return {"type": "categorical", "categories": dtype.categories.tolist()} - - if can_cast_to_int32(dtype, array_values): - return {"type": "int32"} - if can_cast_to_float32(dtype, array_values): - return {"type": "float32"} - if dtype_kind == "f" and not can_cast_to_float32(dtype, array_values): - return {"type": "float64"} - - raise TypeError(f"Annotations of type {dtype} are unsupported.") - - -def can_cast_to_float32(dtype, array_values): - """ - Optimistically returns True signifying that a type downcast to float32 is possible whenever the incoming type is - a float. - - We also handle a special case here where the array is a Series object with integer categorical values AND NaNs. - Since NaNs are floating points in numpy, we upcast the integer array to float32 and return True. - """ - - if dtype.kind == "f": - if not np.can_cast(dtype, np.float32): - logging.warning(f"Type {dtype.name} will be converted to 32 bit float and may lose precision.") - - return True - - if dtype.kind == "O" and array_values.hasnans: - return True - - return False - - -def can_cast_to_int32(dtype, array_values=None): - """ - A type can be cast to 32 bit, overriding the numpy `cast_cast` function if the values in the array that are of - the higher precision type has values that are entirely within the range of the downcast type. - """ - - # Since a NaN is technically a float, any array that contains NaNs cannot be cast to an integer so immediately - # return False. - if array_values.hasnans: - return False - - # If the array is categorical, then we need to order the array values so that functions min and max that occur - # later, can function. They do not function on unordered categories. - ordered_array_values = array_values - if array_values.dtype.name == "category" and not array_values.cat.ordered: - ordered_array_values = array_values.cat.as_ordered() + if dtype.kind == "U": + return (np.dtype(str), {"type": "string"}) if dtype.kind in ["i", "u"]: if np.can_cast(dtype, np.int32): - return True - ii32 = np.iinfo(np.int32) - if ( - not ordered_array_values.empty - and (ordered_array_values.min() >= ii32.min and ordered_array_values.max() <= ii32.max) - or ordered_array_values.empty - ): - return True + return (np.int32, {"type": "int32"}) + + if dtype.kind == "f": + _float64_warning(dtype) + return (np.float32, {"type": "float32"}) + + if dtype.kind == "O" and not dtype.name == "category": + return (np.dtype(str), {"type": "string"}) + + return None + + +def _get_type_info(array: Union[np.ndarray, pd.Series, pd.Index]) -> Tuple[np.dtype, dict]: + """ + Determine encoding type and schema hint from an array. This allows more + flexible casting than may be possible by using just the dtype, as it can + account for category types and array values. + """ + if ( + not isinstance(array, np.ndarray) + and not isinstance(array, pd.Series) + and not isinstance(array, pd.Index) + and not hasattr(array, "dtype") + ): + raise TypeError("Unsupported data type.") + + dtype = array.dtype + + res = _get_type_info_from_dtype(dtype) + if res is not None: + return res + + if dtype.kind == "O": + if dtype.name == "category": + # Sometimes CategoricalDType can be encoded as int or float without further fuss. + # Do not specify the categories in the schema - let the client-side figure it out + # on its own. Utilize Series.to_numpy() to do casting that handles categorical + # NA/NaN (missing or undefined) categories. + if dtype.categories.dtype.kind in ["f", "i", "u"]: + return ( + _get_type_info(array.to_numpy())[0], + {"type": "categorical"}, + ) + else: + return (np.dtype(str), {"type": "categorical", "categories": dtype.categories.to_list()}) + + # all other extension types are str-encoded + return (np.dtype(str), {"type": "string"}) + + if dtype.kind in ["i", "u"] and _can_cast_array_values_to_int32(array): + return (np.int32, {"type": "int32"}) + + if dtype.kind == "f": + _float64_warning(array.dtype) + return (np.float32, {"type": "float32"}) + + raise TypeError(f"Annotations of type {dtype} are unsupported.") + + +def _float64_warning(dtype): + """ + Warn the user if we are down-casting a float64 to float32, and may potentially lose information. + """ + if dtype.kind == "f" and not np.can_cast(dtype, np.float32): + logging.warning(f"Type {dtype.name} will be converted to 32 bit float and may lose precision.") + + +def _can_cast_array_values_to_int32(array: Union[np.ndarray, pd.Series, pd.Index]) -> bool: + """ + Return true if the (U)INT array values can be safely cast to int32. We allow size reducing + casts (ie, int64 to int32) if no actual values require the larger size (ie, actual values + can be represented by the smaller type). + """ + assert array.dtype.kind in ["u", "i"] + + if np.can_cast(array.dtype, np.int32): + return True + + if array.size == 0: + return True + + int32_machine_limits = np.iinfo(np.int32) + if array.min() >= int32_machine_limits.min and array.max() <= int32_machine_limits.max: + return True + return False -def convert_pandas_series_to_numpy(series_to_convert: pd.Series, dtype): - if series_to_convert.hasnans and dtype == np.int32: - logging.error("Cannot convert a pandas Series object to an integer dtype if it contains NaNs.") - - return series_to_convert.to_numpy(dtype) - - def convert_string_to_value(value: str): """convert a string to value with the most appropriate type""" if value.lower() == "true": diff --git a/backend/czi_hosted/common/annotations/hosted_tiledb.py b/backend/czi_hosted/common/annotations/hosted_tiledb.py index f86fec5a..733185a3 100644 --- a/backend/czi_hosted/common/annotations/hosted_tiledb.py +++ b/backend/czi_hosted/common/annotations/hosted_tiledb.py @@ -10,7 +10,7 @@ from flask import current_app from backend.czi_hosted.common.annotations.annotations import Annotations from backend.common.errors import AnnotationCategoryNameError from backend.czi_hosted.common.utils.sanitization_utils import sanitize_values_in_list -from backend.common.utils.type_conversion_utils import get_dtypes_and_schemas_of_dataframe, get_dtype_of_array +from backend.common.utils.type_conversion_utils import get_dtypes_and_schemas_of_dataframe, get_encoding_dtype_of_array from backend.czi_hosted.db.cellxgene_orm import Annotation @@ -143,7 +143,7 @@ class AnnotationsHostedTileDB(Annotations): # convert to tiledb datatypes for col in df: - df[col] = df[col].astype(get_dtype_of_array(df[col])) + df[col] = df[col].astype(get_encoding_dtype_of_array(df[col])) tiledb.from_pandas(uri, df, sparse=True) else: uri = "" diff --git a/backend/czi_hosted/common/utils/cxg_generation_utils.py b/backend/czi_hosted/common/utils/cxg_generation_utils.py index 3f3ee48e..c3c1c867 100644 --- a/backend/czi_hosted/common/utils/cxg_generation_utils.py +++ b/backend/czi_hosted/common/utils/cxg_generation_utils.py @@ -3,7 +3,7 @@ import json import numpy as np import tiledb -from backend.common.utils.type_conversion_utils import get_dtype_of_array, get_dtype_and_schema_of_array +from backend.common.utils.type_conversion_utils import get_encoding_dtype_of_array, get_dtype_and_schema_of_array def convert_dictionary_to_cxg_group(cxg_container, metadata_dict, group_metadata_name="cxg_group_metadata"): @@ -47,7 +47,7 @@ def convert_dataframe_to_cxg_array(cxg_container, dataframe_name, dataframe, ind ] ) attrs = [ - tiledb.Attr(name=column, dtype=get_dtype_of_array(dataframe[column]), filters=tiledb_filter) + tiledb.Attr(name=column, dtype=get_encoding_dtype_of_array(dataframe[column]), filters=tiledb_filter) for column in dataframe ] domain = tiledb.Domain( diff --git a/backend/czi_hosted/data_anndata/anndata_adaptor.py b/backend/czi_hosted/data_anndata/anndata_adaptor.py index efd7404a..8badd8de 100644 --- a/backend/czi_hosted/data_anndata/anndata_adaptor.py +++ b/backend/czi_hosted/data_anndata/anndata_adaptor.py @@ -66,11 +66,11 @@ class AnndataAdaptor(DataAdaptor): @staticmethod def _create_unique_column_name(df, col_name_prefix): - """ given the columns of a dataframe, and a name prefix, return a column name which - does not exist in the dataframe, AND which is prefixed by `prefix` + """given the columns of a dataframe, and a name prefix, return a column name which + does not exist in the dataframe, AND which is prefixed by `prefix` - The approach is to append a numeric suffix, starting at zero and increasing by - one, until an unused name is found (eg, prefix_0, prefix_1, ...). + The approach is to append a numeric suffix, starting at zero and increasing by + one, until an unused name is found (eg, prefix_0, prefix_1, ...). """ suffix = 0 while f"{col_name_prefix}{suffix}" in df: @@ -124,7 +124,11 @@ class AnndataAdaptor(DataAdaptor): def _create_schema(self): self.schema = { - "dataframe": {"nObs": self.cell_count, "nVar": self.gene_count, "type": str(self.data.X.dtype)}, + "dataframe": { + "nObs": self.cell_count, + "nVar": self.gene_count, + **get_schema_type_hint_of_array(self.data.X), + }, "annotations": { "obs": {"index": self.parameters.get("obs_names"), "columns": []}, "var": {"index": self.parameters.get("var_names"), "columns": []}, @@ -201,10 +205,10 @@ class AnndataAdaptor(DataAdaptor): self.parameters.update({"diffexp_may_be_slow": True}) def _is_valid_layout(self, arr): - """ return True if this layout data is a valid array for front-end presentation: - * ndarray, dtype float/int/uint - * with shape (n_obs, >= 2) - * with all values finite or NaN (no +Inf or -Inf) + """return True if this layout data is a valid array for front-end presentation: + * ndarray, dtype float/int/uint + * with shape (n_obs, >= 2) + * with all values finite or NaN (no +Inf or -Inf) """ is_valid = type(arr) == np.ndarray and arr.dtype.kind in "fiu" is_valid = is_valid and arr.shape[0] == self.data.n_obs and arr.shape[1] >= 2 diff --git a/backend/czi_hosted/data_cxg/cxg_adaptor.py b/backend/czi_hosted/data_cxg/cxg_adaptor.py index d20a5edd..a0643935 100644 --- a/backend/czi_hosted/data_cxg/cxg_adaptor.py +++ b/backend/czi_hosted/data_cxg/cxg_adaptor.py @@ -210,7 +210,8 @@ class CxgAdaptor(DataAdaptor): if lfc_cutoff is None: lfc_cutoff = self.dataset_config.diffexp__lfc_cutoff return diffexp_cxg.diffexp_ttest( - adaptor=self, maskA=maskA, maskB=maskB, top_n=top_n, diffexp_lfc_cutoff=lfc_cutoff) + adaptor=self, maskA=maskA, maskB=maskB, top_n=top_n, diffexp_lfc_cutoff=lfc_cutoff + ) def get_colors(self): if self.cxg_version == "0.0": @@ -357,7 +358,7 @@ class CxgAdaptor(DataAdaptor): shape = self.get_shape() dtype = self.get_X_array_dtype() - dataframe = {"nObs": shape[0], "nVar": shape[1], "type": dtype.name} + dataframe = {"nObs": shape[0], "nVar": shape[1], **get_schema_type_hint_from_dtype(dtype)} annotations = {} for ax in ("obs", "var"): diff --git a/backend/server/data_anndata/anndata_adaptor.py b/backend/server/data_anndata/anndata_adaptor.py index 0c6308f6..d0f108ac 100644 --- a/backend/server/data_anndata/anndata_adaptor.py +++ b/backend/server/data_anndata/anndata_adaptor.py @@ -125,7 +125,11 @@ class AnndataAdaptor(DataAdaptor): def _create_schema(self): self.schema = { - "dataframe": {"nObs": self.cell_count, "nVar": self.gene_count, "type": str(self.data.X.dtype)}, + "dataframe": { + "nObs": self.cell_count, + "nVar": self.gene_count, + **get_schema_type_hint_of_array(self.data.X), + }, "annotations": { "obs": {"index": self.parameters.get("obs_names"), "columns": []}, "var": {"index": self.parameters.get("var_names"), "columns": []}, @@ -335,7 +339,7 @@ class AnndataAdaptor(DataAdaptor): """return the approximate distribution of the X matrix.""" if self.X_approximate_distribution is None: """Not yet evaluated.""" - assert(self.dataset_config.X_approximate_distribution == "auto") + assert self.dataset_config.X_approximate_distribution == "auto" self.data = self.data.to_memory() # loads data self.X_approximate_distribution = estimate_distribution.estimate_approximate_distribution(self.data.X) diff --git a/backend/test/fixtures/pbmc3k_64.h5ad b/backend/test/fixtures/pbmc3k_64.h5ad new file mode 100644 index 00000000..3a6a4e52 Binary files /dev/null and b/backend/test/fixtures/pbmc3k_64.h5ad differ diff --git a/backend/test/test_common/unit/fbs/test_matrix.py b/backend/test/test_common/unit/fbs/test_matrix.py index ece1c918..a8e854f9 100644 --- a/backend/test/test_common/unit/fbs/test_matrix.py +++ b/backend/test/test_common/unit/fbs/test_matrix.py @@ -2,16 +2,20 @@ import unittest import pandas as pd import numpy as np from scipy import sparse +from parameterized import parameterized_class +import json -import backend.test.decode_fbs as decode_fbs +from backend.test import decode_fbs from backend.common.fbs.matrix import encode_matrix_fbs, decode_matrix_fbs +from backend.common.utils.type_conversion_utils import get_dtypes_and_schemas_of_dataframe +import backend.common.fbs as fbs class FbsTests(unittest.TestCase): - """Test Case for Matrix FBS data encode/decode """ + """Test Case for Matrix FBS data encode/decode""" def test_encode_boundary(self): - """ test various boundary checks """ + """test various boundary checks""" # row indexing is unsupported with self.assertRaises(ValueError): @@ -46,7 +50,7 @@ class FbsTests(unittest.TestCase): "d": pd.Series(["x", "y", "z", "x", "y", "z", "a", "x", "y", "z"], dtype="category"), } ) - expected_types = ((np.ndarray, np.float32), (np.ndarray, np.int32), (np.ndarray, np.uint32), (list, None)) + expected_types = ((np.ndarray, np.float32), (np.ndarray, np.int32), (np.ndarray, np.int32), (list, None)) fbs = encode_matrix_fbs(matrix=df, row_idx=None, col_idx=df.columns) self.fbs_checks(fbs, (10, 4), expected_types, ["a", "b", "c", "d"]) @@ -80,3 +84,133 @@ class FbsTests(unittest.TestCase): self.assertTrue(np.all(dfSrc[c] == dfDst[c])) else: self.assertEqual(dfSrc[c], dfDst[c]) + + +""" +Test type consistency between FBS encoding and the underlying schema hint. + +Basic assertion: the FBS type returned by encode_matrix_fbs() will be consistent +with the schema hint returned by type_conversion_utils (which is in turn used +to create the client schema). + +The following test cases are all dicts which contain the following keys: + - dataframe - the dataframe used as input for encode_matrix_fbs + - expected_fbs_types - upon success, dict of FBS column types expected (eg, Float32Array) + - expected_schema_hints - upon success, dict of schema hint +All are keyed by column name. +""" + +# simple tests that we convert all ints to int32 +int_dtypes = [np.dtype(d) for d in [np.int8, np.uint8, np.int16, np.uint16, np.int32, np.uint32, np.int64, np.uint64]] +int_test_cases = [ + { + "dataframe": pd.DataFrame({dtype.name: np.zeros((10,), dtype=dtype) for dtype in int_dtypes}), + "expected_fbs_types": dict( + [(dtype.name, fbs.NetEncoding.TypedArray.TypedArray.Int32Array) for dtype in int_dtypes] + ), + "expected_schema_hints": dict([(dtype.name, {"type": "int32"}) for dtype in int_dtypes]), + } +] + +# simple tests that we convert all floats to float32 +float_dtypes = [np.dtype(d) for d in [np.float16, np.float32, np.float64]] +float_test_cases = [ + { + "dataframe": pd.DataFrame({dtype.name: np.zeros((10,), dtype=dtype) for dtype in float_dtypes}), + "expected_fbs_types": dict( + [(dtype.name, fbs.NetEncoding.TypedArray.TypedArray.Float32Array) for dtype in float_dtypes] + ), + "expected_schema_hints": dict([(dtype.name, {"type": "float32"}) for dtype in float_dtypes]), + } +] + +# boolean - should be encoded as an uint32 +bool_dtypes = [np.dtype(d) for d in [np.bool_, bool]] +bool_test_cases = [ + { + "dataframe": pd.DataFrame({dtype.name: np.ones((10,), dtype=dtype) for dtype in bool_dtypes}), + "expected_fbs_types": dict( + [(dtype.name, fbs.NetEncoding.TypedArray.TypedArray.Uint32Array) for dtype in bool_dtypes] + ), + "expected_schema_hints": dict([(dtype.name, {"type": "boolean"}) for dtype in bool_dtypes]), + } +] + +cat_test_cases = [ + { + "dataframe": pd.DataFrame({"a": pd.Series(["a", "b", "c", "a", "b", "c"], dtype="category")}), + "expected_fbs_types": {"a": fbs.NetEncoding.TypedArray.TypedArray.JSONEncodedArray}, + "expected_schema_hints": {"a": {"type": "categorical", "categories": ["a", "b", "c"]}}, + }, + { + "dataframe": pd.DataFrame( + {"a": pd.Series(["a", "b", "c", "a", "b", "c"], dtype="category").cat.remove_categories("b")} + ), + "expected_fbs_types": {"a": fbs.NetEncoding.TypedArray.TypedArray.JSONEncodedArray}, + "expected_schema_hints": {"a": {"type": "categorical", "categories": ["a", "c"]}}, + }, + { + "dataframe": pd.DataFrame({"a": pd.Series(np.arange(0, 10, dtype=np.int64), dtype="category")}), + "expected_fbs_types": {"a": fbs.NetEncoding.TypedArray.TypedArray.Int32Array}, + "expected_schema_hints": {"a": {"type": "categorical"}}, + }, + { + "dataframe": pd.DataFrame( + {"a": pd.Series(np.arange(0, 10, dtype=np.int64), dtype="category").cat.remove_categories(2)} + ), + "expected_fbs_types": {"a": fbs.NetEncoding.TypedArray.TypedArray.Float32Array}, + "expected_schema_hints": {"a": {"type": "categorical"}}, + }, + { + "dataframe": pd.DataFrame({"a": pd.Series(np.arange(0, 10, dtype=np.float64), dtype="category")}), + "expected_fbs_types": {"a": fbs.NetEncoding.TypedArray.TypedArray.Float32Array}, + "expected_schema_hints": {"a": {"type": "categorical"}}, + }, + { + "dataframe": pd.DataFrame( + {"a": pd.Series(np.arange(0, 10, dtype=np.float64), dtype="category").cat.remove_categories(2)} + ), + "expected_fbs_types": {"a": fbs.NetEncoding.TypedArray.TypedArray.Float32Array}, + "expected_schema_hints": {"a": {"type": "categorical"}}, + }, +] + + +test_cases = [ + *int_test_cases, + *float_test_cases, + *bool_test_cases, + *cat_test_cases, +] + + +@parameterized_class(test_cases) +class TestTypeConversionConsistency(unittest.TestCase): + def test_type_conversion_consistency(self): + self.assertEqual(self.dataframe.shape[1], len(self.expected_fbs_types)) + self.assertEqual(self.dataframe.shape[1], len(self.expected_schema_hints)) + + buf = encode_matrix_fbs(matrix=self.dataframe, col_idx=self.dataframe.columns) + encoding_dtypes, schema_hints = get_dtypes_and_schemas_of_dataframe(self.dataframe) + + # check schema hints + # print(schema_hints) + # print(self.expected_schema_hints) + self.assertEqual(schema_hints, self.expected_schema_hints) + + # inspect the FBS types + matrix = fbs.NetEncoding.Matrix.Matrix.GetRootAsMatrix(buf, 0) + columns_length = matrix.ColumnsLength() + self.assertEqual(columns_length, self.dataframe.shape[1]) + + self.assertEqual(matrix.ColIndexType(), fbs.NetEncoding.TypedArray.TypedArray.JSONEncodedArray) + col_labels_arr = fbs.NetEncoding.JSONEncodedArray.JSONEncodedArray() + col_labels_arr.Init(matrix.ColIndex().Bytes, matrix.ColIndex().Pos) + col_index_labels = json.loads(col_labels_arr.DataAsNumpy().tobytes().decode("utf-8")) + self.assertEqual(len(col_index_labels), self.dataframe.shape[1]) + + for col_idx in range(0, columns_length): + col_label = col_index_labels[col_idx] + col = matrix.Columns(col_idx) + col_type = col.UType() + self.assertEqual(self.expected_fbs_types[col_label], col_type) diff --git a/backend/test/test_common/unit/utils/test_type_conversion_utils.py b/backend/test/test_common/unit/utils/test_type_conversion_utils.py index 70fba125..deacaa29 100644 --- a/backend/test/test_common/unit/utils/test_type_conversion_utils.py +++ b/backend/test/test_common/unit/utils/test_type_conversion_utils.py @@ -1,179 +1,22 @@ import unittest -from time import time -from unittest.mock import patch +import logging +from parameterized import parameterized_class import numpy as np +import pandas as pd from pandas import Series, DataFrame +from scipy import sparse from backend.common.utils.type_conversion_utils import ( - can_cast_to_float32, - can_cast_to_int32, - get_dtype_of_array, + get_encoding_dtype_of_array, get_schema_type_hint_of_array, get_dtypes_and_schemas_of_dataframe, - convert_pandas_series_to_numpy, + get_dtype_and_schema_of_array, + get_schema_type_hint_from_dtype, ) class TestTypeConversionUtils(unittest.TestCase): - def test__can_cast_to_float32__string_is_false(self): - array_to_convert = Series(data=["1", "2", "3"], dtype=str) - - can_cast = can_cast_to_float32(array_to_convert.dtype, array_to_convert) - - self.assertFalse(can_cast) - - def test__can_cast_to_float32__float64_is_true_warning_outputted(self): - array_to_convert = Series(data=[1, 2, 3], dtype=np.dtype(np.float64)) - - with self.assertLogs(level="WARN") as logger: - can_cast = can_cast_to_float32(array_to_convert.dtype, array_to_convert) - self.assertIn("may lose precision", logger.output[0]) - - self.assertTrue(can_cast) - - @patch("logging.warning") - def test__can_cast_to_float32__float32_is_false(self, mock_log_warning): - array_to_convert = Series(data=[1, 2, 3], dtype=np.dtype(np.float32)) - - can_cast = can_cast_to_float32(array_to_convert.dtype, array_to_convert) - - self.assertTrue(can_cast) - assert not mock_log_warning.called - - def test__can_cast_to_float32__categorical_float64_is_false(self): - array_to_convert = Series(data=[1.1, 2.2, 3.3], dtype="category") - - can_cast = can_cast_to_float32(array_to_convert.dtype, array_to_convert) - - self.assertFalse(can_cast) - - def test__can_cast_to_float32__categorical_int64_with_nans_is_true(self): - array_to_convert = Series(data=[1, 2, np.NaN], dtype="category") - - can_cast = can_cast_to_float32(array_to_convert.dtype, array_to_convert) - - self.assertTrue(can_cast) - - def test__can_cast_to_float_32__float_32_with_nans_is_true(self): - array_to_convert = Series(data=[1, 2, np.NaN], dtype=np.dtype(np.float32)) - - can_cast = can_cast_to_float32(array_to_convert.dtype, array_to_convert) - - self.assertTrue(can_cast) - - def test__can_cast_to_int32__string_is_false(self): - array_to_convert = Series(data=["1", "2", "3"], dtype=str) - - can_cast = can_cast_to_int32(array_to_convert.dtype, array_to_convert) - - self.assertFalse(can_cast) - - def test__can_cast_to_int32__int64_is_true(self): - array_to_convert = Series(data=[1, 2, 3], dtype=np.dtype(np.int64)) - - can_cast = can_cast_to_int32(array_to_convert.dtype, array_to_convert) - - self.assertTrue(can_cast) - - def test__can_cast_to_int32__int16_is_true(self): - array_to_convert = Series(data=[1, 2, 3], dtype=np.dtype(np.int16)) - - can_cast = can_cast_to_int32(array_to_convert.dtype, array_to_convert) - - self.assertTrue(can_cast) - - def test__can_cast_to_int32__int64_with_large_value_is_false(self): - array_to_convert = Series(data=[3000000000, 2, 3], dtype=np.dtype(np.int64)) - - can_cast = can_cast_to_int32(array_to_convert.dtype, array_to_convert) - - self.assertFalse(can_cast) - - def test__can_cast_to_int32__int64_with_nans_is_false(self): - array_to_convert = Series(data=[np.NaN, "2", "3"], dtype="category") - - can_cast = can_cast_to_int32(array_to_convert.dtype, array_to_convert) - - self.assertFalse(can_cast) - - def test__get_dtype_of_array__supported_dtypes_return_as_expected(self): - types = [np.float32, np.int32, np.bool_, str] - expected_dtypes = [np.float32, np.int32, np.uint8, str] - - for test_type_index in range(len(types)): - with self.subTest( - f"Testing get_dtype_of_array with type {types[test_type_index].__name__}", i=test_type_index - ): - array = Series(data=[], dtype=types[test_type_index]) - self.assertEqual(get_dtype_of_array(array), expected_dtypes[test_type_index]) - - def test__get_dtype_of_array__categories_return_as_expected(self): - array = Series(data=["a", "b", "c"], dtype="category") - expected_dtype = str - - actual_dtype = get_dtype_of_array(array) - - self.assertEqual(expected_dtype, actual_dtype) - - def test__get_dtype_of_array__unordered_integer_categories_return_as_expected(self): - array = Series(data=[2, 3, 1, 3, 1, 2], dtype="category") - expected_dtype = np.int32 - - actual_dtype = get_dtype_of_array(array) - - self.assertEqual(expected_dtype, actual_dtype) - - def test__get_dtype_of_array__castable_dtypes_return_as_expected(self): - types = [np.float64, np.int64] - expected_dtypes = [np.float32, np.int32] - - for test_type_index in range(len(types)): - with self.subTest( - f"Testing get_dtype_of_array with castable type {types[test_type_index].__name__}", i=test_type_index - ): - array = Series(data=[], dtype=types[test_type_index]) - self.assertEqual(get_dtype_of_array(array), expected_dtypes[test_type_index]) - - def test__get_dtype_of_array__unsupported_type_raises_exception(self): - unsupported_array = Series(list([time() for _ in range(2)]), dtype="datetime64[ns]") - - with self.assertRaises(TypeError) as exception_context: - get_dtype_of_array(unsupported_array) - - self.assertIn("unsupported", str(exception_context.exception)) - - def test__get_schema_type_hint_of_array__supported_dtypes_return_as_expected(self): - types = [np.float32, np.int32, np.bool_, str] - expected_schema_hints = [{"type": "float32"}, {"type": "int32"}, {"type": "boolean"}, {"type": "string"}] - - for test_type_index in range(len(types)): - with self.subTest( - f"Testing get_schema_type_hint_of_array with type {types[test_type_index].__name__}", i=test_type_index - ): - array = Series(data=[], dtype=types[test_type_index]) - self.assertEqual(get_schema_type_hint_of_array(array), expected_schema_hints[test_type_index]) - - def test__get_schema_type_hint_of_array__categories_return_as_expected(self): - array = Series(data=["a", "b", "b"], dtype="category") - expected_schema_hint = {"type": "categorical", "categories": ["a", "b"]} - - actual_schema_hint = get_schema_type_hint_of_array(array) - - self.assertEqual(expected_schema_hint, actual_schema_hint) - - def test__get_schema_type_hint_of_array__castable_dtypes_return_as_expected(self): - types = [np.float64, np.int64] - expected_schema_hints = [{"type": "float32"}, {"type": "int32"}] - - for test_type_index in range(len(types)): - with self.subTest( - f"Testing get_schema_type_hint_of_array with castable type {types[test_type_index].__name__}", - i=test_type_index, - ): - array = Series(data=[], dtype=types[test_type_index]) - self.assertEqual(get_schema_type_hint_of_array(array), expected_schema_hints[test_type_index]) - def test__get_dtypes_and_schemas_of_dataframe__dtype_and_schema_returns_as_expected(self): float_array = Series(data=[1, 2, 3], dtype=np.dtype(np.float64)) category_array = Series(data=["a", "b", "b"], dtype="category") @@ -190,28 +33,292 @@ class TestTypeConversionUtils(unittest.TestCase): self.assertEqual(expected_data_types_dict, actual_dataframe_data_types) self.assertEqual(expected_schema_type_hints_dict, actual_dataframe_schema_type_hints) - def test__convert_pandas_series_to_numpy__categorical_float64_to_float64_with_nans(self): - expected_float_array = np.array([1.1, 2.2, np.NaN], dtype=np.float64) - float_series = Series(data=[1.1, 2.2, np.NaN], dtype="category") + def test__get_schema_type_hint_from_dtype(self): + self.assertEqual(get_schema_type_hint_from_dtype(np.dtype(np.bool_)), {"type": "boolean"}) - actual_float_array = convert_pandas_series_to_numpy(float_series, np.float64) + for dtype in [np.int8, np.int8, np.int16, np.uint16, np.int32]: + self.assertEqual(get_schema_type_hint_from_dtype(np.dtype(dtype)), {"type": "int32"}) + for dtype in [np.uint32, np.int64, np.uint64]: + with self.assertRaises(TypeError): + get_schema_type_hint_from_dtype(np.dtype(dtype)) - np.testing.assert_equal(expected_float_array, actual_float_array) + for dtype in [np.float16, np.float32, np.float64]: + self.assertEqual(get_schema_type_hint_from_dtype(np.dtype(dtype)), {"type": "float32"}) - def test__convert_pandas_series_to_numpy__float64_to_float64(self): - expected_float_array = np.array([1.1, 2.2], dtype=np.float64) - float_series = Series(data=[1.1, 2.2], dtype=np.dtype(np.float64)) + for dtype in [np.dtype(object), np.dtype(str)]: + self.assertEqual(get_schema_type_hint_from_dtype(dtype), {"type": "string"}) - actual_float_array = convert_pandas_series_to_numpy(float_series, np.float64) - np.testing.assert_equal(expected_float_array, actual_float_array) +# Credit: https://stackoverflow.com/questions/35871815/python-3-unit-testing-assert-logger-not-called/64774103#64774103 +class AssertNoLog: + def assertNoLogs(self, logger, level): + """functions as a context manager. To be introduced in python 3.10""" - def test__convert_pandas_series_to_numpy__int64_to_int32_with_nans_throws_error(self): - int_series = Series(data=[1, 2, np.NaN], dtype="category") + class AssertNoLogsContext(unittest.TestCase): + def __init__(self, logger, level): + self.logger = logger + self.level = level + self.context = self.assertLogs(logger, level) - with self.assertLogs(level="ERROR") as logger: - convert_pandas_series_to_numpy(int_series, np.int32) + def __enter__(self): + """enter self.assertLogs as context manager, and log something""" + self.initial_logmsg = "sole message" + self.cm = self.context.__enter__() + self.logger.log(self.level, self.initial_logmsg) + return self.cm - self.assertIn( - "Cannot convert a pandas Series object to an integer dtype if it contains NaNs", logger.output[0] - ) + def __exit__(self, exc_type, exc_val, exc_tb): + """cleanup logs, and then check nothing extra was logged""" + # assertLogs.__exit__ should never fail because of initial msg + self.context.__exit__(exc_type, exc_val, exc_tb) + if len(self.cm.output) > 1: + """override any exception passed to __exit__""" + self.context._raiseFailure( + "logs of level {} or higher triggered on {} : {}".format( + logging.getLevelName(self.level), self.logger.name, self.cm.output[1:] + ) + ) + + return AssertNoLogsContext(logger, level) + + +""" +See table of expected cases in type_conversion_utils.py. + +This probes all edge cases. Each case is a dict containing keys: + - data - the array to be introspected + - throws - if not None, the expected Error (eg, TypeError) + - expected_encoding_dtype - upon success + - expected_schema_hint - upon success + - logs - if not None, specify expected log output +""" + +bool_OK_cases = [ + { + "data": data, + "expected_encoding_dtype": np.uint8, + "expected_schema_hint": {"type": "boolean"}, + } + for data in [ + np.array([0, 1, 0, 1], dtype=np.bool_), + pd.Series(np.array([0, 1, 0, 1], dtype=np.bool_)), + # pd.Index with bools doesn't really make any sense...and becomes dtype=object + ] +] + +int_OK_cases = [ + { + "data": data, + "expected_encoding_dtype": np.int32, + "expected_schema_hint": {"type": "int32"}, + } + for dtype in [np.int8, np.uint8, np.int16, np.uint16, np.int32, np.uint32, np.int64, np.uint64] + for data in [ + np.arange(0, 1000, dtype=dtype), + pd.Series(np.arange(0, 1000, dtype=dtype)), + pd.Index(np.arange(0, 1000, dtype=dtype)), + sparse.csr_matrix((10, 100), dtype=dtype), + ] +] + +float_OK_cases = [ + { + "data": data, + "expected_encoding_dtype": np.float32, + "expected_schema_hint": {"type": "float32"}, + "logs": None if data.dtype != np.float64 else {"level": logging.WARNING, "output": "may lose precision"}, + } + for dtype in [np.float16, np.float32, np.float64] + for data in [ + np.arange(-128, 1000, dtype=dtype), + pd.Series(np.arange(-128, 1000, dtype=dtype)), + pd.Index(np.arange(-129, 1000, dtype=dtype)), + np.array([-np.nan, np.NINF, -1, np.NZERO, 0, np.PZERO, 1, np.PINF, np.nan], dtype=dtype), + np.array([np.finfo(dtype).min, 0, np.finfo(dtype).max], dtype=dtype), + sparse.csr_matrix((10, 100), dtype=dtype), + ] +] + + +numeric_ERR_cases = [ + { + "data": data, + "throws": TypeError, + } + for data in [ + np.array([np.iinfo(np.int64).min, np.iinfo(np.int64).max], dtype=np.int64), + np.array([np.iinfo(np.uint64).min, np.iinfo(np.uint64).max], dtype=np.uint64), + np.array([np.iinfo(np.uint32).min, np.iinfo(np.uint32).max], dtype=np.uint32), + ] +] + + +string_OK_cases = [ + { + "data": data, + "expected_encoding_dtype": np.dtype(str), + "expected_schema_hint": {"type": "string"}, + } + for data in [ + np.array(["a", "b", "c"]), + np.array(["a", "b", "c"], dtype="object"), + pd.Series(["a", "b", "c"]), + pd.Index(["a", "b", "c"]), + np.array(["a", [], {}, None, True, False, 383.2], dtype="object"), + ] +] + +category_nonnumeric_OK_cases = [ + { + "data": data, + "expected_encoding_dtype": np.dtype(str), + "expected_schema_hint": {"type": "categorical", "categories": data.dtype.categories.to_list()}, + } + for data in [ + pd.Series(["a", "b", "c"], dtype="category"), + pd.Series(["a", "b", "c", 0, 1, 2], dtype="category"), + pd.Series(["a", "b", "c"], dtype="category").cat.remove_categories(["b"]), + pd.Series(["a", "b", "c", 0, 1, 2], dtype="category").cat.remove_categories(["b", 0]), + ] +] + +category_numeric_OK_cases = [ + # numeric, no NA/NaN, int + *[ + { + "data": data, + "expected_encoding_dtype": np.int32, + "expected_schema_hint": {"type": "categorical"}, + } + for dtype in [np.int8, np.uint8, np.int16, np.uint16, np.int32, np.uint32, np.int64, np.uint64] + for data in [ + pd.Series(np.array([0, 1, 2], dtype=dtype), dtype="category"), + ] + ], + # numeric, no NA/NaN, float + *[ + { + "data": data, + "expected_encoding_dtype": np.float32, + "expected_schema_hint": {"type": "categorical"}, + "logs": {"level": logging.WARNING, "output": "may lose precision"}, + } + for dtype in [np.float16, np.float32, np.float64] + for data in [ + pd.Series(np.array([0, 1, 2], dtype=dtype), dtype="category"), + pd.Series(np.array([0, 1, 2], dtype=dtype), dtype="category").cat.remove_categories([1]), + pd.Categorical(np.array([0, 1, 2], dtype=dtype)), + ] + ], + # numeric, has NA-induced cast to float32 + *[ + { + "data": data, + "expected_encoding_dtype": np.float32, + "expected_schema_hint": {"type": "categorical"}, + "logs": {"level": logging.WARNING, "output": "may lose precision"}, + } + for dtype in [ + np.int8, + np.uint8, + np.int16, + np.uint16, + np.int32, + np.uint32, + np.int64, + np.uint64, + np.float16, + np.float32, + np.float64, + ] + for data in [ + pd.Series(np.array([0, 1, 2], dtype=dtype), dtype="category").cat.remove_categories([1]), + pd.Categorical(np.array([0, 1, 2], dtype=dtype), categories=np.array([0, 1], dtype=dtype)), + ] + ], +] + +category_ERR_cases = [ + # catch expected categorical exceptions for Int64(etc) that have large values + { + "data": data, + "throws": TypeError, + } + for data in [ + pd.Categorical(np.array([np.iinfo(np.int64).min, np.iinfo(np.int64).max], dtype=np.int64)), + pd.Categorical(np.array([np.iinfo(np.uint64).min, np.iinfo(np.uint64).max], dtype=np.uint64)), + pd.Categorical(np.array([np.iinfo(np.uint32).min, np.iinfo(np.uint32).max], dtype=np.uint32)), + ] +] + +object_OK_cases = [ + { + "data": data, + "expected_encoding_dtype": np.dtype(str), + "expected_schema_hint": {"type": "string"}, + } + for data in [ + np.array(["a", True, 1, [], {}], dtype="object"), + pd.Series(["a", True, 1, [], {}], dtype="object"), + pd.Index(["a", True, 1, [], {}], dtype="object"), + ] +] + +err_cases = [ + {"data": np.array, "throws": TypeError} + for data in [ + np.ones((10,), dtype=np.complex64), + np.ones((10,), dtype=np.complex128), + np.array([b"foobar"], dtype=np.bytes_), + np.ones((10,), dtype=np.void), + np.arange("2005-02", "2005-03", dtype="datetime64[D]"), + np.arange("2005-02", "2005-03", dtype="datetime64[D]") - np.datetime64("2008-01-01"), + [], + {}, + ] +] + +test_cases = [ + *bool_OK_cases, + *int_OK_cases, + *float_OK_cases, + *numeric_ERR_cases, + *string_OK_cases, + *category_nonnumeric_OK_cases, + *category_numeric_OK_cases, + *category_ERR_cases, + *object_OK_cases, + *err_cases, +] + + +@parameterized_class(test_cases) +class TestTypeInference(unittest.TestCase, AssertNoLog): + def test_type_inference(self): + throws = getattr(self, "throws", None) + if throws: + with self.assertRaises(throws): + get_dtype_and_schema_of_array(self.data) + with self.assertRaises(throws): + get_encoding_dtype_of_array(self.data) + with self.assertRaises(throws): + get_schema_type_hint_of_array(self.data) + + else: + logs = getattr(self, "logs", None) + if logs is not None: + with self.assertLogs(level=logs["level"]) as logger: + encoding_dtype, schema_hint = get_dtype_and_schema_of_array(self.data) + self.assertEqual(encoding_dtype, self.expected_encoding_dtype) + self.assertEqual(schema_hint, self.expected_schema_hint) + self.assertIn(logs["output"], logger.output[0]) + + else: + with self.assertNoLogs(logging.getLogger(), logging.WARNING): + encoding_dtype, schema_hint = get_dtype_and_schema_of_array(self.data) + self.assertEqual(encoding_dtype, self.expected_encoding_dtype) + self.assertEqual(schema_hint, self.expected_schema_hint) + + # also test the other public API + self.assertEqual(get_encoding_dtype_of_array(self.data), self.expected_encoding_dtype) + self.assertEqual(get_schema_type_hint_of_array(self.data), self.expected_schema_hint) diff --git a/backend/test/test_czi_hosted/unit/data_common/fbs/test_matrix.py b/backend/test/test_czi_hosted/unit/data_common/fbs/test_matrix.py index 9b80d042..21389fe0 100644 --- a/backend/test/test_czi_hosted/unit/data_common/fbs/test_matrix.py +++ b/backend/test/test_czi_hosted/unit/data_common/fbs/test_matrix.py @@ -46,7 +46,7 @@ class FbsTests(unittest.TestCase): "d": pd.Series(["x", "y", "z", "x", "y", "z", "a", "x", "y", "z"], dtype="category"), } ) - expected_types = ((np.ndarray, np.float32), (np.ndarray, np.int32), (np.ndarray, np.uint32), (list, None)) + expected_types = ((np.ndarray, np.float32), (np.ndarray, np.int32), (np.ndarray, np.int32), (list, None)) fbs = encode_matrix_fbs(matrix=df, row_idx=None, col_idx=df.columns) self.fbs_checks(fbs, (10, 4), expected_types, ["a", "b", "c", "d"]) diff --git a/backend/test/test_server/unit/common/test_api.py b/backend/test/test_server/unit/common/test_api.py index 84f79413..4ae65912 100644 --- a/backend/test/test_server/unit/common/test_api.py +++ b/backend/test/test_server/unit/common/test_api.py @@ -6,9 +6,13 @@ from http import HTTPStatus import tempfile from os import path import hashlib +from os.path import basename, splitext import pandas as pd import requests +import numpy as np + +from parameterized import parameterized_class import backend.test.decode_fbs as decode_fbs from backend.server.data_common.matrix_loader import MatrixDataType @@ -44,6 +48,14 @@ class EndPoints(object): len(result_data["schema"]["annotations"]["obs"]["columns"]), 6 if self.ANNOTATIONS_ENABLED else 5 ) + # Check that all schema types are legal + legal_types = ["boolean", "string", "categorical", "float32", "int32"] + self.assertEqual(result_data["schema"]["dataframe"]["type"], "float32") + for column in result_data["schema"]["annotations"]["obs"]["columns"]: + self.assertIn(column["type"], legal_types) + for column in result_data["schema"]["annotations"]["var"]["columns"]: + self.assertIn(column["type"], legal_types) + def test_config(self): endpoint = "config" url = f"{self.URL_BASE}{endpoint}" @@ -52,7 +64,12 @@ class EndPoints(object): self.assertEqual(result.headers["Content-Type"], "application/json") result_data = result.json() self.assertIn("library_versions", result_data["config"]) - self.assertEqual(result_data["config"]["displayNames"]["dataset"], "pbmc3k") + + if hasattr(self, "data_locator"): + title = splitext(basename(self.data_locator))[0] + else: + title = "pbmc3k" + self.assertEqual(result_data["config"]["displayNames"]["dataset"], title) self.assertIsNotNone(result_data["config"]["parameters"]) def test_get_layout_fbs(self): @@ -72,6 +89,8 @@ class EndPoints(object): ) self.assertIsNone(df["row_idx"]) self.assertEqual(len(df["columns"]), df["n_cols"]) + for column in df["columns"]: + self.assertEqual(column.dtype, np.float32) def test_bad_filter(self): endpoint = "data/var" @@ -98,6 +117,9 @@ class EndPoints(object): [obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain"] + (["cluster-test"] if self.ANNOTATIONS_ENABLED else []), ) + for column in df["columns"]: + if type(column) is np.ndarray: + self.assertIn(column.dtype, [np.float32, np.int32]) def test_get_annotations_obs_keys_fbs(self): endpoint = "annotations/obs" @@ -137,6 +159,9 @@ class EndPoints(object): self.assertEqual(len(df["columns"]), df["n_cols"]) var_index_col_name = self.schema["schema"]["annotations"]["var"]["index"] self.assertCountEqual(df["col_idx"], [var_index_col_name, "n_cells"]) + for column in df["columns"]: + if type(column) is np.ndarray: + self.assertIn(column.dtype, [np.float32, np.int32]) def test_get_annotations_var_keys_fbs(self): endpoint = "annotations/var" @@ -207,6 +232,9 @@ class EndPoints(object): self.assertIsNone(df["row_idx"]) self.assertEqual(len(df["columns"]), df["n_cols"]) self.assertListEqual(df["col_idx"].tolist(), [0, 1, 4]) + for column in df["columns"]: + if type(column) is np.ndarray: + self.assertIn(column.dtype, [np.float32, np.int32]) def test_data_get_filter_fbs(self): index_col_name = self.schema["schema"]["annotations"]["var"]["index"] @@ -220,6 +248,9 @@ class EndPoints(object): df = decode_fbs.decode_matrix_FBS(result.content) self.assertEqual(df["n_rows"], 2638) self.assertEqual(df["n_cols"], 1) + for column in df["columns"]: + if type(column) is np.ndarray: + self.assertIn(column.dtype, [np.float32, np.int32]) def test_data_get_unknown_filter_fbs(self): index_col_name = self.schema["schema"]["annotations"]["var"]["index"] @@ -246,6 +277,9 @@ class EndPoints(object): df = decode_fbs.decode_matrix_FBS(result.content) self.assertEqual(df["n_rows"], 2638) self.assertEqual(df["n_cols"], 1) + for column in df["columns"]: + if type(column) is np.ndarray: + self.assertIn(column.dtype, [np.float32, np.int32]) def test_colors(self): endpoint = "colors" @@ -347,6 +381,14 @@ class EndPointsAnnotations(EndPoints): self.assertTrue(matching_columns[0]["writable"]) +@parameterized_class( + [ + {"data_locator": f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad"}, + {"data_locator": f"{FIXTURES_ROOT}/pbmc3k_64.h5ad"}, + {"data_locator": f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad"}, + {"data_locator": f"{FIXTURES_ROOT}/pbmc3k-CSR-gz.h5ad"}, + ] +) class EndPointsAnndata(unittest.TestCase, EndPoints): """Test Case for endpoints""" @@ -355,10 +397,13 @@ class EndPointsAnndata(unittest.TestCase, EndPoints): @classmethod def setUpClass(cls): + if cls == EndPointsAnndata: + raise unittest.SkipTest("`parameterized_class` bug") + cls._setupClass( cls, [ - f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad", + cls.data_locator, "--disable-annotations", "--disable-gene-sets-save", ], @@ -385,8 +430,8 @@ class EndPointsAnndata(unittest.TestCase, EndPoints): self.assertEqual(result.status_code, HTTPStatus.OK) self.assertEqual(result.headers["Content-Type"], "application/json") result_data = result.json() - self.assertEqual(len(result_data['positive']), 7) - self.assertEqual(len(result_data['negative']), 7) + self.assertEqual(len(result_data["positive"]), 7) + self.assertEqual(len(result_data["negative"]), 7) def test_diff_exp_indices(self): endpoint = "diffexp/obs" @@ -401,8 +446,8 @@ class EndPointsAnndata(unittest.TestCase, EndPoints): self.assertEqual(result.status_code, HTTPStatus.OK) self.assertEqual(result.headers["Content-Type"], "application/json") result_data = result.json() - self.assertEqual(len(result_data['positive']), 10) - self.assertEqual(len(result_data['negative']), 10) + self.assertEqual(len(result_data["positive"]), 10) + self.assertEqual(len(result_data["negative"]), 10) def test_get_summaryvar(self): index_col_name = self.schema["schema"]["annotations"]["var"]["index"] @@ -561,23 +606,23 @@ class EndPointsAnnDataGenesets(unittest.TestCase, EndPoints): "geneset_description": "", "geneset_name": "summary test", }, - {'genes': [], 'geneset_description': '', 'geneset_name': 'geneset_to_delete'}, - {'genes': [], 'geneset_description': '', 'geneset_name': 'geneset_to_edit'}, + {"genes": [], "geneset_description": "", "geneset_name": "geneset_to_delete"}, + {"genes": [], "geneset_description": "", "geneset_name": "geneset_to_edit"}, { - 'genes': [{'gene_description': '', 'gene_symbol': 'RER1'}], - 'geneset_description': '', - 'geneset_name': 'fill_this_geneset' + "genes": [{"gene_description": "", "gene_symbol": "RER1"}], + "geneset_description": "", + "geneset_name": "fill_this_geneset", }, { - 'genes': [{'gene_description': '', 'gene_symbol': 'SIK1'}], - 'geneset_description': '', - 'geneset_name': 'empty_this_geneset' + "genes": [{"gene_description": "", "gene_symbol": "SIK1"}], + "geneset_description": "", + "geneset_name": "empty_this_geneset", }, { - 'genes': [{'gene_description': '', 'gene_symbol': 'SIK1'}], - 'geneset_description': '', - 'geneset_name': 'brush_this_gene' - } + "genes": [{"gene_description": "", "gene_symbol": "SIK1"}], + "geneset_description": "", + "geneset_name": "brush_this_gene", + }, ], "tid": 0, }, @@ -680,7 +725,7 @@ brush_this_gene,,SIK1,\r self.assertEqual(result.json(), test3) def test_put_genesets_malformed(self): - """ test malformed submissions that we expect the backend to catch/tolerate """ + """test malformed submissions that we expect the backend to catch/tolerate""" endpoint = "genesets" url = f"{self.URL_BASE}{endpoint}" @@ -690,7 +735,7 @@ brush_this_gene,,SIK1,\r tid = original_data["tid"] def test_case(test, expected_code, original_data): - """ check for expected error AND that no change was made to the original state """ + """check for expected error AND that no change was made to the original state""" result = self.session.put(url, json=test) self.assertEqual(result.status_code, expected_code) result = self.session.get(url, headers={"Accept": "application/json"}) diff --git a/backend/test/test_server/unit/data_anndata/test_anndata_adaptor.py b/backend/test/test_server/unit/data_anndata/test_anndata_adaptor.py index 01666eb2..19ba07aa 100644 --- a/backend/test/test_server/unit/data_anndata/test_anndata_adaptor.py +++ b/backend/test/test_server/unit/data_anndata/test_anndata_adaptor.py @@ -36,6 +36,7 @@ Test the anndata adaptor using the pbmc3k data set. (f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad", True, "normal"), (f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad", True, "normal"), (f"{FIXTURES_ROOT}/pbmc3k-CSR-gz.h5ad", True, "normal"), + (f"{FIXTURES_ROOT}/pbmc3k_64.h5ad", False, "auto"), # 64 bit conversion tests ], ) class AdaptorTest(unittest.TestCase): diff --git a/client/src/annoMatrix/loader.ts b/client/src/annoMatrix/loader.ts index e2e3dd54..b3a1f4a9 100644 --- a/client/src/annoMatrix/loader.ts +++ b/client/src/annoMatrix/loader.ts @@ -272,7 +272,7 @@ export default class AnnoMatrixLoader extends AnnoMatrix { } const buffer = await promiseThrottle.priorityAdd(priority, doRequest); - const result = matrixFBSToDataframe(buffer); + let result = matrixFBSToDataframe(buffer); if (!result || result.isEmpty()) throw Error("Unknown field/col"); const whereCacheUpdate = _whereCacheCreate( @@ -281,18 +281,61 @@ export default class AnnoMatrixLoader extends AnnoMatrix { result.colIndex.labels() ); - if (field === "obs") { - /* cough, cough - see comment on the function called */ - _normalizeCategoricalSchema( - this.schema.annotations.obsByName[query], - result.col(query) - ); - } + result = _responseTypeNormalization(field, query, this.schema, result); return [whereCacheUpdate, result]; } } +// @ts-expect-error ts-migrate(7006) +function _responseTypeNormalization(field, query, schema, response) { + /* + Schema-driven type normalization - there are a number of assumptions the front-end + makes about data typing, eg, that the schema will contain all categories in a categorical + column, that booleans are a JS array of true/false, etc. + + The OTA format does not follow precisely the same conventions. This routine implements + the front-end conventions given the schema and an OTA data column. + */ + if (field === "obs" || field === "var") { + response = _booleanCast(field, query, schema, response); + } + if (field === "obs") { + /* + Note: this must be performed after any possible changes to string, boolean or categorical + columns. This routine relies on having access to any casts or other data transformations + made in this routine, above, in order to correctly determine schema updates. + */ + _normalizeCategoricalSchema( + schema.annotations.obsByName[query], + response.col(query) + ); + } + return response; +} + +// @ts-expect-error ts-migrate(7006) +function _booleanCast(field, query, schema, response) { + /* + Boolean columns may be transmitted as an int [0/1] or bool [true/false]. + Force to JS Array of bool. + */ + if (field === "obs" || field === "var") { + // @ts-expect-error ts-migrate(7006) + response = response.mapColumns((colData, colIdx) => { + const colLabel = response.colIndex.getLabel(colIdx); + const colSchema = _getColumnSchema(schema, field, colLabel); + if (colSchema?.type === "boolean") { + const nColData = new Array(colData.length); + for (let i = 0; i < colData.length; i += 1) nColData[i] = !!colData[i]; + return nColData; + } + return colData; + }); + } + return response; +} + /* Utility functions below */ diff --git a/client/src/annoMatrix/schema.ts b/client/src/annoMatrix/schema.ts index 02fd636c..769d222c 100644 --- a/client/src/annoMatrix/schema.ts +++ b/client/src/annoMatrix/schema.ts @@ -73,6 +73,14 @@ export function _isContinuousType(schema) { // @ts-expect-error ts-migrate(7006) FIXME: Parameter 'colSchema' implicitly has an 'any' type... Remove this comment to see the full error message export function _normalizeCategoricalSchema(colSchema, col) { + /* + Ensure all enum schema types have a categories array, that + the categories array contains all unique values in the data + array, AND that the array is sorted. + + Note that the back-end will not always set this hint, so we + must assume it may be incorrect and/or missing. + */ const { type, writable } = colSchema; if ( type === "string" ||