fork/cellxgene
1
0
Fork 0
mirror of https://github.com/chanzuckerberg/cellxgene.git synced 2026-09-24 07:48:12 +08:00
Code Issues Packages Projects Releases Wiki Activity

clean up type inferencing (#2332)

Browse Source 
* unit tests for 64 bit conversion

* clean up type handling

* type inference tests

* more type inference fixes

* use schema to determine user intent for data typing

* stop using deprecated API

* fbs type encoding test

* add missing test

* add more tests

* correctly infer X type for CXG adaptor

* lint

* fix typo

* ts migration

* cleanup from PR review

* lint

* PR review changes
This commit is contained in:
Bruce Martin
2021-07-28 15:10:12 -07:00
committed by GitHub
parent 1140676106
commit 32f60a1547
15 changed files with 730 additions and 346 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/common/fbs/matrix.py
+7 -3
View File
@@ -5,6 +5,8 @@ import pandas as pd
from flatbuffers import Builder
from scipy import sparse
from backend.common.utils.type_conversion_utils import get_encoding_dtype_of_array
import backend.common.fbs.NetEncoding.Column as Column
import backend.common.fbs.NetEncoding.Float32Array as Float32Array
import backend.common.fbs.NetEncoding.Float64Array as Float64Array
@@ -14,6 +16,7 @@ import backend.common.fbs.NetEncoding.Matrix as Matrix
import backend.common.fbs.NetEncoding.TypedArray as TypedArray
import backend.common.fbs.NetEncoding.Uint32Array as Uint32Array
# Serialization helper
def serialize_column(builder, typed_arr):
""" Serialize NetEncoding.Column """
@@ -84,7 +87,7 @@ def serialize_typed_array(builder, source_array, encoding_info):
def column_encoding(arr):
column_encoding_type_map = {
# array protocol string: ( array_type, as_type )
np.dtype(np.float64).str: (TypedArray.TypedArray.Float64Array, np.float64),
np.dtype(np.float64).str: (TypedArray.TypedArray.Float32Array, np.float32),
np.dtype(np.float32).str: (TypedArray.TypedArray.Float32Array, np.float32),
np.dtype(np.float16).str: (TypedArray.TypedArray.Float32Array, np.float32),
np.dtype(np.int8).str: (TypedArray.TypedArray.Int32Array, np.int32),
@@ -98,7 +101,8 @@ def column_encoding(arr):
}
column_encoding_default = (TypedArray.TypedArray.JSONEncodedArray, "json")
return column_encoding_type_map.get(arr.dtype.str, column_encoding_default)
encoding_dtype = np.dtype(get_encoding_dtype_of_array(arr))
return column_encoding_type_map.get(encoding_dtype.str, column_encoding_default)
def index_encoding(arr):
@@ -198,7 +202,7 @@ def deserialize_typed_array(tarr):
arr.Init(u.Bytes, u.Pos)
narr = arr.DataAsNumpy()
if u_type == TypedArray.TypedArray.JSONEncodedArray:
narr = json.loads(narr.tostring().decode("utf-8"))
narr = json.loads(narr.tobytes().decode("utf-8"))
return narr
backend/common/utils/type_conversion_utils.py
+145 -112
View File
@@ -1,8 +1,42 @@
from typing import Union, Tuple
import logging
import numpy as np
import pandas as pd
"""
These routines drive all type inference for the schema generation and the
FBS (REST OTA) encoding. They are also used for CXG generation.
H5AD Type REST REST
(ndarray, Series, Index) FBS encoding schema type ERROR/exceptions
---------------------------- -------------- --------------- ----------------------
bool_/bool uint8 boolean
(u)int8, (u)int16, int32 int32 int32
uint32, (u)int64 int32 int32 CHECKS value bounds
float16, float32, float64 float32 float32[0]
categorical[T is numeric[4]]:
hasna = False T categorical[1]
hasna = True float32 categorical[1] CHECKS value bounds
categorical[T not numeric] JSON/str categorical[1,2]
(other object) JSON/str string
(all other) Always an ERROR[3]
Notes:
[0] IEEE format, includes non-finite numbers (NaN, Inf, ...)
[1] with NO categories enumerated (client side does it to handle rounding)
[2] NA (undefined) categories are assigned a JSON null value
[3] Includes all other numpy types: datetime, complex, etc.
[4] means float, int, uint (dtype.kind in ['i','u','f'])
"""
def get_dtypes_and_schemas_of_dataframe(dataframe: pd.DataFrame):
dtypes_by_column_name = {}
@@ -17,133 +51,132 @@ def get_dtypes_and_schemas_of_dataframe(dataframe: pd.DataFrame):
return dtypes_by_column_name, schema_type_hints_by_column_name
def get_dtype_of_array(array: pd.Series):
return get_dtype_and_schema_of_array(array)[0]
def get_encoding_dtype_of_array(array: Union[np.ndarray, pd.Series, pd.Index]) -> np.dtype:
return _get_type_info(array)[0]
def get_schema_type_hint_of_array(array: pd.Series):
return get_dtype_and_schema_of_array(array)[1]
def get_schema_type_hint_of_array(array: Union[np.ndarray, pd.Series, pd.Index]) -> dict:
return _get_type_info(array)[1]
def get_dtype_and_schema_of_array(array: pd.Series):
return (
get_dtype_from_dtype(array.dtype, array_values=array),
get_schema_type_hint_from_dtype(array.dtype, array_values=array),
)
def get_dtype_and_schema_of_array(array: Union[np.ndarray, pd.Series, pd.Index]) -> Tuple[np.dtype, dict]:
"""Return tuple (encoding_dtype, schema_type_hint)"""
return _get_type_info(array)
def get_dtype_from_dtype(dtype, array_values=None):
def get_schema_type_hint_from_dtype(dtype) -> dict:
res = _get_type_info_from_dtype(dtype)
if res is None:
raise TypeError(f"Annotations of type {dtype} are unsupported.")
else:
return res[1]
def _get_type_info_from_dtype(dtype) -> Union[Tuple[np.dtype, dict], None]:
"""
Given a data type, finds the equivalent data type that the array should be encoded as. Notably, this is relevant
for 64 bit values which will get downcast to 32 bit.
Best-effort to determine encoding type and schema hint from a dtype.
If this is not possible, or the type is unsupported, return None.
This should be a subset of the cases which are supported by
_get_type_info(). The latter should be preferred if the array (values)
are available for typing.
"""
if dtype.kind == "b":
return (np.uint8, {"type": "boolean"})
dtype_name = dtype.name
dtype_kind = dtype.kind
if dtype_name == "bool":
return np.uint8
if dtype_name == "object" and dtype_kind == "O":
return str
if dtype_name == "category":
return get_dtype_from_dtype(dtype.categories.dtype, array_values)
if can_cast_to_int32(dtype, array_values):
return np.int32
if can_cast_to_float32(dtype, array_values):
return np.float32
if not can_cast_to_float32(dtype, array_values):
return np.float64
raise TypeError(f"Annotations of type {dtype} are unsupported.")
def get_schema_type_hint_from_dtype(dtype, array_values=None):
"""
Returns a dictionary that contains type hints about the data type given, especially if the data type is 64 bit
and will be downcast to 32 bit.
"""
dtype_name = dtype.name
dtype_kind = dtype.kind
if dtype == np.float32 or dtype == np.int32:
return {"type": dtype_name}
if dtype_name == "bool":
return {"type": "boolean"}
if dtype_name == "object" and dtype_kind == "O":
return {"type": "string"}
if dtype_name == "category":
return {"type": "categorical", "categories": dtype.categories.tolist()}
if can_cast_to_int32(dtype, array_values):
return {"type": "int32"}
if can_cast_to_float32(dtype, array_values):
return {"type": "float32"}
if dtype_kind == "f" and not can_cast_to_float32(dtype, array_values):
return {"type": "float64"}
raise TypeError(f"Annotations of type {dtype} are unsupported.")
def can_cast_to_float32(dtype, array_values):
"""
Optimistically returns True signifying that a type downcast to float32 is possible whenever the incoming type is
a float.
We also handle a special case here where the array is a Series object with integer categorical values AND NaNs.
Since NaNs are floating points in numpy, we upcast the integer array to float32 and return True.
"""
if dtype.kind == "f":
if not np.can_cast(dtype, np.float32):
logging.warning(f"Type {dtype.name} will be converted to 32 bit float and may lose precision.")
return True
if dtype.kind == "O" and array_values.hasnans:
return True
return False
def can_cast_to_int32(dtype, array_values=None):
"""
A type can be cast to 32 bit, overriding the numpy `cast_cast` function if the values in the array that are of
the higher precision type has values that are entirely within the range of the downcast type.
"""
# Since a NaN is technically a float, any array that contains NaNs cannot be cast to an integer so immediately
# return False.
if array_values.hasnans:
return False
# If the array is categorical, then we need to order the array values so that functions min and max that occur
# later, can function. They do not function on unordered categories.
ordered_array_values = array_values
if array_values.dtype.name == "category" and not array_values.cat.ordered:
ordered_array_values = array_values.cat.as_ordered()
if dtype.kind == "U":
return (np.dtype(str), {"type": "string"})
if dtype.kind in ["i", "u"]:
if np.can_cast(dtype, np.int32):
return True
ii32 = np.iinfo(np.int32)
if (
not ordered_array_values.empty
and (ordered_array_values.min() >= ii32.min and ordered_array_values.max() <= ii32.max)
or ordered_array_values.empty
):
return True
return (np.int32, {"type": "int32"})
if dtype.kind == "f":
_float64_warning(dtype)
return (np.float32, {"type": "float32"})
if dtype.kind == "O" and not dtype.name == "category":
return (np.dtype(str), {"type": "string"})
return None
def _get_type_info(array: Union[np.ndarray, pd.Series, pd.Index]) -> Tuple[np.dtype, dict]:
"""
Determine encoding type and schema hint from an array. This allows more
flexible casting than may be possible by using just the dtype, as it can
account for category types and array values.
"""
if (
not isinstance(array, np.ndarray)
and not isinstance(array, pd.Series)
and not isinstance(array, pd.Index)
and not hasattr(array, "dtype")
):
raise TypeError("Unsupported data type.")
dtype = array.dtype
res = _get_type_info_from_dtype(dtype)
if res is not None:
return res
if dtype.kind == "O":
if dtype.name == "category":
# Sometimes CategoricalDType can be encoded as int or float without further fuss.
# Do not specify the categories in the schema - let the client-side figure it out
# on its own. Utilize Series.to_numpy() to do casting that handles categorical
# NA/NaN (missing or undefined) categories.
if dtype.categories.dtype.kind in ["f", "i", "u"]:
return (
_get_type_info(array.to_numpy())[0],
{"type": "categorical"},
)
else:
return (np.dtype(str), {"type": "categorical", "categories": dtype.categories.to_list()})
# all other extension types are str-encoded
return (np.dtype(str), {"type": "string"})
if dtype.kind in ["i", "u"] and _can_cast_array_values_to_int32(array):
return (np.int32, {"type": "int32"})
if dtype.kind == "f":
_float64_warning(array.dtype)
return (np.float32, {"type": "float32"})
raise TypeError(f"Annotations of type {dtype} are unsupported.")
def _float64_warning(dtype):
"""
Warn the user if we are down-casting a float64 to float32, and may potentially lose information.
"""
if dtype.kind == "f" and not np.can_cast(dtype, np.float32):
logging.warning(f"Type {dtype.name} will be converted to 32 bit float and may lose precision.")
def _can_cast_array_values_to_int32(array: Union[np.ndarray, pd.Series, pd.Index]) -> bool:
"""
Return true if the (U)INT array values can be safely cast to int32. We allow size reducing
casts (ie, int64 to int32) if no actual values require the larger size (ie, actual values
can be represented by the smaller type).
"""
assert array.dtype.kind in ["u", "i"]
if np.can_cast(array.dtype, np.int32):
return True
if array.size == 0:
return True
int32_machine_limits = np.iinfo(np.int32)
if array.min() >= int32_machine_limits.min and array.max() <= int32_machine_limits.max:
return True
return False
def convert_pandas_series_to_numpy(series_to_convert: pd.Series, dtype):
if series_to_convert.hasnans and dtype == np.int32:
logging.error("Cannot convert a pandas Series object to an integer dtype if it contains NaNs.")
return series_to_convert.to_numpy(dtype)
def convert_string_to_value(value: str):
"""convert a string to value with the most appropriate type"""
if value.lower() == "true":
backend/czi_hosted/common/annotations/hosted_tiledb.py
+2 -2
View File
@@ -10,7 +10,7 @@ from flask import current_app
from backend.czi_hosted.common.annotations.annotations import Annotations
from backend.common.errors import AnnotationCategoryNameError
from backend.czi_hosted.common.utils.sanitization_utils import sanitize_values_in_list
from backend.common.utils.type_conversion_utils import get_dtypes_and_schemas_of_dataframe, get_dtype_of_array
from backend.common.utils.type_conversion_utils import get_dtypes_and_schemas_of_dataframe, get_encoding_dtype_of_array
from backend.czi_hosted.db.cellxgene_orm import Annotation
@@ -143,7 +143,7 @@ class AnnotationsHostedTileDB(Annotations):
# convert to tiledb datatypes
for col in df:
df[col] = df[col].astype(get_dtype_of_array(df[col]))
df[col] = df[col].astype(get_encoding_dtype_of_array(df[col]))
tiledb.from_pandas(uri, df, sparse=True)
else:
uri = ""
backend/czi_hosted/common/utils/cxg_generation_utils.py
+2 -2
View File
@@ -3,7 +3,7 @@ import json
import numpy as np
import tiledb
from backend.common.utils.type_conversion_utils import get_dtype_of_array, get_dtype_and_schema_of_array
from backend.common.utils.type_conversion_utils import get_encoding_dtype_of_array, get_dtype_and_schema_of_array
def convert_dictionary_to_cxg_group(cxg_container, metadata_dict, group_metadata_name="cxg_group_metadata"):
@@ -47,7 +47,7 @@ def convert_dataframe_to_cxg_array(cxg_container, dataframe_name, dataframe, ind
]
)
attrs = [
tiledb.Attr(name=column, dtype=get_dtype_of_array(dataframe[column]), filters=tiledb_filter)
tiledb.Attr(name=column, dtype=get_encoding_dtype_of_array(dataframe[column]), filters=tiledb_filter)
for column in dataframe
]
domain = tiledb.Domain(
backend/czi_hosted/data_anndata/anndata_adaptor.py
+13 -9
View File
@@ -66,11 +66,11 @@ class AnndataAdaptor(DataAdaptor):
@staticmethod
def _create_unique_column_name(df, col_name_prefix):
""" given the columns of a dataframe, and a name prefix, return a column name which
does not exist in the dataframe, AND which is prefixed by `prefix`
"""given the columns of a dataframe, and a name prefix, return a column name which
does not exist in the dataframe, AND which is prefixed by `prefix`
The approach is to append a numeric suffix, starting at zero and increasing by
one, until an unused name is found (eg, prefix_0, prefix_1, ...).
The approach is to append a numeric suffix, starting at zero and increasing by
one, until an unused name is found (eg, prefix_0, prefix_1, ...).
"""
suffix = 0
while f"{col_name_prefix}{suffix}" in df:
@@ -124,7 +124,11 @@ class AnndataAdaptor(DataAdaptor):
def _create_schema(self):
self.schema = {
"dataframe": {"nObs": self.cell_count, "nVar": self.gene_count, "type": str(self.data.X.dtype)},
"dataframe": {
"nObs": self.cell_count,
"nVar": self.gene_count,
**get_schema_type_hint_of_array(self.data.X),
},
"annotations": {
"obs": {"index": self.parameters.get("obs_names"), "columns": []},
"var": {"index": self.parameters.get("var_names"), "columns": []},
@@ -201,10 +205,10 @@ class AnndataAdaptor(DataAdaptor):
self.parameters.update({"diffexp_may_be_slow": True})
def _is_valid_layout(self, arr):
""" return True if this layout data is a valid array for front-end presentation:
* ndarray, dtype float/int/uint
* with shape (n_obs, >= 2)
* with all values finite or NaN (no +Inf or -Inf)
"""return True if this layout data is a valid array for front-end presentation:
* ndarray, dtype float/int/uint
* with shape (n_obs, >= 2)
* with all values finite or NaN (no +Inf or -Inf)
"""
is_valid = type(arr) == np.ndarray and arr.dtype.kind in "fiu"
is_valid = is_valid and arr.shape[0] == self.data.n_obs and arr.shape[1] >= 2
backend/czi_hosted/data_cxg/cxg_adaptor.py
+3 -2
View File
@@ -210,7 +210,8 @@ class CxgAdaptor(DataAdaptor):
if lfc_cutoff is None:
lfc_cutoff = self.dataset_config.diffexp__lfc_cutoff
return diffexp_cxg.diffexp_ttest(
adaptor=self, maskA=maskA, maskB=maskB, top_n=top_n, diffexp_lfc_cutoff=lfc_cutoff)
adaptor=self, maskA=maskA, maskB=maskB, top_n=top_n, diffexp_lfc_cutoff=lfc_cutoff
)
def get_colors(self):
if self.cxg_version == "0.0":
@@ -357,7 +358,7 @@ class CxgAdaptor(DataAdaptor):
shape = self.get_shape()
dtype = self.get_X_array_dtype()
dataframe = {"nObs": shape[0], "nVar": shape[1], "type": dtype.name}
dataframe = {"nObs": shape[0], "nVar": shape[1], **get_schema_type_hint_from_dtype(dtype)}
annotations = {}
for ax in ("obs", "var"):
backend/server/data_anndata/anndata_adaptor.py
+6 -2
View File
@@ -125,7 +125,11 @@ class AnndataAdaptor(DataAdaptor):
def _create_schema(self):
self.schema = {
"dataframe": {"nObs": self.cell_count, "nVar": self.gene_count, "type": str(self.data.X.dtype)},
"dataframe": {
"nObs": self.cell_count,
"nVar": self.gene_count,
**get_schema_type_hint_of_array(self.data.X),
},
"annotations": {
"obs": {"index": self.parameters.get("obs_names"), "columns": []},
"var": {"index": self.parameters.get("var_names"), "columns": []},
@@ -335,7 +339,7 @@ class AnndataAdaptor(DataAdaptor):
"""return the approximate distribution of the X matrix."""
if self.X_approximate_distribution is None:
"""Not yet evaluated."""
assert(self.dataset_config.X_approximate_distribution == "auto")
assert self.dataset_config.X_approximate_distribution == "auto"
self.data = self.data.to_memory() # loads data
self.X_approximate_distribution = estimate_distribution.estimate_approximate_distribution(self.data.X)
backend/test/fixtures/pbmc3k_64.h5ad
Vendored
BIN
View File
Binary file not shown.
backend/test/test_common/unit/fbs/test_matrix.py
+138 -4
View File
@@ -2,16 +2,20 @@ import unittest
import pandas as pd
import numpy as np
from scipy import sparse
from parameterized import parameterized_class
import json
import backend.test.decode_fbs as decode_fbs
from backend.test import decode_fbs
from backend.common.fbs.matrix import encode_matrix_fbs, decode_matrix_fbs
from backend.common.utils.type_conversion_utils import get_dtypes_and_schemas_of_dataframe
import backend.common.fbs as fbs
class FbsTests(unittest.TestCase):
"""Test Case for Matrix FBS data encode/decode """
"""Test Case for Matrix FBS data encode/decode"""
def test_encode_boundary(self):
""" test various boundary checks """
"""test various boundary checks"""
# row indexing is unsupported
with self.assertRaises(ValueError):
@@ -46,7 +50,7 @@ class FbsTests(unittest.TestCase):
"d": pd.Series(["x", "y", "z", "x", "y", "z", "a", "x", "y", "z"], dtype="category"),
}
)
expected_types = ((np.ndarray, np.float32), (np.ndarray, np.int32), (np.ndarray, np.uint32), (list, None))
expected_types = ((np.ndarray, np.float32), (np.ndarray, np.int32), (np.ndarray, np.int32), (list, None))
fbs = encode_matrix_fbs(matrix=df, row_idx=None, col_idx=df.columns)
self.fbs_checks(fbs, (10, 4), expected_types, ["a", "b", "c", "d"])
@@ -80,3 +84,133 @@ class FbsTests(unittest.TestCase):
self.assertTrue(np.all(dfSrc[c] == dfDst[c]))
else:
self.assertEqual(dfSrc[c], dfDst[c])
"""
Test type consistency between FBS encoding and the underlying schema hint.
Basic assertion: the FBS type returned by encode_matrix_fbs() will be consistent
with the schema hint returned by type_conversion_utils (which is in turn used
to create the client schema).
The following test cases are all dicts which contain the following keys:
- dataframe - the dataframe used as input for encode_matrix_fbs
- expected_fbs_types - upon success, dict of FBS column types expected (eg, Float32Array)
- expected_schema_hints - upon success, dict of schema hint
All are keyed by column name.
"""
# simple tests that we convert all ints to int32
int_dtypes = [np.dtype(d) for d in [np.int8, np.uint8, np.int16, np.uint16, np.int32, np.uint32, np.int64, np.uint64]]
int_test_cases = [
{
"dataframe": pd.DataFrame({dtype.name: np.zeros((10,), dtype=dtype) for dtype in int_dtypes}),
"expected_fbs_types": dict(
[(dtype.name, fbs.NetEncoding.TypedArray.TypedArray.Int32Array) for dtype in int_dtypes]
),
"expected_schema_hints": dict([(dtype.name, {"type": "int32"}) for dtype in int_dtypes]),
}
]
# simple tests that we convert all floats to float32
float_dtypes = [np.dtype(d) for d in [np.float16, np.float32, np.float64]]
float_test_cases = [
{
"dataframe": pd.DataFrame({dtype.name: np.zeros((10,), dtype=dtype) for dtype in float_dtypes}),
"expected_fbs_types": dict(
[(dtype.name, fbs.NetEncoding.TypedArray.TypedArray.Float32Array) for dtype in float_dtypes]
),
"expected_schema_hints": dict([(dtype.name, {"type": "float32"}) for dtype in float_dtypes]),
}
]
# boolean - should be encoded as an uint32
bool_dtypes = [np.dtype(d) for d in [np.bool_, bool]]
bool_test_cases = [
{
"dataframe": pd.DataFrame({dtype.name: np.ones((10,), dtype=dtype) for dtype in bool_dtypes}),
"expected_fbs_types": dict(
[(dtype.name, fbs.NetEncoding.TypedArray.TypedArray.Uint32Array) for dtype in bool_dtypes]
),
"expected_schema_hints": dict([(dtype.name, {"type": "boolean"}) for dtype in bool_dtypes]),
}
]
cat_test_cases = [
{
"dataframe": pd.DataFrame({"a": pd.Series(["a", "b", "c", "a", "b", "c"], dtype="category")}),
"expected_fbs_types": {"a": fbs.NetEncoding.TypedArray.TypedArray.JSONEncodedArray},
"expected_schema_hints": {"a": {"type": "categorical", "categories": ["a", "b", "c"]}},
},
{
"dataframe": pd.DataFrame(
{"a": pd.Series(["a", "b", "c", "a", "b", "c"], dtype="category").cat.remove_categories("b")}
),
"expected_fbs_types": {"a": fbs.NetEncoding.TypedArray.TypedArray.JSONEncodedArray},
"expected_schema_hints": {"a": {"type": "categorical", "categories": ["a", "c"]}},
},
{
"dataframe": pd.DataFrame({"a": pd.Series(np.arange(0, 10, dtype=np.int64), dtype="category")}),
"expected_fbs_types": {"a": fbs.NetEncoding.TypedArray.TypedArray.Int32Array},
"expected_schema_hints": {"a": {"type": "categorical"}},
},
{
"dataframe": pd.DataFrame(
{"a": pd.Series(np.arange(0, 10, dtype=np.int64), dtype="category").cat.remove_categories(2)}
),
"expected_fbs_types": {"a": fbs.NetEncoding.TypedArray.TypedArray.Float32Array},
"expected_schema_hints": {"a": {"type": "categorical"}},
},
{
"dataframe": pd.DataFrame({"a": pd.Series(np.arange(0, 10, dtype=np.float64), dtype="category")}),
"expected_fbs_types": {"a": fbs.NetEncoding.TypedArray.TypedArray.Float32Array},
"expected_schema_hints": {"a": {"type": "categorical"}},
},
{
"dataframe": pd.DataFrame(
{"a": pd.Series(np.arange(0, 10, dtype=np.float64), dtype="category").cat.remove_categories(2)}
),
"expected_fbs_types": {"a": fbs.NetEncoding.TypedArray.TypedArray.Float32Array},
"expected_schema_hints": {"a": {"type": "categorical"}},
},
]
test_cases = [
*int_test_cases,
*float_test_cases,
*bool_test_cases,
*cat_test_cases,
]
@parameterized_class(test_cases)
class TestTypeConversionConsistency(unittest.TestCase):
def test_type_conversion_consistency(self):
self.assertEqual(self.dataframe.shape[1], len(self.expected_fbs_types))
self.assertEqual(self.dataframe.shape[1], len(self.expected_schema_hints))
buf = encode_matrix_fbs(matrix=self.dataframe, col_idx=self.dataframe.columns)
encoding_dtypes, schema_hints = get_dtypes_and_schemas_of_dataframe(self.dataframe)
# check schema hints
# print(schema_hints)
# print(self.expected_schema_hints)
self.assertEqual(schema_hints, self.expected_schema_hints)
# inspect the FBS types
matrix = fbs.NetEncoding.Matrix.Matrix.GetRootAsMatrix(buf, 0)
columns_length = matrix.ColumnsLength()
self.assertEqual(columns_length, self.dataframe.shape[1])
self.assertEqual(matrix.ColIndexType(), fbs.NetEncoding.TypedArray.TypedArray.JSONEncodedArray)
col_labels_arr = fbs.NetEncoding.JSONEncodedArray.JSONEncodedArray()
col_labels_arr.Init(matrix.ColIndex().Bytes, matrix.ColIndex().Pos)
col_index_labels = json.loads(col_labels_arr.DataAsNumpy().tobytes().decode("utf-8"))
self.assertEqual(len(col_index_labels), self.dataframe.shape[1])
for col_idx in range(0, columns_length):
col_label = col_index_labels[col_idx]
col = matrix.Columns(col_idx)
col_type = col.UType()
self.assertEqual(self.expected_fbs_types[col_label], col_type)
backend/test/test_common/unit/utils/test_type_conversion_utils.py
+288 -181
View File
@@ -1,179 +1,22 @@
import unittest
from time import time
from unittest.mock import patch
import logging
from parameterized import parameterized_class
import numpy as np
import pandas as pd
from pandas import Series, DataFrame
from scipy import sparse
from backend.common.utils.type_conversion_utils import (
can_cast_to_float32,
can_cast_to_int32,
get_dtype_of_array,
get_encoding_dtype_of_array,
get_schema_type_hint_of_array,
get_dtypes_and_schemas_of_dataframe,
convert_pandas_series_to_numpy,
get_dtype_and_schema_of_array,
get_schema_type_hint_from_dtype,
)
class TestTypeConversionUtils(unittest.TestCase):
def test__can_cast_to_float32__string_is_false(self):
array_to_convert = Series(data=["1", "2", "3"], dtype=str)
can_cast = can_cast_to_float32(array_to_convert.dtype, array_to_convert)
self.assertFalse(can_cast)
def test__can_cast_to_float32__float64_is_true_warning_outputted(self):
array_to_convert = Series(data=[1, 2, 3], dtype=np.dtype(np.float64))
with self.assertLogs(level="WARN") as logger:
can_cast = can_cast_to_float32(array_to_convert.dtype, array_to_convert)
self.assertIn("may lose precision", logger.output[0])
self.assertTrue(can_cast)
@patch("logging.warning")
def test__can_cast_to_float32__float32_is_false(self, mock_log_warning):
array_to_convert = Series(data=[1, 2, 3], dtype=np.dtype(np.float32))
can_cast = can_cast_to_float32(array_to_convert.dtype, array_to_convert)
self.assertTrue(can_cast)
assert not mock_log_warning.called
def test__can_cast_to_float32__categorical_float64_is_false(self):
array_to_convert = Series(data=[1.1, 2.2, 3.3], dtype="category")
can_cast = can_cast_to_float32(array_to_convert.dtype, array_to_convert)
self.assertFalse(can_cast)
def test__can_cast_to_float32__categorical_int64_with_nans_is_true(self):
array_to_convert = Series(data=[1, 2, np.NaN], dtype="category")
can_cast = can_cast_to_float32(array_to_convert.dtype, array_to_convert)
self.assertTrue(can_cast)
def test__can_cast_to_float_32__float_32_with_nans_is_true(self):
array_to_convert = Series(data=[1, 2, np.NaN], dtype=np.dtype(np.float32))
can_cast = can_cast_to_float32(array_to_convert.dtype, array_to_convert)
self.assertTrue(can_cast)
def test__can_cast_to_int32__string_is_false(self):
array_to_convert = Series(data=["1", "2", "3"], dtype=str)
can_cast = can_cast_to_int32(array_to_convert.dtype, array_to_convert)
self.assertFalse(can_cast)
def test__can_cast_to_int32__int64_is_true(self):
array_to_convert = Series(data=[1, 2, 3], dtype=np.dtype(np.int64))
can_cast = can_cast_to_int32(array_to_convert.dtype, array_to_convert)
self.assertTrue(can_cast)
def test__can_cast_to_int32__int16_is_true(self):
array_to_convert = Series(data=[1, 2, 3], dtype=np.dtype(np.int16))
can_cast = can_cast_to_int32(array_to_convert.dtype, array_to_convert)
self.assertTrue(can_cast)
def test__can_cast_to_int32__int64_with_large_value_is_false(self):
array_to_convert = Series(data=[3000000000, 2, 3], dtype=np.dtype(np.int64))
can_cast = can_cast_to_int32(array_to_convert.dtype, array_to_convert)
self.assertFalse(can_cast)
def test__can_cast_to_int32__int64_with_nans_is_false(self):
array_to_convert = Series(data=[np.NaN, "2", "3"], dtype="category")
can_cast = can_cast_to_int32(array_to_convert.dtype, array_to_convert)
self.assertFalse(can_cast)
def test__get_dtype_of_array__supported_dtypes_return_as_expected(self):
types = [np.float32, np.int32, np.bool_, str]
expected_dtypes = [np.float32, np.int32, np.uint8, str]
for test_type_index in range(len(types)):
with self.subTest(
f"Testing get_dtype_of_array with type {types[test_type_index].__name__}", i=test_type_index
):
array = Series(data=[], dtype=types[test_type_index])
self.assertEqual(get_dtype_of_array(array), expected_dtypes[test_type_index])
def test__get_dtype_of_array__categories_return_as_expected(self):
array = Series(data=["a", "b", "c"], dtype="category")
expected_dtype = str
actual_dtype = get_dtype_of_array(array)
self.assertEqual(expected_dtype, actual_dtype)
def test__get_dtype_of_array__unordered_integer_categories_return_as_expected(self):
array = Series(data=[2, 3, 1, 3, 1, 2], dtype="category")
expected_dtype = np.int32
actual_dtype = get_dtype_of_array(array)
self.assertEqual(expected_dtype, actual_dtype)
def test__get_dtype_of_array__castable_dtypes_return_as_expected(self):
types = [np.float64, np.int64]
expected_dtypes = [np.float32, np.int32]
for test_type_index in range(len(types)):
with self.subTest(
f"Testing get_dtype_of_array with castable type {types[test_type_index].__name__}", i=test_type_index
):
array = Series(data=[], dtype=types[test_type_index])
self.assertEqual(get_dtype_of_array(array), expected_dtypes[test_type_index])
def test__get_dtype_of_array__unsupported_type_raises_exception(self):
unsupported_array = Series(list([time() for _ in range(2)]), dtype="datetime64[ns]")
with self.assertRaises(TypeError) as exception_context:
get_dtype_of_array(unsupported_array)
self.assertIn("unsupported", str(exception_context.exception))
def test__get_schema_type_hint_of_array__supported_dtypes_return_as_expected(self):
types = [np.float32, np.int32, np.bool_, str]
expected_schema_hints = [{"type": "float32"}, {"type": "int32"}, {"type": "boolean"}, {"type": "string"}]
for test_type_index in range(len(types)):
with self.subTest(
f"Testing get_schema_type_hint_of_array with type {types[test_type_index].__name__}", i=test_type_index
):
array = Series(data=[], dtype=types[test_type_index])
self.assertEqual(get_schema_type_hint_of_array(array), expected_schema_hints[test_type_index])
def test__get_schema_type_hint_of_array__categories_return_as_expected(self):
array = Series(data=["a", "b", "b"], dtype="category")
expected_schema_hint = {"type": "categorical", "categories": ["a", "b"]}
actual_schema_hint = get_schema_type_hint_of_array(array)
self.assertEqual(expected_schema_hint, actual_schema_hint)
def test__get_schema_type_hint_of_array__castable_dtypes_return_as_expected(self):
types = [np.float64, np.int64]
expected_schema_hints = [{"type": "float32"}, {"type": "int32"}]
for test_type_index in range(len(types)):
with self.subTest(
f"Testing get_schema_type_hint_of_array with castable type {types[test_type_index].__name__}",
i=test_type_index,
):
array = Series(data=[], dtype=types[test_type_index])
self.assertEqual(get_schema_type_hint_of_array(array), expected_schema_hints[test_type_index])
def test__get_dtypes_and_schemas_of_dataframe__dtype_and_schema_returns_as_expected(self):
float_array = Series(data=[1, 2, 3], dtype=np.dtype(np.float64))
category_array = Series(data=["a", "b", "b"], dtype="category")
@@ -190,28 +33,292 @@ class TestTypeConversionUtils(unittest.TestCase):
self.assertEqual(expected_data_types_dict, actual_dataframe_data_types)
self.assertEqual(expected_schema_type_hints_dict, actual_dataframe_schema_type_hints)
def test__convert_pandas_series_to_numpy__categorical_float64_to_float64_with_nans(self):
expected_float_array = np.array([1.1, 2.2, np.NaN], dtype=np.float64)
float_series = Series(data=[1.1, 2.2, np.NaN], dtype="category")
def test__get_schema_type_hint_from_dtype(self):
self.assertEqual(get_schema_type_hint_from_dtype(np.dtype(np.bool_)), {"type": "boolean"})
actual_float_array = convert_pandas_series_to_numpy(float_series, np.float64)
for dtype in [np.int8, np.int8, np.int16, np.uint16, np.int32]:
self.assertEqual(get_schema_type_hint_from_dtype(np.dtype(dtype)), {"type": "int32"})
for dtype in [np.uint32, np.int64, np.uint64]:
with self.assertRaises(TypeError):
get_schema_type_hint_from_dtype(np.dtype(dtype))
np.testing.assert_equal(expected_float_array, actual_float_array)
for dtype in [np.float16, np.float32, np.float64]:
self.assertEqual(get_schema_type_hint_from_dtype(np.dtype(dtype)), {"type": "float32"})
def test__convert_pandas_series_to_numpy__float64_to_float64(self):
expected_float_array = np.array([1.1, 2.2], dtype=np.float64)
float_series = Series(data=[1.1, 2.2], dtype=np.dtype(np.float64))
for dtype in [np.dtype(object), np.dtype(str)]:
self.assertEqual(get_schema_type_hint_from_dtype(dtype), {"type": "string"})
actual_float_array = convert_pandas_series_to_numpy(float_series, np.float64)
np.testing.assert_equal(expected_float_array, actual_float_array)
# Credit: https://stackoverflow.com/questions/35871815/python-3-unit-testing-assert-logger-not-called/64774103#64774103
class AssertNoLog:
def assertNoLogs(self, logger, level):
"""functions as a context manager. To be introduced in python 3.10"""
def test__convert_pandas_series_to_numpy__int64_to_int32_with_nans_throws_error(self):
int_series = Series(data=[1, 2, np.NaN], dtype="category")
class AssertNoLogsContext(unittest.TestCase):
def __init__(self, logger, level):
self.logger = logger
self.level = level
self.context = self.assertLogs(logger, level)
with self.assertLogs(level="ERROR") as logger:
convert_pandas_series_to_numpy(int_series, np.int32)
def __enter__(self):
"""enter self.assertLogs as context manager, and log something"""
self.initial_logmsg = "sole message"
self.cm = self.context.__enter__()
self.logger.log(self.level, self.initial_logmsg)
return self.cm
self.assertIn(
"Cannot convert a pandas Series object to an integer dtype if it contains NaNs", logger.output[0]
)
def __exit__(self, exc_type, exc_val, exc_tb):
"""cleanup logs, and then check nothing extra was logged"""
# assertLogs.__exit__ should never fail because of initial msg
self.context.__exit__(exc_type, exc_val, exc_tb)
if len(self.cm.output) > 1:
"""override any exception passed to __exit__"""
self.context._raiseFailure(
"logs of level {} or higher triggered on {} : {}".format(
logging.getLevelName(self.level), self.logger.name, self.cm.output[1:]
)
)
return AssertNoLogsContext(logger, level)
"""
See table of expected cases in type_conversion_utils.py.
This probes all edge cases. Each case is a dict containing keys:
- data - the array to be introspected
- throws - if not None, the expected Error (eg, TypeError)
- expected_encoding_dtype - upon success
- expected_schema_hint - upon success
- logs - if not None, specify expected log output
"""
bool_OK_cases = [
{
"data": data,
"expected_encoding_dtype": np.uint8,
"expected_schema_hint": {"type": "boolean"},
}
for data in [
np.array([0, 1, 0, 1], dtype=np.bool_),
pd.Series(np.array([0, 1, 0, 1], dtype=np.bool_)),
# pd.Index with bools doesn't really make any sense...and becomes dtype=object
]
]
int_OK_cases = [
{
"data": data,
"expected_encoding_dtype": np.int32,
"expected_schema_hint": {"type": "int32"},
}
for dtype in [np.int8, np.uint8, np.int16, np.uint16, np.int32, np.uint32, np.int64, np.uint64]
for data in [
np.arange(0, 1000, dtype=dtype),
pd.Series(np.arange(0, 1000, dtype=dtype)),
pd.Index(np.arange(0, 1000, dtype=dtype)),
sparse.csr_matrix((10, 100), dtype=dtype),
]
]
float_OK_cases = [
{
"data": data,
"expected_encoding_dtype": np.float32,
"expected_schema_hint": {"type": "float32"},
"logs": None if data.dtype != np.float64 else {"level": logging.WARNING, "output": "may lose precision"},
}
for dtype in [np.float16, np.float32, np.float64]
for data in [
np.arange(-128, 1000, dtype=dtype),
pd.Series(np.arange(-128, 1000, dtype=dtype)),
pd.Index(np.arange(-129, 1000, dtype=dtype)),
np.array([-np.nan, np.NINF, -1, np.NZERO, 0, np.PZERO, 1, np.PINF, np.nan], dtype=dtype),
np.array([np.finfo(dtype).min, 0, np.finfo(dtype).max], dtype=dtype),
sparse.csr_matrix((10, 100), dtype=dtype),
]
]
numeric_ERR_cases = [
{
"data": data,
"throws": TypeError,
}
for data in [
np.array([np.iinfo(np.int64).min, np.iinfo(np.int64).max], dtype=np.int64),
np.array([np.iinfo(np.uint64).min, np.iinfo(np.uint64).max], dtype=np.uint64),
np.array([np.iinfo(np.uint32).min, np.iinfo(np.uint32).max], dtype=np.uint32),
]
]
string_OK_cases = [
{
"data": data,
"expected_encoding_dtype": np.dtype(str),
"expected_schema_hint": {"type": "string"},
}
for data in [
np.array(["a", "b", "c"]),
np.array(["a", "b", "c"], dtype="object"),
pd.Series(["a", "b", "c"]),
pd.Index(["a", "b", "c"]),
np.array(["a", [], {}, None, True, False, 383.2], dtype="object"),
]
]
category_nonnumeric_OK_cases = [
{
"data": data,
"expected_encoding_dtype": np.dtype(str),
"expected_schema_hint": {"type": "categorical", "categories": data.dtype.categories.to_list()},
}
for data in [
pd.Series(["a", "b", "c"], dtype="category"),
pd.Series(["a", "b", "c", 0, 1, 2], dtype="category"),
pd.Series(["a", "b", "c"], dtype="category").cat.remove_categories(["b"]),
pd.Series(["a", "b", "c", 0, 1, 2], dtype="category").cat.remove_categories(["b", 0]),
]
]
category_numeric_OK_cases = [
# numeric, no NA/NaN, int
*[
{
"data": data,
"expected_encoding_dtype": np.int32,
"expected_schema_hint": {"type": "categorical"},
}
for dtype in [np.int8, np.uint8, np.int16, np.uint16, np.int32, np.uint32, np.int64, np.uint64]
for data in [
pd.Series(np.array([0, 1, 2], dtype=dtype), dtype="category"),
]
],
# numeric, no NA/NaN, float
*[
{
"data": data,
"expected_encoding_dtype": np.float32,
"expected_schema_hint": {"type": "categorical"},
"logs": {"level": logging.WARNING, "output": "may lose precision"},
}
for dtype in [np.float16, np.float32, np.float64]
for data in [
pd.Series(np.array([0, 1, 2], dtype=dtype), dtype="category"),
pd.Series(np.array([0, 1, 2], dtype=dtype), dtype="category").cat.remove_categories([1]),
pd.Categorical(np.array([0, 1, 2], dtype=dtype)),
]
],
# numeric, has NA-induced cast to float32
*[
{
"data": data,
"expected_encoding_dtype": np.float32,
"expected_schema_hint": {"type": "categorical"},
"logs": {"level": logging.WARNING, "output": "may lose precision"},
}
for dtype in [
np.int8,
np.uint8,
np.int16,
np.uint16,
np.int32,
np.uint32,
np.int64,
np.uint64,
np.float16,
np.float32,
np.float64,
]
for data in [
pd.Series(np.array([0, 1, 2], dtype=dtype), dtype="category").cat.remove_categories([1]),
pd.Categorical(np.array([0, 1, 2], dtype=dtype), categories=np.array([0, 1], dtype=dtype)),
]
],
]
category_ERR_cases = [
# catch expected categorical exceptions for Int64(etc) that have large values
{
"data": data,
"throws": TypeError,
}
for data in [
pd.Categorical(np.array([np.iinfo(np.int64).min, np.iinfo(np.int64).max], dtype=np.int64)),
pd.Categorical(np.array([np.iinfo(np.uint64).min, np.iinfo(np.uint64).max], dtype=np.uint64)),
pd.Categorical(np.array([np.iinfo(np.uint32).min, np.iinfo(np.uint32).max], dtype=np.uint32)),
]
]
object_OK_cases = [
{
"data": data,
"expected_encoding_dtype": np.dtype(str),
"expected_schema_hint": {"type": "string"},
}
for data in [
np.array(["a", True, 1, [], {}], dtype="object"),
pd.Series(["a", True, 1, [], {}], dtype="object"),
pd.Index(["a", True, 1, [], {}], dtype="object"),
]
]
err_cases = [
{"data": np.array, "throws": TypeError}
for data in [
np.ones((10,), dtype=np.complex64),
np.ones((10,), dtype=np.complex128),
np.array([b"foobar"], dtype=np.bytes_),
np.ones((10,), dtype=np.void),
np.arange("2005-02", "2005-03", dtype="datetime64[D]"),
np.arange("2005-02", "2005-03", dtype="datetime64[D]") - np.datetime64("2008-01-01"),
[],
{},
]
]
test_cases = [
*bool_OK_cases,
*int_OK_cases,
*float_OK_cases,
*numeric_ERR_cases,
*string_OK_cases,
*category_nonnumeric_OK_cases,
*category_numeric_OK_cases,
*category_ERR_cases,
*object_OK_cases,
*err_cases,
]
@parameterized_class(test_cases)
class TestTypeInference(unittest.TestCase, AssertNoLog):
def test_type_inference(self):
throws = getattr(self, "throws", None)
if throws:
with self.assertRaises(throws):
get_dtype_and_schema_of_array(self.data)
with self.assertRaises(throws):
get_encoding_dtype_of_array(self.data)
with self.assertRaises(throws):
get_schema_type_hint_of_array(self.data)
else:
logs = getattr(self, "logs", None)
if logs is not None:
with self.assertLogs(level=logs["level"]) as logger:
encoding_dtype, schema_hint = get_dtype_and_schema_of_array(self.data)
self.assertEqual(encoding_dtype, self.expected_encoding_dtype)
self.assertEqual(schema_hint, self.expected_schema_hint)
self.assertIn(logs["output"], logger.output[0])
else:
with self.assertNoLogs(logging.getLogger(), logging.WARNING):
encoding_dtype, schema_hint = get_dtype_and_schema_of_array(self.data)
self.assertEqual(encoding_dtype, self.expected_encoding_dtype)
self.assertEqual(schema_hint, self.expected_schema_hint)
# also test the other public API
self.assertEqual(get_encoding_dtype_of_array(self.data), self.expected_encoding_dtype)
self.assertEqual(get_schema_type_hint_of_array(self.data), self.expected_schema_hint)
backend/test/test_czi_hosted/unit/data_common/fbs/test_matrix.py
+1 -1
View File
@@ -46,7 +46,7 @@ class FbsTests(unittest.TestCase):
"d": pd.Series(["x", "y", "z", "x", "y", "z", "a", "x", "y", "z"], dtype="category"),
}
)
expected_types = ((np.ndarray, np.float32), (np.ndarray, np.int32), (np.ndarray, np.uint32), (list, None))
expected_types = ((np.ndarray, np.float32), (np.ndarray, np.int32), (np.ndarray, np.int32), (list, None))
fbs = encode_matrix_fbs(matrix=df, row_idx=None, col_idx=df.columns)
self.fbs_checks(fbs, (10, 4), expected_types, ["a", "b", "c", "d"])
backend/test/test_server/unit/common/test_api.py
+65 -20
View File
@@ -6,9 +6,13 @@ from http import HTTPStatus
import tempfile
from os import path
import hashlib
from os.path import basename, splitext
import pandas as pd
import requests
import numpy as np
from parameterized import parameterized_class
import backend.test.decode_fbs as decode_fbs
from backend.server.data_common.matrix_loader import MatrixDataType
@@ -44,6 +48,14 @@ class EndPoints(object):
len(result_data["schema"]["annotations"]["obs"]["columns"]), 6 if self.ANNOTATIONS_ENABLED else 5
)
# Check that all schema types are legal
legal_types = ["boolean", "string", "categorical", "float32", "int32"]
self.assertEqual(result_data["schema"]["dataframe"]["type"], "float32")
for column in result_data["schema"]["annotations"]["obs"]["columns"]:
self.assertIn(column["type"], legal_types)
for column in result_data["schema"]["annotations"]["var"]["columns"]:
self.assertIn(column["type"], legal_types)
def test_config(self):
endpoint = "config"
url = f"{self.URL_BASE}{endpoint}"
@@ -52,7 +64,12 @@ class EndPoints(object):
self.assertEqual(result.headers["Content-Type"], "application/json")
result_data = result.json()
self.assertIn("library_versions", result_data["config"])
self.assertEqual(result_data["config"]["displayNames"]["dataset"], "pbmc3k")
if hasattr(self, "data_locator"):
title = splitext(basename(self.data_locator))[0]
else:
title = "pbmc3k"
self.assertEqual(result_data["config"]["displayNames"]["dataset"], title)
self.assertIsNotNone(result_data["config"]["parameters"])
def test_get_layout_fbs(self):
@@ -72,6 +89,8 @@ class EndPoints(object):
)
self.assertIsNone(df["row_idx"])
self.assertEqual(len(df["columns"]), df["n_cols"])
for column in df["columns"]:
self.assertEqual(column.dtype, np.float32)
def test_bad_filter(self):
endpoint = "data/var"
@@ -98,6 +117,9 @@ class EndPoints(object):
[obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain"]
+ (["cluster-test"] if self.ANNOTATIONS_ENABLED else []),
)
for column in df["columns"]:
if type(column) is np.ndarray:
self.assertIn(column.dtype, [np.float32, np.int32])
def test_get_annotations_obs_keys_fbs(self):
endpoint = "annotations/obs"
@@ -137,6 +159,9 @@ class EndPoints(object):
self.assertEqual(len(df["columns"]), df["n_cols"])
var_index_col_name = self.schema["schema"]["annotations"]["var"]["index"]
self.assertCountEqual(df["col_idx"], [var_index_col_name, "n_cells"])
for column in df["columns"]:
if type(column) is np.ndarray:
self.assertIn(column.dtype, [np.float32, np.int32])
def test_get_annotations_var_keys_fbs(self):
endpoint = "annotations/var"
@@ -207,6 +232,9 @@ class EndPoints(object):
self.assertIsNone(df["row_idx"])
self.assertEqual(len(df["columns"]), df["n_cols"])
self.assertListEqual(df["col_idx"].tolist(), [0, 1, 4])
for column in df["columns"]:
if type(column) is np.ndarray:
self.assertIn(column.dtype, [np.float32, np.int32])
def test_data_get_filter_fbs(self):
index_col_name = self.schema["schema"]["annotations"]["var"]["index"]
@@ -220,6 +248,9 @@ class EndPoints(object):
df = decode_fbs.decode_matrix_FBS(result.content)
self.assertEqual(df["n_rows"], 2638)
self.assertEqual(df["n_cols"], 1)
for column in df["columns"]:
if type(column) is np.ndarray:
self.assertIn(column.dtype, [np.float32, np.int32])
def test_data_get_unknown_filter_fbs(self):
index_col_name = self.schema["schema"]["annotations"]["var"]["index"]
@@ -246,6 +277,9 @@ class EndPoints(object):
df = decode_fbs.decode_matrix_FBS(result.content)
self.assertEqual(df["n_rows"], 2638)
self.assertEqual(df["n_cols"], 1)
for column in df["columns"]:
if type(column) is np.ndarray:
self.assertIn(column.dtype, [np.float32, np.int32])
def test_colors(self):
endpoint = "colors"
@@ -347,6 +381,14 @@ class EndPointsAnnotations(EndPoints):
self.assertTrue(matching_columns[0]["writable"])
@parameterized_class(
[
{"data_locator": f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad"},
{"data_locator": f"{FIXTURES_ROOT}/pbmc3k_64.h5ad"},
{"data_locator": f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad"},
{"data_locator": f"{FIXTURES_ROOT}/pbmc3k-CSR-gz.h5ad"},
]
)
class EndPointsAnndata(unittest.TestCase, EndPoints):
"""Test Case for endpoints"""
@@ -355,10 +397,13 @@ class EndPointsAnndata(unittest.TestCase, EndPoints):
@classmethod
def setUpClass(cls):
if cls == EndPointsAnndata:
raise unittest.SkipTest("`parameterized_class` bug")
cls._setupClass(
cls,
[
f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad",
cls.data_locator,
"--disable-annotations",
"--disable-gene-sets-save",
],
@@ -385,8 +430,8 @@ class EndPointsAnndata(unittest.TestCase, EndPoints):
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/json")
result_data = result.json()
self.assertEqual(len(result_data['positive']), 7)
self.assertEqual(len(result_data['negative']), 7)
self.assertEqual(len(result_data["positive"]), 7)
self.assertEqual(len(result_data["negative"]), 7)
def test_diff_exp_indices(self):
endpoint = "diffexp/obs"
@@ -401,8 +446,8 @@ class EndPointsAnndata(unittest.TestCase, EndPoints):
self.assertEqual(result.status_code, HTTPStatus.OK)
self.assertEqual(result.headers["Content-Type"], "application/json")
result_data = result.json()
self.assertEqual(len(result_data['positive']), 10)
self.assertEqual(len(result_data['negative']), 10)
self.assertEqual(len(result_data["positive"]), 10)
self.assertEqual(len(result_data["negative"]), 10)
def test_get_summaryvar(self):
index_col_name = self.schema["schema"]["annotations"]["var"]["index"]
@@ -561,23 +606,23 @@ class EndPointsAnnDataGenesets(unittest.TestCase, EndPoints):
"geneset_description": "",
"geneset_name": "summary test",
},
{'genes': [], 'geneset_description': '', 'geneset_name': 'geneset_to_delete'},
{'genes': [], 'geneset_description': '', 'geneset_name': 'geneset_to_edit'},
{"genes": [], "geneset_description": "", "geneset_name": "geneset_to_delete"},
{"genes": [], "geneset_description": "", "geneset_name": "geneset_to_edit"},
{
'genes': [{'gene_description': '', 'gene_symbol': 'RER1'}],
'geneset_description': '',
'geneset_name': 'fill_this_geneset'
"genes": [{"gene_description": "", "gene_symbol": "RER1"}],
"geneset_description": "",
"geneset_name": "fill_this_geneset",
},
{
'genes': [{'gene_description': '', 'gene_symbol': 'SIK1'}],
'geneset_description': '',
'geneset_name': 'empty_this_geneset'
"genes": [{"gene_description": "", "gene_symbol": "SIK1"}],
"geneset_description": "",
"geneset_name": "empty_this_geneset",
},
{
'genes': [{'gene_description': '', 'gene_symbol': 'SIK1'}],
'geneset_description': '',
'geneset_name': 'brush_this_gene'
}
"genes": [{"gene_description": "", "gene_symbol": "SIK1"}],
"geneset_description": "",
"geneset_name": "brush_this_gene",
},
],
"tid": 0,
},
@@ -680,7 +725,7 @@ brush_this_gene,,SIK1,\r
self.assertEqual(result.json(), test3)
def test_put_genesets_malformed(self):
""" test malformed submissions that we expect the backend to catch/tolerate """
"""test malformed submissions that we expect the backend to catch/tolerate"""
endpoint = "genesets"
url = f"{self.URL_BASE}{endpoint}"
@@ -690,7 +735,7 @@ brush_this_gene,,SIK1,\r
tid = original_data["tid"]
def test_case(test, expected_code, original_data):
""" check for expected error AND that no change was made to the original state """
"""check for expected error AND that no change was made to the original state"""
result = self.session.put(url, json=test)
self.assertEqual(result.status_code, expected_code)
result = self.session.get(url, headers={"Accept": "application/json"})
backend/test/test_server/unit/data_anndata/test_anndata_adaptor.py
+1
View File
@@ -36,6 +36,7 @@ Test the anndata adaptor using the pbmc3k data set.
(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad", True, "normal"),
(f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad", True, "normal"),
(f"{FIXTURES_ROOT}/pbmc3k-CSR-gz.h5ad", True, "normal"),
(f"{FIXTURES_ROOT}/pbmc3k_64.h5ad", False, "auto"), # 64 bit conversion tests
],
)
class AdaptorTest(unittest.TestCase):
client/src/annoMatrix/loader.ts
+51 -8
View File
@@ -272,7 +272,7 @@ export default class AnnoMatrixLoader extends AnnoMatrix {
}
const buffer = await promiseThrottle.priorityAdd(priority, doRequest);
const result = matrixFBSToDataframe(buffer);
let result = matrixFBSToDataframe(buffer);
if (!result || result.isEmpty()) throw Error("Unknown field/col");
const whereCacheUpdate = _whereCacheCreate(
@@ -281,18 +281,61 @@ export default class AnnoMatrixLoader extends AnnoMatrix {
result.colIndex.labels()
);
if (field === "obs") {
/* cough, cough - see comment on the function called */
_normalizeCategoricalSchema(
this.schema.annotations.obsByName[query],
result.col(query)
);
}
result = _responseTypeNormalization(field, query, this.schema, result);
return [whereCacheUpdate, result];
}
}
// @ts-expect-error ts-migrate(7006)
function _responseTypeNormalization(field, query, schema, response) {
/*
Schema-driven type normalization - there are a number of assumptions the front-end
makes about data typing, eg, that the schema will contain all categories in a categorical
column, that booleans are a JS array of true/false, etc.
The OTA format does not follow precisely the same conventions. This routine implements
the front-end conventions given the schema and an OTA data column.
*/
if (field === "obs" || field === "var") {
response = _booleanCast(field, query, schema, response);
}
if (field === "obs") {
/*
Note: this must be performed after any possible changes to string, boolean or categorical
columns. This routine relies on having access to any casts or other data transformations
made in this routine, above, in order to correctly determine schema updates.
*/
_normalizeCategoricalSchema(
schema.annotations.obsByName[query],
response.col(query)
);
}
return response;
}
// @ts-expect-error ts-migrate(7006)
function _booleanCast(field, query, schema, response) {
/*
Boolean columns may be transmitted as an int [0/1] or bool [true/false].
Force to JS Array of bool.
*/
if (field === "obs" || field === "var") {
// @ts-expect-error ts-migrate(7006)
response = response.mapColumns((colData, colIdx) => {
const colLabel = response.colIndex.getLabel(colIdx);
const colSchema = _getColumnSchema(schema, field, colLabel);
if (colSchema?.type === "boolean") {
const nColData = new Array(colData.length);
for (let i = 0; i < colData.length; i += 1) nColData[i] = !!colData[i];
return nColData;
}
return colData;
});
}
return response;
}
/*
Utility functions below
*/
client/src/annoMatrix/schema.ts
+8
View File
@@ -73,6 +73,14 @@ export function _isContinuousType(schema) {
// @ts-expect-error ts-migrate(7006) FIXME: Parameter 'colSchema' implicitly has an 'any' type... Remove this comment to see the full error message
export function _normalizeCategoricalSchema(colSchema, col) {
/*
Ensure all enum schema types have a categories array, that
the categories array contains all unique values in the data
array, AND that the array is sorted.
Note that the back-end will not always set this hint, so we
must assume it may be incorrect and/or missing.
*/
const { type, writable } = colSchema;
if (
type === "string" ||