mirror of
https://github.com/chanzuckerberg/cellxgene.git
synced 2026-10-02 07:38:12 +08:00
Refactor czi_hosted and server into backend directory, pull common code into backend/common, refactor tests (#2102)
* move local_server -> backend/server server-> backend/czi_hosted, pull common code into backend/common update imports, tests and make commands
This commit is contained in:
Madison Dunitz
@@ -0,0 +1 @@
|
||||
|
||||
@@ -0,0 +1,82 @@
|
||||
import unittest
|
||||
import pandas as pd
|
||||
import numpy as np
|
||||
from scipy import sparse
|
||||
|
||||
import backend.test.decode_fbs as decode_fbs
|
||||
from backend.common.fbs.matrix import encode_matrix_fbs, decode_matrix_fbs
|
||||
|
||||
|
||||
class FbsTests(unittest.TestCase):
|
||||
"""Test Case for Matrix FBS data encode/decode """
|
||||
|
||||
def test_encode_boundary(self):
|
||||
""" test various boundary checks """
|
||||
|
||||
# row indexing is unsupported
|
||||
with self.assertRaises(ValueError):
|
||||
encode_matrix_fbs(matrix=pd.DataFrame(), row_idx=[])
|
||||
|
||||
# matrix must be 2D
|
||||
with self.assertRaises(ValueError):
|
||||
encode_matrix_fbs(matrix=np.zeros((3, 2, 1)))
|
||||
with self.assertRaises(ValueError):
|
||||
encode_matrix_fbs(matrix=np.ones((10,)))
|
||||
|
||||
def fbs_checks(self, fbs, dims, expected_types, expected_column_idx):
|
||||
d = decode_fbs.decode_matrix_FBS(fbs)
|
||||
self.assertEqual(d["n_rows"], dims[0])
|
||||
self.assertEqual(d["n_cols"], dims[1])
|
||||
self.assertIsNone(d["row_idx"])
|
||||
self.assertEqual(len(d["columns"]), dims[1])
|
||||
for i in range(0, len(d["columns"])):
|
||||
self.assertEqual(len(d["columns"][i]), dims[0])
|
||||
self.assertIsInstance(d["columns"][i], expected_types[i][0])
|
||||
if expected_types[i][1] is not None:
|
||||
self.assertEqual(d["columns"][i].dtype, expected_types[i][1])
|
||||
if expected_column_idx is not None:
|
||||
self.assertSetEqual(set(expected_column_idx), set(d["col_idx"]))
|
||||
|
||||
def test_encode_DataFrame(self):
|
||||
df = pd.DataFrame(
|
||||
data={
|
||||
"a": np.zeros((10,), dtype=np.float32),
|
||||
"b": np.ones((10,), dtype=np.int64),
|
||||
"c": np.array([i for i in range(0, 10)], dtype=np.uint16),
|
||||
"d": pd.Series(["x", "y", "z", "x", "y", "z", "a", "x", "y", "z"], dtype="category"),
|
||||
}
|
||||
)
|
||||
expected_types = ((np.ndarray, np.float32), (np.ndarray, np.int32), (np.ndarray, np.uint32), (list, None))
|
||||
fbs = encode_matrix_fbs(matrix=df, row_idx=None, col_idx=df.columns)
|
||||
self.fbs_checks(fbs, (10, 4), expected_types, ["a", "b", "c", "d"])
|
||||
|
||||
def test_encode_ndarray(self):
|
||||
arr = np.zeros((3, 2), dtype=np.float32)
|
||||
expected_types = ((np.ndarray, np.float32), (np.ndarray, np.float32), (np.ndarray, np.float32))
|
||||
fbs = encode_matrix_fbs(matrix=arr, row_idx=None, col_idx=None)
|
||||
self.fbs_checks(fbs, (3, 2), expected_types, None)
|
||||
|
||||
def test_encode_sparse(self):
|
||||
csc = sparse.csc_matrix(np.array([[0, 1, 2], [3, 0, 4]]))
|
||||
expected_types = ((np.ndarray, np.int32), (np.ndarray, np.int32), (np.ndarray, np.int32))
|
||||
fbs = encode_matrix_fbs(matrix=csc, row_idx=None, col_idx=None)
|
||||
self.fbs_checks(fbs, (2, 3), expected_types, None)
|
||||
|
||||
def test_roundtrip(self):
|
||||
dfSrc = pd.DataFrame(
|
||||
data={
|
||||
"a": np.zeros((10,), dtype=np.float32),
|
||||
"b": np.ones((10,), dtype=np.int64),
|
||||
"c": np.array([i for i in range(0, 10)], dtype=np.uint16),
|
||||
"d": pd.Series(["x", "y", "z", "x", "y", "z", "a", "x", "y", "z"], dtype="category"),
|
||||
}
|
||||
)
|
||||
dfDst = decode_matrix_fbs(encode_matrix_fbs(matrix=dfSrc, col_idx=dfSrc.columns))
|
||||
self.assertEqual(dfSrc.shape, dfDst.shape)
|
||||
self.assertEqual(set(dfSrc.columns), set(dfDst.columns))
|
||||
for c in dfSrc.columns:
|
||||
self.assertTrue(c in dfDst.columns)
|
||||
if isinstance(dfSrc[c], pd.Series):
|
||||
self.assertTrue(np.all(dfSrc[c] == dfDst[c]))
|
||||
else:
|
||||
self.assertEqual(dfSrc[c], dfDst[c])
|
||||
@@ -0,0 +1,41 @@
|
||||
import unittest
|
||||
|
||||
import anndata
|
||||
|
||||
from backend.common.colors import convert_color_to_hex_format, convert_anndata_category_colors_to_cxg_category_colors
|
||||
from backend.common.errors import ColorFormatException
|
||||
from backend.test import PROJECT_ROOT
|
||||
from backend.test.fixtures.fixtures import pbmc3k_colors
|
||||
|
||||
|
||||
class ColorsTest(unittest.TestCase):
|
||||
""" Test color helper functions """
|
||||
|
||||
def test_convert_color_to_hex_format(self):
|
||||
self.assertEqual(convert_color_to_hex_format("wheat"), "#f5deb3")
|
||||
self.assertEqual(convert_color_to_hex_format("WHEAT"), "#f5deb3")
|
||||
self.assertEqual(convert_color_to_hex_format((245, 222, 179)), "#f5deb3")
|
||||
self.assertEqual(convert_color_to_hex_format([245, 222, 179]), "#f5deb3")
|
||||
self.assertEqual(convert_color_to_hex_format("#f5deb3"), "#f5deb3")
|
||||
self.assertEqual(
|
||||
convert_color_to_hex_format([0.9607843137254902, 0.8705882352941177, 0.7019607843137254]), "#f5deb3"
|
||||
)
|
||||
for bad_input in ["foo", "BAR", "#AABB", "#AABBCCDD", "#AABBGG", (1, 2), [1, 2], (1, 2, 3, 4), [1, 2, 3, 4]]:
|
||||
with self.assertRaises(ColorFormatException):
|
||||
convert_color_to_hex_format(bad_input)
|
||||
|
||||
def test_anndata_colors_to_cxg_colors(self):
|
||||
# test standard behavior
|
||||
adata = self._get_h5ad()
|
||||
self.assertEqual(convert_anndata_category_colors_to_cxg_category_colors(adata), pbmc3k_colors)
|
||||
# test that invalid color formats raise an exception
|
||||
adata.uns["louvain_colors"][0] = "#NOTCOOL"
|
||||
with self.assertRaises(ColorFormatException):
|
||||
convert_anndata_category_colors_to_cxg_category_colors(adata)
|
||||
# test that colors without a matching obs category are skipped
|
||||
adata = self._get_h5ad()
|
||||
del adata.obs["louvain"]
|
||||
self.assertEqual(convert_anndata_category_colors_to_cxg_category_colors(adata), {})
|
||||
|
||||
def _get_h5ad(self):
|
||||
return anndata.read_h5ad(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad")
|
||||
@@ -0,0 +1,217 @@
|
||||
import unittest
|
||||
from time import time
|
||||
from unittest.mock import patch
|
||||
|
||||
import numpy as np
|
||||
from pandas import Series, DataFrame
|
||||
|
||||
from backend.common.utils.type_conversion_utils import (
|
||||
can_cast_to_float32,
|
||||
can_cast_to_int32,
|
||||
get_dtype_of_array,
|
||||
get_schema_type_hint_of_array,
|
||||
get_dtypes_and_schemas_of_dataframe,
|
||||
convert_pandas_series_to_numpy,
|
||||
)
|
||||
|
||||
|
||||
class TestTypeConversionUtils(unittest.TestCase):
|
||||
def test__can_cast_to_float32__string_is_false(self):
|
||||
array_to_convert = Series(data=["1", "2", "3"], dtype=str)
|
||||
|
||||
can_cast = can_cast_to_float32(array_to_convert.dtype, array_to_convert)
|
||||
|
||||
self.assertFalse(can_cast)
|
||||
|
||||
def test__can_cast_to_float32__float64_is_true_warning_outputted(self):
|
||||
array_to_convert = Series(data=[1, 2, 3], dtype=np.dtype(np.float64))
|
||||
|
||||
with self.assertLogs(level="WARN") as logger:
|
||||
can_cast = can_cast_to_float32(array_to_convert.dtype, array_to_convert)
|
||||
self.assertIn("may lose precision", logger.output[0])
|
||||
|
||||
self.assertTrue(can_cast)
|
||||
|
||||
@patch("logging.warning")
|
||||
def test__can_cast_to_float32__float32_is_false(self, mock_log_warning):
|
||||
array_to_convert = Series(data=[1, 2, 3], dtype=np.dtype(np.float32))
|
||||
|
||||
can_cast = can_cast_to_float32(array_to_convert.dtype, array_to_convert)
|
||||
|
||||
self.assertTrue(can_cast)
|
||||
assert not mock_log_warning.called
|
||||
|
||||
def test__can_cast_to_float32__categorical_float64_is_false(self):
|
||||
array_to_convert = Series(data=[1.1, 2.2, 3.3], dtype="category")
|
||||
|
||||
can_cast = can_cast_to_float32(array_to_convert.dtype, array_to_convert)
|
||||
|
||||
self.assertFalse(can_cast)
|
||||
|
||||
def test__can_cast_to_float32__categorical_int64_with_nans_is_true(self):
|
||||
array_to_convert = Series(data=[1, 2, np.NaN], dtype="category")
|
||||
|
||||
can_cast = can_cast_to_float32(array_to_convert.dtype, array_to_convert)
|
||||
|
||||
self.assertTrue(can_cast)
|
||||
|
||||
def test__can_cast_to_float_32__float_32_with_nans_is_true(self):
|
||||
array_to_convert = Series(data=[1, 2, np.NaN], dtype=np.dtype(np.float32))
|
||||
|
||||
can_cast = can_cast_to_float32(array_to_convert.dtype, array_to_convert)
|
||||
|
||||
self.assertTrue(can_cast)
|
||||
|
||||
def test__can_cast_to_int32__string_is_false(self):
|
||||
array_to_convert = Series(data=["1", "2", "3"], dtype=str)
|
||||
|
||||
can_cast = can_cast_to_int32(array_to_convert.dtype, array_to_convert)
|
||||
|
||||
self.assertFalse(can_cast)
|
||||
|
||||
def test__can_cast_to_int32__int64_is_true(self):
|
||||
array_to_convert = Series(data=["1", "2", "3"], dtype=np.dtype(np.int64))
|
||||
|
||||
can_cast = can_cast_to_int32(array_to_convert.dtype, array_to_convert)
|
||||
|
||||
self.assertTrue(can_cast)
|
||||
|
||||
def test__can_cast_to_int32__int16_is_true(self):
|
||||
array_to_convert = Series(data=["1", "2", "3"], dtype=np.dtype(np.int16))
|
||||
|
||||
can_cast = can_cast_to_int32(array_to_convert.dtype, array_to_convert)
|
||||
|
||||
self.assertTrue(can_cast)
|
||||
|
||||
def test__can_cast_to_int32__int64_with_large_value_is_false(self):
|
||||
array_to_convert = Series(data=["3000000000", "2", "3"], dtype=np.dtype(np.int64))
|
||||
|
||||
can_cast = can_cast_to_int32(array_to_convert.dtype, array_to_convert)
|
||||
|
||||
self.assertFalse(can_cast)
|
||||
|
||||
def test__can_cast_to_int32__int64_with_nans_is_false(self):
|
||||
array_to_convert = Series(data=[np.NaN, "2", "3"], dtype="category")
|
||||
|
||||
can_cast = can_cast_to_int32(array_to_convert.dtype, array_to_convert)
|
||||
|
||||
self.assertFalse(can_cast)
|
||||
|
||||
def test__get_dtype_of_array__supported_dtypes_return_as_expected(self):
|
||||
types = [np.float32, np.int32, np.bool_, str]
|
||||
expected_dtypes = [np.float32, np.int32, np.uint8, str]
|
||||
|
||||
for test_type_index in range(len(types)):
|
||||
with self.subTest(
|
||||
f"Testing get_dtype_of_array with type {types[test_type_index].__name__}", i=test_type_index
|
||||
):
|
||||
array = Series(data=[], dtype=types[test_type_index])
|
||||
self.assertEqual(get_dtype_of_array(array), expected_dtypes[test_type_index])
|
||||
|
||||
def test__get_dtype_of_array__categories_return_as_expected(self):
|
||||
array = Series(data=["a", "b", "c"], dtype="category")
|
||||
expected_dtype = str
|
||||
|
||||
actual_dtype = get_dtype_of_array(array)
|
||||
|
||||
self.assertEqual(expected_dtype, actual_dtype)
|
||||
|
||||
def test__get_dtype_of_array__unordered_integer_categories_return_as_expected(self):
|
||||
array = Series(data=[2, 3, 1, 3, 1, 2], dtype="category")
|
||||
expected_dtype = np.int32
|
||||
|
||||
actual_dtype = get_dtype_of_array(array)
|
||||
|
||||
self.assertEqual(expected_dtype, actual_dtype)
|
||||
|
||||
def test__get_dtype_of_array__castable_dtypes_return_as_expected(self):
|
||||
types = [np.float64, np.int64]
|
||||
expected_dtypes = [np.float32, np.int32]
|
||||
|
||||
for test_type_index in range(len(types)):
|
||||
with self.subTest(
|
||||
f"Testing get_dtype_of_array with castable type {types[test_type_index].__name__}", i=test_type_index
|
||||
):
|
||||
array = Series(data=[], dtype=types[test_type_index])
|
||||
self.assertEqual(get_dtype_of_array(array), expected_dtypes[test_type_index])
|
||||
|
||||
def test__get_dtype_of_array__unsupported_type_raises_exception(self):
|
||||
unsupported_array = Series(list([time() for _ in range(2)]), dtype="datetime64[ns]")
|
||||
|
||||
with self.assertRaises(TypeError) as exception_context:
|
||||
get_dtype_of_array(unsupported_array)
|
||||
|
||||
self.assertIn("unsupported", str(exception_context.exception))
|
||||
|
||||
def test__get_schema_type_hint_of_array__supported_dtypes_return_as_expected(self):
|
||||
types = [np.float32, np.int32, np.bool_, str]
|
||||
expected_schema_hints = [{"type": "float32"}, {"type": "int32"}, {"type": "boolean"}, {"type": "string"}]
|
||||
|
||||
for test_type_index in range(len(types)):
|
||||
with self.subTest(
|
||||
f"Testing get_schema_type_hint_of_array with type {types[test_type_index].__name__}", i=test_type_index
|
||||
):
|
||||
array = Series(data=[], dtype=types[test_type_index])
|
||||
self.assertEqual(get_schema_type_hint_of_array(array), expected_schema_hints[test_type_index])
|
||||
|
||||
def test__get_schema_type_hint_of_array__categories_return_as_expected(self):
|
||||
array = Series(data=["a", "b", "b"], dtype="category")
|
||||
expected_schema_hint = {"type": "categorical", "categories": ["a", "b"]}
|
||||
|
||||
actual_schema_hint = get_schema_type_hint_of_array(array)
|
||||
|
||||
self.assertEqual(expected_schema_hint, actual_schema_hint)
|
||||
|
||||
def test__get_schema_type_hint_of_array__castable_dtypes_return_as_expected(self):
|
||||
types = [np.float64, np.int64]
|
||||
expected_schema_hints = [{"type": "float32"}, {"type": "int32"}]
|
||||
|
||||
for test_type_index in range(len(types)):
|
||||
with self.subTest(
|
||||
f"Testing get_schema_type_hint_of_array with castable type {types[test_type_index].__name__}",
|
||||
i=test_type_index,
|
||||
):
|
||||
array = Series(data=[], dtype=types[test_type_index])
|
||||
self.assertEqual(get_schema_type_hint_of_array(array), expected_schema_hints[test_type_index])
|
||||
|
||||
def test__get_dtypes_and_schemas_of_dataframe__dtype_and_schema_returns_as_expected(self):
|
||||
float_array = Series(data=[1, 2, 3], dtype=np.dtype(np.float64))
|
||||
category_array = Series(data=["a", "b", "b"], dtype="category")
|
||||
dataframe = DataFrame({"float_array": float_array, "category_array": category_array})
|
||||
|
||||
expected_data_types_dict = {"float_array": np.float32, "category_array": str}
|
||||
expected_schema_type_hints_dict = {
|
||||
"float_array": {"type": "float32"},
|
||||
"category_array": {"type": "categorical", "categories": ["a", "b"]},
|
||||
}
|
||||
|
||||
actual_dataframe_data_types, actual_dataframe_schema_type_hints = get_dtypes_and_schemas_of_dataframe(dataframe)
|
||||
|
||||
self.assertEqual(expected_data_types_dict, actual_dataframe_data_types)
|
||||
self.assertEqual(expected_schema_type_hints_dict, actual_dataframe_schema_type_hints)
|
||||
|
||||
def test__convert_pandas_series_to_numpy__categorical_float64_to_float64_with_nans(self):
|
||||
expected_float_array = np.array([1.1, 2.2, np.NaN], dtype=np.float64)
|
||||
float_series = Series(data=[1.1, 2.2, np.NaN], dtype="category")
|
||||
|
||||
actual_float_array = convert_pandas_series_to_numpy(float_series, np.float64)
|
||||
|
||||
np.testing.assert_equal(expected_float_array, actual_float_array)
|
||||
|
||||
def test__convert_pandas_series_to_numpy__float64_to_float64(self):
|
||||
expected_float_array = np.array([1.1, 2.2], dtype=np.float64)
|
||||
float_series = Series(data=[1.1, 2.2], dtype=np.dtype(np.float64))
|
||||
|
||||
actual_float_array = convert_pandas_series_to_numpy(float_series, np.float64)
|
||||
|
||||
np.testing.assert_equal(expected_float_array, actual_float_array)
|
||||
|
||||
def test__convert_pandas_series_to_numpy__int64_to_int32_with_nans_throws_error(self):
|
||||
int_series = Series(data=[1, 2, np.NaN], dtype="category")
|
||||
|
||||
with self.assertLogs(level="ERROR") as logger:
|
||||
convert_pandas_series_to_numpy(int_series, np.int32)
|
||||
|
||||
self.assertIn(
|
||||
"Cannot convert a pandas Series object to an integer dtype if it contains NaNs", logger.output[0]
|
||||
)
|
||||
Reference in New Issue
Block a user