From 2cc02a84cbd47704d9fbeeec9b705678ce8ad5fa Mon Sep 17 00:00:00 2001 From: bmccandless Date: Fri, 20 Nov 2020 15:01:53 -0800 Subject: [PATCH] Convert float annotations if possible. (#1987) * Convert float annotations if possible. The client converts all arrays to floats. If a category contains integer labels, and that category is copied, it will contains floats (e.g 1.0 instead of 1). When that category is put back to the server, it fails in the tiledb code, which does not accept floats. The solution is to convert a float category to integer, if possible. #1984 * updates --- client/src/util/stateManager/matrix.js | 10 +++---- server/common/rest.py | 2 +- server/data_common/data_adaptor.py | 17 +++++++++++ .../unit/common/test_writable_annotation.py | 29 +++++++++++++++++++ 4 files changed, 52 insertions(+), 6 deletions(-) diff --git a/client/src/util/stateManager/matrix.js b/client/src/util/stateManager/matrix.js index e9a1e445..52717157 100644 --- a/client/src/util/stateManager/matrix.js +++ b/client/src/util/stateManager/matrix.js @@ -178,26 +178,26 @@ function promoteTypedArray(o) { */ if (isFpTypedArray(o) || Array.isArray(o)) return o; - let TyepdArrayCtor; + let TypedArrayCtor; switch (o.constructor) { case Int8Array: case Uint8Array: case Uint8ClampedArray: case Int16Array: case Uint16Array: - TyepdArrayCtor = Float32Array; + TypedArrayCtor = Float32Array; break; case Int32Array: case Uint32Array: - TyepdArrayCtor = Float64Array; + TypedArrayCtor = Float64Array; break; default: throw new Error("Unexpected data type returned from server."); } - if (o.constructor === TyepdArrayCtor) return o; - return new TyepdArrayCtor(o); + if (o.constructor === TypedArrayCtor) return o; + return new TypedArrayCtor(o); } export function matrixFBSToDataframe(arrayBuffers) { diff --git a/server/common/rest.py b/server/common/rest.py index 6760ee6c..ca140709 100644 --- a/server/common/rest.py +++ b/server/common/rest.py @@ -156,7 +156,7 @@ def annotations_put_fbs_helper(data_adaptor, fbs): new_label_df = decode_matrix_fbs(fbs) if not new_label_df.empty: - data_adaptor.check_new_labels(new_label_df) + new_label_df = data_adaptor.check_new_labels(new_label_df) annotations.write_labels(new_label_df, data_adaptor) diff --git a/server/data_common/data_adaptor.py b/server/data_common/data_adaptor.py index 36af8339..cb1e3c57 100644 --- a/server/data_common/data_adaptor.py +++ b/server/data_common/data_adaptor.py @@ -249,6 +249,23 @@ class DataAdaptor(metaclass=ABCMeta): if labels_df.shape[0] != shape[0]: raise ValueError("Labels file must have same number of rows as data file.") + # This will convert a float column that contains integer data into an integer type. + # This case can occur when a user makes a copy of a category that originally contained integer data. + # The client always copies array data to floats, therefore the copy will contain floats instead of integers. + # float data is not allowed as a categorical type. + if any([np.issubdtype(coltype.type, np.floating) for coltype in labels_df.dtypes]): + labels_df = labels_df.convert_dtypes() + for col, dtype in zip(labels_df, labels_df.dtypes): + if isinstance(dtype, pd.Int32Dtype): + labels_df[col] = labels_df[col].astype("int32") + if isinstance(dtype, pd.Int64Dtype): + labels_df[col] = labels_df[col].astype("int64") + + if any([np.issubdtype(coltype.type, np.floating) for coltype in labels_df.dtypes]): + raise ValueError("Columns may not have floating point types") + + return labels_df + def data_frame_to_fbs_matrix(self, filter, axis): """ Retrieves data 'X' and returns in a flatbuffer Matrix. diff --git a/server/test/unit/common/test_writable_annotation.py b/server/test/unit/common/test_writable_annotation.py index e4e47ac1..00cbd0f3 100644 --- a/server/test/unit/common/test_writable_annotation.py +++ b/server/test/unit/common/test_writable_annotation.py @@ -284,3 +284,32 @@ class WritableAnnotationTest(unittest.TestCase): all_col_schema["cat_B"], {"name": "cat_B", "type": "categorical", "categories": ["label_B"], "writable": True}, ) + + def test_put_float_data(self): + # verify that OBS PUTs (annotation_put_fbs) are accessible via + # GET (annotation_to_fbs_matrix) + + n_rows = self.data.get_shape()[0] + + # verifies that floating point with decimals fail. + fbs = make_fbs({"cat_F_FAIL": pd.Series([1.1] * n_rows, dtype=np.dtype("float"))}) + with self.assertRaises(ValueError) as exception_context: + res = self.annotation_put_fbs(fbs) + self.assertEqual(str(exception_context.exception), "Columns may not have floating point types") + + # verifies that floating point that can be converted to int passes + fbs = make_fbs({"cat_F_PASS": pd.Series([1.0] * n_rows, dtype="float")}) + res = self.annotation_put_fbs(fbs) + self.assertEqual(res, json.dumps({"status": "OK"})) + + # check read_labels + labels = self.annotations.read_labels(None) + fbsAll = self.data.annotation_to_fbs_matrix("obs", None, labels) + schema = schema_get_helper(self.data) + annotations = decode_fbs.decode_matrix_FBS(fbsAll) + self.assertEqual(annotations["n_rows"], n_rows) + all_col_schema = {c["name"]: c for c in schema["annotations"]["obs"]["columns"]} + self.assertEqual( + all_col_schema["cat_F_PASS"], + {"name": "cat_F_PASS", "type": "int32", "writable": True}, + )