From 023ae10822847b62126e83499feb38462f7500c4 Mon Sep 17 00:00:00 2001 From: maniarathi Date: Sat, 19 Jun 2021 09:37:05 -0700 Subject: [PATCH] Update unit tests for CXG conversion to check for actual content rather than file names alone which have changed with the recent 0.9 release of tiledb's python package. (#2249) * Update unit tests for CXG conversion to check for actual content rather than file names alone which have changed with the recent 0.9 release of tiledb's python package. * Some cleanup * Undo a bad line --- .../unit/converters/test_h5ad_data_file.py | 124 ++++++++++-------- 1 file changed, 70 insertions(+), 54 deletions(-) diff --git a/backend/test/test_czi_hosted/unit/converters/test_h5ad_data_file.py b/backend/test/test_czi_hosted/unit/converters/test_h5ad_data_file.py index d0e0c6da..c750ca1e 100644 --- a/backend/test/test_czi_hosted/unit/converters/test_h5ad_data_file.py +++ b/backend/test/test_czi_hosted/unit/converters/test_h5ad_data_file.py @@ -8,6 +8,7 @@ from uuid import uuid4 import anndata import numpy as np from pandas import Series, DataFrame +import tiledb from backend.czi_hosted.common.corpora import CorporaConstants from backend.czi_hosted.converters.h5ad_data_file import H5ADDataFile @@ -129,25 +130,25 @@ class TestH5ADDataFile(unittest.TestCase): h5ad_file = H5ADDataFile(self.sample_h5ad_filename, use_corpora_schema=False) h5ad_file.to_cxg(self.sample_output_directory, 100) - self._validate_expected_generated_list_of_tiledb_files() + self._validate_cxg_and_h5ad_content_match(self.sample_h5ad_filename, self.sample_output_directory, True) def test__to_cxg__simple_anndata_with_corpora_and_sparse(self): h5ad_file = H5ADDataFile(self.sample_h5ad_filename) h5ad_file.to_cxg(self.sample_output_directory, 100) - self._validate_expected_generated_list_of_tiledb_files() + self._validate_cxg_and_h5ad_content_match(self.sample_h5ad_filename, self.sample_output_directory, True) def test__to_cxg__simple_anndata_no_corpora_and_dense(self): h5ad_file = H5ADDataFile(self.sample_h5ad_filename, use_corpora_schema=False) h5ad_file.to_cxg(self.sample_output_directory, 0) - self._validate_expected_generated_list_of_tiledb_files() + self._validate_cxg_and_h5ad_content_match(self.sample_h5ad_filename, self.sample_output_directory, False) def test__to_cxg__simple_anndata_with_corpora_and_dense(self): h5ad_file = H5ADDataFile(self.sample_h5ad_filename) h5ad_file.to_cxg(self.sample_output_directory, 0) - self._validate_expected_generated_list_of_tiledb_files() + self._validate_cxg_and_h5ad_content_match(self.sample_h5ad_filename, self.sample_output_directory, False) def test__to_cxg__with_sparse_column_encoding(self): anndata = self._create_sample_anndata_dataset() @@ -157,69 +158,84 @@ class TestH5ADDataFile(unittest.TestCase): h5ad_file = H5ADDataFile(sparse_with_column_shift_filename) h5ad_file.to_cxg(self.sample_output_directory, 50) - self._validate_expected_generated_list_of_tiledb_files(has_column_encoding=True) + self._validate_cxg_and_h5ad_content_match( + sparse_with_column_shift_filename, self.sample_output_directory, False, has_column_encoding=True + ) # Clean up remove(sparse_with_column_shift_filename) - def _validate_expected_generated_list_of_tiledb_files(self, has_column_encoding=False): - ( - expected_directories, - expected_obs_files, - expected_var_files, - ) = self._get_expected_generated_list_of_tiledb_files() + def _validate_cxg_and_h5ad_content_match(self, h5ad_filename, cxg_directory, is_sparse, has_column_encoding=False): + anndata_object = anndata.read_h5ad(h5ad_filename) - for directory in expected_directories: - self.assertTrue(path.isdir(directory)) + # Array locations + metadata_array_location = f"{cxg_directory}/cxg_group_metadata" + main_x_array_location = f"{cxg_directory}/X" + embedding_array_location = f"{cxg_directory}/emb" + specific_embedding_array_location = f"{self.sample_output_directory}/emb/awesome_embedding" + obs_array_location = f"{cxg_directory}/obs" + var_array_location = f"{cxg_directory}/var" + x_col_shift_array_location = f"{cxg_directory}/X_col_shift" - for obs_file in expected_obs_files: - expected_location_of_obs_file = f"{self.sample_output_directory}/obs/*/{obs_file}" - self.assertTrue(path.isfile(glob(expected_location_of_obs_file)[0])) - - for var_file in expected_var_files: - expected_location_of_var_file = f"{self.sample_output_directory}/var/*/{var_file}" - self.assertTrue(path.isfile(glob(expected_location_of_var_file)[0])) + # Assert CXG structure + self.assertEqual(tiledb.object_type(cxg_directory), "group") + self.assertEqual(tiledb.object_type(obs_array_location), "array") + self.assertEqual(tiledb.object_type(var_array_location), "array") + self.assertEqual(tiledb.object_type(main_x_array_location), "array") + self.assertEqual(tiledb.object_type(embedding_array_location), "group") + self.assertEqual(tiledb.object_type(specific_embedding_array_location), "array") if has_column_encoding: - self.assertTrue(path.isdir(f"{self.sample_output_directory}/X_col_shift")) + self.assertEqual(tiledb.object_type(x_col_shift_array_location), "array") - def _get_expected_generated_list_of_tiledb_files(self): + # Validate metadata + metadata_array = tiledb.DenseArray(metadata_array_location, mode="r") + self.assertIn("cxg_version", metadata_array.meta) - # Expected directories - metadata_directory = f"{self.sample_output_directory}/cxg_group_metadata" - main_x_directory = f"{self.sample_output_directory}/X" - overall_embedding_directory = f"{self.sample_output_directory}/emb" - specific_embedding_directory = f"{self.sample_output_directory}/emb/awesome_embedding" - obs_directory = f"{self.sample_output_directory}/obs" - var_directory = f"{self.sample_output_directory}/var" + # Validate obs index + obs_array = tiledb.DenseArray(obs_array_location, mode="r") + expected_index_data = anndata_object.obs.index.to_numpy() + index_name = json.loads(obs_array.meta["cxg_schema"])["index"] + actual_index_data = obs_array.query(attrs=[index_name])[:][index_name] + self.assertTrue(np.array_equal(expected_index_data, actual_index_data)) - # Obs files - obs_files = [] - obs_files.append("name_0.tdb") - obs_files.append("name_0_var.tdb") - obs_files.append("string_category.tdb") - obs_files.append("string_category_var.tdb") - obs_files.append("float_category.tdb") + # Validate obs columns + expected_columns = list(anndata_object.obs.columns.values) + for column_name in expected_columns: + expected_data = anndata_object.obs[column_name].to_numpy() + actual_data = obs_array.query(attrs=[column_name])[:][column_name] + self.assertTrue(np.array_equal(expected_data, actual_data)) - # Var files - var_files = [] - var_files.append("name_0.tdb") - var_files.append("name_0_var.tdb") - var_files.append("bool_category.tdb") - var_files.append("int_category.tdb") + # Validate var index + var_array = tiledb.DenseArray(var_array_location, mode="r") + expected_index_data = anndata_object.var.index.to_numpy() + index_name = json.loads(var_array.meta["cxg_schema"])["index"] + actual_index_data = var_array.query(attrs=[index_name])[:][index_name] + self.assertTrue(np.array_equal(expected_index_data, actual_index_data)) - return ( - [ - metadata_directory, - main_x_directory, - overall_embedding_directory, - specific_embedding_directory, - obs_directory, - var_directory, - ], - obs_files, - var_files, - ) + # Validate var columns + expected_columns = anndata_object.var.columns.values + for column_name in expected_columns: + expected_data = anndata_object.var[column_name].to_numpy() + actual_data = var_array.query(attrs=[column_name])[:][column_name] + self.assertTrue(np.array_equal(expected_data, actual_data)) + + # Validate embedding + expected_embedding_data = anndata_object.obsm.get("X_awesome_embedding") + embedding_array = tiledb.DenseArray(specific_embedding_array_location, mode="r") + actual_embedding_data = embedding_array[:, 0:2] + self.assertTrue(np.array_equal(expected_embedding_data, actual_embedding_data)) + + # Validate X matrix if not column shifted + if not has_column_encoding: + expected_x_data = anndata_object.X + if is_sparse: + x_array = tiledb.SparseArray(main_x_array_location, mode="r") + actual_x_data = np.reshape(x_array[:, :][""], expected_x_data.shape) + else: + x_array = tiledb.DenseArray(main_x_array_location, mode="r") + actual_x_data = x_array[:, :] + self.assertTrue(np.array_equal(expected_x_data, actual_x_data)) def _write_anndata_to_file(self, anndata): temporary_filename = f"{PROJECT_ROOT}/backend/test/fixtures/{uuid4()}.h5ad"