mirror of
https://github.com/chanzuckerberg/cellxgene.git
synced 2026-09-21 04:28:12 +08:00
269 lines
11 KiB
Python
269 lines
11 KiB
Python
import json
|
|
import unittest
|
|
from glob import glob
|
|
from os import popen, remove, path
|
|
from shutil import rmtree
|
|
from uuid import uuid4
|
|
|
|
import anndata
|
|
import numpy as np
|
|
from pandas import Series, DataFrame
|
|
|
|
from server.common.utils.corpora_constants import CorporaConstants
|
|
from server.converters.h5ad_data_file import H5ADDataFile
|
|
|
|
PROJECT_ROOT = popen("git rev-parse --show-toplevel").read().strip()
|
|
|
|
|
|
class TestH5ADDataFile(unittest.TestCase):
|
|
def setUp(self):
|
|
self.sample_anndata = self._create_sample_anndata_dataset()
|
|
self.sample_h5ad_filename = self._write_anndata_to_file(self.sample_anndata)
|
|
|
|
self.sample_output_directory = path.splitext(self.sample_h5ad_filename)[0] + ".cxg"
|
|
|
|
def tearDown(self):
|
|
if self.sample_h5ad_filename:
|
|
remove(self.sample_h5ad_filename)
|
|
|
|
if path.isdir(self.sample_output_directory):
|
|
rmtree(self.sample_output_directory)
|
|
|
|
def test__create_h5ad_data_file__non_h5ad_raises_exception(self):
|
|
non_h5ad_filename = "my_fancy_dataset.csv"
|
|
|
|
with self.assertRaises(Exception) as exception_context:
|
|
H5ADDataFile(non_h5ad_filename)
|
|
|
|
self.assertIn("File must be an H5AD", str(exception_context.exception))
|
|
|
|
def test__create_h5ad_data_file__assert_warning_outputted_if_dataset_title_or_about_given(self):
|
|
with self.assertLogs(level="WARN") as logger:
|
|
H5ADDataFile(
|
|
self.sample_h5ad_filename,
|
|
dataset_title="My Awesome Dataset",
|
|
dataset_about="http://www.awesomedataset.com",
|
|
use_corpora_schema=False,
|
|
)
|
|
|
|
self.assertIn("will override any metadata that is extracted", logger.output[0])
|
|
|
|
def test__create_h5ad_data_file__reads_anndata_successfully(self):
|
|
h5ad_file = H5ADDataFile(self.sample_h5ad_filename, use_corpora_schema=False)
|
|
|
|
self.assertTrue((h5ad_file.anndata.X == self.sample_anndata.X).all())
|
|
self.assertEqual(
|
|
h5ad_file.anndata.obs.sort_index(inplace=True), self.sample_anndata.obs.sort_index(inplace=True)
|
|
)
|
|
self.assertEqual(
|
|
h5ad_file.anndata.var.sort_index(inplace=True), self.sample_anndata.var.sort_index(inplace=True)
|
|
)
|
|
|
|
for key in h5ad_file.anndata.obsm.keys():
|
|
self.assertIn(key, self.sample_anndata.obsm.keys())
|
|
self.assertTrue((h5ad_file.anndata.obsm[key] == self.sample_anndata.obsm[key]).all())
|
|
|
|
for key in self.sample_anndata.obsm.keys():
|
|
self.assertIn(key, h5ad_file.anndata.obsm.keys())
|
|
self.assertTrue((h5ad_file.anndata.obsm[key] == self.sample_anndata.obsm[key]).all())
|
|
|
|
def test__create_h5ad_data_file__copies_index_of_obs_and_var_to_column(self):
|
|
h5ad_file = H5ADDataFile(self.sample_h5ad_filename, use_corpora_schema=False)
|
|
|
|
# The automatic name chosen for the index should be "name_0"
|
|
self.assertNotIn("name_0", self.sample_anndata.obs.columns)
|
|
self.assertIn("name_0", h5ad_file.obs.columns)
|
|
|
|
self.assertNotIn("name_0", self.sample_anndata.var.columns)
|
|
self.assertIn("name_0", h5ad_file.var.columns)
|
|
|
|
def test__create_h5ad_data_file__no_copy_if_obs_and_var_index_names_specified(self):
|
|
h5ad_file = H5ADDataFile(
|
|
self.sample_h5ad_filename,
|
|
use_corpora_schema=False,
|
|
obs_index_column_name="float_category",
|
|
vars_index_column_name="int_category",
|
|
)
|
|
|
|
self.assertNotIn("name_0", h5ad_file.obs.columns)
|
|
self.assertNotIn("name_0", h5ad_file.var.columns)
|
|
|
|
def test__create_h5ad_data_file__obs_and_var_index_names_specified_not_unique_raises_exception(self):
|
|
|
|
with self.assertRaises(Exception) as exception_context:
|
|
H5ADDataFile(
|
|
self.sample_h5ad_filename,
|
|
use_corpora_schema=False,
|
|
obs_index_column_name="float_category",
|
|
vars_index_column_name="bool_category",
|
|
)
|
|
|
|
self.assertIn("Please prepare data to contain unique values", str(exception_context.exception))
|
|
|
|
def test__create_h5ad_data_file__obs_and_var_index_names_specified_doesnt_exist_raises_exception(self):
|
|
with self.assertRaises(Exception) as exception_context:
|
|
H5ADDataFile(
|
|
self.sample_h5ad_filename,
|
|
use_corpora_schema=False,
|
|
obs_index_column_name="unknown_category",
|
|
vars_index_column_name="i_dont_exist",
|
|
)
|
|
|
|
self.assertIn("does not exist", str(exception_context.exception))
|
|
|
|
def test__create_h5ad_data_file__extract_about_and_title_from_dataset(self):
|
|
h5ad_file = H5ADDataFile(self.sample_h5ad_filename)
|
|
|
|
self.assertEqual(h5ad_file.dataset_title, "random_link_name")
|
|
self.assertEqual(h5ad_file.dataset_about, "www.link.com")
|
|
|
|
def test__create_h5ad_data_file__inputted_dataset_title_and_about_overrides_extracted(self):
|
|
h5ad_file = H5ADDataFile(
|
|
self.sample_h5ad_filename, dataset_about="override_about", dataset_title="override_title"
|
|
)
|
|
|
|
self.assertEqual(h5ad_file.dataset_title, "override_title")
|
|
self.assertEqual(h5ad_file.dataset_about, "override_about")
|
|
|
|
def test__to_cxg__simple_anndata_no_corpora_and_sparse(self):
|
|
h5ad_file = H5ADDataFile(self.sample_h5ad_filename, use_corpora_schema=False)
|
|
h5ad_file.to_cxg(self.sample_output_directory, 100)
|
|
|
|
self._validate_expected_generated_list_of_tiledb_files()
|
|
|
|
def test__to_cxg__simple_anndata_with_corpora_and_sparse(self):
|
|
h5ad_file = H5ADDataFile(self.sample_h5ad_filename)
|
|
h5ad_file.to_cxg(self.sample_output_directory, 100)
|
|
|
|
self._validate_expected_generated_list_of_tiledb_files()
|
|
|
|
def test__to_cxg__simple_anndata_no_corpora_and_dense(self):
|
|
h5ad_file = H5ADDataFile(self.sample_h5ad_filename, use_corpora_schema=False)
|
|
h5ad_file.to_cxg(self.sample_output_directory, 0)
|
|
|
|
self._validate_expected_generated_list_of_tiledb_files()
|
|
|
|
def test__to_cxg__simple_anndata_with_corpora_and_dense(self):
|
|
h5ad_file = H5ADDataFile(self.sample_h5ad_filename)
|
|
h5ad_file.to_cxg(self.sample_output_directory, 0)
|
|
|
|
self._validate_expected_generated_list_of_tiledb_files()
|
|
|
|
def test__to_cxg__with_sparse_column_encoding(self):
|
|
anndata = self._create_sample_anndata_dataset()
|
|
anndata.X = np.ones((3, 4))
|
|
sparse_with_column_shift_filename = self._write_anndata_to_file(anndata)
|
|
|
|
h5ad_file = H5ADDataFile(sparse_with_column_shift_filename)
|
|
h5ad_file.to_cxg(self.sample_output_directory, 50)
|
|
|
|
self._validate_expected_generated_list_of_tiledb_files(has_column_encoding=True)
|
|
|
|
# Clean up
|
|
remove(sparse_with_column_shift_filename)
|
|
|
|
def _validate_expected_generated_list_of_tiledb_files(self, has_column_encoding=False):
|
|
(
|
|
expected_directories,
|
|
expected_obs_files,
|
|
expected_var_files,
|
|
) = self._get_expected_generated_list_of_tiledb_files()
|
|
|
|
for directory in expected_directories:
|
|
self.assertTrue(path.isdir(directory))
|
|
|
|
for obs_file in expected_obs_files:
|
|
expected_location_of_obs_file = f"{self.sample_output_directory}/obs/*/{obs_file}"
|
|
self.assertTrue(path.isfile(glob(expected_location_of_obs_file)[0]))
|
|
|
|
for var_file in expected_var_files:
|
|
expected_location_of_var_file = f"{self.sample_output_directory}/var/*/{var_file}"
|
|
self.assertTrue(path.isfile(glob(expected_location_of_var_file)[0]))
|
|
|
|
if has_column_encoding:
|
|
self.assertTrue(path.isdir(f"{self.sample_output_directory}/X_col_shift"))
|
|
|
|
def _get_expected_generated_list_of_tiledb_files(self):
|
|
|
|
# Expected directories
|
|
metadata_directory = f"{self.sample_output_directory}/cxg_group_metadata"
|
|
main_x_directory = f"{self.sample_output_directory}/X"
|
|
overall_embedding_directory = f"{self.sample_output_directory}/emb"
|
|
specific_embedding_directory = f"{self.sample_output_directory}/emb/awesome_embedding"
|
|
obs_directory = f"{self.sample_output_directory}/obs"
|
|
var_directory = f"{self.sample_output_directory}/var"
|
|
|
|
# Obs files
|
|
obs_files = []
|
|
obs_files.append("name_0.tdb")
|
|
obs_files.append("name_0_var.tdb")
|
|
obs_files.append("string_category.tdb")
|
|
obs_files.append("string_category_var.tdb")
|
|
obs_files.append("float_category.tdb")
|
|
|
|
# Var files
|
|
var_files = []
|
|
var_files.append("name_0.tdb")
|
|
var_files.append("name_0_var.tdb")
|
|
var_files.append("bool_category.tdb")
|
|
var_files.append("int_category.tdb")
|
|
|
|
return (
|
|
[
|
|
metadata_directory,
|
|
main_x_directory,
|
|
overall_embedding_directory,
|
|
specific_embedding_directory,
|
|
obs_directory,
|
|
var_directory,
|
|
],
|
|
obs_files,
|
|
var_files,
|
|
)
|
|
|
|
def _write_anndata_to_file(self, anndata):
|
|
temporary_filename = f"{PROJECT_ROOT}/server/test/fixtures/{uuid4()}.h5ad"
|
|
anndata.write(temporary_filename)
|
|
|
|
return temporary_filename
|
|
|
|
def _create_sample_anndata_dataset(self):
|
|
# Create X
|
|
X = np.random.rand(3, 4)
|
|
|
|
# Create obs
|
|
random_string_category = Series(data=["a", "b", "b"], dtype="category")
|
|
random_float_category = Series(data=[3.2, 1.1, 2.2], dtype=np.float32)
|
|
obs_dataframe = DataFrame(
|
|
data={"string_category": random_string_category, "float_category": random_float_category}
|
|
)
|
|
obs = obs_dataframe
|
|
|
|
# Create vars
|
|
random_int_category = Series(data=[3, 1, 2, 4], dtype=np.int32)
|
|
random_bool_category = Series(data=[True, True, False, True], dtype=np.bool_)
|
|
var_dataframe = DataFrame(data={"int_category": random_int_category, "bool_category": random_bool_category})
|
|
var = var_dataframe
|
|
|
|
# Create embeddings
|
|
random_embedding = np.random.rand(3, 2)
|
|
obsm = {"X_awesome_embedding": random_embedding}
|
|
|
|
# Create uns corpora metadata
|
|
uns = {}
|
|
for metadata_field in CorporaConstants.REQUIRED_SIMPLE_METADATA_FIELDS:
|
|
uns[metadata_field] = "random"
|
|
|
|
for metadata_field in CorporaConstants.OPTIONAL_JSON_ENCODED_METADATA_FIELD:
|
|
uns[metadata_field] = json.dumps({"random_key": "random_value"})
|
|
|
|
# Need to carefully set the corpora schema versions in order for tests to pass.
|
|
uns["version"] = {"corpora_schema_version": "1.0.0", "corpora_encoding_version": "0.1.0"}
|
|
|
|
# Set project links to be a dictionary
|
|
uns["project_links"] = json.dumps(
|
|
[{"link_name": "random_link_name", "link_url": "www.link.com", "link_type": "SUMMARY"}]
|
|
)
|
|
|
|
return anndata.AnnData(X=X, obs=obs, var=var, obsm=obsm, uns=uns)
|