mirror of
https://github.com/chanzuckerberg/cellxgene.git
synced 2026-10-01 20:08:11 +08:00
Move cxgtool into CLI and modularize conversion functions (#1701)
This commit is contained in:
maniarathi
@@ -423,11 +423,7 @@ class EndPointsAnndataAnnotations(unittest.TestCase, EndPointsAnnotations):
|
||||
cls.data, cls.tmp_dir, cls.annotations = data_with_tmp_annotations(
|
||||
MatrixDataType.H5AD, annotations_fixture=True
|
||||
)
|
||||
cls._setupClass(cls, [
|
||||
"--annotations-file",
|
||||
cls.annotations.output_file,
|
||||
cls.data.get_location(),
|
||||
])
|
||||
cls._setupClass(cls, ["--annotations-file", cls.annotations.output_file, cls.data.get_location(), ])
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
import unittest
|
||||
import anndata
|
||||
import json
|
||||
import tempfile
|
||||
import shutil
|
||||
import tempfile
|
||||
import unittest
|
||||
from http import HTTPStatus
|
||||
|
||||
import anndata
|
||||
import requests
|
||||
|
||||
from server.common.corpora import (
|
||||
@@ -104,7 +105,7 @@ class CorporaRESTAPITest(unittest.TestCase):
|
||||
"project_links": json.dumps([
|
||||
{"link_name": "test link", "link_type": "SUMMARY", "link_url": "https://a.u.r.l/"}
|
||||
]),
|
||||
"default_embedding": "X_tsne"
|
||||
"default_embedding": "X_tsne",
|
||||
}
|
||||
adata.uns.update(corpora_props)
|
||||
adata.write(path)
|
||||
|
||||
@@ -0,0 +1,149 @@
|
||||
import json
|
||||
import unittest
|
||||
from os import popen, path, mkdir
|
||||
from shutil import rmtree
|
||||
from uuid import uuid4
|
||||
|
||||
import numpy as np
|
||||
import tiledb
|
||||
from pandas import Series, DataFrame
|
||||
|
||||
from server.common.utils.cxg_generation_utils import (convert_dictionary_to_cxg_group, convert_dataframe_to_cxg_array,
|
||||
convert_ndarray_to_cxg_dense_array, convert_matrix_to_cxg_array)
|
||||
|
||||
PROJECT_ROOT = popen("git rev-parse --show-toplevel").read().strip()
|
||||
|
||||
|
||||
class TestCxgGenerationUtils(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.testing_cxg_temp_directory = f"{PROJECT_ROOT}/server/test/fixtures/{uuid4()}"
|
||||
mkdir(self.testing_cxg_temp_directory)
|
||||
|
||||
def tearDown(self):
|
||||
if path.isdir(self.testing_cxg_temp_directory):
|
||||
rmtree(self.testing_cxg_temp_directory)
|
||||
|
||||
def test__convert_dictionary_to_cxg_group__writes_successfully(self):
|
||||
random_dictionary = {"cookies": "chocolate_chip", "brownies": "chocolate", "cake": "double chocolate"}
|
||||
dictionary_name = "favorite_desserts"
|
||||
expected_array_directory = f"{self.testing_cxg_temp_directory}/{dictionary_name}"
|
||||
|
||||
convert_dictionary_to_cxg_group(self.testing_cxg_temp_directory, random_dictionary,
|
||||
group_metadata_name=dictionary_name)
|
||||
|
||||
array = tiledb.open(expected_array_directory)
|
||||
actual_stored_metadata = dict(array.meta.items())
|
||||
|
||||
self.assertTrue(path.isdir(expected_array_directory))
|
||||
self.assertTrue(isinstance(array, tiledb.DenseArray))
|
||||
self.assertEqual(random_dictionary, actual_stored_metadata)
|
||||
|
||||
def test__convert_dataframe_to_cxg_array__writes_successfully(self):
|
||||
random_int_category = Series(data=[3, 1, 2, 4], dtype=np.int64)
|
||||
random_bool_category = Series(data=[True, True, False, True], dtype=np.bool_)
|
||||
random_dataframe_name = f"random_dataframe_{uuid4()}"
|
||||
random_dataframe = DataFrame(data={"int_category": random_int_category, "bool_category": random_bool_category})
|
||||
|
||||
convert_dataframe_to_cxg_array(self.testing_cxg_temp_directory, random_dataframe_name, random_dataframe,
|
||||
"int_category", tiledb.Ctx())
|
||||
|
||||
expected_array_directory = f"{self.testing_cxg_temp_directory}/{random_dataframe_name}"
|
||||
expected_array_metadata = {
|
||||
"cxg_schema": json.dumps({"int_category": {"type": "int32"}, "bool_category": {"type": "boolean"},
|
||||
"index": "int_category"})}
|
||||
|
||||
actual_stored_dataframe_array = tiledb.open(expected_array_directory)
|
||||
actual_stored_dataframe_metadata = dict(actual_stored_dataframe_array.meta.items())
|
||||
|
||||
self.assertTrue(path.isdir(expected_array_directory))
|
||||
self.assertTrue(isinstance(actual_stored_dataframe_array, tiledb.DenseArray))
|
||||
self.assertDictEqual(expected_array_metadata, actual_stored_dataframe_metadata)
|
||||
self.assertTrue((actual_stored_dataframe_array[0:4]["int_category"] == random_int_category.to_numpy()).all())
|
||||
self.assertTrue((actual_stored_dataframe_array[0:4]["bool_category"] == random_bool_category.to_numpy()).all())
|
||||
|
||||
def test__convert_ndarray_to_cxg_dense_array__writes_successfully(self):
|
||||
ndarray = np.random.rand(3, 2)
|
||||
ndarray_name = f"{self.testing_cxg_temp_directory}/awesome_ndarray_{uuid4()}"
|
||||
|
||||
convert_ndarray_to_cxg_dense_array(ndarray_name, ndarray, tiledb.Ctx())
|
||||
|
||||
actual_stored_array = tiledb.open(ndarray_name)
|
||||
|
||||
self.assertTrue(path.isdir(ndarray_name))
|
||||
self.assertTrue(isinstance(actual_stored_array, tiledb.DenseArray))
|
||||
self.assertTrue((actual_stored_array[:, :] == ndarray).all())
|
||||
|
||||
def test__convert_matrix_to_cxg_array__dense_array_writes_successfully(self):
|
||||
matrix = np.float32(np.random.rand(3, 2))
|
||||
matrix_name = f"{self.testing_cxg_temp_directory}/awesome_matrix_{uuid4()}"
|
||||
|
||||
convert_matrix_to_cxg_array(matrix_name, matrix, False, tiledb.Ctx())
|
||||
|
||||
actual_stored_array = tiledb.open(matrix_name)
|
||||
|
||||
self.assertTrue(path.isdir(matrix_name))
|
||||
self.assertTrue(isinstance(actual_stored_array, tiledb.DenseArray))
|
||||
self.assertTrue((actual_stored_array[:, :] == matrix).all())
|
||||
|
||||
def test__convert_matrix_to_cxg_array__sparse_array_only_store_nonzeros_empty_array(self):
|
||||
matrix = np.zeros([3, 2])
|
||||
matrix_name = f"{self.testing_cxg_temp_directory}/awesome_zero_matrix_{uuid4()}"
|
||||
|
||||
convert_matrix_to_cxg_array(matrix_name, matrix, True, tiledb.Ctx())
|
||||
|
||||
actual_stored_array = tiledb.open(matrix_name)
|
||||
|
||||
self.assertTrue(path.isdir(matrix_name))
|
||||
self.assertTrue(isinstance(actual_stored_array, tiledb.SparseArray))
|
||||
self.assertTrue(actual_stored_array[:, :][''].size == 0)
|
||||
|
||||
def test__convert_matrix_to_cxg_array__sparse_array_only_store_nonzeros(self):
|
||||
matrix = np.zeros([3, 3])
|
||||
matrix[0, 0] = 1
|
||||
matrix[1, 1] = 1
|
||||
matrix[2, 2] = 2
|
||||
matrix_name = f"{self.testing_cxg_temp_directory}/awesome_sparse_matrix_{uuid4()}"
|
||||
|
||||
convert_matrix_to_cxg_array(matrix_name, matrix, True, tiledb.Ctx())
|
||||
|
||||
actual_stored_array = tiledb.open(matrix_name)
|
||||
|
||||
self.assertTrue(path.isdir(matrix_name))
|
||||
self.assertTrue(isinstance(actual_stored_array, tiledb.SparseArray))
|
||||
self.assertTrue(actual_stored_array[0, 0][''] == 1)
|
||||
self.assertTrue(actual_stored_array[1, 1][''] == 1)
|
||||
self.assertTrue(actual_stored_array[2, 2][''] == 2)
|
||||
self.assertTrue(actual_stored_array[:, :][''].size == 3)
|
||||
|
||||
def test__convert_matrix_to_cxg_array__sparse_array_with_column_encoding_empty_array(self):
|
||||
matrix_name = f"{self.testing_cxg_temp_directory}/awesome_column_shift_matrix_{uuid4()}"
|
||||
matrix = np.ones((3, 2))
|
||||
# The column shift will be equal to the matrix since subtracting the column shift from the matrix will create
|
||||
# a matrix of zeros which is sparse.
|
||||
column_shift = np.ones((3, 2))
|
||||
|
||||
convert_matrix_to_cxg_array(matrix_name, matrix, True, tiledb.Ctx(),
|
||||
column_shift_for_sparse_encoding=column_shift)
|
||||
|
||||
actual_stored_array = tiledb.open(matrix_name)
|
||||
|
||||
self.assertTrue(path.isdir(matrix_name))
|
||||
self.assertTrue(isinstance(actual_stored_array, tiledb.SparseArray))
|
||||
self.assertTrue(actual_stored_array[:, :][''].size == 0)
|
||||
|
||||
def test__convert_matrix_to_cxg_array__sparse_array_with_column_encoding_partial_array(self):
|
||||
matrix_name = f"{self.testing_cxg_temp_directory}/awesome_column_shift_matrix_{uuid4()}"
|
||||
matrix = np.ones((2, 2))
|
||||
# Only column shift the first column of ones.
|
||||
column_shift = np.array([[1, 0], [1, 0]])
|
||||
|
||||
convert_matrix_to_cxg_array(matrix_name, matrix, True, tiledb.Ctx(),
|
||||
column_shift_for_sparse_encoding=column_shift)
|
||||
|
||||
actual_stored_array = tiledb.open(matrix_name)
|
||||
|
||||
self.assertTrue(path.isdir(matrix_name))
|
||||
self.assertTrue(isinstance(actual_stored_array, tiledb.SparseArray))
|
||||
self.assertTrue(actual_stored_array[0, 1][''] == 1)
|
||||
self.assertTrue(actual_stored_array[1, 1][''] == 1)
|
||||
self.assertTrue(actual_stored_array[:, :][''].size == 2)
|
||||
@@ -2,10 +2,10 @@ import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
import numpy as np
|
||||
from pandas import Series
|
||||
from pandas import Series, DataFrame
|
||||
|
||||
from server.common.utils.type_conversion_utils import can_cast_to_float32, can_cast_to_int32, get_dtype_of_array, \
|
||||
get_schema_type_hint_of_array
|
||||
get_schema_type_hint_of_array, get_dtypes_and_schemas_of_dataframe
|
||||
|
||||
|
||||
class TestTypeConversionUtils(unittest.TestCase):
|
||||
@@ -119,3 +119,17 @@ class TestTypeConversionUtils(unittest.TestCase):
|
||||
i=test_type_index):
|
||||
array = Series(data=[], dtype=types[test_type_index])
|
||||
self.assertEqual(get_schema_type_hint_of_array(array), expected_schema_hints[test_type_index])
|
||||
|
||||
def test__get_dtypes_and_schemas_of_dataframe__dtype_and_schema_returns_as_expected(self):
|
||||
float_array = Series(data=[1, 2, 3], dtype=np.dtype(np.float64))
|
||||
category_array = Series(data=["a", "b", "b"], dtype="category")
|
||||
dataframe = DataFrame({"float_array": float_array, "category_array": category_array})
|
||||
|
||||
expected_data_types_dict = {"float_array": np.float32, "category_array": np.unicode}
|
||||
expected_schema_type_hints_dict = {"float_array": {"type": "float32"},
|
||||
"category_array": {"type": "categorical", "categories": ["a", "b"]}}
|
||||
|
||||
actual_dataframe_data_types, actual_dataframe_schema_type_hints = get_dtypes_and_schemas_of_dataframe(dataframe)
|
||||
|
||||
self.assertEqual(expected_data_types_dict, actual_dataframe_data_types)
|
||||
self.assertEqual(expected_schema_type_hints_dict, actual_dataframe_schema_type_hints)
|
||||
|
||||
@@ -1,14 +1,16 @@
|
||||
import os
|
||||
import tempfile
|
||||
import unittest
|
||||
from server.data_common.matrix_loader import MatrixDataLoader
|
||||
from server.test import PROJECT_ROOT, app_config, FIXTURES_ROOT
|
||||
|
||||
import numpy as np
|
||||
|
||||
import server.compute.diffexp_cxg as diffexp_cxg
|
||||
import server.compute.diffexp_generic as diffexp_generic
|
||||
from server.converters.cxgtool import write_cxg, create_cxg_group_metadata
|
||||
from server.test.performance.create_test_matrix import create_test_h5ad
|
||||
from server.converters.h5ad_data_file import H5ADDataFile
|
||||
from server.data_common.fbs.matrix import encode_matrix_fbs, decode_matrix_fbs
|
||||
import numpy as np
|
||||
import tempfile
|
||||
import os
|
||||
from server.data_common.matrix_loader import MatrixDataLoader
|
||||
from server.test import PROJECT_ROOT, app_config, FIXTURES_ROOT
|
||||
from server.test.performance.create_test_matrix import create_test_h5ad
|
||||
|
||||
|
||||
class DiffExpTest(unittest.TestCase):
|
||||
@@ -98,21 +100,22 @@ class DiffExpTest(unittest.TestCase):
|
||||
def sparse_diffexp(self, apply_col_shift):
|
||||
with tempfile.TemporaryDirectory() as dirname:
|
||||
# create a sparse matrix
|
||||
h5adfile = os.path.join(dirname, "sparse.h5ad")
|
||||
create_test_h5ad(h5adfile, 2000, 2000, 10, apply_col_shift)
|
||||
adaptor_anndata = self.load_dataset(h5adfile, extra_dataset_config=dict(embeddings__names=[]))
|
||||
adata = adaptor_anndata.data
|
||||
h5adfile_path = os.path.join(dirname, "sparse.h5ad")
|
||||
create_test_h5ad(h5adfile_path, 2000, 2000, 10, apply_col_shift)
|
||||
|
||||
h5ad_file_to_convert = H5ADDataFile(h5adfile_path, use_corpora_schema=False)
|
||||
|
||||
sparsename = os.path.join(dirname, "sparse.cxg")
|
||||
cxg_group_metadata = create_cxg_group_metadata(adata=adata, basefname="sparse.h5ad", title="sparse",)
|
||||
write_cxg(adata=adata, container=sparsename, cxg_group_metadata=cxg_group_metadata, sparse_threshold=11)
|
||||
h5ad_file_to_convert.to_cxg(sparsename, 11, True)
|
||||
|
||||
adaptor_anndata = self.load_dataset(h5adfile_path, extra_dataset_config=dict(embeddings__names=[]))
|
||||
|
||||
adaptor_sparse = self.load_dataset(sparsename)
|
||||
assert adaptor_sparse.open_array("X").schema.sparse
|
||||
assert adaptor_sparse.has_array("X_col_shift") == apply_col_shift
|
||||
|
||||
densename = os.path.join(dirname, "dense.cxg")
|
||||
cxg_group_metadata = create_cxg_group_metadata(adata=adata, basefname="dense.h5ad", title="dense",)
|
||||
write_cxg(adata=adata, container=densename, cxg_group_metadata=cxg_group_metadata, sparse_threshold=0)
|
||||
h5ad_file_to_convert.to_cxg(densename, True, 0)
|
||||
adaptor_dense = self.load_dataset(densename)
|
||||
assert not adaptor_dense.open_array("X").schema.sparse
|
||||
assert not adaptor_dense.has_array("X_col_shift")
|
||||
|
||||
@@ -1,41 +0,0 @@
|
||||
import shutil
|
||||
import unittest
|
||||
|
||||
import anndata
|
||||
|
||||
from server.common.data_locator import DataLocator
|
||||
from server.converters.cxgtool import write_cxg, create_cxg_group_metadata
|
||||
from server.data_cxg.cxg_adaptor import CxgAdaptor
|
||||
from server.test import PROJECT_ROOT, app_config, random_string
|
||||
from server.test.fixtures.fixtures import pbmc3k_colors
|
||||
|
||||
|
||||
class TestCxgAdaptor(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self.fixtures = []
|
||||
|
||||
def tearDown(self) -> None:
|
||||
try:
|
||||
for data_locator in self.fixtures:
|
||||
print("REMOVING ", data_locator)
|
||||
shutil.rmtree(data_locator)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
|
||||
def test_cxg_category_colors(self):
|
||||
data = self.convert_pbmc3k(extract_colors=True)
|
||||
self.assertEqual(data.get_colors(), pbmc3k_colors)
|
||||
data = self.convert_pbmc3k(extract_colors=False)
|
||||
self.assertEqual(data.get_colors(), {})
|
||||
|
||||
def convert_pbmc3k(self, **kwargs):
|
||||
rand_str = random_string(8)
|
||||
data_locator = f"/tmp/test_{rand_str}.cxg"
|
||||
self.fixtures.append(data_locator)
|
||||
source_h5ad = anndata.read_h5ad(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad")
|
||||
cxg_group_metadata = create_cxg_group_metadata(
|
||||
adata=source_h5ad, basefname="pbmc3k.h5ad", title="pbmc3k", **kwargs
|
||||
)
|
||||
write_cxg(adata=source_h5ad, container=data_locator, cxg_group_metadata=cxg_group_metadata)
|
||||
config = app_config(data_locator)
|
||||
return CxgAdaptor(DataLocator(data_locator), config)
|
||||
@@ -0,0 +1,235 @@
|
||||
import json
|
||||
import unittest
|
||||
from glob import glob
|
||||
from os import popen, remove, path
|
||||
from shutil import rmtree
|
||||
from uuid import uuid4
|
||||
|
||||
import anndata
|
||||
import numpy as np
|
||||
from pandas import Series, DataFrame
|
||||
|
||||
from server.common.utils.corpora_constants import CorporaConstants
|
||||
from server.converters.h5ad_data_file import H5ADDataFile
|
||||
|
||||
PROJECT_ROOT = popen("git rev-parse --show-toplevel").read().strip()
|
||||
|
||||
|
||||
class TestH5ADDataFile(unittest.TestCase):
|
||||
|
||||
def setUp(self):
|
||||
self.sample_anndata = self._create_sample_anndata_dataset()
|
||||
self.sample_h5ad_filename = self._write_anndata_to_file(self.sample_anndata)
|
||||
|
||||
self.sample_output_directory = path.splitext(self.sample_h5ad_filename)[0] + ".cxg"
|
||||
|
||||
def tearDown(self):
|
||||
if self.sample_h5ad_filename:
|
||||
remove(self.sample_h5ad_filename)
|
||||
|
||||
if path.isdir(self.sample_output_directory):
|
||||
rmtree(self.sample_output_directory)
|
||||
|
||||
def test__create_h5ad_data_file__non_h5ad_raises_exception(self):
|
||||
non_h5ad_filename = "my_fancy_dataset.csv"
|
||||
|
||||
with self.assertRaises(Exception) as exception_context:
|
||||
H5ADDataFile(non_h5ad_filename)
|
||||
|
||||
self.assertIn("File must be an H5AD", str(exception_context.exception))
|
||||
|
||||
def test__create_h5ad_data_file__assert_warning_outputted_if_dataset_title_or_about_given(self):
|
||||
with self.assertLogs(level="WARN") as logger:
|
||||
H5ADDataFile(self.sample_h5ad_filename, dataset_title="My Awesome Dataset",
|
||||
dataset_about="http://www.awesomedataset.com", use_corpora_schema=False)
|
||||
|
||||
self.assertIn("will override any metadata that is extracted", logger.output[0])
|
||||
|
||||
def test__create_h5ad_data_file__reads_anndata_successfully(self):
|
||||
h5ad_file = H5ADDataFile(self.sample_h5ad_filename, use_corpora_schema=False)
|
||||
|
||||
self.assertTrue((h5ad_file.anndata.X == self.sample_anndata.X).all())
|
||||
self.assertEqual(h5ad_file.anndata.obs.sort_index(inplace=True),
|
||||
self.sample_anndata.obs.sort_index(inplace=True))
|
||||
self.assertEqual(h5ad_file.anndata.var.sort_index(inplace=True),
|
||||
self.sample_anndata.var.sort_index(inplace=True))
|
||||
|
||||
for key in h5ad_file.anndata.obsm.keys():
|
||||
self.assertIn(key, self.sample_anndata.obsm.keys())
|
||||
self.assertTrue((h5ad_file.anndata.obsm[key] == self.sample_anndata.obsm[key]).all())
|
||||
|
||||
for key in self.sample_anndata.obsm.keys():
|
||||
self.assertIn(key, h5ad_file.anndata.obsm.keys())
|
||||
self.assertTrue((h5ad_file.anndata.obsm[key] == self.sample_anndata.obsm[key]).all())
|
||||
|
||||
def test__create_h5ad_data_file__copies_index_of_obs_and_var_to_column(self):
|
||||
h5ad_file = H5ADDataFile(self.sample_h5ad_filename, use_corpora_schema=False)
|
||||
|
||||
# The automatic name chosen for the index should be "name_0"
|
||||
self.assertNotIn("name_0", self.sample_anndata.obs.columns)
|
||||
self.assertIn("name_0", h5ad_file.obs.columns)
|
||||
|
||||
self.assertNotIn("name_0", self.sample_anndata.var.columns)
|
||||
self.assertIn("name_0", h5ad_file.var.columns)
|
||||
|
||||
def test__create_h5ad_data_file__no_copy_if_obs_and_var_index_names_specified(self):
|
||||
h5ad_file = H5ADDataFile(self.sample_h5ad_filename, use_corpora_schema=False,
|
||||
obs_index_column_name="float_category", vars_index_column_name="int_category")
|
||||
|
||||
self.assertNotIn("name_0", h5ad_file.obs.columns)
|
||||
self.assertNotIn("name_0", h5ad_file.var.columns)
|
||||
|
||||
def test__create_h5ad_data_file__obs_and_var_index_names_specified_not_unique_raises_exception(self):
|
||||
|
||||
with self.assertRaises(Exception) as exception_context:
|
||||
H5ADDataFile(self.sample_h5ad_filename, use_corpora_schema=False,
|
||||
obs_index_column_name="float_category", vars_index_column_name="bool_category")
|
||||
|
||||
self.assertIn("Please prepare data to contain unique values", str(exception_context.exception))
|
||||
|
||||
def test__create_h5ad_data_file__obs_and_var_index_names_specified_doesnt_exist_raises_exception(self):
|
||||
with self.assertRaises(Exception) as exception_context:
|
||||
H5ADDataFile(self.sample_h5ad_filename, use_corpora_schema=False,
|
||||
obs_index_column_name="unknown_category", vars_index_column_name="i_dont_exist")
|
||||
|
||||
self.assertIn("does not exist", str(exception_context.exception))
|
||||
|
||||
def test__create_h5ad_data_file__extract_about_and_title_from_dataset(self):
|
||||
h5ad_file = H5ADDataFile(self.sample_h5ad_filename)
|
||||
|
||||
self.assertEqual(h5ad_file.dataset_title, "random_link_name")
|
||||
self.assertEqual(h5ad_file.dataset_about, "www.link.com")
|
||||
|
||||
def test__create_h5ad_data_file__inputted_dataset_title_and_about_overrides_extracted(self):
|
||||
h5ad_file = H5ADDataFile(self.sample_h5ad_filename, dataset_about="override_about",
|
||||
dataset_title="override_title")
|
||||
|
||||
self.assertEqual(h5ad_file.dataset_title, "override_title")
|
||||
self.assertEqual(h5ad_file.dataset_about, "override_about")
|
||||
|
||||
def test__to_cxg__simple_anndata_no_corpora_and_sparse(self):
|
||||
h5ad_file = H5ADDataFile(self.sample_h5ad_filename, use_corpora_schema=False)
|
||||
h5ad_file.to_cxg(self.sample_output_directory, 100)
|
||||
|
||||
self._validate_expected_generated_list_of_tiledb_files()
|
||||
|
||||
def test__to_cxg__simple_anndata_with_corpora_and_sparse(self):
|
||||
h5ad_file = H5ADDataFile(self.sample_h5ad_filename)
|
||||
h5ad_file.to_cxg(self.sample_output_directory, 100)
|
||||
|
||||
self._validate_expected_generated_list_of_tiledb_files()
|
||||
|
||||
def test__to_cxg__simple_anndata_no_corpora_and_dense(self):
|
||||
h5ad_file = H5ADDataFile(self.sample_h5ad_filename, use_corpora_schema=False)
|
||||
h5ad_file.to_cxg(self.sample_output_directory, 0)
|
||||
|
||||
self._validate_expected_generated_list_of_tiledb_files()
|
||||
|
||||
def test__to_cxg__simple_anndata_with_corpora_and_dense(self):
|
||||
h5ad_file = H5ADDataFile(self.sample_h5ad_filename)
|
||||
h5ad_file.to_cxg(self.sample_output_directory, 0)
|
||||
|
||||
self._validate_expected_generated_list_of_tiledb_files()
|
||||
|
||||
def test__to_cxg__with_sparse_column_encoding(self):
|
||||
anndata = self._create_sample_anndata_dataset()
|
||||
anndata.X = np.ones((3, 4))
|
||||
sparse_with_column_shift_filename = self._write_anndata_to_file(anndata)
|
||||
|
||||
h5ad_file = H5ADDataFile(sparse_with_column_shift_filename)
|
||||
h5ad_file.to_cxg(self.sample_output_directory, 50)
|
||||
|
||||
self._validate_expected_generated_list_of_tiledb_files(has_column_encoding=True)
|
||||
|
||||
# Clean up
|
||||
remove(sparse_with_column_shift_filename)
|
||||
|
||||
def _validate_expected_generated_list_of_tiledb_files(self, has_column_encoding=False):
|
||||
expected_directories, expected_obs_files, expected_var_files = \
|
||||
self._get_expected_generated_list_of_tiledb_files()
|
||||
|
||||
for directory in expected_directories:
|
||||
self.assertTrue(path.isdir(directory))
|
||||
|
||||
for obs_file in expected_obs_files:
|
||||
expected_location_of_obs_file = f"{self.sample_output_directory}/obs/*/{obs_file}"
|
||||
self.assertTrue(path.isfile(glob(expected_location_of_obs_file)[0]))
|
||||
|
||||
for var_file in expected_var_files:
|
||||
expected_location_of_var_file = f"{self.sample_output_directory}/var/*/{var_file}"
|
||||
self.assertTrue(path.isfile(glob(expected_location_of_var_file)[0]))
|
||||
|
||||
if has_column_encoding:
|
||||
self.assertTrue(path.isdir(f"{self.sample_output_directory}/X_col_shift"))
|
||||
|
||||
def _get_expected_generated_list_of_tiledb_files(self):
|
||||
|
||||
# Expected directories
|
||||
metadata_directory = f"{self.sample_output_directory}/cxg_group_metadata"
|
||||
main_x_directory = f"{self.sample_output_directory}/X"
|
||||
overall_embedding_directory = f"{self.sample_output_directory}/emb"
|
||||
specific_embedding_directory = f"{self.sample_output_directory}/emb/awesome_embedding"
|
||||
obs_directory = f"{self.sample_output_directory}/obs"
|
||||
var_directory = f"{self.sample_output_directory}/var"
|
||||
|
||||
# Obs files
|
||||
obs_files = []
|
||||
obs_files.append("name_0.tdb")
|
||||
obs_files.append("name_0_var.tdb")
|
||||
obs_files.append("string_category.tdb")
|
||||
obs_files.append("string_category_var.tdb")
|
||||
obs_files.append("float_category.tdb")
|
||||
|
||||
# Var files
|
||||
var_files = []
|
||||
var_files.append("name_0.tdb")
|
||||
var_files.append("name_0_var.tdb")
|
||||
var_files.append("bool_category.tdb")
|
||||
var_files.append("int_category.tdb")
|
||||
|
||||
return [metadata_directory, main_x_directory, overall_embedding_directory, specific_embedding_directory,
|
||||
obs_directory, var_directory], obs_files, var_files
|
||||
|
||||
def _write_anndata_to_file(self, anndata):
|
||||
temporary_filename = f"{PROJECT_ROOT}/server/test/fixtures/{uuid4()}.h5ad"
|
||||
anndata.write(temporary_filename)
|
||||
|
||||
return temporary_filename
|
||||
|
||||
def _create_sample_anndata_dataset(self):
|
||||
# Create X
|
||||
X = np.random.rand(3, 4)
|
||||
|
||||
# Create obs
|
||||
random_string_category = Series(data=["a", "b", "b"], dtype="category")
|
||||
random_float_category = Series(data=[3.2, 1.1, 2.2], dtype=np.float32)
|
||||
obs_dataframe = DataFrame(
|
||||
data={"string_category": random_string_category, "float_category": random_float_category})
|
||||
obs = obs_dataframe
|
||||
|
||||
# Create vars
|
||||
random_int_category = Series(data=[3, 1, 2, 4], dtype=np.int32)
|
||||
random_bool_category = Series(data=[True, True, False, True], dtype=np.bool_)
|
||||
var_dataframe = DataFrame(data={"int_category": random_int_category, "bool_category": random_bool_category})
|
||||
var = var_dataframe
|
||||
|
||||
# Create embeddings
|
||||
random_embedding = np.random.rand(3, 2)
|
||||
obsm = {"X_awesome_embedding": random_embedding}
|
||||
|
||||
# Create uns corpora metadata
|
||||
uns = {}
|
||||
for metadata_field in CorporaConstants.REQUIRED_SIMPLE_METADATA_FIELDS:
|
||||
uns[metadata_field] = "random"
|
||||
|
||||
for metadata_field in CorporaConstants.REQUIRED_JSON_ENCODED_METADATA_FIELD:
|
||||
uns[metadata_field] = json.dumps({"random_key": "random_value"})
|
||||
|
||||
# Need to carefully set the corpora schema versions in order for tests to pass.
|
||||
uns["version"] = {"corpora_schema_version": "1.0.0", "corpora_encoding_version": "0.1.0"}
|
||||
|
||||
# Set project links to be a dictionary
|
||||
uns["project_links"] = json.dumps(
|
||||
[{"link_name": "random_link_name", "link_url": "www.link.com", "link_type": "SUMMARY"}])
|
||||
|
||||
return anndata.AnnData(X=X, obs=obs, var=var, obsm=obsm, uns=uns)
|
||||
Reference in New Issue
Block a user