fork/cellxgene
1
0
Fork 0
mirror of https://github.com/chanzuckerberg/cellxgene.git synced 2026-10-01 06:08:11 +08:00
Code Issues Packages Projects Releases Wiki Activity

Move cxgtool into CLI and modularize conversion functions (#1701)

Browse Source
This commit is contained in:
maniarathi
2020-08-17 17:28:29 -07:00
committed by GitHub
parent 1acb8e4a6f
commit 994c20c094
17 changed files with 1032 additions and 758 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/test/unit/common/utils/test_cxg_generation_utils.py
+149
View File
@@ -0,0 +1,149 @@
import json
import unittest
from os import popen, path, mkdir
from shutil import rmtree
from uuid import uuid4
import numpy as np
import tiledb
from pandas import Series, DataFrame
from server.common.utils.cxg_generation_utils import (convert_dictionary_to_cxg_group, convert_dataframe_to_cxg_array,
convert_ndarray_to_cxg_dense_array, convert_matrix_to_cxg_array)
PROJECT_ROOT = popen("git rev-parse --show-toplevel").read().strip()
class TestCxgGenerationUtils(unittest.TestCase):
def setUp(self):
self.testing_cxg_temp_directory = f"{PROJECT_ROOT}/server/test/fixtures/{uuid4()}"
mkdir(self.testing_cxg_temp_directory)
def tearDown(self):
if path.isdir(self.testing_cxg_temp_directory):
rmtree(self.testing_cxg_temp_directory)
def test__convert_dictionary_to_cxg_group__writes_successfully(self):
random_dictionary = {"cookies": "chocolate_chip", "brownies": "chocolate", "cake": "double chocolate"}
dictionary_name = "favorite_desserts"
expected_array_directory = f"{self.testing_cxg_temp_directory}/{dictionary_name}"
convert_dictionary_to_cxg_group(self.testing_cxg_temp_directory, random_dictionary,
group_metadata_name=dictionary_name)
array = tiledb.open(expected_array_directory)
actual_stored_metadata = dict(array.meta.items())
self.assertTrue(path.isdir(expected_array_directory))
self.assertTrue(isinstance(array, tiledb.DenseArray))
self.assertEqual(random_dictionary, actual_stored_metadata)
def test__convert_dataframe_to_cxg_array__writes_successfully(self):
random_int_category = Series(data=[3, 1, 2, 4], dtype=np.int64)
random_bool_category = Series(data=[True, True, False, True], dtype=np.bool_)
random_dataframe_name = f"random_dataframe_{uuid4()}"
random_dataframe = DataFrame(data={"int_category": random_int_category, "bool_category": random_bool_category})
convert_dataframe_to_cxg_array(self.testing_cxg_temp_directory, random_dataframe_name, random_dataframe,
"int_category", tiledb.Ctx())
expected_array_directory = f"{self.testing_cxg_temp_directory}/{random_dataframe_name}"
expected_array_metadata = {
"cxg_schema": json.dumps({"int_category": {"type": "int32"}, "bool_category": {"type": "boolean"},
"index": "int_category"})}
actual_stored_dataframe_array = tiledb.open(expected_array_directory)
actual_stored_dataframe_metadata = dict(actual_stored_dataframe_array.meta.items())
self.assertTrue(path.isdir(expected_array_directory))
self.assertTrue(isinstance(actual_stored_dataframe_array, tiledb.DenseArray))
self.assertDictEqual(expected_array_metadata, actual_stored_dataframe_metadata)
self.assertTrue((actual_stored_dataframe_array[0:4]["int_category"] == random_int_category.to_numpy()).all())
self.assertTrue((actual_stored_dataframe_array[0:4]["bool_category"] == random_bool_category.to_numpy()).all())
def test__convert_ndarray_to_cxg_dense_array__writes_successfully(self):
ndarray = np.random.rand(3, 2)
ndarray_name = f"{self.testing_cxg_temp_directory}/awesome_ndarray_{uuid4()}"
convert_ndarray_to_cxg_dense_array(ndarray_name, ndarray, tiledb.Ctx())
actual_stored_array = tiledb.open(ndarray_name)
self.assertTrue(path.isdir(ndarray_name))
self.assertTrue(isinstance(actual_stored_array, tiledb.DenseArray))
self.assertTrue((actual_stored_array[:, :] == ndarray).all())
def test__convert_matrix_to_cxg_array__dense_array_writes_successfully(self):
matrix = np.float32(np.random.rand(3, 2))
matrix_name = f"{self.testing_cxg_temp_directory}/awesome_matrix_{uuid4()}"
convert_matrix_to_cxg_array(matrix_name, matrix, False, tiledb.Ctx())
actual_stored_array = tiledb.open(matrix_name)
self.assertTrue(path.isdir(matrix_name))
self.assertTrue(isinstance(actual_stored_array, tiledb.DenseArray))
self.assertTrue((actual_stored_array[:, :] == matrix).all())
def test__convert_matrix_to_cxg_array__sparse_array_only_store_nonzeros_empty_array(self):
matrix = np.zeros([3, 2])
matrix_name = f"{self.testing_cxg_temp_directory}/awesome_zero_matrix_{uuid4()}"
convert_matrix_to_cxg_array(matrix_name, matrix, True, tiledb.Ctx())
actual_stored_array = tiledb.open(matrix_name)
self.assertTrue(path.isdir(matrix_name))
self.assertTrue(isinstance(actual_stored_array, tiledb.SparseArray))
self.assertTrue(actual_stored_array[:, :][''].size == 0)
def test__convert_matrix_to_cxg_array__sparse_array_only_store_nonzeros(self):
matrix = np.zeros([3, 3])
matrix[0, 0] = 1
matrix[1, 1] = 1
matrix[2, 2] = 2
matrix_name = f"{self.testing_cxg_temp_directory}/awesome_sparse_matrix_{uuid4()}"
convert_matrix_to_cxg_array(matrix_name, matrix, True, tiledb.Ctx())
actual_stored_array = tiledb.open(matrix_name)
self.assertTrue(path.isdir(matrix_name))
self.assertTrue(isinstance(actual_stored_array, tiledb.SparseArray))
self.assertTrue(actual_stored_array[0, 0][''] == 1)
self.assertTrue(actual_stored_array[1, 1][''] == 1)
self.assertTrue(actual_stored_array[2, 2][''] == 2)
self.assertTrue(actual_stored_array[:, :][''].size == 3)
def test__convert_matrix_to_cxg_array__sparse_array_with_column_encoding_empty_array(self):
matrix_name = f"{self.testing_cxg_temp_directory}/awesome_column_shift_matrix_{uuid4()}"
matrix = np.ones((3, 2))
# The column shift will be equal to the matrix since subtracting the column shift from the matrix will create
# a matrix of zeros which is sparse.
column_shift = np.ones((3, 2))
convert_matrix_to_cxg_array(matrix_name, matrix, True, tiledb.Ctx(),
column_shift_for_sparse_encoding=column_shift)
actual_stored_array = tiledb.open(matrix_name)
self.assertTrue(path.isdir(matrix_name))
self.assertTrue(isinstance(actual_stored_array, tiledb.SparseArray))
self.assertTrue(actual_stored_array[:, :][''].size == 0)
def test__convert_matrix_to_cxg_array__sparse_array_with_column_encoding_partial_array(self):
matrix_name = f"{self.testing_cxg_temp_directory}/awesome_column_shift_matrix_{uuid4()}"
matrix = np.ones((2, 2))
# Only column shift the first column of ones.
column_shift = np.array([[1, 0], [1, 0]])
convert_matrix_to_cxg_array(matrix_name, matrix, True, tiledb.Ctx(),
column_shift_for_sparse_encoding=column_shift)
actual_stored_array = tiledb.open(matrix_name)
self.assertTrue(path.isdir(matrix_name))
self.assertTrue(isinstance(actual_stored_array, tiledb.SparseArray))
self.assertTrue(actual_stored_array[0, 1][''] == 1)
self.assertTrue(actual_stored_array[1, 1][''] == 1)
self.assertTrue(actual_stored_array[:, :][''].size == 2)
server/test/unit/common/utils/test_type_conversion_utils.py
+16 -2
View File
@@ -2,10 +2,10 @@ import unittest
from unittest.mock import patch
import numpy as np
from pandas import Series
from pandas import Series, DataFrame
from server.common.utils.type_conversion_utils import can_cast_to_float32, can_cast_to_int32, get_dtype_of_array, \
get_schema_type_hint_of_array
get_schema_type_hint_of_array, get_dtypes_and_schemas_of_dataframe
class TestTypeConversionUtils(unittest.TestCase):
@@ -119,3 +119,17 @@ class TestTypeConversionUtils(unittest.TestCase):
i=test_type_index):
array = Series(data=[], dtype=types[test_type_index])
self.assertEqual(get_schema_type_hint_of_array(array), expected_schema_hints[test_type_index])
def test__get_dtypes_and_schemas_of_dataframe__dtype_and_schema_returns_as_expected(self):
float_array = Series(data=[1, 2, 3], dtype=np.dtype(np.float64))
category_array = Series(data=["a", "b", "b"], dtype="category")
dataframe = DataFrame({"float_array": float_array, "category_array": category_array})
expected_data_types_dict = {"float_array": np.float32, "category_array": np.unicode}
expected_schema_type_hints_dict = {"float_array": {"type": "float32"},
"category_array": {"type": "categorical", "categories": ["a", "b"]}}
actual_dataframe_data_types, actual_dataframe_schema_type_hints = get_dtypes_and_schemas_of_dataframe(dataframe)
self.assertEqual(expected_data_types_dict, actual_dataframe_data_types)
self.assertEqual(expected_schema_type_hints_dict, actual_dataframe_schema_type_hints)