Refactor czi_hosted and server into backend directory, pull common code into backend/common, refactor tests (#2102)

* move local_server -> backend/server server-> backend/czi_hosted, pull common code into backend/common update imports, tests and make commands
2026-10-03 21:28:12 +08:00 · 2021-03-26 00:27:07 -05:00
parent e6e358ddc8
commit 78c9d24ed4
425 changed files with 734 additions and 5317 deletions
@@ -0,0 +1,158 @@
+import json
+import unittest
+from os import path, mkdir
+from shutil import rmtree
+from uuid import uuid4
+
+import numpy as np
+import tiledb
+from pandas import Series, DataFrame
+
+from backend.czi_hosted.common.utils.cxg_generation_utils import (
+    convert_dictionary_to_cxg_group,
+    convert_dataframe_to_cxg_array,
+    convert_ndarray_to_cxg_dense_array,
+    convert_matrix_to_cxg_array,
+)
+from backend.test import FIXTURES_ROOT
+
+
+class TestCxgGenerationUtils(unittest.TestCase):
+    def setUp(self):
+        self.testing_cxg_temp_directory = f"{FIXTURES_ROOT}/{uuid4()}"
+        mkdir(self.testing_cxg_temp_directory)
+
+    def tearDown(self):
+        if path.isdir(self.testing_cxg_temp_directory):
+            rmtree(self.testing_cxg_temp_directory)
+
+    def test__convert_dictionary_to_cxg_group__writes_successfully(self):
+        random_dictionary = {"cookies": "chocolate_chip", "brownies": "chocolate", "cake": "double chocolate"}
+        dictionary_name = "favorite_desserts"
+        expected_array_directory = f"{self.testing_cxg_temp_directory}/{dictionary_name}"
+
+        convert_dictionary_to_cxg_group(
+            self.testing_cxg_temp_directory, random_dictionary, group_metadata_name=dictionary_name
+        )
+
+        array = tiledb.open(expected_array_directory)
+        actual_stored_metadata = dict(array.meta.items())
+
+        self.assertTrue(path.isdir(expected_array_directory))
+        self.assertTrue(isinstance(array, tiledb.DenseArray))
+        self.assertEqual(random_dictionary, actual_stored_metadata)
+
+    def test__convert_dataframe_to_cxg_array__writes_successfully(self):
+        random_int_category = Series(data=[3, 1, 2, 4], dtype=np.int64)
+        random_bool_category = Series(data=[True, True, False, True], dtype=np.bool_)
+        random_dataframe_name = f"random_dataframe_{uuid4()}"
+        random_dataframe = DataFrame(data={"int_category": random_int_category, "bool_category": random_bool_category})
+
+        convert_dataframe_to_cxg_array(
+            self.testing_cxg_temp_directory, random_dataframe_name, random_dataframe, "int_category", tiledb.Ctx()
+        )
+
+        expected_array_directory = f"{self.testing_cxg_temp_directory}/{random_dataframe_name}"
+        expected_array_metadata = {
+            "cxg_schema": json.dumps(
+                {"int_category": {"type": "int32"}, "bool_category": {"type": "boolean"}, "index": "int_category"}
+            )
+        }
+
+        actual_stored_dataframe_array = tiledb.open(expected_array_directory)
+        actual_stored_dataframe_metadata = dict(actual_stored_dataframe_array.meta.items())
+
+        self.assertTrue(path.isdir(expected_array_directory))
+        self.assertTrue(isinstance(actual_stored_dataframe_array, tiledb.DenseArray))
+        self.assertDictEqual(expected_array_metadata, actual_stored_dataframe_metadata)
+        self.assertTrue((actual_stored_dataframe_array[0:4]["int_category"] == random_int_category.to_numpy()).all())
+        self.assertTrue((actual_stored_dataframe_array[0:4]["bool_category"] == random_bool_category.to_numpy()).all())
+
+    def test__convert_ndarray_to_cxg_dense_array__writes_successfully(self):
+        ndarray = np.random.rand(3, 2)
+        ndarray_name = f"{self.testing_cxg_temp_directory}/awesome_ndarray_{uuid4()}"
+
+        convert_ndarray_to_cxg_dense_array(ndarray_name, ndarray, tiledb.Ctx())
+
+        actual_stored_array = tiledb.open(ndarray_name)
+
+        self.assertTrue(path.isdir(ndarray_name))
+        self.assertTrue(isinstance(actual_stored_array, tiledb.DenseArray))
+        self.assertTrue((actual_stored_array[:, :] == ndarray).all())
+
+    def test__convert_matrix_to_cxg_array__dense_array_writes_successfully(self):
+        matrix = np.float32(np.random.rand(3, 2))
+        matrix_name = f"{self.testing_cxg_temp_directory}/awesome_matrix_{uuid4()}"
+
+        convert_matrix_to_cxg_array(matrix_name, matrix, False, tiledb.Ctx())
+
+        actual_stored_array = tiledb.open(matrix_name)
+
+        self.assertTrue(path.isdir(matrix_name))
+        self.assertTrue(isinstance(actual_stored_array, tiledb.DenseArray))
+        self.assertTrue((actual_stored_array[:, :] == matrix).all())
+
+    def test__convert_matrix_to_cxg_array__sparse_array_only_store_nonzeros_empty_array(self):
+        matrix = np.zeros([3, 2])
+        matrix_name = f"{self.testing_cxg_temp_directory}/awesome_zero_matrix_{uuid4()}"
+
+        convert_matrix_to_cxg_array(matrix_name, matrix, True, tiledb.Ctx())
+
+        actual_stored_array = tiledb.open(matrix_name)
+
+        self.assertTrue(path.isdir(matrix_name))
+        self.assertTrue(isinstance(actual_stored_array, tiledb.SparseArray))
+        self.assertTrue(actual_stored_array[:, :][""].size == 0)
+
+    def test__convert_matrix_to_cxg_array__sparse_array_only_store_nonzeros(self):
+        matrix = np.zeros([3, 3])
+        matrix[0, 0] = 1
+        matrix[1, 1] = 1
+        matrix[2, 2] = 2
+        matrix_name = f"{self.testing_cxg_temp_directory}/awesome_sparse_matrix_{uuid4()}"
+
+        convert_matrix_to_cxg_array(matrix_name, matrix, True, tiledb.Ctx())
+
+        actual_stored_array = tiledb.open(matrix_name)
+
+        self.assertTrue(path.isdir(matrix_name))
+        self.assertTrue(isinstance(actual_stored_array, tiledb.SparseArray))
+        self.assertTrue(actual_stored_array[0, 0][""] == 1)
+        self.assertTrue(actual_stored_array[1, 1][""] == 1)
+        self.assertTrue(actual_stored_array[2, 2][""] == 2)
+        self.assertTrue(actual_stored_array[:, :][""].size == 3)
+
+    def test__convert_matrix_to_cxg_array__sparse_array_with_column_encoding_empty_array(self):
+        matrix_name = f"{self.testing_cxg_temp_directory}/awesome_column_shift_matrix_{uuid4()}"
+        matrix = np.ones((3, 2))
+        # The column shift will be equal to the matrix since subtracting the column shift from the matrix will create
+        # a matrix of zeros which is sparse.
+        column_shift = np.ones((3, 2))
+
+        convert_matrix_to_cxg_array(
+            matrix_name, matrix, True, tiledb.Ctx(), column_shift_for_sparse_encoding=column_shift
+        )
+
+        actual_stored_array = tiledb.open(matrix_name)
+
+        self.assertTrue(path.isdir(matrix_name))
+        self.assertTrue(isinstance(actual_stored_array, tiledb.SparseArray))
+        self.assertTrue(actual_stored_array[:, :][""].size == 0)
+
+    def test__convert_matrix_to_cxg_array__sparse_array_with_column_encoding_partial_array(self):
+        matrix_name = f"{self.testing_cxg_temp_directory}/awesome_column_shift_matrix_{uuid4()}"
+        matrix = np.ones((2, 2))
+        # Only column shift the first column of ones.
+        column_shift = np.array([[1, 0], [1, 0]])
+
+        convert_matrix_to_cxg_array(
+            matrix_name, matrix, True, tiledb.Ctx(), column_shift_for_sparse_encoding=column_shift
+        )
+
+        actual_stored_array = tiledb.open(matrix_name)
+
+        self.assertTrue(path.isdir(matrix_name))
+        self.assertTrue(isinstance(actual_stored_array, tiledb.SparseArray))
+        self.assertTrue(actual_stored_array[0, 1][""] == 1)
+        self.assertTrue(actual_stored_array[1, 1][""] == 1)
+        self.assertTrue(actual_stored_array[:, :][""].size == 2)
@@ -0,0 +1,66 @@
+import unittest
+
+import numpy as np
+
+from backend.czi_hosted.common.utils.matrix_utils import is_matrix_sparse, get_column_shift_encode_for_matrix
+
+
+class TestMatrixUtils(unittest.TestCase):
+    def test__is_matrix_sparse__zero_and_one_hundred_percent_threshold(self):
+        matrix = np.array([1, 2, 3])
+
+        self.assertFalse(is_matrix_sparse(matrix, 0))
+        self.assertTrue(is_matrix_sparse(matrix, 100))
+
+    def test__is_matrix_sparse__partially_populated_sparse_matrix_returns_true(self):
+        matrix = np.zeros([3, 4])
+        matrix[2][3] = 1.0
+        matrix[1][1] = 2.2
+
+        self.assertTrue(is_matrix_sparse(matrix, 50))
+
+    def test__is_matrix_sparse__partially_populated_dense_matrix_returns_false(self):
+        matrix = np.zeros([2, 2])
+        matrix[0][0] = 1.0
+        matrix[0][1] = 2.2
+        matrix[1][1] = 3.7
+
+        self.assertFalse(is_matrix_sparse(matrix, 50))
+
+    def test__is_matrix_sparse__giant_matrix_returns_false_early(self):
+        matrix = np.ones([20000, 20])
+
+        with self.assertLogs(level="INFO") as logger:
+            self.assertFalse(is_matrix_sparse(matrix, 1))
+
+            # Because the function returns early a log will output the _estimate_ instead of the _exact_ percentage of
+            # non-zero elements in the matrix.
+            self.assertIn("Percentage of non-zero elements (estimate)", logger.output[0])
+
+    def test__is_matrix_sparse_with_column_shift_encoding__regular_sparse_returns_true(self):
+        matrix = np.zeros([2, 2])
+        matrix[0][0] = 1.0
+
+        self.assertIsNotNone(get_column_shift_encode_for_matrix(matrix, 50))
+
+    def test__is_matrix_sparse_with_column_shift_encoding__column_shift_returns_same_value(self):
+        matrix = np.ones([2, 2])
+        expected_column_shift = [1, 1]
+
+        actual_column_shift = get_column_shift_encode_for_matrix(matrix, 50)
+        self.assertTrue((expected_column_shift == actual_column_shift).all())
+
+    def test__is_matrix_sparse_with_column_shift_encoding__impossible_column_shift_returns_none(self):
+        matrix = np.array([[1, 2], [3, 4]])
+
+        self.assertIsNone(get_column_shift_encode_for_matrix(matrix, 50))
+
+    def test__is_matrix_sparse_with_column_shift_encoding__giant_matrix_returns_false_early(self):
+        matrix = np.random.rand(20000, 20)
+
+        with self.assertLogs(level="INFO") as logger:
+            self.assertFalse(is_matrix_sparse(matrix, 1))
+
+            # Because the function returns early a log will output the _estimate_ instead of the _exact_ percentage of
+            # non-zero elements in the matrix.
+            self.assertIn("Percentage of non-zero elements (estimate)", logger.output[0])
@@ -0,0 +1,55 @@
+import unittest
+
+from backend.czi_hosted.common.utils.sanitization_utils import sanitize_values_in_list, sanitize_keys_in_dictionary
+
+
+class TestSanitizationUtils(unittest.TestCase):
+    def test__sanitize_values_in_list__not_strings_raises_exception(self):
+        keys_to_sanitize = [1, 2, 3]
+
+        with self.assertRaises(Exception) as exception_context:
+            sanitize_values_in_list(keys_to_sanitize)
+
+        self.assertIn("must contain all strings", str(exception_context.exception))
+
+    def test__sanitize_values_in_list__not_all_strings_raises_exception(self):
+        keys_to_sanitize = ["1", "2", 3]
+
+        with self.assertRaises(Exception) as exception_context:
+            sanitize_values_in_list(keys_to_sanitize)
+
+        self.assertIn("must contain all strings", str(exception_context.exception))
+
+    def test__sanitize_values_in_list__replace_non_ascii_character_with_underscore(self):
+        keys_to_sanitize = ["abc.", "~abc", "a~b/c"]
+        expected_sanitized_keys_dict = dict(zip(keys_to_sanitize, ["abc_", "_abc", "a_b_c"]))
+
+        actual_sanitized_keys_dict = sanitize_values_in_list(keys_to_sanitize)
+
+        self.assertEqual(expected_sanitized_keys_dict, actual_sanitized_keys_dict)
+
+    def test__sanitize_keys_in_dictionary__replace_non_ascii_character_with_underscore(self):
+        dictionary_to_sanitize = {"abc.": 3, "~abc": 4, "a~b/c": 5}
+        expected_sanitized_dict = {"abc_": 3, "_abc": 4, "a_b_c": 5}
+
+        actual_sanitized_dict = dictionary_to_sanitize
+        sanitize_keys_in_dictionary(actual_sanitized_dict)
+
+        self.assertEqual(expected_sanitized_dict, actual_sanitized_dict)
+
+    def test__sanitize_keys_in_dictionary__non_string_key_raises_exception(self):
+        dictionary_to_sanitize = {4: 3, "~abc": 4, "a~b/c": 5}
+
+        with self.assertRaises(Exception) as exception_context:
+            sanitize_keys_in_dictionary(dictionary_to_sanitize)
+
+        self.assertIn("must contain all strings", str(exception_context.exception))
+
+    def test__sanitize_keys_in_dictionary__replace_only_some_keys(self):
+        dictionary_to_sanitize = {"abc": 3, "~abc": 4, "a~b/c": 5}
+        expected_sanitized_dict = {"abc": 3, "_abc": 4, "a_b_c": 5}
+
+        actual_sanitized_dict = dictionary_to_sanitize
+        sanitize_keys_in_dictionary(actual_sanitized_dict)
+
+        self.assertEqual(expected_sanitized_dict, actual_sanitized_dict)
@@ -0,0 +1,34 @@
+import os
+import shutil
+import unittest
+
+from backend.common.utils.utils import import_plugins
+from backend.test import PROJECT_ROOT, random_string
+
+
+class TestPlugins(unittest.TestCase):
+    """ Test plugin import functionality """
+
+    plugins_dir = f"{PROJECT_ROOT}/backend/test/test_czi_hosted/unit/plugins"
+    test_plugin_path = f"{plugins_dir}/foo.py"
+    secret = random_string(8)
+
+    @classmethod
+    def setUpClass(cls) -> None:
+        if not os.path.isdir(cls.plugins_dir):
+            os.mkdir(cls.plugins_dir)
+        with open(cls.test_plugin_path, "w") as fh:
+            fh.write(f'SECRET = "{cls.secret}"\n')
+
+    @classmethod
+    def tearDownClass(cls) -> None:
+        if os.path.isdir(cls.plugins_dir):
+            shutil.rmtree(cls.plugins_dir)
+
+    def test_import_plugins(self):
+        self.assertTrue(os.path.isfile(self.test_plugin_path))
+        loaded_modules = import_plugins("backend.test.test_czi_hosted.unit.plugins")
+        # test that import plugins found the file
+        self.assertEqual(["backend.test.test_czi_hosted.unit.plugins.foo"], [ele.__name__ for ele in loaded_modules])
+        # test that the module was properly executed
+        self.assertEqual(self.secret, loaded_modules[0].SECRET)