Refactoring cxg utility classes in preparation for CXG conversion tooling (#1739)

server/test/__init__.py

@@ -1,22 +1,21 @@
+import os
 import random
 import shutil
 import string
 import tempfile
-import requests
 import time
-import os
-from subprocess import Popen
-from os import path, popen
 from contextlib import contextmanager
+from os import path, popen
+from subprocess import Popen
 
 import pandas as pd
+import requests
 
 from server.common.annotations.hosted_tiledb import AnnotationsHostedTileDB
 from server.common.annotations.local_file_csv import AnnotationsLocalFile
-
-from server.common.data_locator import DataLocator
 from server.common.app_config import AppConfig, DEFAULT_SERVER_PORT
-from server.common.utils import find_available_port
+from server.common.data_locator import DataLocator
+from server.common.utils.utils import find_available_port
 from server.data_common.fbs.matrix import encode_matrix_fbs
 from server.data_common.matrix_loader import MatrixDataLoader, MatrixDataType
 from server.db.db_utils import DbUtils
@@ -137,7 +136,7 @@ def start_test_server(command_line_args=[], app_config=None):
     yaml config file, which this server will read and parse.
     """
 
-    start = random.randint(DEFAULT_SERVER_PORT, 2**16 - 1)
+    start = random.randint(DEFAULT_SERVER_PORT, 2 ** 16 - 1)
     port = int(os.environ.get("CXG_SERVER_PORT", start))
     port = find_available_port("localhost", port)
     command = ["cellxgene", "--no-upgrade-check", "launch", "--verbose", "--port=%d" % port] + command_line_args

server/test/unit/common/test_app_config.py

@@ -3,10 +3,11 @@ import unittest
 from unittest import mock
 from unittest.mock import patch
 
+import requests
+
 from server.common.app_config import AppConfig
 from server.common.errors import ConfigurationError
 from server.test import PROJECT_ROOT, test_server, FIXTURES_ROOT
-import requests
 
 
 # NOTE, there are more tests that should be written for AppConfig.
@@ -119,7 +120,6 @@ class AppConfigTest(unittest.TestCase):
         config = AppConfig()
 
         with self.assertLogs(level="INFO") as logger:
-
             from server.common.aws_secret_utils import handle_config_from_secret
             # should not throw error
             # "AttributeError: 'XConfig' object has no attribute 'x'"
@@ -133,4 +133,3 @@ class AppConfigTest(unittest.TestCase):
             self.assertEqual(config.server_config.app__flask_secret_key, "mock_flask_secret")
             self.assertEqual(config.server_config.authentication__params_oauth__client_secret, "mock_oauth_secret")
             self.assertEqual(config.default_dataset_config.user_annotations__hosted_tiledb_array__db_uri, "mock_db_uri")
-

server/test/unit/common/utils/test_matrix_utils.py

@@ -0,0 +1,67 @@
+import unittest
+
+import numpy as np
+
+from server.common.utils.matrix_utils import is_matrix_sparse, get_column_shift_encode_for_matrix
+
+
+class TestMatrixUtils(unittest.TestCase):
+
+    def test__is_matrix_sparse__zero_and_one_hundred_percent_threshold(self):
+        matrix = np.array([1, 2, 3])
+
+        self.assertFalse(is_matrix_sparse(matrix, 0))
+        self.assertTrue(is_matrix_sparse(matrix, 100))
+
+    def test__is_matrix_sparse__partially_populated_sparse_matrix_returns_true(self):
+        matrix = np.zeros([3, 4])
+        matrix[2][3] = 1.0
+        matrix[1][1] = 2.2
+
+        self.assertTrue(is_matrix_sparse(matrix, 50))
+
+    def test__is_matrix_sparse__partially_populated_dense_matrix_returns_false(self):
+        matrix = np.zeros([2, 2])
+        matrix[0][0] = 1.0
+        matrix[0][1] = 2.2
+        matrix[1][1] = 3.7
+
+        self.assertFalse(is_matrix_sparse(matrix, 50))
+
+    def test__is_matrix_sparse__giant_matrix_returns_false_early(self):
+        matrix = np.ones([20000, 20])
+
+        with self.assertLogs(level="INFO") as logger:
+            self.assertFalse(is_matrix_sparse(matrix, 1))
+
+            # Because the function returns early a log will output the _estimate_ instead of the _exact_ percentage of
+            # non-zero elements in the matrix.
+            self.assertIn("Percentage of non-zero elements (estimate)", logger.output[0])
+
+    def test__is_matrix_sparse_with_column_shift_encoding__regular_sparse_returns_true(self):
+        matrix = np.zeros([2, 2])
+        matrix[0][0] = 1.0
+
+        self.assertIsNotNone(get_column_shift_encode_for_matrix(matrix, 50))
+
+    def test__is_matrix_sparse_with_column_shift_encoding__column_shift_returns_same_value(self):
+        matrix = np.ones([2, 2])
+        expected_column_shift = [1, 1]
+
+        actual_column_shift = get_column_shift_encode_for_matrix(matrix, 50)
+        self.assertTrue((expected_column_shift == actual_column_shift).all())
+
+    def test__is_matrix_sparse_with_column_shift_encoding__impossible_column_shift_returns_none(self):
+        matrix = np.array([[1, 2], [3, 4]])
+
+        self.assertIsNone(get_column_shift_encode_for_matrix(matrix, 50))
+
+    def test__is_matrix_sparse_with_column_shift_encoding__giant_matrix_returns_false_early(self):
+        matrix = np.random.rand(20000, 20)
+
+        with self.assertLogs(level="INFO") as logger:
+            self.assertFalse(is_matrix_sparse(matrix, 1))
+
+            # Because the function returns early a log will output the _estimate_ instead of the _exact_ percentage of
+            # non-zero elements in the matrix.
+            self.assertIn("Percentage of non-zero elements (estimate)", logger.output[0])

server/test/unit/common/utils/test_sanitization_utils.py

@@ -0,0 +1,56 @@
+import unittest
+
+from server.common.utils.sanitization_utils import sanitize_values_in_list, sanitize_keys_in_dictionary
+
+
+class TestSanitizationUtils(unittest.TestCase):
+
+    def test__sanitize_values_in_list__not_strings_raises_exception(self):
+        keys_to_sanitize = [1, 2, 3]
+
+        with self.assertRaises(Exception) as exception_context:
+            sanitize_values_in_list(keys_to_sanitize)
+
+        self.assertIn("must contain all strings", str(exception_context.exception))
+
+    def test__sanitize_values_in_list__not_all_strings_raises_exception(self):
+        keys_to_sanitize = ["1", "2", 3]
+
+        with self.assertRaises(Exception) as exception_context:
+            sanitize_values_in_list(keys_to_sanitize)
+
+        self.assertIn("must contain all strings", str(exception_context.exception))
+
+    def test__sanitize_values_in_list__replace_non_ascii_character_with_underscore(self):
+        keys_to_sanitize = ["abc.", "~abc", "a~b/c"]
+        expected_sanitized_keys_dict = dict(zip(keys_to_sanitize, ["abc_", "_abc", "a_b_c"]))
+
+        actual_sanitized_keys_dict = sanitize_values_in_list(keys_to_sanitize)
+
+        self.assertEqual(expected_sanitized_keys_dict, actual_sanitized_keys_dict)
+
+    def test__sanitize_keys_in_dictionary__replace_non_ascii_character_with_underscore(self):
+        dictionary_to_sanitize = {"abc.": 3, "~abc": 4, "a~b/c": 5}
+        expected_sanitized_dict = {"abc_": 3, "_abc": 4, "a_b_c": 5}
+
+        actual_sanitized_dict = dictionary_to_sanitize
+        sanitize_keys_in_dictionary(actual_sanitized_dict)
+
+        self.assertEqual(expected_sanitized_dict, actual_sanitized_dict)
+
+    def test__sanitize_keys_in_dictionary__non_string_key_raises_exception(self):
+        dictionary_to_sanitize = {4: 3, "~abc": 4, "a~b/c": 5}
+
+        with self.assertRaises(Exception) as exception_context:
+            sanitize_keys_in_dictionary(dictionary_to_sanitize)
+
+        self.assertIn("must contain all strings", str(exception_context.exception))
+
+    def test__sanitize_keys_in_dictionary__replace_only_some_keys(self):
+        dictionary_to_sanitize = {"abc": 3, "~abc": 4, "a~b/c": 5}
+        expected_sanitized_dict = {"abc": 3, "_abc": 4, "a_b_c": 5}
+
+        actual_sanitized_dict = dictionary_to_sanitize
+        sanitize_keys_in_dictionary(actual_sanitized_dict)
+
+        self.assertEqual(expected_sanitized_dict, actual_sanitized_dict)

server/test/unit/common/utils/test_type_conversion_utils.py

@@ -0,0 +1,121 @@
+import unittest
+from unittest.mock import patch
+
+import numpy as np
+from pandas import Series
+
+from server.common.utils.type_conversion_utils import can_cast_to_float32, can_cast_to_int32, get_dtype_of_array, \
+    get_schema_type_hint_of_array
+
+
+class TestTypeConversionUtils(unittest.TestCase):
+
+    def test__can_cast_to_float32__string_is_false(self):
+        array_to_convert = Series(data=["1", "2", "3"], dtype=str)
+
+        can_cast = can_cast_to_float32(array_to_convert.dtype)
+
+        self.assertFalse(can_cast)
+
+    def test__can_cast_to_float32__int_is_true_warning_outputted(self):
+        array_to_convert = Series(data=[1, 2, 3], dtype=np.dtype(np.float64))
+
+        with self.assertLogs(level="WARN") as logger:
+            can_cast = can_cast_to_float32(array_to_convert.dtype)
+            self.assertIn("may lose precision", logger.output[0])
+
+        self.assertTrue(can_cast)
+
+    @patch("logging.warning")
+    def test__can_cast_to_float64__int_is_false(self, mock_log_warning):
+        array_to_convert = Series(data=[1, 2, 3], dtype=np.dtype(np.float32))
+
+        can_cast = can_cast_to_float32(array_to_convert.dtype)
+
+        self.assertTrue(can_cast)
+        assert not mock_log_warning.called
+
+    def test__can_cast_to_int32__string_is_false(self):
+        array_to_convert = Series(data=["1", "2", "3"], dtype=str)
+
+        can_cast = can_cast_to_int32(array_to_convert.dtype, array_to_convert)
+
+        self.assertFalse(can_cast)
+
+    def test__can_cast_to_int32__int64_is_true(self):
+        array_to_convert = Series(data=["1", "2", "3"], dtype=np.dtype(np.int64))
+
+        can_cast = can_cast_to_int32(array_to_convert.dtype, array_to_convert)
+
+        self.assertTrue(can_cast)
+
+    def test__can_cast_to_int32__int16_is_true(self):
+        array_to_convert = Series(data=["1", "2", "3"], dtype=np.dtype(np.int16))
+
+        can_cast = can_cast_to_int32(array_to_convert.dtype, array_to_convert)
+
+        self.assertTrue(can_cast)
+
+    def test__can_cast_to_int32__int64_with_large_value_is_false(self):
+        array_to_convert = Series(data=["3000000000", "2", "3"], dtype=np.dtype(np.int64))
+
+        can_cast = can_cast_to_int32(array_to_convert.dtype, array_to_convert)
+
+        self.assertFalse(can_cast)
+
+    def test__get_dtype_of_array__supported_dtypes_return_as_expected(self):
+        types = [np.float32, np.int32, np.bool_, str]
+        expected_dtypes = [np.float32, np.int32, np.uint8, np.unicode]
+
+        for test_type_index in range(len(types)):
+            with self.subTest(f"Testing get_dtype_of_array with type {types[test_type_index].__name__}",
+                              i=test_type_index):
+                array = Series(data=[], dtype=types[test_type_index])
+                self.assertEqual(get_dtype_of_array(array), expected_dtypes[test_type_index])
+
+    def test__get_schema_type_hint_of_array__supported_dtypes_return_as_expected(self):
+        types = [np.float32, np.int32, np.bool_, str]
+        expected_schema_hints = [{"type": "float32"}, {"type": "int32"}, {"type": "boolean"}, {"type": "string"}]
+
+        for test_type_index in range(len(types)):
+            with self.subTest(f"Testing get_schema_type_hint_of_array with type {types[test_type_index].__name__}",
+                              i=test_type_index):
+                array = Series(data=[], dtype=types[test_type_index])
+                self.assertEqual(get_schema_type_hint_of_array(array), expected_schema_hints[test_type_index])
+
+    def test__get_dtype_of_array__categories_return_as_expected(self):
+        array = Series(data=["a", "b", "c"], dtype="category")
+        expected_dtype = np.unicode
+
+        actual_dtype = get_dtype_of_array(array)
+
+        self.assertEqual(expected_dtype, actual_dtype)
+
+    def test__get_schema_type_hint_of_array__categories_return_as_expected(self):
+        array = Series(data=["a", "b", "b"], dtype="category")
+        expected_schema_hint = {"type": "categorical", "categories": ["a", "b"]}
+
+        actual_schema_hint = get_schema_type_hint_of_array(array)
+
+        self.assertEqual(expected_schema_hint, actual_schema_hint)
+
+    def test__get_dtype_of_array__castable_dtypes_return_as_expected(self):
+        types = [np.float64, np.int64]
+        expected_dtypes = [np.float32, np.int32]
+
+        for test_type_index in range(len(types)):
+            with self.subTest(f"Testing get_dtype_of_array with castable type {types[test_type_index].__name__}",
+                              i=test_type_index):
+                array = Series(data=[], dtype=types[test_type_index])
+                self.assertEqual(get_dtype_of_array(array), expected_dtypes[test_type_index])
+
+    def test__get_schema_type_hint_of_array__castable_dtypes_return_as_expected(self):
+        types = [np.float64, np.int64]
+        expected_schema_hints = [{"type": "float32"}, {"type": "int32"}]
+
+        for test_type_index in range(len(types)):
+            with self.subTest(
+                    f"Testing get_schema_type_hint_of_array with castable type {types[test_type_index].__name__}",
+                    i=test_type_index):
+                array = Series(data=[], dtype=types[test_type_index])
+                self.assertEqual(get_schema_type_hint_of_array(array), expected_schema_hints[test_type_index])

server/test/unit/common/utils/test_utils.py

@@ -2,7 +2,7 @@ import os
 import shutil
 import unittest
 
-from server.common.utils import import_plugins
+from server.common.utils.utils import import_plugins
 from server.test import PROJECT_ROOT, random_string
 
 

server/test/unit/data_anndata/test_nan_anndata_adaptor.py

@@ -22,8 +22,9 @@ class NaNTest(unittest.TestCase):
             self.data._create_schema()
 
     def test_load(self):
-        with self.assertWarns(UserWarning):
+        with self.assertLogs(level="WARN") as logger:
             self.data = AnndataAdaptor(self.data_locator, self.config)
+            self.assertTrue(logger.output)
 
     def test_init(self):
         self.assertEqual(self.data.cell_count, 100)