Reorganize the server testing directory (#1705)

server/test/unit/data_common/fbs/test_matrix.py

@@ -0,0 +1,82 @@
+import unittest
+import pandas as pd
+import numpy as np
+from scipy import sparse
+
+import server.test.unit.decode_fbs as decode_fbs
+from server.data_common.fbs.matrix import encode_matrix_fbs, decode_matrix_fbs
+
+
+class FbsTests(unittest.TestCase):
+    """Test Case for Matrix FBS data encode/decode """
+
+    def test_encode_boundary(self):
+        """ test various boundary checks """
+
+        # row indexing is unsupported
+        with self.assertRaises(ValueError):
+            encode_matrix_fbs(matrix=pd.DataFrame(), row_idx=[])
+
+        # matrix must be 2D
+        with self.assertRaises(ValueError):
+            encode_matrix_fbs(matrix=np.zeros((3, 2, 1)))
+        with self.assertRaises(ValueError):
+            encode_matrix_fbs(matrix=np.ones((10,)))
+
+    def fbs_checks(self, fbs, dims, expected_types, expected_column_idx):
+        d = decode_fbs.decode_matrix_FBS(fbs)
+        self.assertEqual(d["n_rows"], dims[0])
+        self.assertEqual(d["n_cols"], dims[1])
+        self.assertIsNone(d["row_idx"])
+        self.assertEqual(len(d["columns"]), dims[1])
+        for i in range(0, len(d["columns"])):
+            self.assertEqual(len(d["columns"][i]), dims[0])
+            self.assertIsInstance(d["columns"][i], expected_types[i][0])
+            if expected_types[i][1] is not None:
+                self.assertEqual(d["columns"][i].dtype, expected_types[i][1])
+        if expected_column_idx is not None:
+            self.assertSetEqual(set(expected_column_idx), set(d["col_idx"]))
+
+    def test_encode_DataFrame(self):
+        df = pd.DataFrame(
+            data={
+                "a": np.zeros((10,), dtype=np.float32),
+                "b": np.ones((10,), dtype=np.int64),
+                "c": np.array([i for i in range(0, 10)], dtype=np.uint16),
+                "d": pd.Series(["x", "y", "z", "x", "y", "z", "a", "x", "y", "z"], dtype="category"),
+            }
+        )
+        expected_types = ((np.ndarray, np.float32), (np.ndarray, np.int32), (np.ndarray, np.uint32), (list, None))
+        fbs = encode_matrix_fbs(matrix=df, row_idx=None, col_idx=df.columns)
+        self.fbs_checks(fbs, (10, 4), expected_types, ["a", "b", "c", "d"])
+
+    def test_encode_ndarray(self):
+        arr = np.zeros((3, 2), dtype=np.float32)
+        expected_types = ((np.ndarray, np.float32), (np.ndarray, np.float32), (np.ndarray, np.float32))
+        fbs = encode_matrix_fbs(matrix=arr, row_idx=None, col_idx=None)
+        self.fbs_checks(fbs, (3, 2), expected_types, None)
+
+    def test_encode_sparse(self):
+        csc = sparse.csc_matrix(np.array([[0, 1, 2], [3, 0, 4]]))
+        expected_types = ((np.ndarray, np.int32), (np.ndarray, np.int32), (np.ndarray, np.int32))
+        fbs = encode_matrix_fbs(matrix=csc, row_idx=None, col_idx=None)
+        self.fbs_checks(fbs, (2, 3), expected_types, None)
+
+    def test_roundtrip(self):
+        dfSrc = pd.DataFrame(
+            data={
+                "a": np.zeros((10,), dtype=np.float32),
+                "b": np.ones((10,), dtype=np.int64),
+                "c": np.array([i for i in range(0, 10)], dtype=np.uint16),
+                "d": pd.Series(["x", "y", "z", "x", "y", "z", "a", "x", "y", "z"], dtype="category"),
+            }
+        )
+        dfDst = decode_matrix_fbs(encode_matrix_fbs(matrix=dfSrc, col_idx=dfSrc.columns))
+        self.assertEqual(dfSrc.shape, dfDst.shape)
+        self.assertEqual(set(dfSrc.columns), set(dfDst.columns))
+        for c in dfSrc.columns:
+            self.assertTrue(c in dfDst.columns)
+            if isinstance(dfSrc[c], pd.Series):
+                self.assertTrue(np.all(dfSrc[c] == dfDst[c]))
+            else:
+                self.assertEqual(dfSrc[c], dfDst[c])

server/test/unit/data_common/test_matrix_loader.py

@@ -0,0 +1,126 @@
+import os
+import shutil
+import tempfile
+import time
+import unittest
+
+from server.common.app_config import AppConfig
+from server.common.errors import DatasetAccessError
+from server.data_common.matrix_loader import MatrixDataCacheManager
+from server.test import FIXTURES_ROOT
+
+
+class MatrixCacheTest(unittest.TestCase):
+    def setup(self):
+        pass
+
+    def make_temporay_datasets(self, dirname, num):
+        source = f"{FIXTURES_ROOT}/pbmc3k.cxg"
+        for i in range(num):
+            target = os.path.join(dirname, str(i) + ".cxg")
+            shutil.copytree(source, target)
+
+    def use_dataset(self, matrix_cache, dirname, app_config, dataset_index):
+        with matrix_cache.data_adaptor(None, os.path.join(dirname, str(dataset_index) + ".cxg"), app_config) as adaptor:
+            pass
+        return adaptor
+
+    def use_dataset_with_error(self, matrix_cache, dirname, app_config, dataset_index):
+        try:
+            with matrix_cache.data_adaptor(None, os.path.join(dirname, str(dataset_index) + ".cxg"), app_config):
+                raise DatasetAccessError("something bad happened")
+        except DatasetAccessError:
+            # the MatrixDataCacheManager rethrows the exception, so catch and ignore
+            pass
+
+    def get_datasets(self, matrix_cache, dirname):
+        datasets = matrix_cache.datasets
+        result = {}
+        for k, v in datasets.items():
+            # filter out the dirname and the .cxg from the name
+            newk = int(k[1][len(dirname) + 1: -4])
+            result[newk] = v
+
+        return result
+
+    def check_datasets(self, matrix_cache, dirname, expected):
+        res = self.get_datasets(matrix_cache, dirname)
+        actual = res.keys()
+        self.assertSetEqual(set(actual), set(expected))
+
+    def test_basic(self):
+        with tempfile.TemporaryDirectory() as dirname:
+            self.make_temporay_datasets(dirname, 5)
+            app_config = AppConfig()
+            m = MatrixDataCacheManager(max_cached=3, timelimit_s=None)
+
+            # should have only dataset 0
+            self.use_dataset(m, dirname, app_config, 0)
+            self.check_datasets(m, dirname, [0])
+
+            # should have datasets 0, 1
+            self.use_dataset(m, dirname, app_config, 1)
+            self.check_datasets(m, dirname, [0, 1])
+
+            # should have datasets 0, 1, 2
+            self.use_dataset(m, dirname, app_config, 2)
+            self.check_datasets(m, dirname, [0, 1, 2])
+
+            # should have datasets 1, 2, 3
+            self.use_dataset(m, dirname, app_config, 3)
+            self.check_datasets(m, dirname, [1, 2, 3])
+
+            # use dataset 1, making is more recent than dataset 2
+            self.use_dataset(m, dirname, app_config, 1)
+            self.check_datasets(m, dirname, [1, 2, 3])
+
+            # use dataset 4, should have 1,3,4
+            self.use_dataset(m, dirname, app_config, 4)
+            self.check_datasets(m, dirname, [1, 3, 4])
+
+            # use dataset 4 a few more times, get the count to 3
+            self.use_dataset(m, dirname, app_config, 4)
+            self.use_dataset(m, dirname, app_config, 4)
+
+            datasets = self.get_datasets(m, dirname)
+            self.assertEqual(datasets[1].num_access, 2)
+            self.assertEqual(datasets[3].num_access, 1)
+            self.assertEqual(datasets[4].num_access, 3)
+
+    def test_timelimit(self):
+        with tempfile.TemporaryDirectory() as dirname:
+            self.make_temporay_datasets(dirname, 2)
+
+            app_config = AppConfig()
+            m = MatrixDataCacheManager(max_cached=3, timelimit_s=1)
+
+            adaptor = self.use_dataset(m, dirname, app_config, 0)
+            adaptor1 = self.use_dataset(m, dirname, app_config, 0)
+            self.assertTrue(adaptor is adaptor1)
+
+            # wait until the timelimit expires and check that there is a new adaptor
+            time.sleep(1.1)
+            adaptor2 = self.use_dataset(m, dirname, app_config, 0)
+            self.assertTrue(adaptor is not adaptor2)
+            self.check_datasets(m, dirname, [0])
+
+            # now load a different dataset and see if dataset 0 gets evicted
+            time.sleep(1.1)
+            self.use_dataset(m, dirname, app_config, 1)
+            self.check_datasets(m, dirname, [1])
+
+    def test_access_error(self):
+        with tempfile.TemporaryDirectory() as dirname:
+            self.make_temporay_datasets(dirname, 1)
+
+            app_config = AppConfig()
+            m = MatrixDataCacheManager(max_cached=3, timelimit_s=1)
+
+            # use the 0 datasets
+            self.use_dataset(m, dirname, app_config, 0)
+            self.check_datasets(m, dirname, [0])
+
+            # use the 0 datasets, but this time a DatasetAccessError is raised.
+            # verify that dataset is removed from the cache.
+            self.use_dataset_with_error(m, dirname, app_config, 0)
+            self.check_datasets(m, dirname, [])