Reorganize the server testing directory (#1705)

server/test/unit/auth/test_auth.py

@@ -0,0 +1,147 @@
+import unittest
+
+import requests
+
+from server.common.app_config import AppConfig
+from server.test import FIXTURES_ROOT, test_server
+
+
+class AuthTest(unittest.TestCase):
+    def setUp(self):
+        self.dataset_dataroot = FIXTURES_ROOT
+
+    def test_auth_none(self):
+        c = AppConfig()
+        c.update_server_config(
+            authentication__type=None, multi_dataset__dataroot=self.dataset_dataroot
+        )
+        c.update_default_dataset_config(user_annotations__enable=False)
+
+        c.complete_config()
+
+        with test_server(app_config=c) as server:
+            session = requests.Session()
+            r = session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/config")
+            data_config = r.json()
+            assert "authentication" not in data_config["config"]
+
+    def test_auth_session(self):
+        c = AppConfig()
+        c.update_server_config(
+            authentication__type="session", multi_dataset__dataroot=self.dataset_dataroot
+        )
+        c.update_default_dataset_config(user_annotations__enable=True)
+        c.complete_config()
+
+        with test_server(app_config=c) as server:
+            session = requests.Session()
+            r = session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/config")
+            data_config = r.json()
+            assert data_config["config"]["authentication"]["is_authenticated"]
+            assert not data_config["config"]["authentication"]["requires_client_login"]
+            assert data_config["config"]["authentication"]["username"] == "anonymous"
+
+    def test_auth_test(self):
+        c = AppConfig()
+        c.update_server_config(authentication__type="test")
+        c.update_server_config(
+            multi_dataset__dataroot=dict(
+                a1=dict(dataroot=self.dataset_dataroot, base_url="auth"),
+                a2=dict(dataroot=self.dataset_dataroot, base_url="no-auth"),
+            )
+        )
+
+        # specialize the configs
+        c.add_dataroot_config("a1", app__authentication_enable=True, user_annotations__enable=True)
+        c.add_dataroot_config("a2", app__authentication_enable=False, user_annotations__enable=False)
+
+        c.complete_config()
+
+        with test_server(app_config=c) as server:
+            session = requests.Session()
+
+            # auth datasets
+            r = session.get(f"{server}/auth/pbmc3k.cxg/api/v0.2/config")
+            data_config = r.json()
+            assert not data_config["config"]["authentication"]["is_authenticated"]
+            assert data_config["config"]["authentication"]["requires_client_login"]
+            assert data_config["config"]["authentication"]["username"] is None
+            assert data_config["config"]["parameters"]["annotations"]
+
+            login_uri = data_config["config"]["authentication"]["login"]
+            logout_uri = data_config["config"]["authentication"]["logout"]
+
+            assert login_uri == "/login?dataset=auth/pbmc3k.cxg"
+            assert logout_uri == "/logout?dataset=auth/pbmc3k.cxg"
+
+            r = session.get(f"{server}/{login_uri}")
+            # check that the login redirect worked
+            assert r.history[0].status_code == 302
+            assert r.url == f"{server}/auth/pbmc3k.cxg/"
+
+            r = session.get(f"{server}/auth/pbmc3k.cxg/api/v0.2/config")
+            data_config = r.json()
+            assert data_config["config"]["authentication"]["is_authenticated"]
+            assert data_config["config"]["authentication"]["username"] == "test_account"
+            assert data_config["config"]["parameters"]["annotations"]
+
+            r = session.get(f"{server}/{logout_uri}")
+            # check that the logout redirect worked
+            assert r.history[0].status_code == 302
+            assert r.url == f"{server}/auth/pbmc3k.cxg/"
+            r = session.get(f"{server}/auth/pbmc3k.cxg/api/v0.2/config")
+            data_config = r.json()
+            assert not data_config["config"]["authentication"]["is_authenticated"]
+            assert data_config["config"]["authentication"]["username"] is None
+            assert data_config["config"]["parameters"]["annotations"]
+
+            # no-auth datasets
+            r = session.get(f"{server}/no-auth/pbmc3k.cxg/api/v0.2/config")
+            data_config = r.json()
+            assert "authentication" not in data_config["config"]
+            assert not data_config["config"]["parameters"]["annotations"]
+
+    def test_auth_test_single(self):
+        c = AppConfig()
+        c.update_server_config(
+            authentication__type="test",
+            single_dataset__datapath=f"{self.dataset_dataroot}/pbmc3k.cxg")
+
+        c.complete_config()
+
+        with test_server(app_config=c) as server:
+            session = requests.Session()
+
+            r = session.get(f"{server}/api/v0.2/config")
+            data_config = r.json()
+            assert not data_config["config"]["authentication"]["is_authenticated"]
+            assert data_config["config"]["authentication"]["requires_client_login"]
+            assert data_config["config"]["authentication"]["username"] is None
+            assert data_config["config"]["parameters"]["annotations"]
+
+            login_uri = data_config["config"]["authentication"]["login"]
+            logout_uri = data_config["config"]["authentication"]["logout"]
+
+            assert login_uri == "/login"
+            assert logout_uri == "/logout"
+
+            r = session.get(f"{server}/{login_uri}")
+            # check that the login redirect worked
+            assert r.history[0].status_code == 302
+            assert r.url == f"{server}/"
+
+            r = session.get(f"{server}/api/v0.2/config")
+            data_config = r.json()
+            assert data_config["config"]["authentication"]["is_authenticated"]
+            assert data_config["config"]["authentication"]["username"] == "test_account"
+            assert data_config["config"]["parameters"]["annotations"]
+
+            r = session.get(f"{server}/{logout_uri}")
+            # check that the logout redirect worked
+            assert r.history[0].status_code == 302
+            assert r.url == f"{server}/"
+            r = session.get(f"{server}/api/v0.2/config")
+            data_config = r.json()
+            assert not data_config["config"]["authentication"]["is_authenticated"]
+            assert data_config["config"]["authentication"]["username"] is None
+            assert data_config["config"]["parameters"]["annotations"]

server/test/unit/cli/test_prepare.py

@@ -0,0 +1,15 @@
+import unittest
+
+import pandas as pd
+
+from server.cli.prepare import make_index_unique
+
+
+class CLIPrepareTests(unittest.TestCase):
+    """ Test cases for CLI prepare logic """
+
+    def test_make_index_unique(self):
+        index = pd.Index(["SNORD113", "SNORD113", "SNORD113-1"])
+        result = make_index_unique(index)
+        expected = pd.Index(["SNORD113", "SNORD113-2", "SNORD113-1"])
+        self.assertTrue(all(left == right for left, right in zip(result.values, expected.values)))

server/test/unit/cli/test_upgrade.py

@@ -0,0 +1,28 @@
+import unittest
+
+from server.cli.upgrade import validate_version_str, split_version, version_gt
+
+
+class CLIUpgradeTests(unittest.TestCase):
+    """ Test cases for CLI logic """
+
+    def test_validate_version_str(self):
+        self.assertTrue(validate_version_str("0.1.2"))
+        self.assertTrue(validate_version_str("0.1.2-RC", release_only=False))
+        self.assertFalse(validate_version_str("0.1"))
+        self.assertFalse(validate_version_str("0.1.2.3"))
+        self.assertFalse(validate_version_str("0.1.2-RC"))
+
+    def test_split_version_str(self):
+        self.assertEqual(split_version("0.1.2"), [0, 1, 2])
+        with self.assertRaises(AttributeError):
+            split_version("0.1")
+
+    def test_assert_verstion_gt(self):
+        self.assertTrue(version_gt("1.0.0", "0.1.1"))
+        self.assertTrue(version_gt("0.1.0", "0.0.1"))
+        self.assertTrue(version_gt("0.0.1", "0.0.0"))
+        self.assertFalse(version_gt("0.0.0", "0.0.0"))
+        self.assertFalse(version_gt("0.0.0", "0.0.1"))
+        self.assertFalse(version_gt("0.0.1", "0.1.0"))
+        self.assertFalse(version_gt("0.1.1", "1.0.0"))

server/test/unit/common/test_api.py

@@ -0,0 +1,455 @@
+import shutil
+import time
+import unittest
+from http import HTTPStatus
+
+import pandas as pd
+import requests
+
+import server.test.unit.decode_fbs as decode_fbs
+from server.data_common.matrix_loader import MatrixDataType
+from server.test import (data_with_tmp_annotations, make_fbs, PROJECT_ROOT, FIXTURES_ROOT, start_test_server,
+                         stop_test_server)
+from server.test.fixtures.fixtures import pbmc3k_colors
+
+BAD_FILTER = {"filter": {"obs": {"annotation_value": [{"name": "xyz"}]}}}
+
+
+# TODO (mweiden): remove ANNOTATIONS_ENABLED and Annotation subclasses when annotations are no longer experimental
+
+
+class EndPoints(object):
+    ANNOTATIONS_ENABLED = True
+
+    def test_initialize(self):
+        endpoint = "schema"
+        url = f"{self.URL_BASE}{endpoint}"
+        result = self.session.get(url)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/json")
+        result_data = result.json()
+        self.assertEqual(result_data["schema"]["dataframe"]["nObs"], 2638)
+        self.assertEqual(len(result_data["schema"]["annotations"]["obs"]), 2)
+        self.assertEqual(
+            len(result_data["schema"]["annotations"]["obs"]["columns"]), 6 if self.ANNOTATIONS_ENABLED else 5
+        )
+
+    def test_config(self):
+        endpoint = "config"
+        url = f"{self.URL_BASE}{endpoint}"
+        result = self.session.get(url)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/json")
+        result_data = result.json()
+        self.assertIn("library_versions", result_data["config"])
+        self.assertEqual(result_data["config"]["displayNames"]["dataset"], "pbmc3k")
+        self.assertEqual(len(result_data["config"]["features"]), 5)
+
+    def test_get_layout_fbs(self):
+        endpoint = "layout/obs"
+        url = f"{self.URL_BASE}{endpoint}"
+        header = {"Accept": "application/octet-stream"}
+        result = self.session.get(url, headers=header)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
+        df = decode_fbs.decode_matrix_FBS(result.content)
+        self.assertEqual(df["n_rows"], 2638)
+        self.assertEqual(df["n_cols"], 8)
+        self.assertIsNotNone(df["columns"])
+        self.assertSetEqual(
+            set(df["col_idx"]),
+            {"pca_0", "pca_1", "tsne_0", "tsne_1", "umap_0", "umap_1", "draw_graph_fr_0", "draw_graph_fr_1"},
+        )
+        self.assertIsNone(df["row_idx"])
+        self.assertEqual(len(df["columns"]), df["n_cols"])
+
+    def test_put_layout_fbs(self):
+        # first check that re-embedding is turned on
+        result = self.session.get(f"{self.URL_BASE}config")
+        config_data = result.json()
+        re_embed = config_data["config"]["parameters"]["enable-reembedding"]
+        if not re_embed:
+            return
+        # attempt to reembed with umap over 100 cells.
+        endpoint = "layout/obs"
+        url = f"{self.URL_BASE}{endpoint}"
+        data = {}
+        data["filter"] = {}
+        data["filter"]["obs"] = {}
+        data["filter"]["obs"]["index"] = list(range(100))
+        data["method"] = "umap"
+        result = self.session.put(url, json=data)
+
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        result_data = result.json()
+        self.assertIsInstance(result_data, dict)
+        self.assertEqual(result_data["type"], "float32")
+        self.assertTrue(result_data["name"].startswith("reembed:umap_"))
+        self.assertIsInstance(result_data["dims"], list)
+        self.assertEqual(len(result_data["dims"]), 2)
+        dims = result_data["dims"]
+        self.assertTrue(dims[0].startswith("reembed:umap_") and dims[0].endswith("_0"))
+        self.assertTrue(dims[1].startswith("reembed:umap_") and dims[1].endswith("_1"))
+
+    def test_bad_filter(self):
+        endpoint = "data/var"
+        url = f"{self.URL_BASE}{endpoint}"
+        result = self.session.put(url, json=BAD_FILTER)
+        self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST)
+
+    def test_get_annotations_obs_fbs(self):
+        endpoint = "annotations/obs"
+        url = f"{self.URL_BASE}{endpoint}"
+        header = {"Accept": "application/octet-stream"}
+        result = self.session.get(url, headers=header)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
+        df = decode_fbs.decode_matrix_FBS(result.content)
+        self.assertEqual(df["n_rows"], 2638)
+        self.assertEqual(df["n_cols"], 6 if self.ANNOTATIONS_ENABLED else 5)
+        self.assertIsNotNone(df["columns"])
+        self.assertIsNone(df["row_idx"])
+        self.assertEqual(len(df["columns"]), df["n_cols"])
+        obs_index_col_name = self.schema["schema"]["annotations"]["obs"]["index"]
+        self.assertCountEqual(
+            df["col_idx"],
+            [obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain"]
+            + (["cluster-test"] if self.ANNOTATIONS_ENABLED else []),
+        )
+
+    def test_get_annotations_obs_keys_fbs(self):
+        endpoint = "annotations/obs"
+        query = "annotation-name=n_genes&annotation-name=percent_mito"
+        url = f"{self.URL_BASE}{endpoint}?{query}"
+        header = {"Accept": "application/octet-stream"}
+        result = self.session.get(url, headers=header)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
+        df = decode_fbs.decode_matrix_FBS(result.content)
+        self.assertEqual(df["n_rows"], 2638)
+        self.assertEqual(df["n_cols"], 2)
+        self.assertIsNotNone(df["columns"])
+        self.assertIsNone(df["row_idx"])
+        self.assertEqual(len(df["columns"]), df["n_cols"])
+        self.assertCountEqual(df["col_idx"], ["n_genes", "percent_mito"])
+
+    def test_get_annotations_obs_error(self):
+        endpoint = "annotations/obs"
+        query = "annotation-name=notakey"
+        url = f"{self.URL_BASE}{endpoint}?{query}"
+        result = self.session.get(url)
+        self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST)
+
+    def test_diff_exp(self):
+        endpoint = "diffexp/obs"
+        url = f"{self.URL_BASE}{endpoint}"
+        params = {
+            "mode": "topN",
+            "set1": {"filter": {"obs": {"annotation_value": [{"name": "louvain", "values": ["NK cells"]}]}}},
+            "set2": {"filter": {"obs": {"annotation_value": [{"name": "louvain", "values": ["CD8 T cells"]}]}}},
+            "count": 7,
+        }
+        result = self.session.post(url, json=params)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/json")
+        result_data = result.json()
+        self.assertEqual(len(result_data), 7)
+
+    def test_diff_exp_indices(self):
+        endpoint = "diffexp/obs"
+        url = f"{self.URL_BASE}{endpoint}"
+        params = {
+            "mode": "topN",
+            "count": 10,
+            "set1": {"filter": {"obs": {"index": [[0, 500]]}}},
+            "set2": {"filter": {"obs": {"index": [[500, 1000]]}}},
+        }
+        result = self.session.post(url, json=params)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/json")
+        result_data = result.json()
+        self.assertEqual(len(result_data), 10)
+
+    def test_get_annotations_var_fbs(self):
+        endpoint = "annotations/var"
+        url = f"{self.URL_BASE}{endpoint}"
+        header = {"Accept": "application/octet-stream"}
+        result = self.session.get(url, headers=header)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
+        df = decode_fbs.decode_matrix_FBS(result.content)
+        self.assertEqual(df["n_rows"], 1838)
+        self.assertEqual(df["n_cols"], 2)
+        self.assertIsNotNone(df["columns"])
+        self.assertIsNone(df["row_idx"])
+        self.assertEqual(len(df["columns"]), df["n_cols"])
+        var_index_col_name = self.schema["schema"]["annotations"]["var"]["index"]
+        self.assertCountEqual(df["col_idx"], [var_index_col_name, "n_cells"])
+
+    def test_get_annotations_var_keys_fbs(self):
+        endpoint = "annotations/var"
+        query = "annotation-name=n_cells"
+        url = f"{self.URL_BASE}{endpoint}?{query}"
+        header = {"Accept": "application/octet-stream"}
+        result = self.session.get(url, headers=header)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
+        df = decode_fbs.decode_matrix_FBS(result.content)
+        self.assertEqual(df["n_rows"], 1838)
+        self.assertEqual(df["n_cols"], 1)
+        self.assertIsNotNone(df["columns"])
+        self.assertIsNone(df["row_idx"])
+        self.assertEqual(len(df["columns"]), df["n_cols"])
+        self.assertCountEqual(df["col_idx"], ["n_cells"])
+
+    def test_get_annotations_var_error(self):
+        endpoint = "annotations/var"
+        query = "annotation-name=notakey"
+        url = f"{self.URL_BASE}{endpoint}?{query}"
+        result = self.session.get(url)
+        self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST)
+
+    def test_data_mimetype_error(self):
+        endpoint = "data/var"
+        header = {"Accept": "xxx"}
+        url = f"{self.URL_BASE}{endpoint}"
+        result = self.session.put(url, headers=header)
+        self.assertEqual(result.status_code, HTTPStatus.NOT_ACCEPTABLE)
+
+    def test_fbs_default(self):
+        endpoint = "data/var"
+        url = f"{self.URL_BASE}{endpoint}"
+        result = self.session.put(url)
+        self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST)
+
+        filter = {"filter": {"var": {"index": [0, 1, 4]}}}
+        result = self.session.put(url, json=filter)
+        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
+
+    def test_data_put_fbs(self):
+        endpoint = "data/var"
+        url = f"{self.URL_BASE}{endpoint}"
+        header = {"Accept": "application/octet-stream"}
+        result = self.session.put(url, headers=header)
+        self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST)
+
+    def test_data_get_fbs(self):
+        endpoint = "data/var"
+        url = f"{self.URL_BASE}{endpoint}"
+        header = {"Accept": "application/octet-stream"}
+        result = self.session.get(url, headers=header)
+        self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST)
+
+    def test_data_put_filter_fbs(self):
+        endpoint = "data/var"
+        url = f"{self.URL_BASE}{endpoint}"
+        header = {"Accept": "application/octet-stream"}
+        filter = {"filter": {"var": {"index": [0, 1, 4]}}}
+        result = self.session.put(url, headers=header, json=filter)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
+        df = decode_fbs.decode_matrix_FBS(result.content)
+        self.assertEqual(df["n_rows"], 2638)
+        self.assertEqual(df["n_cols"], 3)
+        self.assertIsNotNone(df["columns"])
+        self.assertIsNone(df["row_idx"])
+        self.assertEqual(len(df["columns"]), df["n_cols"])
+        self.assertListEqual(df["col_idx"].tolist(), [0, 1, 4])
+
+    def test_data_get_filter_fbs(self):
+        index_col_name = self.schema["schema"]["annotations"]["var"]["index"]
+        endpoint = "data/var"
+        query = f"var:{index_col_name}=SIK1"
+        url = f"{self.URL_BASE}{endpoint}?{query}"
+        header = {"Accept": "application/octet-stream"}
+        result = self.session.get(url, headers=header)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
+        df = decode_fbs.decode_matrix_FBS(result.content)
+        self.assertEqual(df["n_rows"], 2638)
+        self.assertEqual(df["n_cols"], 1)
+
+    def test_data_get_unknown_filter_fbs(self):
+        index_col_name = self.schema["schema"]["annotations"]["var"]["index"]
+        endpoint = "data/var"
+        query = f"var:{index_col_name}=UNKNOWN"
+        url = f"{self.URL_BASE}{endpoint}?{query}"
+        header = {"Accept": "application/octet-stream"}
+        result = self.session.get(url, headers=header)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
+        df = decode_fbs.decode_matrix_FBS(result.content)
+        self.assertEqual(df["n_rows"], 2638)
+        self.assertEqual(df["n_cols"], 0)
+
+    def test_data_put_single_var(self):
+        endpoint = "data/var"
+        url = f"{self.URL_BASE}{endpoint}"
+        header = {"Accept": "application/octet-stream"}
+        index_col_name = self.schema["schema"]["annotations"]["var"]["index"]
+        var_filter = {"filter": {"var": {"annotation_value": [{"name": index_col_name, "values": ["RER1"]}]}}}
+        result = self.session.put(url, headers=header, json=var_filter)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
+        df = decode_fbs.decode_matrix_FBS(result.content)
+        self.assertEqual(df["n_rows"], 2638)
+        self.assertEqual(df["n_cols"], 1)
+
+    def test_colors(self):
+        endpoint = "colors"
+        url = f"{self.URL_BASE}{endpoint}"
+        result = self.session.get(url)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/json")
+        result_data = result.json()
+        self.assertEqual(result_data, pbmc3k_colors)
+
+    def test_static(self):
+        endpoint = "static"
+        file = "assets/favicon.ico"
+        url = f"{self.server}/{endpoint}/{file}"
+        result = self.session.get(url)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+
+    def _setupClass(child_class, command_line):
+        child_class.ps, child_class.server = start_test_server(command_line)
+        child_class.URL_BASE = f"{child_class.server}/api/v0.2/"
+        child_class.session = requests.Session()
+        for i in range(90):
+            try:
+                result = child_class.session.get(f"{child_class.URL_BASE}schema")
+                child_class.schema = result.json()
+            except requests.exceptions.ConnectionError:
+                time.sleep(1)
+
+
+class EndPointsAnnotations(EndPoints):
+    def test_get_schema_existing_writable(self):
+        self._test_get_schema_writable("cluster-test")
+
+    def test_get_user_annotations_existing_obs_keys_fbs(self):
+        self._test_get_user_annotations_obs_keys_fbs(
+            "cluster-test", {"unassigned", "one", "two", "three", "four", "five", "six", "seven"},
+        )
+
+    def test_put_user_annotations_obs_fbs(self):
+        endpoint = "annotations/obs"
+        query = "annotation-collection-name=test_annotations"
+        url = f"{self.URL_BASE}{endpoint}?{query}"
+        n_rows = self.data.get_shape()[0]
+        fbs = make_fbs({"cat_A": pd.Series(["label_A"] * n_rows, dtype="category")})
+        result = self.session.put(url, data=fbs)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/json")
+        self.assertEqual(result.json(), {"status": "OK"})
+        self._test_get_schema_writable("cat_A")
+        self._test_get_user_annotations_obs_keys_fbs("cat_A", {"label_A"})
+
+    def _test_get_user_annotations_obs_keys_fbs(self, annotation_name, columns):
+        endpoint = "annotations/obs"
+        query = f"annotation-name={annotation_name}"
+        url = f"{self.URL_BASE}{endpoint}?{query}"
+        header = {"Accept": "application/octet-stream"}
+        result = self.session.get(url, headers=header)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
+        df = decode_fbs.decode_matrix_FBS(result.content)
+        self.assertEqual(df["n_rows"], 2638)
+        self.assertEqual(df["n_cols"], 1)
+        self.assertListEqual(df["col_idx"], [annotation_name])
+        self.assertEqual(set(df["columns"][0]), columns)
+        self.assertIsNone(df["row_idx"])
+        self.assertEqual(len(df["columns"]), df["n_cols"])
+
+    def _test_get_schema_writable(self, cluster_name):
+        endpoint = "schema"
+        url = f"{self.URL_BASE}{endpoint}"
+        result = self.session.get(url)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/json")
+        result_data = result.json()
+        columns = result_data["schema"]["annotations"]["obs"]["columns"]
+        matching_columns = [c for c in columns if c["name"] == cluster_name]
+        self.assertEqual(len(matching_columns), 1)
+        self.assertTrue(matching_columns[0]["writable"])
+
+
+class EndPointsAnndata(unittest.TestCase, EndPoints):
+    """Test Case for endpoints"""
+
+    ANNOTATIONS_ENABLED = False
+
+    @classmethod
+    def setUpClass(cls):
+        cls._setupClass(cls, [
+            f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad",
+            "--disable-annotations",
+            "--experimental-enable-reembedding",
+        ])
+
+    @classmethod
+    def tearDownClass(cls):
+        stop_test_server(cls.ps)
+
+    @property
+    def annotations_enabled(self):
+        return False
+
+
+class EndPointsCxg(unittest.TestCase, EndPoints):
+    """Test Case for endpoints"""
+
+    ANNOTATIONS_ENABLED = False
+
+    @classmethod
+    def setUpClass(cls):
+        cls._setupClass(cls, [
+            f"{FIXTURES_ROOT}/pbmc3k.cxg",
+            "--disable-annotations",
+        ])
+
+    @classmethod
+    def tearDownClass(cls):
+        stop_test_server(cls.ps)
+
+
+class EndPointsAnndataAnnotations(unittest.TestCase, EndPointsAnnotations):
+    """Test Case for endpoints"""
+
+    ANNOTATIONS_ENABLED = True
+
+    @classmethod
+    def setUpClass(cls):
+        cls.data, cls.tmp_dir, cls.annotations = data_with_tmp_annotations(
+            MatrixDataType.H5AD, annotations_fixture=True
+        )
+        cls._setupClass(cls, [
+            "--annotations-file",
+            cls.annotations.output_file,
+            cls.data.get_location(),
+        ])
+
+    @classmethod
+    def tearDownClass(cls):
+        shutil.rmtree(cls.tmp_dir)
+        stop_test_server(cls.ps)
+
+
+class EndPointsCxgAnnotations(unittest.TestCase, EndPointsAnnotations):
+    """Test Case for endpoints"""
+
+    ANNOTATIONS_ENABLED = True
+
+    @classmethod
+    def setUpClass(cls):
+        cls.data, cls.tmp_dir, cls.annotations = data_with_tmp_annotations(MatrixDataType.CXG, annotations_fixture=True)
+        cls._setupClass(cls, [
+            "--annotations-file",
+            cls.annotations.output_file,
+            cls.data.get_location(),
+        ])
+
+    @classmethod
+    def tearDownClass(cls):
+        shutil.rmtree(cls.tmp_dir)
+        stop_test_server(cls.ps)

server/test/unit/common/test_app_config.py

@@ -0,0 +1,100 @@
+import unittest
+from server.common.app_config import AppConfig
+from server.common.errors import ConfigurationError
+from server.test import PROJECT_ROOT, test_server, FIXTURES_ROOT
+import requests
+
+# NOTE, there are more tests that should be written for AppConfig.
+# this is just a start.
+
+
+class AppConfigTest(unittest.TestCase):
+    def test_update(self):
+        c = AppConfig()
+        c.update_server_config(app__verbose=True, multi_dataset__dataroot="datadir")
+        v = c.server_config.changes_from_default()
+        self.assertCountEqual(v, [("app__verbose", True, False), ("multi_dataset__dataroot", "datadir", None)])
+
+        c = AppConfig()
+        c.update_default_dataset_config(app__scripts=(), app__inline_scripts=())
+        v = c.server_config.changes_from_default()
+        self.assertCountEqual(v, [])
+
+        c = AppConfig()
+        c.update_default_dataset_config(app__scripts=[], app__inline_scripts=[])
+        v = c.default_dataset_config.changes_from_default()
+        self.assertCountEqual(v, [])
+
+        c = AppConfig()
+        c.update_default_dataset_config(app__scripts=("a", "b"), app__inline_scripts=["c", "d"])
+        v = c.default_dataset_config.changes_from_default()
+        self.assertCountEqual(v, [("app__scripts", ["a", "b"], []), ("app__inline_scripts", ["c", "d"], [])])
+
+    def test_multi_dataset(self):
+
+        c = AppConfig()
+        # test for illegal url_dataroots
+        for illegal in ("../b", "!$*", "\\n", "", "(bad)"):
+            c.update_server_config(
+                multi_dataset__dataroot={"tag": {"base_url": illegal, "dataroot": "{PROJECT_ROOT}/example-dataset"}}
+            )
+            with self.assertRaises(ConfigurationError):
+                c.complete_config()
+
+        # test for legal url_dataroots
+        for legal in ("d", "this.is-okay_", "a/b"):
+            c.update_server_config(
+                multi_dataset__dataroot={"tag": {"base_url": legal, "dataroot": "{PROJECT_ROOT}/example-dataset"}}
+            )
+            c.complete_config()
+
+        # test that multi dataroots work end to end
+        c.update_server_config(
+            multi_dataset__dataroot=dict(
+                s1=dict(dataroot=f"{PROJECT_ROOT}/example-dataset", base_url="set1/1/2"),
+                s2=dict(dataroot=f"{FIXTURES_ROOT}", base_url="set2"),
+                s3=dict(dataroot=f"{FIXTURES_ROOT}", base_url="set3"),
+            )
+        )
+
+        # Change this default to test if the dataroot overrides below work.
+        c.update_default_dataset_config(app__about_legal_tos="tos_default.html")
+
+        # specialize the configs for set1
+        c.add_dataroot_config(
+            "s1", user_annotations__enable=False, diffexp__enable=True, app__about_legal_tos="tos_set1.html"
+        )
+
+        # specialize the configs for set2
+        c.add_dataroot_config(
+            "s2", user_annotations__enable=True, diffexp__enable=False, app__about_legal_tos="tos_set2.html"
+        )
+
+        # no specializations for set3 (they get the default dataset config)
+        c.complete_config()
+
+        with test_server(app_config=c) as server:
+            session = requests.Session()
+
+            r = session.get(f"{server}/set1/1/2/pbmc3k.h5ad/api/v0.2/config")
+            data_config = r.json()
+            assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
+            assert data_config["config"]["parameters"]["annotations"] is False
+            assert data_config["config"]["parameters"]["disable-diffexp"] is False
+            assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_set1.html"
+
+            r = session.get(f"{server}/set2/pbmc3k.cxg/api/v0.2/config")
+            data_config = r.json()
+            assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
+            assert data_config["config"]["parameters"]["annotations"] is True
+            assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_set2.html"
+
+            r = session.get(f"{server}/set3/pbmc3k.cxg/api/v0.2/config")
+            data_config = r.json()
+            assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
+            assert data_config["config"]["parameters"]["annotations"] is True
+            assert data_config["config"]["parameters"]["disable-diffexp"] is False
+            assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_default.html"
+
+            r = session.get(f"{server}/health")
+            assert r.json()["status"] == "pass"

server/test/unit/common/test_colors.py

@@ -0,0 +1,40 @@
+import unittest
+
+import anndata
+from server.common.colors import convert_color_to_hex_format, convert_anndata_category_colors_to_cxg_category_colors
+from server.common.errors import ColorFormatException
+from server.test import PROJECT_ROOT
+from server.test.fixtures.fixtures import pbmc3k_colors
+
+
+class ColorsTest(unittest.TestCase):
+    """ Test color helper functions """
+
+    def test_convert_color_to_hex_format(self):
+        self.assertEqual(convert_color_to_hex_format("wheat"), "#f5deb3")
+        self.assertEqual(convert_color_to_hex_format("WHEAT"), "#f5deb3")
+        self.assertEqual(convert_color_to_hex_format((245, 222, 179)), "#f5deb3")
+        self.assertEqual(convert_color_to_hex_format([245, 222, 179]), "#f5deb3")
+        self.assertEqual(convert_color_to_hex_format("#f5deb3"), "#f5deb3")
+        self.assertEqual(
+            convert_color_to_hex_format([0.9607843137254902, 0.8705882352941177, 0.7019607843137254]), "#f5deb3"
+        )
+        for bad_input in ["foo", "BAR", "#AABB", "#AABBCCDD", "#AABBGG", (1, 2), [1, 2], (1, 2, 3, 4), [1, 2, 3, 4]]:
+            with self.assertRaises(ColorFormatException):
+                convert_color_to_hex_format(bad_input)
+
+    def test_anndata_colors_to_cxg_colors(self):
+        # test standard behavior
+        adata = self._get_h5ad()
+        self.assertEqual(convert_anndata_category_colors_to_cxg_category_colors(adata), pbmc3k_colors)
+        # test that invalid color formats raise an exception
+        adata.uns["louvain_colors"][0] = "#NOTCOOL"
+        with self.assertRaises(ColorFormatException):
+            convert_anndata_category_colors_to_cxg_category_colors(adata)
+        # test that colors without a matching obs category are skipped
+        adata = self._get_h5ad()
+        del adata.obs["louvain"]
+        self.assertEqual(convert_anndata_category_colors_to_cxg_category_colors(adata), {})
+
+    def _get_h5ad(self):
+        return anndata.read_h5ad(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad")

server/test/unit/common/test_corpora.py

@@ -0,0 +1,146 @@
+import unittest
+import anndata
+import json
+import tempfile
+import shutil
+from http import HTTPStatus
+import requests
+
+from server.common.corpora import (
+    corpora_get_versions_from_anndata,
+    corpora_is_version_supported,
+    corpora_get_props_from_anndata,
+)
+from server.test import PROJECT_ROOT, start_test_server, stop_test_server
+
+VERSION = "v0.2"
+
+
+class CorporaAPITest(unittest.TestCase):
+    def test_corpora_get_versions_from_anndata(self):
+        adata = self._get_h5ad()
+
+        if "version" in adata.uns:
+            del adata.uns["version"]
+        self.assertIsNone(corpora_get_versions_from_anndata(adata))
+
+        # something bogus
+        adata.uns["version"] = 99
+        self.assertIsNone(corpora_get_versions_from_anndata(adata))
+
+        # something legit
+        adata.uns["version"] = {"corpora_schema_version": "0.0.0", "corpora_encoding_version": "9.9.9"}
+        self.assertEqual(corpora_get_versions_from_anndata(adata), ["0.0.0", "9.9.9"])
+
+    def test_corpora_is_version_supported(self):
+        self.assertTrue(corpora_is_version_supported("1.0.0", "0.1.0"))
+        self.assertFalse(corpora_is_version_supported("0.0.0", "0.1.0"))
+        self.assertFalse(corpora_is_version_supported("1.0.0", "0.0.0"))
+
+    def test_corpora_get_props_from_anndata(self):
+        adata = self._get_h5ad()
+
+        if "version" in adata.uns:
+            del adata.uns["version"]
+        self.assertIsNone(corpora_get_props_from_anndata(adata))
+
+        # something bogus
+        adata.uns["version"] = 99
+        self.assertIsNone(corpora_get_props_from_anndata(adata))
+
+        # unsupported version, but missing required values
+        adata.uns["version"] = {"corpora_schema_version": "99.0.0", "corpora_encoding_version": "32.1.0"}
+        with self.assertRaises(ValueError):
+            corpora_get_props_from_anndata(adata)
+
+        # legit version, but missing required values
+        adata.uns["version"] = {"corpora_schema_version": "1.0.0", "corpora_encoding_version": "0.1.0"}
+        with self.assertRaises(KeyError):
+            corpora_get_props_from_anndata(adata)
+
+        some_fields = {
+            "version": {"corpora_schema_version": "1.0.0", "corpora_encoding_version": "0.1.0"},
+            "title": "title",
+            "layer_descriptions": "layer_descriptions",
+            "organism": "organism",
+            "organism_ontology_term_id": "organism_ontology_term_id",
+            "project_name": "project_name",
+            "project_description": "project_description",
+            "contributors": json.dumps([{"contributors": "contributors"}]),
+            "project_links": json.dumps([{"link_name": "link_name", "link_url": "link_url", "link_type": "SUMMARY"}]),
+        }
+        for k in some_fields:
+            adata.uns[k] = some_fields[k]
+        some_fields["contributors"] = json.loads(some_fields["contributors"])
+        some_fields["project_links"] = json.loads(some_fields["project_links"])
+        self.assertEqual(corpora_get_props_from_anndata(adata), some_fields)
+
+    def _get_h5ad(self):
+        return anndata.read_h5ad(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad")
+
+
+class CorporaRESTAPITest(unittest.TestCase):
+    """ Confirm endpoints reflect Corpora-specific features """
+
+    @classmethod
+    def setCorporaFields(cls, path):
+        adata = anndata.read_h5ad(path)
+        corpora_props = {
+            "version": {
+                "corpora_schema_version": "1.0.0",
+                "corpora_encoding_version": "0.1.0"
+            },
+            "title": "PBMC3K",
+            "contributors": json.dumps([
+                {"name": "name"}
+            ]),
+            "layer_descriptions": {
+                "X": "raw counts"
+            },
+            "organism": "human",
+            "organism_ontology_term_id": "unknown",
+            "project_name": "test project",
+            "project_description": "test description",
+            "project_links": json.dumps([
+                {"link_name": "test link", "link_type": "SUMMARY", "link_url": "https://a.u.r.l/"}
+            ]),
+            "default_embedding": "X_tsne"
+        }
+        adata.uns.update(corpora_props)
+        adata.write(path)
+
+    @classmethod
+    def setUpClass(cls):
+        cls.tmp_dir = tempfile.TemporaryDirectory()
+        src = f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad"
+        dst = f"{cls.tmp_dir.name}/pbmc3k.h5ad"
+        shutil.copyfile(src, dst)
+        cls.setCorporaFields(dst)
+        cls.ps, cls.server = start_test_server([dst])
+
+    @classmethod
+    def tearDownClass(cls):
+        stop_test_server(cls.ps)
+        cls.tmp_dir.cleanup()
+
+    def setUp(self):
+        self.session = requests.Session()
+        self.url_base = f"{self.server}/api/{VERSION}/"
+
+    def test_config(self):
+        endpoint = "config"
+        url = f"{self.url_base}{endpoint}"
+        result = self.session.get(url)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/json")
+
+        result_data = result.json()
+        self.assertIsInstance(result_data["config"]["corpora_props"], dict)
+        self.assertIsInstance(result_data["config"]["parameters"], dict)
+
+        corpora_props = result_data["config"]["corpora_props"]
+        parameters = result_data["config"]["parameters"]
+
+        self.assertEqual(corpora_props["version"]["corpora_schema_version"], "1.0.0")
+        self.assertEqual(corpora_props["organism"], "human")
+        self.assertEqual(parameters["default_embedding"], "tsne")

server/test/unit/common/test_nan_rest.py

@@ -0,0 +1,61 @@
+from http import HTTPStatus
+import unittest
+import math
+from server.test import start_test_server, stop_test_server
+
+import server.test.unit.decode_fbs as decode_fbs
+
+import requests
+
+VERSION = "v0.2"
+BAD_FILTER = {"filter": {"obs": {"annotation_value": [{"name": "xyz"}]}}}
+
+
+class WithNaNs(unittest.TestCase):
+    """Test Case for endpoints"""
+
+    @classmethod
+    def setUpClass(cls):
+        cls.ps, cls.server = start_test_server(["test/fixtures/nan.h5ad"])
+
+    @classmethod
+    def tearDownClass(cls):
+        stop_test_server(cls.ps)
+
+    def setUp(self):
+        self.session = requests.Session()
+        self.url_base = f"{self.server}/api/{VERSION}/"
+
+    def test_initialize(self):
+        endpoint = "schema"
+        url = f"{self.url_base}{endpoint}"
+        result = self.session.get(url)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+
+    def test_data(self):
+        endpoint = "data/var"
+        url = f"{self.url_base}{endpoint}"
+        filter = {"filter": {"var": {"index": [[0, 20]]}}}
+        result = self.session.put(url, json=filter)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
+        df = decode_fbs.decode_matrix_FBS(result.content)
+        self.assertTrue(math.isnan(df["columns"][3][3]))
+
+    def test_annotation_obs(self):
+        endpoint = "annotations/obs"
+        url = f"{self.url_base}{endpoint}"
+        result = self.session.get(url)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
+        df = decode_fbs.decode_matrix_FBS(result.content)
+        self.assertTrue(math.isnan(df["columns"][2][0]))
+
+    def test_annotation_var(self):
+        endpoint = "annotations/var"
+        url = f"{self.url_base}{endpoint}"
+        result = self.session.get(url)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
+        df = decode_fbs.decode_matrix_FBS(result.content)
+        self.assertTrue(math.isnan(df["columns"][2][0]))

server/test/unit/common/test_rest.py

@@ -0,0 +1,79 @@
+import unittest
+from urllib.parse import parse_qs
+from werkzeug.datastructures import MultiDict
+from server.common.rest import _query_parameter_to_filter
+from server.common.errors import FilterError
+
+
+def _qsparse(qs):
+    """ emulate what Flask/Werkzeug do to our QS """
+    return MultiDict(parse_qs(qs))
+
+
+class FilterParseTests(unittest.TestCase):
+    """ Test cases for various filter parsing """
+
+    def test_queryparam_to_filter_parse(self):
+        # categories
+        self.assertEqual(
+            _query_parameter_to_filter(_qsparse("obs:foo=bar&var:baz=133&var:baz=A&obs:baz=foo")),
+            {
+                "obs": {"annotation_value": [{"name": "foo", "values": ["bar"]}, {"name": "baz", "values": ["foo"]}]},
+                "var": {"annotation_value": [{"name": "baz", "values": ["133", "A"]}]},
+            },
+        )
+
+        # ranges
+        self.assertEqual(
+            _query_parameter_to_filter(_qsparse("obs:A=1,99&obs:B=*,100&obs:C=0,*")),
+            {
+                "obs": {
+                    "annotation_value": [
+                        {"name": "A", "min": 1, "max": 99.0},
+                        {"name": "B", "max": 100.0},
+                        {"name": "C", "min": 0.0},
+                    ]
+                },
+            },
+        )
+
+        # combo
+        self.assertEqual(
+            _query_parameter_to_filter(_qsparse("var:B=YES&var:A=1,99&var:B=NO")),
+            {
+                "var": {
+                    "annotation_value": [
+                        {"name": "B", "values": ["YES", "NO"]},
+                        {"name": "A", "min": 1.0, "max": 99.0},
+                    ]
+                },
+            },
+        )
+
+    def test_queryparam_to_filter_escaping(self):
+        self.assertEqual(
+            _query_parameter_to_filter(_qsparse("obs:var=%2521%252C%253AOK%253D&obs:A%2521=YO")),
+            {"obs": {"annotation_value": [{"name": "var", "values": ["!,:OK="]}, {"name": "A!", "values": ["YO"]}]}},
+        )
+
+    def test_queryparam_to_filter_errors(self):
+
+        # should raise FilterError
+        filter_errors = [
+            "foo=bar",  # no axis
+            "X=&Y=3",  # no value
+            "X&Y=3",  # no value
+            "moo:foo=bar",  # bad axis
+            "obs:x=1,A",  # non-numeric range
+            "var:X=1,2&var:X=3,4",  # duplicate ranges
+            "var:Y=,",
+            "var:Y=2,",
+            "var:Y=,5",
+            "var:Y=*,",
+            "var:Y=,*",
+            "var:Y=*,*",
+        ]
+
+        for qs in filter_errors:
+            with self.assertRaises(FilterError):
+                _query_parameter_to_filter(_qsparse(qs))

server/test/unit/common/test_utils.py

@@ -0,0 +1,34 @@
+import os
+import shutil
+import unittest
+
+from server.common.utils import import_plugins
+from server.test import PROJECT_ROOT, random_string
+
+
+class TestPlugins(unittest.TestCase):
+    """ Test plugin import functionality """
+
+    plugins_dir = f"{PROJECT_ROOT}/server/test/plugins"
+    test_plugin_path = f"{plugins_dir}/foo.py"
+    secret = random_string(8)
+
+    @classmethod
+    def setUpClass(cls) -> None:
+        if not os.path.isdir(cls.plugins_dir):
+            os.mkdir(cls.plugins_dir)
+        with open(cls.test_plugin_path, "w") as fh:
+            fh.write(f'SECRET = "{cls.secret}"\n')
+
+    @classmethod
+    def tearDownClass(cls) -> None:
+        if os.path.isdir(cls.plugins_dir):
+            shutil.rmtree(cls.plugins_dir)
+
+    def test_import_plugins(self):
+        self.assertTrue(os.path.isfile(self.test_plugin_path))
+        loaded_modules = import_plugins("server.test.plugins")
+        # test that import plugins found the file
+        self.assertEqual(["server.test.plugins.foo"], [ele.__name__ for ele in loaded_modules])
+        # test that the module was properly executed
+        self.assertEqual(self.secret, loaded_modules[0].SECRET)

server/test/unit/common/test_writable_annotation.py

@@ -0,0 +1,156 @@
+import json
+from os import path, listdir
+import unittest
+import server.test.unit.decode_fbs as decode_fbs
+import shutil
+
+import numpy as np
+import pandas as pd
+
+from server.common.rest import schema_get_helper, annotations_put_fbs_helper
+from server.test import data_with_tmp_annotations, make_fbs
+from server.data_common.matrix_loader import MatrixDataType
+
+
+class WritableAnnotationTest(unittest.TestCase):
+    def setUp(self):
+        self.data, self.tmp_dir, self.annotations = data_with_tmp_annotations(MatrixDataType.H5AD)
+        self.data.dataset_config.user_annotations = self.annotations
+
+    def tearDown(self):
+        shutil.rmtree(self.tmp_dir)
+
+    def annotation_put_fbs(self, fbs):
+        annotations_put_fbs_helper(self.data, fbs)
+        res = json.dumps({"status": "OK"})
+        return res
+
+    def test_error_checks(self):
+        # verify that the expected errors are generated
+        n_rows = self.data.get_shape()[0]
+        fbs_bad = make_fbs({"louvain": pd.Series(["undefined"] * n_rows, dtype="category")})
+
+        # ensure we catch attempt to overwrite non-writable data
+        with self.assertRaises(KeyError):
+            self.annotation_put_fbs(fbs_bad)
+
+    def test_write_to_file(self):
+        # verify the file is written as expected
+        n_rows = self.data.get_shape()[0]
+        fbs = make_fbs(
+            {
+                "cat_A": pd.Series(["label_A"] * n_rows, dtype="category"),
+                "cat_B": pd.Series(["label_B"] * n_rows, dtype="category"),
+            }
+        )
+        res = self.annotation_put_fbs(fbs)
+        self.assertEqual(res, json.dumps({"status": "OK"}))
+        self.assertTrue(path.exists(self.annotations.output_file))
+        df = pd.read_csv(self.annotations.output_file, index_col=0, header=0, comment="#")
+        self.assertEqual(df.shape, (n_rows, 2))
+        self.assertEqual(set(df.columns), {"cat_A", "cat_B"})
+        self.assertTrue(self.data.original_obs_index.equals(df.index))
+        self.assertTrue(np.all(df["cat_A"] == ["label_A"] * n_rows))
+        self.assertTrue(np.all(df["cat_B"] == ["label_B"] * n_rows))
+
+        # verify complete overwrite on second attempt, AND rotation occurs
+        fbs = make_fbs(
+            {
+                "cat_A": pd.Series(["label_A1"] * n_rows, dtype="category"),
+                "cat_C": pd.Series(["label_C"] * n_rows, dtype="category"),
+            }
+        )
+        res = self.annotation_put_fbs(fbs)
+        self.assertEqual(res, json.dumps({"status": "OK"}))
+        self.assertTrue(path.exists(self.annotations.output_file))
+        df = pd.read_csv(self.annotations.output_file, index_col=0, header=0, comment="#")
+        self.assertEqual(set(df.columns), {"cat_A", "cat_C"})
+        self.assertTrue(np.all(df["cat_A"] == ["label_A1"] * n_rows))
+        self.assertTrue(np.all(df["cat_C"] == ["label_C"] * n_rows))
+
+        # rotation
+        name, ext = path.splitext(self.annotations.output_file)
+        backup_dir = f"{name}-backups"
+        self.assertTrue(path.isdir(backup_dir))
+        found_files = listdir(backup_dir)
+        self.assertEqual(len(found_files), 1)
+
+    def test_file_rotation_to_max_9(self):
+        # verify we stop rotation at 9
+        n_rows = self.data.get_shape()[0]
+        fbs = make_fbs(
+            {
+                "cat_A": pd.Series(["label_A"] * n_rows, dtype="category"),
+                "cat_B": pd.Series(["label_B"] * n_rows, dtype="category"),
+            }
+        )
+        for i in range(0, 11):
+            res = self.annotation_put_fbs(fbs)
+            self.assertEqual(res, json.dumps({"status": "OK"}))
+
+        name, ext = path.splitext(self.annotations.output_file)
+        backup_dir = f"{name}-backups"
+        self.assertTrue(path.isdir(backup_dir))
+        found_files = listdir(backup_dir)
+        self.assertTrue(len(found_files) <= 9)
+
+    def test_put_get_roundtrip(self):
+        # verify that OBS PUTs (annotation_put_fbs) are accessible via
+        # GET (annotation_to_fbs_matrix)
+
+        n_rows = self.data.get_shape()[0]
+        fbs = make_fbs(
+            {
+                "cat_A": pd.Series(["label_A"] * n_rows, dtype="category"),
+                "cat_B": pd.Series(["label_B"] * n_rows, dtype="category"),
+            }
+        )
+
+        # put
+        res = self.annotation_put_fbs(fbs)
+        self.assertEqual(res, json.dumps({"status": "OK"}))
+
+        # get
+        labels = self.annotations.read_labels(None)
+        fbsAll = self.data.annotation_to_fbs_matrix("obs", None, labels)
+        schema = schema_get_helper(self.data)
+        annotations = decode_fbs.decode_matrix_FBS(fbsAll)
+        obs_index_col_name = schema["annotations"]["obs"]["index"]
+        self.assertEqual(annotations["n_rows"], n_rows)
+        self.assertEqual(annotations["n_cols"], 7)
+        self.assertIsNone(annotations["row_idx"])
+        self.assertEqual(
+            annotations["col_idx"],
+            [obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain", "cat_A", "cat_B"],
+        )
+        col_idx = annotations["col_idx"]
+        self.assertEqual(annotations["columns"][col_idx.index("cat_A")], ["label_A"] * n_rows)
+        self.assertEqual(annotations["columns"][col_idx.index("cat_B")], ["label_B"] * n_rows)
+
+        # verify the schema was updated
+        all_col_schema = {c["name"]: c for c in schema["annotations"]["obs"]["columns"]}
+        self.assertEqual(
+            all_col_schema["cat_A"],
+            {"name": "cat_A", "type": "categorical", "categories": ["label_A"], "writable": True},
+        )
+        self.assertEqual(
+            all_col_schema["cat_B"],
+            {"name": "cat_B", "type": "categorical", "categories": ["label_B"], "writable": True},
+        )
+
+    def test_config(self):
+        features = self.data.get_features(self.annotations)
+
+        # test each for singular presence and accuracy of available flag
+        def check_feature(method, path, available):
+            feature = list(
+                filter(lambda f: f.method == method and f.path == path and f.available == available, features)
+            )
+            self.assertIsNotNone(feature)
+            self.assertEqual(len(feature), 1)
+
+        check_feature("POST", "/cluster/", False)
+        check_feature("POST", "/diffexp/", self.data.dataset_config.diffexp__enable)
+        check_feature("GET", "/layout/obs", True)
+        check_feature("PUT", "/layout/obs", self.data.dataset_config.embeddings__enable_reembedding)
+        check_feature("PUT", "/annotations/obs", True)

server/test/unit/compute/test_diffexp_cxg.py

@@ -0,0 +1,153 @@
+import unittest
+from server.data_common.matrix_loader import MatrixDataLoader
+from server.test import PROJECT_ROOT, app_config, FIXTURES_ROOT
+import server.compute.diffexp_cxg as diffexp_cxg
+import server.compute.diffexp_generic as diffexp_generic
+from server.converters.cxgtool import write_cxg, create_cxg_group_metadata
+from server.test.performance.create_test_matrix import create_test_h5ad
+from server.data_common.fbs.matrix import encode_matrix_fbs, decode_matrix_fbs
+import numpy as np
+import tempfile
+import os
+
+
+class DiffExpTest(unittest.TestCase):
+    """Tests the diffexp returns the expected results for one test case, using different
+    adaptor types and different algorithms."""
+
+    def load_dataset(self, path, extra_server_config={}, extra_dataset_config={}):
+        config = app_config(path, extra_server_config=extra_server_config, extra_dataset_config=extra_dataset_config)
+        loader = MatrixDataLoader(path)
+        adaptor = loader.open(config)
+        return adaptor
+
+    def get_mask(self, adaptor, start, stride):
+        """Simple function to return a mask or rows"""
+        rows = adaptor.get_shape()[0]
+        sel = list(range(start, rows, stride))
+        mask = np.zeros(rows, dtype=bool)
+        mask[sel] = True
+        return mask
+
+    def compare_diffexp_results(self, results, expects):
+        self.assertEqual(len(results), len(expects))
+        for result, expect in zip(results, expects):
+            self.assertEqual(result[0], expect[0])
+            self.assertTrue(np.isclose(result[1], expect[1], 1e-6, 1e-4))
+            self.assertTrue(np.isclose(result[2], expect[2], 1e-6, 1e-4))
+            self.assertTrue(np.isclose(result[3], expect[3], 1e-6, 1e-4))
+
+    def check_1_10_2_10(self, results):
+        """Checks the results for a specific set of rows selections"""
+        expects = [
+            [956, 0.016060986, 0.0008649321884808977, 1.0],
+            [1124, 0.96602094, 0.0011717216548271284, 1.0],
+            [1809, 1.1110606, 0.0019304405196777848, 1.0],
+            [1712, -0.5525154, 0.0051788902660723345, 1.0],
+            [1754, 0.5201581, 0.005691734062127954, 1.0],
+            [948, 1.6390722, 0.006622111055981219, 1.0],
+            [1810, 0.78618884, 0.007055917428377063, 1.0],
+            [779, 1.5241305, 0.007202934422407284, 1.0],
+            [1575, 1.0317602, 0.007830310753043345, 1.0],
+            [576, 0.97873515, 0.008272092578813124, 1.0],
+        ]
+        self.compare_diffexp_results(results, expects)
+
+    def get_X_col(self, adaptor, cols):
+        varmask = np.zeros(adaptor.get_shape()[1], dtype=bool)
+        varmask[cols] = True
+        return adaptor.get_X_array(None, varmask)
+
+    def test_anndata_default(self):
+        """Test an anndata adaptor with its default diffexp algorithm (diffexp_generic)"""
+        adaptor = self.load_dataset(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad")
+        maskA = self.get_mask(adaptor, 1, 10)
+        maskB = self.get_mask(adaptor, 2, 10)
+        results = adaptor.compute_diffexp_ttest(maskA, maskB, 10)
+        self.check_1_10_2_10(results)
+
+    def test_cxg_default(self):
+        """Test a cxg adaptor with its default diffexp algorithm (diffexp_cxg)"""
+        adaptor = self.load_dataset(f"{FIXTURES_ROOT}/pbmc3k.cxg")
+        maskA = self.get_mask(adaptor, 1, 10)
+        maskB = self.get_mask(adaptor, 2, 10)
+
+        # run it through the adaptor
+        results = adaptor.compute_diffexp_ttest(maskA, maskB, 10)
+        self.check_1_10_2_10(results)
+
+        # run it directly
+        results = diffexp_cxg.diffexp_ttest(adaptor, maskA, maskB, 10)
+        self.check_1_10_2_10(results)
+
+    def test_cxg_generic(self):
+        """Test a cxg adaptor with the generic adaptor"""
+        adaptor = self.load_dataset(f"{FIXTURES_ROOT}/pbmc3k.cxg")
+        maskA = self.get_mask(adaptor, 1, 10)
+        maskB = self.get_mask(adaptor, 2, 10)
+        # run it directly
+        results = diffexp_generic.diffexp_ttest(adaptor, maskA, maskB, 10)
+        self.check_1_10_2_10(results)
+
+    def test_cxg_sparse(self):
+        self.sparse_diffexp(False)
+
+    def test_cxg_sparse_col_shift(self):
+        self.sparse_diffexp(True)
+
+    def sparse_diffexp(self, apply_col_shift):
+        with tempfile.TemporaryDirectory() as dirname:
+            # create a sparse matrix
+            h5adfile = os.path.join(dirname, "sparse.h5ad")
+            create_test_h5ad(h5adfile, 2000, 2000, 10, apply_col_shift)
+            adaptor_anndata = self.load_dataset(h5adfile, extra_dataset_config=dict(embeddings__names=[]))
+            adata = adaptor_anndata.data
+
+            sparsename = os.path.join(dirname, "sparse.cxg")
+            cxg_group_metadata = create_cxg_group_metadata(adata=adata, basefname="sparse.h5ad", title="sparse",)
+            write_cxg(adata=adata, container=sparsename, cxg_group_metadata=cxg_group_metadata, sparse_threshold=11)
+            adaptor_sparse = self.load_dataset(sparsename)
+            assert adaptor_sparse.open_array("X").schema.sparse
+            assert adaptor_sparse.has_array("X_col_shift") == apply_col_shift
+
+            densename = os.path.join(dirname, "dense.cxg")
+            cxg_group_metadata = create_cxg_group_metadata(adata=adata, basefname="dense.h5ad", title="dense",)
+            write_cxg(adata=adata, container=densename, cxg_group_metadata=cxg_group_metadata, sparse_threshold=0)
+            adaptor_dense = self.load_dataset(densename)
+            assert not adaptor_dense.open_array("X").schema.sparse
+            assert not adaptor_dense.has_array("X_col_shift")
+
+            maskA = self.get_mask(adaptor_anndata, 1, 10)
+            maskB = self.get_mask(adaptor_anndata, 2, 10)
+
+            diffexp_results_anndata = diffexp_generic.diffexp_ttest(adaptor_anndata, maskA, maskB, 10)
+            diffexp_results_sparse = diffexp_cxg.diffexp_ttest(adaptor_sparse, maskA, maskB, 10)
+            diffexp_results_dense = diffexp_cxg.diffexp_ttest(adaptor_dense, maskA, maskB, 10)
+
+            self.compare_diffexp_results(diffexp_results_anndata, diffexp_results_sparse)
+            self.compare_diffexp_results(diffexp_results_anndata, diffexp_results_dense)
+
+            topcols = np.array([x[0] for x in diffexp_results_anndata])
+            cols_anndata = self.get_X_col(adaptor_anndata, topcols)
+            cols_sparse = self.get_X_col(adaptor_sparse, topcols)
+            cols_dense = self.get_X_col(adaptor_dense, topcols)
+            assert cols_anndata.shape[0] == adaptor_sparse.get_shape()[0]
+            assert cols_anndata.shape[1] == len(diffexp_results_anndata)
+
+            def convert(mat, cols):
+                return decode_matrix_fbs(encode_matrix_fbs(mat, col_idx=cols)).to_numpy()
+
+            cols_anndata = convert(cols_anndata, topcols)
+            cols_sparse = convert(cols_sparse, topcols)
+            cols_dense = convert(cols_dense, topcols)
+
+            x = adaptor_sparse.get_X_array()
+            assert x.shape == adaptor_sparse.get_shape()
+
+            for row in range(cols_anndata.shape[0]):
+                for col in range(cols_anndata.shape[1]):
+                    vanndata = cols_anndata[row][col]
+                    vsparse = cols_sparse[row][col]
+                    vdense = cols_dense[row][col]
+                    self.assertTrue(np.isclose(vanndata, vsparse, 1e-6, 1e-6))
+                    self.assertTrue(np.isclose(vanndata, vdense, 1e-6, 1e-6))

server/test/unit/converters/test_cxgtool.py

@@ -0,0 +1,41 @@
+import shutil
+import unittest
+
+import anndata
+
+from server.common.data_locator import DataLocator
+from server.converters.cxgtool import write_cxg, create_cxg_group_metadata
+from server.data_cxg.cxg_adaptor import CxgAdaptor
+from server.test import PROJECT_ROOT, app_config, random_string
+from server.test.fixtures.fixtures import pbmc3k_colors
+
+
+class TestCxgAdaptor(unittest.TestCase):
+    def setUp(self) -> None:
+        self.fixtures = []
+
+    def tearDown(self) -> None:
+        try:
+            for data_locator in self.fixtures:
+                print("REMOVING ", data_locator)
+                shutil.rmtree(data_locator)
+        except FileNotFoundError:
+            pass
+
+    def test_cxg_category_colors(self):
+        data = self.convert_pbmc3k(extract_colors=True)
+        self.assertEqual(data.get_colors(), pbmc3k_colors)
+        data = self.convert_pbmc3k(extract_colors=False)
+        self.assertEqual(data.get_colors(), {})
+
+    def convert_pbmc3k(self, **kwargs):
+        rand_str = random_string(8)
+        data_locator = f"/tmp/test_{rand_str}.cxg"
+        self.fixtures.append(data_locator)
+        source_h5ad = anndata.read_h5ad(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad")
+        cxg_group_metadata = create_cxg_group_metadata(
+            adata=source_h5ad, basefname="pbmc3k.h5ad", title="pbmc3k", **kwargs
+        )
+        write_cxg(adata=source_h5ad, container=data_locator, cxg_group_metadata=cxg_group_metadata)
+        config = app_config(data_locator)
+        return CxgAdaptor(DataLocator(data_locator), config)

server/test/unit/data_anndata/test_anndata_adaptor.py

@@ -0,0 +1,249 @@
+import json
+import sys
+import time
+import unittest
+
+import numpy as np
+import pandas as pd
+import pytest
+from parameterized import parameterized_class
+
+import server.test.unit.decode_fbs as decode_fbs
+from server.common.data_locator import DataLocator
+from server.common.errors import FilterError
+from server.data_anndata.anndata_adaptor import AnndataAdaptor
+from server.test import PROJECT_ROOT, app_config, FIXTURES_ROOT
+from server.test.fixtures.fixtures import pbmc3k_colors
+
+"""
+Test the anndata adaptor using the pbmc3k data set.
+"""
+
+
+@parameterized_class(
+    ("data_locator", "backed"),
+    [
+        (f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad", False),
+        (f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad", False),
+        (f"{FIXTURES_ROOT}/pbmc3k-CSR-gz.h5ad", False),
+        (f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad", True),
+        (f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad", True),
+        (f"{FIXTURES_ROOT}/pbmc3k-CSR-gz.h5ad", True),
+    ],
+)
+class AdaptorTest(unittest.TestCase):
+    def setUp(self):
+        config = app_config(self.data_locator, self.backed)
+        self.data = AnndataAdaptor(DataLocator(self.data_locator), config)
+
+    def test_init(self):
+        self.assertEqual(self.data.cell_count, 2638)
+        self.assertEqual(self.data.gene_count, 1838)
+        epsilon = 0.000_005
+        self.assertTrue(self.data.data.X[0, 0] - -0.171_469_51 < epsilon)
+
+    def test_mandatory_annotations(self):
+        obs_index_col_name = self.data.get_schema()["annotations"]["obs"]["index"]
+        self.assertIn(obs_index_col_name, self.data.data.obs)
+        self.assertEqual(list(self.data.data.obs.index), list(range(2638)))
+        var_index_col_name = self.data.get_schema()["annotations"]["var"]["index"]
+        self.assertIn(var_index_col_name, self.data.data.var)
+        self.assertEqual(list(self.data.data.var.index), list(range(1838)))
+
+    @pytest.mark.filterwarnings("ignore:Anndata data matrix")
+    def test_data_type(self):
+        # don't run the test on the more exotic data types, as they don't
+        # support the astype() interface (used by this test, but not underlying app)
+        if isinstance(self.data.data.X, np.ndarray):
+            self.data.data.X = self.data.data.X.astype("float64")
+            with self.assertWarns(UserWarning):
+                self.data._validate_data_types()
+
+    def test_filter_idx(self):
+        filter_ = {"filter": {"var": {"index": [1, 99, [200, 300]]}}}
+        fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
+        data = decode_fbs.decode_matrix_FBS(fbs)
+        self.assertEqual(data["n_rows"], 2638)
+        self.assertEqual(data["n_cols"], 102)
+
+    def test_filter_complex(self):
+        filter_ = {
+            "filter": {"var": {"annotation_value": [{"name": "n_cells", "min": 10}], "index": [1, 99, [200, 300]]}}
+        }
+        fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
+        data = decode_fbs.decode_matrix_FBS(fbs)
+        self.assertEqual(data["n_rows"], 2638)
+        self.assertEqual(data["n_cols"], 91)
+
+    def test_obs_and_var_names(self):
+        self.assertEqual(np.sum(self.data.data.var[self.data.get_schema()["annotations"]["var"]["index"]].isna()), 0)
+        self.assertEqual(np.sum(self.data.data.obs[self.data.get_schema()["annotations"]["obs"]["index"]].isna()), 0)
+
+    def test_get_colors(self):
+        self.assertEqual(self.data.get_colors(), pbmc3k_colors)
+
+    def test_get_schema(self):
+        with open(f"{FIXTURES_ROOT}/schema.json") as fh:
+            schema = json.load(fh)
+            self.assertDictEqual(self.data.get_schema(), schema)
+
+    def test_schema_produces_error(self):
+        self.data.data.obs["time"] = pd.Series(
+            list([time.time() for i in range(self.data.cell_count)]), dtype="datetime64[ns]",
+        )
+        with pytest.raises(TypeError):
+            self.data._create_schema()
+
+    def test_config(self):
+        features = self.data.get_features(annotations=None)
+
+        # test each for singular presence and accuracy of available flag
+        def check_feature(method, path, available):
+            feature = list(
+                filter(lambda f: f.method == method and f.path == path and f.available == available, features)
+            )
+            self.assertIsNotNone(feature)
+            self.assertEqual(len(feature), 1)
+
+        check_feature("POST", "/cluster/", False)
+        check_feature("POST", "/diffexp/", self.data.dataset_config.diffexp__enable)
+        check_feature("GET", "/layout/obs", True)
+        check_feature("PUT", "/layout/obs", self.data.dataset_config.embeddings__enable_reembedding)
+        check_feature("PUT", "/annotations/obs", False)
+
+    def test_layout(self):
+        fbs = self.data.layout_to_fbs_matrix(fields=None)
+        layout = decode_fbs.decode_matrix_FBS(fbs)
+        self.assertEqual(layout["n_cols"], 6)
+        self.assertEqual(layout["n_rows"], 2638)
+
+        X = layout["columns"][0]
+        self.assertTrue((X >= 0).all() and (X <= 1).all())
+        Y = layout["columns"][1]
+        self.assertTrue((Y >= 0).all() and (Y <= 1).all())
+
+    def test_layout_fields(self):
+        """ X_pca, X_tsne, X_umap are available """
+        fbs = self.data.layout_to_fbs_matrix(["pca"])
+        layout = decode_fbs.decode_matrix_FBS(fbs)
+        self.assertEqual(layout["n_cols"], 2)
+        self.assertEqual(layout["n_rows"], 2638)
+        self.assertCountEqual(layout["col_idx"], ["pca_0", "pca_1"])
+
+        fbs = self.data.layout_to_fbs_matrix(["tsne", "pca"])
+        layout = decode_fbs.decode_matrix_FBS(fbs)
+        self.assertEqual(layout["n_cols"], 4)
+        self.assertEqual(layout["n_rows"], 2638)
+        self.assertCountEqual(layout["col_idx"], ["tsne_0", "tsne_1", "pca_0", "pca_1"])
+
+    def test_annotations(self):
+        fbs = self.data.annotation_to_fbs_matrix("obs")
+        annotations = decode_fbs.decode_matrix_FBS(fbs)
+        self.assertEqual(annotations["n_rows"], 2638)
+        self.assertEqual(annotations["n_cols"], 5)
+        obs_index_col_name = self.data.get_schema()["annotations"]["obs"]["index"]
+        self.assertEqual(
+            annotations["col_idx"], [obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain"],
+        )
+
+        fbs = self.data.annotation_to_fbs_matrix("var")
+        annotations = decode_fbs.decode_matrix_FBS(fbs)
+        self.assertEqual(annotations["n_rows"], 1838)
+        self.assertEqual(annotations["n_cols"], 2)
+        var_index_col_name = self.data.get_schema()["annotations"]["var"]["index"]
+        self.assertEqual(annotations["col_idx"], [var_index_col_name, "n_cells"])
+
+    def test_annotation_fields(self):
+        fbs = self.data.annotation_to_fbs_matrix("obs", ["n_genes", "n_counts"])
+        annotations = decode_fbs.decode_matrix_FBS(fbs)
+        self.assertEqual(annotations["n_rows"], 2638)
+        self.assertEqual(annotations["n_cols"], 2)
+
+        var_index_col_name = self.data.get_schema()["annotations"]["var"]["index"]
+        fbs = self.data.annotation_to_fbs_matrix("var", [var_index_col_name])
+        annotations = decode_fbs.decode_matrix_FBS(fbs)
+        self.assertEqual(annotations["n_rows"], 1838)
+        self.assertEqual(annotations["n_cols"], 1)
+
+    def test_diffexp_topN(self):
+        f1 = {"filter": {"obs": {"index": [[0, 500]]}}}
+        f2 = {"filter": {"obs": {"index": [[500, 1000]]}}}
+        result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"]))
+        self.assertEqual(len(result), 10)
+        result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"], 20))
+        self.assertEqual(len(result), 20)
+
+    def test_data_frame(self):
+        f1 = {"var": {"index": [[0, 10]]}}
+        fbs = self.data.data_frame_to_fbs_matrix(f1, "var")
+        data = decode_fbs.decode_matrix_FBS(fbs)
+        self.assertEqual(data["n_rows"], 2638)
+        self.assertEqual(data["n_cols"], 10)
+
+        with self.assertRaises(ValueError):
+            self.data.data_frame_to_fbs_matrix(None, "obs")
+
+    def test_filtered_data_frame(self):
+        filter_ = {"filter": {"var": {"annotation_value": [{"name": "n_cells", "min": 100}]}}}
+        fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
+        data = decode_fbs.decode_matrix_FBS(fbs)
+        self.assertEqual(data["n_rows"], 2638)
+        self.assertEqual(data["n_cols"], 1040)
+
+        filter_ = {"filter": {"obs": {"annotation_value": [{"name": "n_counts", "min": 3000}]}}}
+        with self.assertRaises(FilterError):
+            self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
+
+    def test_data_named_gene(self):
+        var_index_col_name = self.data.get_schema()["annotations"]["var"]["index"]
+        filter_ = {"filter": {"var": {"annotation_value": [{"name": var_index_col_name, "values": ["RER1"]}]}}}
+        fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
+        data = decode_fbs.decode_matrix_FBS(fbs)
+        self.assertEqual(data["n_rows"], 2638)
+        self.assertEqual(data["n_cols"], 1)
+        self.assertEqual(data["col_idx"], [4])
+
+        filter_ = {
+            "filter": {"var": {"annotation_value": [{"name": var_index_col_name, "values": ["SPEN", "TYMP", "PRMT2"]}]}}
+        }
+        fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
+        data = decode_fbs.decode_matrix_FBS(fbs)
+        self.assertEqual(data["n_rows"], 2638)
+        self.assertEqual(data["n_cols"], 3)
+        self.assertTrue((data["col_idx"] == [15, 1818, 1837]).all())
+
+    def test_compute_embedding(self):
+        filter = {"obs": {"index": [[0, 100]]}}
+
+        # Verify that we correctly handle the case where we lack scanpy
+        import unittest.mock
+
+        with unittest.mock.patch.dict(sys.modules, {"scanpy": None}):
+            with self.assertRaises(NotImplementedError):
+                self.data.compute_embedding("umap", filter)
+
+        # if we happen to have scanpy, test the full API, else punt
+        import importlib
+
+        scanpy_spec = importlib.util.find_spec("scanpy")
+        if scanpy_spec is None:
+            print("Skipping compute_embedding test as ScanPy not installed")
+            return
+
+        # this feature is unsupported in backed mode, and we expect an error
+        if self.data.data.isbacked:
+            with self.assertRaises(NotImplementedError):
+                self.data.compute_embedding("umap", filter)
+            return
+
+        schema = self.data.compute_embedding("umap", filter)
+
+        self.assertIsInstance(schema["name"], str)
+        name = schema["name"]
+        self.assertEqual(schema["type"], "float32")
+        self.assertEqual(schema["dims"], [f"{name}_0", f"{name}_1"])
+
+        emb = self.data.data.obsm[f"X_{name}"]
+        self.assertEqual(emb.shape, (2638, 2))
+        self.assertTrue(np.isfinite(emb[0:100]).all())
+        self.assertTrue(np.isnan(emb[100:]).all())

server/test/unit/data_anndata/test_anndata_adaptor_data_load.py

@@ -0,0 +1,79 @@
+import unittest
+import json
+
+from server.data_anndata.anndata_adaptor import AnndataAdaptor
+from server.common.data_locator import DataLocator
+from server.common.app_config import AppConfig
+from server.test import PROJECT_ROOT
+
+
+class DataLoadAdaptorTest(unittest.TestCase):
+    """
+    Test file loading, including deferred loading/update.
+    """
+
+    def setUp(self):
+        self.data_file = DataLocator(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad")
+        config = AppConfig()
+        config.update_server_config(single_dataset__datapath=self.data_file.path)
+        config.complete_config()
+        self.data = AnndataAdaptor(self.data_file, config)
+
+    def test_delayed_load_data(self):
+        self.data._create_schema()
+        self.assertEqual(self.data.cell_count, 2638)
+        self.assertEqual(self.data.gene_count, 1838)
+        epsilon = 0.000_005
+        self.assertTrue(self.data.data.X[0, 0] - -0.171_469_51 < epsilon)
+
+    def test_diffexp_topN(self):
+        f1 = {"filter": {"obs": {"index": [[0, 500]]}}}
+        f2 = {"filter": {"obs": {"index": [[500, 1000]]}}}
+        result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"]))
+        self.assertEqual(len(result), 10)
+        result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"], 20))
+        self.assertEqual(len(result), 20)
+
+
+class DataLocatorAdaptorTest(unittest.TestCase):
+    """
+    Test various types of data locators we expect to consume
+    """
+
+    def get_basic_config(self):
+        config = AppConfig()
+        config.update_server_config(
+            single_dataset__obs_names=None, single_dataset__var_names=None,
+        )
+        config.update_default_dataset_config(
+            embeddings__names=["umap"], presentation__max_categories=100, diffexp__lfc_cutoff=0.01,
+        )
+        return config
+
+    def stdAsserts(self, data):
+        """ run these each time we load the data """
+        self.assertIsNotNone(data)
+        self.assertEqual(data.cell_count, 2638)
+        self.assertEqual(data.gene_count, 1838)
+
+    def test_posix_file(self):
+        locator = DataLocator("../example-dataset/pbmc3k.h5ad")
+        config = self.get_basic_config()
+        config.update_server_config(single_dataset__datapath=locator.path)
+        config.complete_config()
+        data = AnndataAdaptor(locator, config)
+        self.stdAsserts(data)
+
+    def test_url_https(self):
+        url = "https://raw.githubusercontent.com/chanzuckerberg/cellxgene/main/example-dataset/pbmc3k.h5ad"
+        locator = DataLocator(url)
+        config = self.get_basic_config()
+        data = AnndataAdaptor(locator, config)
+        self.stdAsserts(data)
+
+    def test_url_http(self):
+        url = "http://raw.githubusercontent.com/chanzuckerberg/cellxgene/main/example-dataset/pbmc3k.h5ad"
+        locator = DataLocator(url)
+        config = self.get_basic_config()
+        data = AnndataAdaptor(locator, config)
+        self.stdAsserts(data)

server/test/unit/data_anndata/test_nan_anndata_adaptor.py

@@ -0,0 +1,63 @@
+import math
+import unittest
+import warnings
+
+import pytest
+
+import server.test.unit.decode_fbs as decode_fbs
+from server.common.data_locator import DataLocator
+from server.common.errors import FilterError
+from server.data_anndata.anndata_adaptor import AnndataAdaptor
+from server.test import app_config, FIXTURES_ROOT
+
+
+class NaNTest(unittest.TestCase):
+    def setUp(self):
+        self.data_locator = DataLocator(f"{FIXTURES_ROOT}/nan.h5ad")
+        self.config = app_config(self.data_locator.path)
+
+        with warnings.catch_warnings():
+            warnings.simplefilter("ignore", category=UserWarning)
+            self.data = AnndataAdaptor(self.data_locator, self.config)
+            self.data._create_schema()
+
+    def test_load(self):
+        with self.assertWarns(UserWarning):
+            self.data = AnndataAdaptor(self.data_locator, self.config)
+
+    def test_init(self):
+        self.assertEqual(self.data.cell_count, 100)
+        self.assertEqual(self.data.gene_count, 100)
+        epsilon = 0.000_005
+        self.assertTrue(self.data.data.X[0, 0] - -0.171_469_51 < epsilon)
+
+    def test_dataframe(self):
+        data_frame_var = decode_fbs.decode_matrix_FBS(self.data.data_frame_to_fbs_matrix(None, "var"))
+        self.assertIsNotNone(data_frame_var)
+        self.assertEqual(data_frame_var["n_rows"], 100)
+        self.assertEqual(data_frame_var["n_cols"], 100)
+        self.assertTrue(math.isnan(data_frame_var["columns"][3][3]))
+
+        with pytest.raises(FilterError):
+            self.data.data_frame_to_fbs_matrix("an erroneous filter", "var")
+        with pytest.raises(FilterError):
+            filter_ = {"filter": {"obs": {"index": [1, 99, [200, 300]]}}}
+            self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
+
+    def test_dataframe_obs_not_implemented(self):
+        with self.assertRaises(ValueError) as cm:
+            decode_fbs.decode_matrix_FBS(self.data.data_frame_to_fbs_matrix(None, "obs"))
+        self.assertIsNotNone(cm.exception)
+
+    def test_annotation(self):
+        annotations = decode_fbs.decode_matrix_FBS(self.data.annotation_to_fbs_matrix("obs"))
+        obs_index_col_name = self.data.schema["annotations"]["obs"]["index"]
+        self.assertEqual(annotations["col_idx"], [obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain"])
+        self.assertEqual(annotations["n_rows"], 100)
+        self.assertTrue(math.isnan(annotations["columns"][2][0]))
+
+        annotations = decode_fbs.decode_matrix_FBS(self.data.annotation_to_fbs_matrix("var"))
+        var_index_col_name = self.data.schema["annotations"]["var"]["index"]
+        self.assertEqual(annotations["col_idx"], [var_index_col_name, "n_cells", "var_with_nans"])
+        self.assertEqual(annotations["n_rows"], 100)
+        self.assertTrue(math.isnan(annotations["columns"][2][0]))

server/test/unit/data_common/fbs/test_matrix.py

@@ -0,0 +1,82 @@
+import unittest
+import pandas as pd
+import numpy as np
+from scipy import sparse
+
+import server.test.unit.decode_fbs as decode_fbs
+from server.data_common.fbs.matrix import encode_matrix_fbs, decode_matrix_fbs
+
+
+class FbsTests(unittest.TestCase):
+    """Test Case for Matrix FBS data encode/decode """
+
+    def test_encode_boundary(self):
+        """ test various boundary checks """
+
+        # row indexing is unsupported
+        with self.assertRaises(ValueError):
+            encode_matrix_fbs(matrix=pd.DataFrame(), row_idx=[])
+
+        # matrix must be 2D
+        with self.assertRaises(ValueError):
+            encode_matrix_fbs(matrix=np.zeros((3, 2, 1)))
+        with self.assertRaises(ValueError):
+            encode_matrix_fbs(matrix=np.ones((10,)))
+
+    def fbs_checks(self, fbs, dims, expected_types, expected_column_idx):
+        d = decode_fbs.decode_matrix_FBS(fbs)
+        self.assertEqual(d["n_rows"], dims[0])
+        self.assertEqual(d["n_cols"], dims[1])
+        self.assertIsNone(d["row_idx"])
+        self.assertEqual(len(d["columns"]), dims[1])
+        for i in range(0, len(d["columns"])):
+            self.assertEqual(len(d["columns"][i]), dims[0])
+            self.assertIsInstance(d["columns"][i], expected_types[i][0])
+            if expected_types[i][1] is not None:
+                self.assertEqual(d["columns"][i].dtype, expected_types[i][1])
+        if expected_column_idx is not None:
+            self.assertSetEqual(set(expected_column_idx), set(d["col_idx"]))
+
+    def test_encode_DataFrame(self):
+        df = pd.DataFrame(
+            data={
+                "a": np.zeros((10,), dtype=np.float32),
+                "b": np.ones((10,), dtype=np.int64),
+                "c": np.array([i for i in range(0, 10)], dtype=np.uint16),
+                "d": pd.Series(["x", "y", "z", "x", "y", "z", "a", "x", "y", "z"], dtype="category"),
+            }
+        )
+        expected_types = ((np.ndarray, np.float32), (np.ndarray, np.int32), (np.ndarray, np.uint32), (list, None))
+        fbs = encode_matrix_fbs(matrix=df, row_idx=None, col_idx=df.columns)
+        self.fbs_checks(fbs, (10, 4), expected_types, ["a", "b", "c", "d"])
+
+    def test_encode_ndarray(self):
+        arr = np.zeros((3, 2), dtype=np.float32)
+        expected_types = ((np.ndarray, np.float32), (np.ndarray, np.float32), (np.ndarray, np.float32))
+        fbs = encode_matrix_fbs(matrix=arr, row_idx=None, col_idx=None)
+        self.fbs_checks(fbs, (3, 2), expected_types, None)
+
+    def test_encode_sparse(self):
+        csc = sparse.csc_matrix(np.array([[0, 1, 2], [3, 0, 4]]))
+        expected_types = ((np.ndarray, np.int32), (np.ndarray, np.int32), (np.ndarray, np.int32))
+        fbs = encode_matrix_fbs(matrix=csc, row_idx=None, col_idx=None)
+        self.fbs_checks(fbs, (2, 3), expected_types, None)
+
+    def test_roundtrip(self):
+        dfSrc = pd.DataFrame(
+            data={
+                "a": np.zeros((10,), dtype=np.float32),
+                "b": np.ones((10,), dtype=np.int64),
+                "c": np.array([i for i in range(0, 10)], dtype=np.uint16),
+                "d": pd.Series(["x", "y", "z", "x", "y", "z", "a", "x", "y", "z"], dtype="category"),
+            }
+        )
+        dfDst = decode_matrix_fbs(encode_matrix_fbs(matrix=dfSrc, col_idx=dfSrc.columns))
+        self.assertEqual(dfSrc.shape, dfDst.shape)
+        self.assertEqual(set(dfSrc.columns), set(dfDst.columns))
+        for c in dfSrc.columns:
+            self.assertTrue(c in dfDst.columns)
+            if isinstance(dfSrc[c], pd.Series):
+                self.assertTrue(np.all(dfSrc[c] == dfDst[c]))
+            else:
+                self.assertEqual(dfSrc[c], dfDst[c])

server/test/unit/data_common/test_matrix_loader.py

@@ -0,0 +1,126 @@
+import os
+import shutil
+import tempfile
+import time
+import unittest
+
+from server.common.app_config import AppConfig
+from server.common.errors import DatasetAccessError
+from server.data_common.matrix_loader import MatrixDataCacheManager
+from server.test import FIXTURES_ROOT
+
+
+class MatrixCacheTest(unittest.TestCase):
+    def setup(self):
+        pass
+
+    def make_temporay_datasets(self, dirname, num):
+        source = f"{FIXTURES_ROOT}/pbmc3k.cxg"
+        for i in range(num):
+            target = os.path.join(dirname, str(i) + ".cxg")
+            shutil.copytree(source, target)
+
+    def use_dataset(self, matrix_cache, dirname, app_config, dataset_index):
+        with matrix_cache.data_adaptor(None, os.path.join(dirname, str(dataset_index) + ".cxg"), app_config) as adaptor:
+            pass
+        return adaptor
+
+    def use_dataset_with_error(self, matrix_cache, dirname, app_config, dataset_index):
+        try:
+            with matrix_cache.data_adaptor(None, os.path.join(dirname, str(dataset_index) + ".cxg"), app_config):
+                raise DatasetAccessError("something bad happened")
+        except DatasetAccessError:
+            # the MatrixDataCacheManager rethrows the exception, so catch and ignore
+            pass
+
+    def get_datasets(self, matrix_cache, dirname):
+        datasets = matrix_cache.datasets
+        result = {}
+        for k, v in datasets.items():
+            # filter out the dirname and the .cxg from the name
+            newk = int(k[1][len(dirname) + 1: -4])
+            result[newk] = v
+
+        return result
+
+    def check_datasets(self, matrix_cache, dirname, expected):
+        res = self.get_datasets(matrix_cache, dirname)
+        actual = res.keys()
+        self.assertSetEqual(set(actual), set(expected))
+
+    def test_basic(self):
+        with tempfile.TemporaryDirectory() as dirname:
+            self.make_temporay_datasets(dirname, 5)
+            app_config = AppConfig()
+            m = MatrixDataCacheManager(max_cached=3, timelimit_s=None)
+
+            # should have only dataset 0
+            self.use_dataset(m, dirname, app_config, 0)
+            self.check_datasets(m, dirname, [0])
+
+            # should have datasets 0, 1
+            self.use_dataset(m, dirname, app_config, 1)
+            self.check_datasets(m, dirname, [0, 1])
+
+            # should have datasets 0, 1, 2
+            self.use_dataset(m, dirname, app_config, 2)
+            self.check_datasets(m, dirname, [0, 1, 2])
+
+            # should have datasets 1, 2, 3
+            self.use_dataset(m, dirname, app_config, 3)
+            self.check_datasets(m, dirname, [1, 2, 3])
+
+            # use dataset 1, making is more recent than dataset 2
+            self.use_dataset(m, dirname, app_config, 1)
+            self.check_datasets(m, dirname, [1, 2, 3])
+
+            # use dataset 4, should have 1,3,4
+            self.use_dataset(m, dirname, app_config, 4)
+            self.check_datasets(m, dirname, [1, 3, 4])
+
+            # use dataset 4 a few more times, get the count to 3
+            self.use_dataset(m, dirname, app_config, 4)
+            self.use_dataset(m, dirname, app_config, 4)
+
+            datasets = self.get_datasets(m, dirname)
+            self.assertEqual(datasets[1].num_access, 2)
+            self.assertEqual(datasets[3].num_access, 1)
+            self.assertEqual(datasets[4].num_access, 3)
+
+    def test_timelimit(self):
+        with tempfile.TemporaryDirectory() as dirname:
+            self.make_temporay_datasets(dirname, 2)
+
+            app_config = AppConfig()
+            m = MatrixDataCacheManager(max_cached=3, timelimit_s=1)
+
+            adaptor = self.use_dataset(m, dirname, app_config, 0)
+            adaptor1 = self.use_dataset(m, dirname, app_config, 0)
+            self.assertTrue(adaptor is adaptor1)
+
+            # wait until the timelimit expires and check that there is a new adaptor
+            time.sleep(1.1)
+            adaptor2 = self.use_dataset(m, dirname, app_config, 0)
+            self.assertTrue(adaptor is not adaptor2)
+            self.check_datasets(m, dirname, [0])
+
+            # now load a different dataset and see if dataset 0 gets evicted
+            time.sleep(1.1)
+            self.use_dataset(m, dirname, app_config, 1)
+            self.check_datasets(m, dirname, [1])
+
+    def test_access_error(self):
+        with tempfile.TemporaryDirectory() as dirname:
+            self.make_temporay_datasets(dirname, 1)
+
+            app_config = AppConfig()
+            m = MatrixDataCacheManager(max_cached=3, timelimit_s=1)
+
+            # use the 0 datasets
+            self.use_dataset(m, dirname, app_config, 0)
+            self.check_datasets(m, dirname, [0])
+
+            # use the 0 datasets, but this time a DatasetAccessError is raised.
+            # verify that dataset is removed from the cache.
+            self.use_dataset_with_error(m, dirname, app_config, 0)
+            self.check_datasets(m, dirname, [])

server/test/unit/data_cxg/test_cxg_adaptor.py

@@ -0,0 +1,19 @@
+import unittest
+
+from server.common.data_locator import DataLocator
+from server.data_cxg.cxg_adaptor import CxgAdaptor
+from server.test import FIXTURES_ROOT, app_config
+from server.test.fixtures.fixtures import pbmc3k_colors
+
+
+class TestCxgAdaptor(unittest.TestCase):
+    def test_get_colors(self):
+        data = self.get_data("pbmc3k.cxg")
+        self.assertDictEqual(data.get_colors(), pbmc3k_colors)
+        data = self.get_data("pbmc3k_v0.cxg")
+        self.assertDictEqual(data.get_colors(), dict())
+
+    def get_data(self, fixture):
+        data_locator = f"{FIXTURES_ROOT}/{fixture}"
+        config = app_config(data_locator)
+        return CxgAdaptor(DataLocator(data_locator), config)

server/test/unit/decode_fbs.py

@@ -0,0 +1,60 @@
+"""
+Code to decode, for testing purposes, the flatbuffer encoded blobs.
+This code will need to be updated if fbs/matrix.fbs changes.
+For more information, see fbs/matrix.fbs and server/data_common/fbs/
+"""
+import json
+
+import server.data_common.fbs.NetEncoding.Float32Array as Float32Array
+import server.data_common.fbs.NetEncoding.Float64Array as Float64Array
+import server.data_common.fbs.NetEncoding.Int32Array as Int32Array
+import server.data_common.fbs.NetEncoding.JSONEncodedArray as JSONEncodedArray
+import server.data_common.fbs.NetEncoding.Matrix as Matrix
+import server.data_common.fbs.NetEncoding.TypedArray as TypedArray
+import server.data_common.fbs.NetEncoding.Uint32Array as Uint32Array
+
+
+def decode_typed_array(tarr):
+    type_map = {
+        TypedArray.TypedArray.Uint32Array: Uint32Array.Uint32Array,
+        TypedArray.TypedArray.Int32Array: Int32Array.Int32Array,
+        TypedArray.TypedArray.Float32Array: Float32Array.Float32Array,
+        TypedArray.TypedArray.Float64Array: Float64Array.Float64Array,
+        TypedArray.TypedArray.JSONEncodedArray: JSONEncodedArray.JSONEncodedArray,
+    }
+    (u_type, u) = tarr
+    if u_type == TypedArray.TypedArray.NONE:
+        return None
+
+    TarType = type_map.get(u_type, None)
+    assert TarType is not None
+
+    arr = TarType()
+    arr.Init(u.Bytes, u.Pos)
+    narr = arr.DataAsNumpy()
+    if u_type == TypedArray.TypedArray.JSONEncodedArray:
+        narr = json.loads(narr.tostring().decode("utf-8"))
+    return narr
+
+
+def decode_matrix_FBS(buf):
+    """
+    Given a FBS Matrix, return an decoded Python dict containing
+    same info in native format.
+    NOTE / TODO: row_idx not currently implemented
+    """
+    df = Matrix.Matrix.GetRootAsMatrix(buf, 0)
+    n_rows = df.NRows()
+    n_cols = df.NCols()
+
+    columns_length = df.ColumnsLength()
+
+    decoded_columns = []
+    for col_idx in range(0, columns_length):
+        col = df.Columns(col_idx)
+        tarr = (col.UType(), col.U())
+        decoded_columns.append(decode_typed_array(tarr))
+
+    cidx = decode_typed_array((df.ColIndexType(), df.ColIndex()))
+
+    return {"n_rows": n_rows, "n_cols": n_cols, "columns": decoded_columns, "col_idx": cidx, "row_idx": None}

server/test/unit/eb/test_eb.py

@@ -0,0 +1,54 @@
+import unittest
+import tempfile
+import requests
+import subprocess
+from server.test import PROJECT_ROOT, FIXTURES_ROOT
+from server.common.app_config import AppConfig
+from contextlib import contextmanager
+import time
+
+
+@contextmanager
+def run_eb_app(tempdirname):
+    ps = subprocess.Popen(["python", "artifact.dir/application.py"], cwd=tempdirname)
+    server = "http://localhost:5000"
+    for _ in range(10):
+        try:
+            requests.get(f"{server}/health")
+            break
+        except requests.exceptions.ConnectionError:
+            time.sleep(1)
+
+    try:
+        yield server
+    finally:
+        try:
+            ps.terminate()
+        except ProcessLookupError:
+            pass
+
+
+class Elastic_Beanstalk_Test(unittest.TestCase):
+    def test_run(self):
+
+        tempdir = tempfile.TemporaryDirectory(dir=f"{PROJECT_ROOT}/server")
+        tempdirname = tempdir.name
+
+        c = AppConfig()
+        # test that eb works
+        c.update_server_config(
+            multi_dataset__dataroot=f"{FIXTURES_ROOT}", app__flask_secret_key="open sesame"
+        )
+
+        c.complete_config()
+        c.write_config(f"{tempdirname}/config.yaml")
+
+        subprocess.check_call(f"git ls-files . | cpio -pdm {tempdirname}", cwd=f"{PROJECT_ROOT}/server/eb", shell=True)
+        subprocess.check_call(["make", "build"], cwd=tempdirname)
+
+        with run_eb_app(tempdirname) as server:
+            session = requests.Session()
+
+            r = session.get(f"{server}/d/pbmc3k.cxg/api/v0.2/config")
+            data_config = r.json()
+            assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"