move common code into server, update tests and makefile (#2425)

* move common code into server, update tests and makefile

remove backend directory, refactor

update smoke tests

test/unit/__init__.py

@@ -0,0 +1,164 @@
+import os
+import random
+import shutil
+import tempfile
+import time
+from contextlib import contextmanager
+from os import path
+from subprocess import Popen
+
+import pandas as pd
+import requests
+
+from server.common.annotations.local_file_csv import AnnotationsLocalFile
+from server.common.config import DEFAULT_SERVER_PORT
+from server.common.config.app_config import AppConfig
+from server.common.fbs.matrix import encode_matrix_fbs
+from server.common.utils.data_locator import DataLocator
+from server.common.utils.utils import find_available_port
+from server.data_common.matrix_loader import MatrixDataType, MatrixDataLoader
+from test import PROJECT_ROOT
+
+
+def data_with_tmp_annotations(ext: MatrixDataType, annotations_fixture=False):
+    tmp_dir = tempfile.mkdtemp()
+    annotations_file = path.join(tmp_dir, "test_annotations.csv")
+    if annotations_fixture:
+        shutil.copyfile(f"{PROJECT_ROOT}/test/fixtures/pbmc3k-annotations.csv", annotations_file)
+    fname = {
+        MatrixDataType.H5AD: f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad",
+    }[ext]
+    data_locator = DataLocator(fname)
+    config = AppConfig()
+    config.update_server_config(
+        app__flask_secret_key="secret",
+        single_dataset__obs_names=None,
+        single_dataset__var_names=None,
+        single_dataset__datapath=data_locator.path,
+    )
+    config.update_dataset_config(
+        embeddings__names=["umap"],
+        presentation__max_categories=100,
+        diffexp__lfc_cutoff=0.01,
+    )
+
+    config.complete_config()
+    data = MatrixDataLoader(data_locator.abspath()).open(config)
+    anno_config = {
+        "user-annotations": True,
+        "genesets-save": False,
+    }
+    annotations = AnnotationsLocalFile(anno_config, None, annotations_file, None)
+    return data, tmp_dir, annotations
+
+
+def make_fbs(data):
+    df = pd.DataFrame(data)
+    return encode_matrix_fbs(matrix=df, row_idx=None, col_idx=df.columns)
+
+
+def skip_if(condition, reason: str):
+    def decorator(f):
+        def wraps(self, *args, **kwargs):
+            if condition(self):
+                self.skipTest(reason)
+            else:
+                f(self, *args, **kwargs)
+
+        return wraps
+
+    return decorator
+
+
+def app_config(data_locator, backed=False, extra_server_config={}, extra_dataset_config={}):
+    config = AppConfig()
+    config.update_server_config(
+        app__flask_secret_key="secret",
+        single_dataset__obs_names=None,
+        single_dataset__var_names=None,
+        adaptor__anndata_adaptor__backed=backed,
+        single_dataset__datapath=data_locator,
+        limits__diffexp_cellcount_max=None,
+        limits__column_request_max=None,
+    )
+    config.update_dataset_config(
+        embeddings__names=["umap", "tsne", "pca"], presentation__max_categories=100, diffexp__lfc_cutoff=0.01
+    )
+    config.update_server_config(**extra_server_config)
+    config.update_dataset_config(**extra_dataset_config)
+    config.complete_config()
+    return config
+
+
+def start_test_server(command_line_args=[], app_config=None, env=None):
+    """
+    Command line arguments can be passed in, as well as an app_config.
+    This function is meant to be used like this, for example:
+
+    with unit(...) as server:
+        r = requests.get(f"{server}/...")
+        // check r
+
+    where the server can be accessed within the context, and is terminated when
+    the context is exited.
+    The port is automatically set using find_available_port, unless passed in as a command line arg.
+    The verbose flag is automatically set to True.
+    If an app_config is provided, then this function writes a temporary
+    yaml config file, which this server will read and parse.
+    """
+
+    command = ["cellxgene", "--no-upgrade-check", "launch", "--verbose"]
+    if "-p" in command_line_args:
+        port = int(command_line_args[command_line_args.index("-p") + 1])
+    elif "--port" in command_line_args:
+        port = int(command_line_args[command_line_args.index("--port") + 1])
+    else:
+        start = random.randint(DEFAULT_SERVER_PORT, 2 ** 16 - 1)
+        port = int(os.environ.get("CXG_SERVER_PORT", start))
+        port = find_available_port("localhost", port)
+        command += ["--port=%d" % port]
+
+    command += command_line_args
+
+    tempdir = None
+    if app_config:
+        tempdir = tempfile.TemporaryDirectory()
+        config_file = os.path.join(tempdir.name, "config.yaml")
+        app_config.write_config(config_file)
+        command.extend(["-c", config_file])
+
+    server = f"http://localhost:{port}"
+    ps = Popen(command, env=env)
+
+    for _ in range(10):
+        try:
+            requests.get(f"{server}/health")
+            break
+        except requests.exceptions.ConnectionError:
+            time.sleep(1)
+
+    if tempdir:
+        tempdir.cleanup()
+
+    return ps, server
+
+
+def stop_test_server(ps):
+    try:
+        ps.terminate()
+    except ProcessLookupError:
+        pass
+
+
+@contextmanager
+def test_server(command_line_args=[], app_config=None, env=None):
+    """A context to run the cellxgene server."""
+
+    ps, server = start_test_server(command_line_args, app_config, env)
+    try:
+        yield server
+    finally:
+        try:
+            stop_test_server(ps)
+        except ProcessLookupError:
+            pass

test/unit/cli/test_launch.py

@@ -0,0 +1,27 @@
+import filecmp
+import os
+import shutil
+import unittest
+
+import yaml
+
+from server.default_config import default_config
+from test import FIXTURES_ROOT
+
+
+class CLIPLaunchTests(unittest.TestCase):
+    tmp_dir = os.path.join(FIXTURES_ROOT, "dump_configs")
+
+    @classmethod
+    def setUpClass(cls) -> None:
+        os.mkdir(cls.tmp_dir)
+
+    @classmethod
+    def tearDownClass(cls) -> None:
+        shutil.rmtree(cls.tmp_dir)
+
+    def test_dump_default_config(self):
+        os.system(f"cellxgene launch --dump-default-config > {self.tmp_dir}/test_config_dump.txt")
+        with open(f"{self.tmp_dir}/expected_config_dump.txt", "w") as expected_config:
+            expected_config.write(yaml.dump(default_config))
+        filecmp.cmp(f"{self.tmp_dir}/expected_config_dump.txt", f"{self.tmp_dir}/test_config_dump.txt")

test/unit/cli/test_prepare.py

@@ -0,0 +1,15 @@
+import unittest
+
+import pandas as pd
+
+from server.cli.prepare import make_index_unique
+
+
+class CLIPrepareTests(unittest.TestCase):
+    """ Test cases for CLI prepare logic """
+
+    def test_make_index_unique(self):
+        index = pd.Index(["SNORD113", "SNORD113", "SNORD113-1"])
+        result = make_index_unique(index)
+        expected = pd.Index(["SNORD113", "SNORD113-2", "SNORD113-1"])
+        self.assertTrue(all(left == right for left, right in zip(result.values, expected.values)))

test/unit/cli/test_upgrade.py

@@ -0,0 +1,28 @@
+import unittest
+
+from server.cli.upgrade import validate_version_str, split_version, version_gt
+
+
+class CLIUpgradeTests(unittest.TestCase):
+    """ Test cases for CLI logic """
+
+    def test_validate_version_str(self):
+        self.assertTrue(validate_version_str("0.1.2"))
+        self.assertTrue(validate_version_str("0.1.2-RC", release_only=False))
+        self.assertFalse(validate_version_str("0.1"))
+        self.assertFalse(validate_version_str("0.1.2.3"))
+        self.assertFalse(validate_version_str("0.1.2-RC"))
+
+    def test_split_version_str(self):
+        self.assertEqual(split_version("0.1.2"), [0, 1, 2])
+        with self.assertRaises(AttributeError):
+            split_version("0.1")
+
+    def test_assert_verstion_gt(self):
+        self.assertTrue(version_gt("1.0.0", "0.1.1"))
+        self.assertTrue(version_gt("0.1.0", "0.0.1"))
+        self.assertTrue(version_gt("0.0.1", "0.0.0"))
+        self.assertFalse(version_gt("0.0.0", "0.0.0"))
+        self.assertFalse(version_gt("0.0.0", "0.0.1"))
+        self.assertFalse(version_gt("0.0.1", "0.1.0"))
+        self.assertFalse(version_gt("0.1.1", "1.0.0"))

test/unit/common/config/__init__.py

@@ -0,0 +1,212 @@
+import os
+import shutil
+import unittest
+import random
+from unittest import mock
+import yaml
+
+from test import FIXTURES_ROOT
+
+
+def mockenv(**envvars):
+    return mock.patch.dict(os.environ, envvars)
+
+
+class ConfigTests(unittest.TestCase):
+    tmp_fixtures_directory = os.path.join(FIXTURES_ROOT, "tmp_dir")
+
+    @classmethod
+    def tearDownClass(cls) -> None:
+        shutil.rmtree(cls.tmp_fixtures_directory)
+
+    @classmethod
+    def setUpClass(cls) -> None:
+        os.makedirs(cls.tmp_fixtures_directory)
+
+    def custom_server_config(
+        self,
+        verbose="false",
+        debug="false",
+        host="localhost",
+        port="null",
+        open_browser="false",
+        force_https="false",
+        flask_secret_key="secret",
+        generate_cache_control_headers="false",
+        insecure_test_environment="false",
+        index="false",
+        allowed_matrix_types=[],
+        max_cached_datasets=5,
+        timelimit_s=5,
+        dataset_datapath="null",
+        obs_names="null",
+        var_names="null",
+        about="null",
+        title="null",
+        data_locater_region_name="us-east-1",
+        anndata_backed="false",
+        column_request_max=32,
+        diffexp_cellcount_max="null",
+        config_file_name="server_config.yaml",
+    ):
+        configfile = os.path.join(self.tmp_fixtures_directory, config_file_name)
+        server_config_outline_path = os.path.join(FIXTURES_ROOT, "server_config_outline.py")
+        with open(server_config_outline_path, "r") as config_skeleton:
+            config = config_skeleton.read()
+            server_config = eval(config)
+        with open(configfile, "w") as server_config_file:
+            server_config_file.write(server_config)
+        return configfile
+
+    def custom_app_config(
+        self,
+        verbose="false",
+        debug="false",
+        host="localhost",
+        port="null",
+        open_browser="false",
+        force_https="false",
+        flask_secret_key="secret",
+        generate_cache_control_headers="false",
+        index="false",
+        allowed_matrix_types=[],
+        max_cached_datasets=5,
+        timelimit_s=5,
+        dataset_datapath="null",
+        obs_names="null",
+        var_names="null",
+        about="null",
+        title="null",
+        data_locater_region_name="us-east-1",
+        anndata_backed="false",
+        column_request_max=32,
+        diffexp_cellcount_max="null",
+        scripts=[],
+        inline_scripts=[],
+        max_categories=1000,
+        custom_colors="true",
+        enable_users_annotations="true",
+        annotation_type="local_file_csv",
+        db_uri="null",
+        hosted_file_directory="null",
+        local_file_csv_directory="null",
+        local_file_csv_file="null",
+        local_file_csv_gene_sets_file="null",
+        gene_sets_readonly="false",
+        embedding_names=[],
+        enable_difexp="true",
+        lfc_cutoff=0.01,
+        top_n=10,
+        environment=None,
+        X_approximate_distribution="auto",
+        config_file_name="app_config.yml",
+    ):
+        random_num = random.randrange(999999)
+        configfile = os.path.join(self.tmp_fixtures_directory, config_file_name)
+        server_config = self.custom_server_config(
+            verbose=verbose,
+            debug=debug,
+            host=host,
+            port=port,
+            open_browser=open_browser,
+            force_https=force_https,
+            flask_secret_key=flask_secret_key,
+            generate_cache_control_headers=generate_cache_control_headers,
+            index=index,
+            allowed_matrix_types=allowed_matrix_types,
+            max_cached_datasets=max_cached_datasets,
+            timelimit_s=timelimit_s,
+            dataset_datapath=dataset_datapath,
+            obs_names=obs_names,
+            var_names=var_names,
+            about=about,
+            title=title,
+            data_locater_region_name=data_locater_region_name,
+            anndata_backed=anndata_backed,
+            column_request_max=column_request_max,
+            diffexp_cellcount_max=diffexp_cellcount_max,
+            config_file_name=f"temp_server_config_{random_num}.yml",
+        )
+        dataset_config = self.custom_dataset_config(
+            scripts=scripts,
+            inline_scripts=inline_scripts,
+            max_categories=max_categories,
+            custom_colors=custom_colors,
+            enable_users_annotations=enable_users_annotations,
+            annotation_type=annotation_type,
+            db_uri=db_uri,
+            hosted_file_directory=hosted_file_directory,
+            local_file_csv_directory=local_file_csv_directory,
+            local_file_csv_file=local_file_csv_file,
+            local_file_csv_gene_sets_file=local_file_csv_gene_sets_file,
+            gene_sets_readonly=gene_sets_readonly,
+            embedding_names=embedding_names,
+            enable_difexp=enable_difexp,
+            lfc_cutoff=lfc_cutoff,
+            top_n=top_n,
+            X_approximate_distribution=X_approximate_distribution,
+            config_file_name=f"temp_dataset_config_{random_num}.yml",
+        )
+        external_config = self.custom_external_config(
+            environment=environment,
+            config_file_name=f"temp_external_config_{random_num}.yml",
+        )
+
+        with open(configfile, "w") as app_config_file:
+            app_config_file.write(open(server_config).read())
+            app_config_file.write(open(dataset_config).read())
+            app_config_file.write(open(external_config).read())
+
+        return configfile
+
+    def custom_dataset_config(
+        self,
+        scripts=[],
+        inline_scripts=[],
+        max_categories=1000,
+        custom_colors="true",
+        enable_users_annotations="true",
+        annotation_type="local_file_csv",
+        db_uri="null",
+        hosted_file_directory="null",
+        local_file_csv_directory="null",
+        local_file_csv_file="null",
+        local_file_csv_gene_sets_file="null",
+        gene_sets_readonly="false",
+        embedding_names=[],
+        enable_difexp="true",
+        lfc_cutoff=0.01,
+        top_n=10,
+        X_approximate_distribution="auto",
+        config_file_name="dataset_config.yml",
+    ):
+        configfile = os.path.join(self.tmp_fixtures_directory, config_file_name)
+        dataset_config_outline_path = os.path.join(FIXTURES_ROOT, "dataset_config_outline.py")
+        with open(dataset_config_outline_path, "r") as config_skeleton:
+            config = config_skeleton.read()
+            dataset_config = eval(config)
+        with open(configfile, "w") as dataset_config_file:
+            dataset_config_file.write(dataset_config)
+
+        return configfile
+
+    def custom_external_config(
+        self,
+        environment=None,
+        config_file_name="external_config.yaml",
+    ):
+        # set to the default if environment is None
+        if environment is None:
+            environment = [
+                dict(name="CXG_SECRET_KEY", path=["server", "app", "flask_secret_key"], required=False),
+            ]
+        external_config = {
+            "external": {
+                "environment": environment,
+            }
+        }
+
+        configfile = os.path.join(self.tmp_fixtures_directory, config_file_name)
+        with open(configfile, "w") as external_config_file:
+            yaml.dump(external_config, external_config_file)
+        return configfile

test/unit/common/config/test_app_config.py

@@ -0,0 +1,153 @@
+import os
+import tempfile
+import unittest
+
+import yaml
+
+from server.default_config import default_config
+from server.common.config.app_config import AppConfig
+from server.common.errors import ConfigurationError
+from test.unit.common.config import ConfigTests
+from test import FIXTURES_ROOT, H5AD_FIXTURE
+
+
+class AppConfigTest(ConfigTests):
+    def setUp(self):
+        self.config_file_name = f"{unittest.TestCase.id(self).split('.')[-1]}.yml"
+        self.config = AppConfig()
+        self.config.update_server_config(app__flask_secret_key="secret")
+        self.config.update_server_config(single_dataset__datapath=H5AD_FIXTURE)
+        self.server_config = self.config.server_config
+        self.config.complete_config()
+
+        message_list = []
+
+        def noop(message):
+            message_list.append(message)
+
+        messagefn = noop
+        self.context = dict(messagefn=messagefn, messages=message_list)
+
+    def get_config(self, **kwargs):
+        file_name = self.custom_app_config(
+            dataset_datapath=H5AD_FIXTURE, config_file_name=self.config_file_name, **kwargs
+        )
+        config = AppConfig()
+        config.update_from_config_file(file_name)
+        return config
+
+    def test_get_default_config_correctly_reads_default_config_file(self):
+        app_default_config = AppConfig().default_config
+
+        expected_config = yaml.load(default_config, Loader=yaml.Loader)
+
+        server_config = app_default_config["server"]
+        dataset_config = app_default_config["dataset"]
+
+        expected_server_config = expected_config["server"]
+        expected_dataset_config = expected_config["dataset"]
+
+        self.assertDictEqual(app_default_config, expected_config)
+        self.assertDictEqual(server_config, expected_server_config)
+        self.assertDictEqual(dataset_config, expected_dataset_config)
+
+    def test_get_dataset_config_returns_dataset_config_for_single_datasets(self):
+        datapath = f"{FIXTURES_ROOT}/1e4dfec4-c0b2-46ad-a04e-ff3ffb3c0a8f.h5ad"
+        file_name = self.custom_app_config(dataset_datapath=datapath, config_file_name=self.config_file_name)
+        config = AppConfig()
+        config.update_from_config_file(file_name)
+
+        self.assertEqual(config.get_dataset_config(), config.dataset_config)
+
+    def test_update_server_config_updates_server_config_and_config_status(self):
+        config = self.get_config()
+        config.complete_config()
+        config.check_config()
+        config.update_server_config(single_dataset__datapath=H5AD_FIXTURE)
+        with self.assertRaises(ConfigurationError):
+            config.server_config.check_config()
+
+    def test_write_config_outputs_yaml_with_all_config_vars(self):
+        config = self.get_config()
+        config.write_config(f"{FIXTURES_ROOT}/tmp_dir/write_config.yml")
+        with open(f"{FIXTURES_ROOT}/tmp_dir/{self.config_file_name}", "r") as default_config:
+            default_config_yml = yaml.safe_load(default_config)
+
+        with open(f"{FIXTURES_ROOT}/tmp_dir/write_config.yml", "r") as output_config:
+            output_config_yml = yaml.safe_load(output_config)
+        self.maxDiff = None
+        self.assertEqual(default_config_yml, output_config_yml)
+
+    def test_update_app_config(self):
+        config = AppConfig()
+        config.update_server_config(app__verbose=True, single_dataset__datapath="datapath")
+        vars = config.server_config.changes_from_default()
+        self.assertCountEqual(vars, [("app__verbose", True, False), ("single_dataset__datapath", "datapath", None)])
+
+        config = AppConfig()
+        config.update_dataset_config(app__scripts=(), app__inline_scripts=())
+        vars = config.server_config.changes_from_default()
+        self.assertCountEqual(vars, [])
+
+        config = AppConfig()
+        config.update_dataset_config(app__scripts=[], app__inline_scripts=[])
+        vars = config.dataset_config.changes_from_default()
+        self.assertCountEqual(vars, [])
+
+        config = AppConfig()
+        config.update_dataset_config(app__scripts=("a", "b"), app__inline_scripts=["c", "d"])
+        vars = config.dataset_config.changes_from_default()
+        self.assertCountEqual(vars, [("app__scripts", ["a", "b"], []), ("app__inline_scripts", ["c", "d"], [])])
+
+    def test_configfile_no_server_section(self):
+        # test a config file without a dataset section
+
+        with tempfile.TemporaryDirectory() as tempdir:
+            configfile = os.path.join(tempdir, "config.yaml")
+            with open(configfile, "w") as fconfig:
+                config = """
+                dataset:
+                    user_annotations:
+                        enable: false
+                """
+                fconfig.write(config)
+
+            app_config = AppConfig()
+            app_config.update_from_config_file(configfile)
+            server_changes = app_config.server_config.changes_from_default()
+            dataset_changes = app_config.dataset_config.changes_from_default()
+            self.assertEqual(server_changes, [])
+            self.assertEqual(dataset_changes, [("user_annotations__enable", False, True)])
+
+    def test_simple_update_single_config_from_path_and_value(self):
+        """Update a simple config parameter"""
+
+        config = AppConfig()
+        config.server_config.single_dataset__datapath = "my/data/path"
+
+        # test simple value in server
+        config.update_single_config_from_path_and_value(["server", "app", "flask_secret_key"], "mysecret")
+        self.assertEqual(config.server_config.app__flask_secret_key, "mysecret")
+
+        # test simple value in default dataset
+        config.update_single_config_from_path_and_value(
+            ["dataset", "user_annotations"],
+            "dummy_location",
+        )
+
+        # error checking
+        bad_paths = [
+            (
+                ["dataset", "does", "not", "exist"],
+                "unknown config parameter at path: '['dataset', 'does', 'not', 'exist']'",
+            ),
+            (["does", "not", "exist"], "path must start with 'server', or 'dataset'"),
+            ([], "path must start with 'server', or 'dataset'"),
+            ([1, 2, 3], "path must be a list of strings, got '[1, 2, 3]'"),
+            ("string", "path must be a list of strings, got 'string'"),
+        ]
+        for bad_path, error_message in bad_paths:
+            with self.assertRaises(ConfigurationError) as config_error:
+                config.update_single_config_from_path_and_value(bad_path, "value")
+
+            self.assertEqual(config_error.exception.message, error_message)

test/unit/common/config/test_base_config.py

@@ -0,0 +1,62 @@
+import unittest
+
+from server.common.config.app_config import AppConfig
+from test import H5AD_FIXTURE
+from server.common.errors import ConfigurationError
+from test.unit.common.config import ConfigTests
+
+
+class BaseConfigTest(ConfigTests):
+    def setUp(self):
+        self.config_file_name = f"{unittest.TestCase.id(self).split('.')[-1]}.yml"
+        self.config = AppConfig()
+        self.config.update_server_config(app__flask_secret_key="secret")
+        self.config.update_server_config(single_dataset__datapath=H5AD_FIXTURE)
+        self.server_config = self.config.server_config
+        self.config.complete_config()
+
+        message_list = []
+
+        def noop(message):
+            message_list.append(message)
+
+        messagefn = noop
+        self.context = dict(messagefn=messagefn, messages=message_list)
+
+    def get_config(self, **kwargs):
+        file_name = self.custom_app_config(
+            dataset_datapath=f"{H5AD_FIXTURE}", config_file_name=self.config_file_name, **kwargs
+        )
+        config = AppConfig()
+        config.update_from_config_file(file_name)
+        return config
+
+    def test_mapping_creation_returns_map_of_server_and_dataset_config(self):
+        config = AppConfig()
+        mapping = config.dataset_config.create_mapping(config.default_config)
+        self.assertIsNotNone(mapping["server__app__verbose"])
+        self.assertIsNotNone(mapping["dataset__presentation__max_categories"])
+
+    def test_changes_from_default_returns_list_of_nondefault_config_values(self):
+        config = self.get_config(verbose="true", lfc_cutoff=0.05)
+        server_changes = config.server_config.changes_from_default()
+        dataset_changes = config.dataset_config.changes_from_default()
+
+        self.assertEqual(
+            server_changes,
+            [
+                ("app__verbose", True, False),
+                ("app__flask_secret_key", "secret", None),
+                ("single_dataset__datapath", H5AD_FIXTURE, None),
+                ("data_locator__s3__region_name", "us-east-1", True),
+            ],
+        )
+        self.assertEqual(dataset_changes, [("diffexp__lfc_cutoff", 0.05, 0.01)])
+
+    def test_check_config_throws_error_if_attr_has_not_been_checked(self):
+        config = self.get_config(verbose="true")
+        config.complete_config()
+        config.check_config()
+        config.update_server_config(app__verbose=False)
+        with self.assertRaises(ConfigurationError):
+            config.check_config()

test/unit/common/config/test_dataset_config.py

@@ -0,0 +1,132 @@
+import os
+import tempfile
+
+import unittest
+from unittest.mock import patch
+
+from server.common.annotations.local_file_csv import AnnotationsLocalFile
+from server.common.config.app_config import AppConfig
+from server.common.config.base_config import BaseConfig
+from test import H5AD_FIXTURE
+
+from server.common.errors import ConfigurationError
+from test.unit.common.config import ConfigTests
+
+
+class TestDatasetConfig(ConfigTests):
+    def setUp(self):
+        self.config_file_name = f"{unittest.TestCase.id(self).split('.')[-1]}.yml"
+        self.config = AppConfig()
+        self.config.update_server_config(app__flask_secret_key="secret")
+        self.config.update_server_config(single_dataset__datapath=H5AD_FIXTURE)
+        self.dataset_config = self.config.dataset_config
+        self.config.complete_config()
+        message_list = []
+
+        def noop(message):
+            message_list.append(message)
+
+        messagefn = noop
+        self.context = dict(messagefn=messagefn, messages=message_list)
+
+    def get_config(self, **kwargs):
+        file_name = self.custom_app_config(dataset_datapath=H5AD_FIXTURE, **kwargs)
+        config = AppConfig()
+        config.update_from_config_file(file_name)
+        return config
+
+    def test_init_datatset_config_sets_vars_from_config(self):
+        config = AppConfig()
+        self.assertEqual(config.dataset_config.presentation__max_categories, 1000)
+        self.assertEqual(config.dataset_config.user_annotations__type, "local_file_csv")
+        self.assertEqual(config.dataset_config.diffexp__lfc_cutoff, 0.01)
+
+    @patch("server.common.config.dataset_config.BaseConfig.validate_correct_type_of_configuration_attribute")
+    def test_complete_config_checks_all_attr(self, mock_check_attrs):
+        mock_check_attrs.side_effect = BaseConfig.validate_correct_type_of_configuration_attribute()
+        self.dataset_config.complete_config(self.context)
+        self.assertIsNotNone(self.config.server_config.data_adaptor)
+        self.assertEqual(mock_check_attrs.call_count, 16)
+
+    def test_app_sets_script_vars(self):
+        config = self.get_config(scripts=["path/to/script"])
+        config.dataset_config.handle_app()
+
+        self.assertEqual(config.dataset_config.app__scripts, [{"src": "path/to/script"}])
+
+        config = self.get_config(scripts=[{"src": "path/to/script", "more": "different/script/path"}])
+        config.dataset_config.handle_app()
+        self.assertEqual(
+            config.dataset_config.app__scripts, [{"src": "path/to/script", "more": "different/script/path"}]
+        )
+
+        config = self.get_config(scripts=["path/to/script", "different/script/path"])
+        config.dataset_config.handle_app()
+        # TODO @madison -- is this the desired functionality?
+        self.assertEqual(
+            config.dataset_config.app__scripts, [{"src": "path/to/script"}, {"src": "different/script/path"}]
+        )
+
+        config = self.get_config(scripts=[{"more": "different/script/path"}])
+        with self.assertRaises(ConfigurationError):
+            config.dataset_config.handle_app()
+
+    def test_handle_user_annotations__instantiates_user_annotations_class_correctly(self):
+        config = self.get_config(
+            enable_users_annotations="true", annotation_type="local_file_csv"
+        )
+        config.server_config.complete_config(self.context)
+        config.dataset_config.handle_user_annotations(self.context)
+        self.assertIsInstance(config.dataset_config.user_annotations, AnnotationsLocalFile)
+
+        config = self.get_config(
+            enable_users_annotations="true", annotation_type="NOT_REAL"
+        )
+        config.server_config.complete_config(self.context)
+        with self.assertRaises(ConfigurationError):
+            config.dataset_config.handle_user_annotations(self.context)
+
+    def test_handle_local_file_csv_annotations__sets_dir_if_not_passed_in(self):
+        config = self.get_config(
+            enable_users_annotations="true", annotation_type="local_file_csv"
+        )
+        config.server_config.complete_config(self.context)
+        config.dataset_config.handle_local_file_csv_annotations(self.context)
+        self.assertIsInstance(config.dataset_config.user_annotations, AnnotationsLocalFile)
+        cwd = os.getcwd()
+        self.assertEqual(config.dataset_config.user_annotations._get_output_dir(), cwd)
+
+    def test_handle_diffexp__raises_warning_for_large_datasets(self):
+        config = self.get_config(lfc_cutoff=0.02, enable_difexp="true", top_n=15)
+        config.server_config.complete_config(self.context)
+        config.dataset_config.handle_diffexp(self.context)
+        self.assertEqual(len(self.context["messages"]), 1)
+
+    def test_configfile_with_specialization(self):
+        # test that per_dataset_config config load the default config, then the specialized config
+
+        with tempfile.TemporaryDirectory() as tempdir:
+            configfile = os.path.join(tempdir, "config.yaml")
+            with open(configfile, "w") as fconfig:
+                config = """
+                server:
+                    single_dataset:
+                        datapath: fake_datapath
+                dataset:
+                    user_annotations:
+                        enable: false
+                        type: local_file_csv
+                        local_file_csv:
+                            file: fake_file
+                            directory: fake_dir
+                """
+                fconfig.write(config)
+
+            app_config = AppConfig()
+            app_config.update_from_config_file(configfile)
+
+            test_config = app_config.dataset_config
+
+            # test config from default
+            self.assertEqual(test_config.user_annotations__type, "local_file_csv")
+            self.assertEqual(test_config.user_annotations__local_file_csv__file, "fake_file")

test/unit/common/config/test_external_config.py

@@ -0,0 +1,90 @@
+import os
+
+import requests
+
+from server.common.config.app_config import AppConfig
+from server.common.errors import ConfigurationError
+from server.common.utils.type_conversion_utils import convert_string_to_value
+
+from test import FIXTURES_ROOT
+from test.unit import test_server
+from test.unit.common.config import ConfigTests
+
+
+class TestExternalConfig(ConfigTests):
+    def test_type_convert(self):
+        # The values from environment variables are returned as strings.
+        # These values need to be converted to the proper types.
+
+        self.assertEqual(convert_string_to_value("1"), int(1))
+        self.assertEqual(convert_string_to_value("1.1"), float(1.1))
+        self.assertEqual(convert_string_to_value("string"), "string")
+        self.assertEqual(convert_string_to_value("true"), True)
+        self.assertEqual(convert_string_to_value("True"), True)
+        self.assertEqual(convert_string_to_value("false"), False)
+        self.assertEqual(convert_string_to_value("False"), False)
+        self.assertEqual(convert_string_to_value("null"), None)
+        self.assertEqual(convert_string_to_value("None"), None)
+        self.assertEqual(convert_string_to_value("{'a':10, 'b':'string'}"), dict(a=int(10), b="string"))
+
+    def test_environment_variable(self):
+        configfile = self.custom_external_config(
+            environment=[
+                dict(name="DATAPATH", path=["server", "single_dataset", "datapath"], required=True),
+                dict(name="DIFFEXP", path=["dataset", "diffexp", "enable"], required=True),
+            ],
+            config_file_name="environment_external_config.yaml",
+        )
+
+        env = os.environ
+        env["DATAPATH"] = f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad"
+        env["DIFFEXP"] = "False"
+        with test_server(command_line_args=["-c", configfile], env=env) as server:
+            session = requests.Session()
+            response = session.get(f"{server}/api/v0.2/config")
+            data_config = response.json()
+            self.assertEqual(data_config["config"]["displayNames"]["dataset"], "pbmc3k-CSC-gz")
+            self.assertTrue(data_config["config"]["parameters"]["disable-diffexp"])
+
+        env["DATAPATH"] = f"{FIXTURES_ROOT}/a95c59b4-7f5d-4b80-ad53-a694834ca18b.h5ad"
+        env["DIFFEXP"] = "True"
+        with test_server(command_line_args=["-c", configfile], env=env) as server:
+            session = requests.Session()
+            response = session.get(f"{server}/api/v0.2/config")
+            data_config = response.json()
+            self.assertEqual(data_config["config"]["displayNames"]["dataset"], "a95c59b4-7f5d-4b80-ad53-a694834ca18b")
+            self.assertFalse(data_config["config"]["parameters"]["disable-diffexp"])
+
+    def test_environment_variable_errors(self):
+
+        # no name
+        app_config = AppConfig()
+        app_config.external_config.environment = [dict(required=True, path=["this", "is", "a", "path"])]
+        with self.assertRaises(ConfigurationError) as config_error:
+            app_config.complete_config()
+        self.assertEqual(config_error.exception.message, "environment: 'name' is missing")
+
+        # required has wrong type
+        app_config = AppConfig()
+        app_config.external_config.environment = [
+            dict(name="myenvar", required="optional", path=["this", "is", "a", "path"])
+        ]
+        with self.assertRaises(ConfigurationError) as config_error:
+            app_config.complete_config()
+        self.assertEqual(config_error.exception.message, "environment: 'required' must be a bool")
+
+        # no path
+        app_config = AppConfig()
+        app_config.external_config.environment = [dict(name="myenvar", required=True)]
+        with self.assertRaises(ConfigurationError) as config_error:
+            app_config.complete_config()
+        self.assertEqual(config_error.exception.message, "environment: 'path' is missing")
+
+        # required environment variable is not set
+        app_config = AppConfig()
+        app_config.external_config.environment = [
+            dict(name="THIS_ENV_IS_NOT_SET", required=True, path=["this", "is", "a", "path"])
+        ]
+        with self.assertRaises(ConfigurationError) as config_error:
+            app_config.complete_config()
+        self.assertEqual(config_error.exception.message, "required environment variable 'THIS_ENV_IS_NOT_SET' not set")

test/unit/common/config/test_server_config.py

@@ -0,0 +1,111 @@
+import os
+import unittest
+from unittest import mock
+from unittest.mock import patch
+
+from server.common.config.base_config import BaseConfig
+from test import H5AD_FIXTURE
+
+from server.common.config.app_config import AppConfig
+from server.common.errors import ConfigurationError
+from test.unit.common.config import ConfigTests
+
+
+def mockenv(**envvars):
+    return mock.patch.dict(os.environ, envvars)
+
+
+class TestServerConfig(ConfigTests):
+    def setUp(self):
+        self.config_file_name = f"{unittest.TestCase.id(self).split('.')[-1]}.yml"
+        self.config = AppConfig()
+        self.config.update_server_config(app__flask_secret_key="secret")
+        self.config.update_server_config(single_dataset__datapath=H5AD_FIXTURE)
+        self.server_config = self.config.server_config
+        self.config.complete_config()
+
+        message_list = []
+
+        def noop(message):
+            message_list.append(message)
+
+        messagefn = noop
+        self.context = dict(messagefn=messagefn, messages=message_list)
+
+    def get_config(self, **kwargs):
+        file_name = self.custom_app_config(
+            dataset_datapath=f"{H5AD_FIXTURE}", config_file_name=self.config_file_name, **kwargs
+        )
+        config = AppConfig()
+        config.update_from_config_file(file_name)
+        return config
+
+    def test_init_raises_error_if_default_config_is_invalid(self):
+        invalid_config = self.get_config(port="not_valid")
+        with self.assertRaises(ConfigurationError):
+            invalid_config.complete_config()
+
+    @patch("server.common.config.server_config.BaseConfig.validate_correct_type_of_configuration_attribute")
+    def test_complete_config_checks_all_attr(self, mock_check_attrs):
+        mock_check_attrs.side_effect = BaseConfig.validate_correct_type_of_configuration_attribute()
+        self.server_config.complete_config(self.context)
+        self.assertEqual(mock_check_attrs.call_count, 19)
+
+    def test_handle_app__throws_error_if_port_doesnt_exist(self):
+        config = self.get_config(port=99999999)
+        with self.assertRaises(ConfigurationError):
+            config.server_config.handle_app(self.context)
+
+    @patch("server.common.config.server_config.discover_s3_region_name")
+    def test_handle_data_locator_works_for_default_types(self, mock_discover_region_name):
+        mock_discover_region_name.return_value = None
+        # Default config
+        self.assertEqual(self.config.server_config.data_locator__s3__region_name, None)
+        # hard coded
+        config = self.get_config()
+        self.assertEqual(config.server_config.data_locator__s3__region_name, "us-east-1")
+        # incorrectly formatted
+        datapath = "s3://shouldnt/work"
+        file_name = self.custom_app_config(
+            dataset_datapath=datapath, config_file_name=self.config_file_name, data_locater_region_name="true"
+        )
+        config = AppConfig()
+        config.update_from_config_file(file_name)
+        with self.assertRaises(ConfigurationError):
+            config.server_config.handle_data_locator()
+
+    def test_handle_app___can_use_envar_port(self):
+        config = self.get_config(port=24)
+        self.assertEqual(config.server_config.app__port, 24)
+
+        # Note if the port is set in the config file it will NOT be overwritten by a different envvar
+        os.environ["CXG_SERVER_PORT"] = "4008"
+        self.config = AppConfig()
+        self.config.update_server_config(app__flask_secret_key="secret")
+        self.config.server_config.handle_app(self.context)
+        self.assertEqual(self.config.server_config.app__port, 4008)
+        del os.environ["CXG_SERVER_PORT"]
+
+    def test_handle_app__can_get_secret_key_from_envvar_or_config_file_with_envvar_given_preference(self):
+        config = self.get_config(flask_secret_key="KEY_FROM_FILE")
+        self.assertEqual(config.server_config.app__flask_secret_key, "KEY_FROM_FILE")
+
+        os.environ["CXG_SECRET_KEY"] = "KEY_FROM_ENV"
+        config.external_config.handle_environment(self.context)
+        self.assertEqual(config.server_config.app__flask_secret_key, "KEY_FROM_ENV")
+
+    def test_config_for_single_dataset(self):
+        file_name = self.custom_app_config(config_file_name="single_dataset.yml", dataset_datapath=f"{H5AD_FIXTURE}")
+        config = AppConfig()
+        config.update_from_config_file(file_name)
+        config.server_config.handle_single_dataset(self.context)
+
+        file_name = self.custom_app_config(
+            config_file_name="single_dataset_with_about.yml",
+            about="www.cziscience.com",
+            dataset_datapath=f"{H5AD_FIXTURE}",
+        )
+        config = AppConfig()
+        config.update_from_config_file(file_name)
+        with self.assertRaises(ConfigurationError):
+            config.server_config.handle_single_dataset(self.context)

test/unit/common/test_api.py

@@ -0,0 +1,872 @@
+import shutil
+import time
+import unittest
+import zlib
+from http import HTTPStatus
+import tempfile
+from os import path
+import hashlib
+from os.path import basename, splitext
+
+import pandas as pd
+import requests
+import numpy as np
+
+from parameterized import parameterized_class
+
+import test.decode_fbs as decode_fbs
+from server.data_common.matrix_loader import MatrixDataType
+from test.unit import (
+    data_with_tmp_annotations,
+    make_fbs,
+    start_test_server,
+    stop_test_server,
+)
+from test.fixtures.fixtures import pbmc3k_colors
+from test import PROJECT_ROOT, FIXTURES_ROOT
+
+BAD_FILTER = {"filter": {"obs": {"annotation_value": [{"name": "xyz"}]}}}
+
+
+# TODO (mweiden): remove ANNOTATIONS_ENABLED and Annotation subclasses when annotations are no longer experimental
+
+
+class EndPoints(object):
+    ANNOTATIONS_ENABLED = True
+    GENESETS_READONLY = False
+
+    def test_initialize(self):
+        endpoint = "schema"
+        url = f"{self.URL_BASE}{endpoint}"
+        result = self.session.get(url)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/json")
+        result_data = result.json()
+        self.assertEqual(result_data["schema"]["dataframe"]["nObs"], 2638)
+        self.assertEqual(len(result_data["schema"]["annotations"]["obs"]), 2)
+        self.assertEqual(
+            len(result_data["schema"]["annotations"]["obs"]["columns"]), 6 if self.ANNOTATIONS_ENABLED else 5
+        )
+
+        # Check that all schema types are legal
+        legal_types = ["boolean", "string", "categorical", "float32", "int32"]
+        self.assertEqual(result_data["schema"]["dataframe"]["type"], "float32")
+        for column in result_data["schema"]["annotations"]["obs"]["columns"]:
+            self.assertIn(column["type"], legal_types)
+        for column in result_data["schema"]["annotations"]["var"]["columns"]:
+            self.assertIn(column["type"], legal_types)
+
+    def test_config(self):
+        endpoint = "config"
+        url = f"{self.URL_BASE}{endpoint}"
+        result = self.session.get(url)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/json")
+        result_data = result.json()
+        self.assertIn("library_versions", result_data["config"])
+
+        if hasattr(self, "data_locator"):
+            title = splitext(basename(self.data_locator))[0]
+        else:
+            title = "pbmc3k"
+        self.assertEqual(result_data["config"]["displayNames"]["dataset"], title)
+        self.assertIsNotNone(result_data["config"]["parameters"])
+
+    def test_get_layout_fbs(self):
+        endpoint = "layout/obs"
+        url = f"{self.URL_BASE}{endpoint}"
+        header = {"Accept": "application/octet-stream"}
+        result = self.session.get(url, headers=header)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
+        df = decode_fbs.decode_matrix_FBS(result.content)
+        self.assertEqual(df["n_rows"], 2638)
+        self.assertEqual(df["n_cols"], 8)
+        self.assertIsNotNone(df["columns"])
+        self.assertSetEqual(
+            set(df["col_idx"]),
+            {"pca_0", "pca_1", "tsne_0", "tsne_1", "umap_0", "umap_1", "draw_graph_fr_0", "draw_graph_fr_1"},
+        )
+        self.assertIsNone(df["row_idx"])
+        self.assertEqual(len(df["columns"]), df["n_cols"])
+        for column in df["columns"]:
+            self.assertEqual(column.dtype, np.float32)
+
+    def test_bad_filter(self):
+        endpoint = "data/var"
+        url = f"{self.URL_BASE}{endpoint}"
+        result = self.session.put(url, json=BAD_FILTER)
+        self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST)
+
+    def test_get_annotations_obs_fbs(self):
+        endpoint = "annotations/obs"
+        url = f"{self.URL_BASE}{endpoint}"
+        header = {"Accept": "application/octet-stream"}
+        result = self.session.get(url, headers=header)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
+        df = decode_fbs.decode_matrix_FBS(result.content)
+        self.assertEqual(df["n_rows"], 2638)
+        self.assertEqual(df["n_cols"], 6 if self.ANNOTATIONS_ENABLED else 5)
+        self.assertIsNotNone(df["columns"])
+        self.assertIsNone(df["row_idx"])
+        self.assertEqual(len(df["columns"]), df["n_cols"])
+        obs_index_col_name = self.schema["schema"]["annotations"]["obs"]["index"]
+        self.assertCountEqual(
+            df["col_idx"],
+            [obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain"]
+            + (["cluster-test"] if self.ANNOTATIONS_ENABLED else []),
+        )
+        for column in df["columns"]:
+            if type(column) is np.ndarray:
+                self.assertIn(column.dtype, [np.float32, np.int32])
+
+    def test_get_annotations_obs_keys_fbs(self):
+        endpoint = "annotations/obs"
+        query = "annotation-name=n_genes&annotation-name=percent_mito"
+        url = f"{self.URL_BASE}{endpoint}?{query}"
+        header = {"Accept": "application/octet-stream"}
+        result = self.session.get(url, headers=header)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
+        df = decode_fbs.decode_matrix_FBS(result.content)
+        self.assertEqual(df["n_rows"], 2638)
+        self.assertEqual(df["n_cols"], 2)
+        self.assertIsNotNone(df["columns"])
+        self.assertIsNone(df["row_idx"])
+        self.assertEqual(len(df["columns"]), df["n_cols"])
+        self.assertCountEqual(df["col_idx"], ["n_genes", "percent_mito"])
+
+    def test_get_annotations_obs_error(self):
+        endpoint = "annotations/obs"
+        query = "annotation-name=notakey"
+        url = f"{self.URL_BASE}{endpoint}?{query}"
+        result = self.session.get(url)
+        self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST)
+
+    def test_get_annotations_var_fbs(self):
+        endpoint = "annotations/var"
+        url = f"{self.URL_BASE}{endpoint}"
+        header = {"Accept": "application/octet-stream"}
+        result = self.session.get(url, headers=header)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
+        df = decode_fbs.decode_matrix_FBS(result.content)
+        self.assertEqual(df["n_rows"], 1838)
+        self.assertEqual(df["n_cols"], 2)
+        self.assertIsNotNone(df["columns"])
+        self.assertIsNone(df["row_idx"])
+        self.assertEqual(len(df["columns"]), df["n_cols"])
+        var_index_col_name = self.schema["schema"]["annotations"]["var"]["index"]
+        self.assertCountEqual(df["col_idx"], [var_index_col_name, "n_cells"])
+        for column in df["columns"]:
+            if type(column) is np.ndarray:
+                self.assertIn(column.dtype, [np.float32, np.int32])
+
+    def test_get_annotations_var_keys_fbs(self):
+        endpoint = "annotations/var"
+        query = "annotation-name=n_cells"
+        url = f"{self.URL_BASE}{endpoint}?{query}"
+        header = {"Accept": "application/octet-stream"}
+        result = self.session.get(url, headers=header)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
+        df = decode_fbs.decode_matrix_FBS(result.content)
+        self.assertEqual(df["n_rows"], 1838)
+        self.assertEqual(df["n_cols"], 1)
+        self.assertIsNotNone(df["columns"])
+        self.assertIsNone(df["row_idx"])
+        self.assertEqual(len(df["columns"]), df["n_cols"])
+        self.assertCountEqual(df["col_idx"], ["n_cells"])
+
+    def test_get_annotations_var_error(self):
+        endpoint = "annotations/var"
+        query = "annotation-name=notakey"
+        url = f"{self.URL_BASE}{endpoint}?{query}"
+        result = self.session.get(url)
+        self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST)
+
+    def test_data_mimetype_error(self):
+        endpoint = "data/var"
+        header = {"Accept": "xxx"}
+        url = f"{self.URL_BASE}{endpoint}"
+        result = self.session.put(url, headers=header)
+        self.assertEqual(result.status_code, HTTPStatus.NOT_ACCEPTABLE)
+
+    def test_fbs_default(self):
+        endpoint = "data/var"
+        url = f"{self.URL_BASE}{endpoint}"
+        result = self.session.put(url)
+        self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST)
+
+        filter = {"filter": {"var": {"index": [0, 1, 4]}}}
+        result = self.session.put(url, json=filter)
+        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
+
+    def test_data_put_fbs(self):
+        endpoint = "data/var"
+        url = f"{self.URL_BASE}{endpoint}"
+        header = {"Accept": "application/octet-stream"}
+        result = self.session.put(url, headers=header)
+        self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST)
+
+    def test_data_get_fbs(self):
+        endpoint = "data/var"
+        url = f"{self.URL_BASE}{endpoint}"
+        header = {"Accept": "application/octet-stream"}
+        result = self.session.get(url, headers=header)
+        self.assertEqual(result.status_code, HTTPStatus.BAD_REQUEST)
+
+    def test_data_put_filter_fbs(self):
+        endpoint = "data/var"
+        url = f"{self.URL_BASE}{endpoint}"
+        header = {"Accept": "application/octet-stream"}
+        filter = {"filter": {"var": {"index": [0, 1, 4]}}}
+        result = self.session.put(url, headers=header, json=filter)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
+        df = decode_fbs.decode_matrix_FBS(result.content)
+        self.assertEqual(df["n_rows"], 2638)
+        self.assertEqual(df["n_cols"], 3)
+        self.assertIsNotNone(df["columns"])
+        self.assertIsNone(df["row_idx"])
+        self.assertEqual(len(df["columns"]), df["n_cols"])
+        self.assertListEqual(df["col_idx"].tolist(), [0, 1, 4])
+        for column in df["columns"]:
+            if type(column) is np.ndarray:
+                self.assertIn(column.dtype, [np.float32, np.int32])
+
+    def test_data_get_filter_fbs(self):
+        index_col_name = self.schema["schema"]["annotations"]["var"]["index"]
+        endpoint = "data/var"
+        query = f"var:{index_col_name}=SIK1"
+        url = f"{self.URL_BASE}{endpoint}?{query}"
+        header = {"Accept": "application/octet-stream"}
+        result = self.session.get(url, headers=header)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
+        df = decode_fbs.decode_matrix_FBS(result.content)
+        self.assertEqual(df["n_rows"], 2638)
+        self.assertEqual(df["n_cols"], 1)
+        for column in df["columns"]:
+            if type(column) is np.ndarray:
+                self.assertIn(column.dtype, [np.float32, np.int32])
+
+    def test_data_get_unknown_filter_fbs(self):
+        index_col_name = self.schema["schema"]["annotations"]["var"]["index"]
+        endpoint = "data/var"
+        query = f"var:{index_col_name}=UNKNOWN"
+        url = f"{self.URL_BASE}{endpoint}?{query}"
+        header = {"Accept": "application/octet-stream"}
+        result = self.session.get(url, headers=header)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
+        df = decode_fbs.decode_matrix_FBS(result.content)
+        self.assertEqual(df["n_rows"], 2638)
+        self.assertEqual(df["n_cols"], 0)
+
+    def test_data_put_single_var(self):
+        endpoint = "data/var"
+        url = f"{self.URL_BASE}{endpoint}"
+        header = {"Accept": "application/octet-stream"}
+        index_col_name = self.schema["schema"]["annotations"]["var"]["index"]
+        var_filter = {"filter": {"var": {"annotation_value": [{"name": index_col_name, "values": ["RER1"]}]}}}
+        result = self.session.put(url, headers=header, json=var_filter)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
+        df = decode_fbs.decode_matrix_FBS(result.content)
+        self.assertEqual(df["n_rows"], 2638)
+        self.assertEqual(df["n_cols"], 1)
+        for column in df["columns"]:
+            if type(column) is np.ndarray:
+                self.assertIn(column.dtype, [np.float32, np.int32])
+
+    def test_colors(self):
+        endpoint = "colors"
+        url = f"{self.URL_BASE}{endpoint}"
+        result = self.session.get(url)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/json")
+        result_data = result.json()
+        self.assertEqual(result_data, pbmc3k_colors)
+
+    def test_static(self):
+        endpoint = "static"
+        file = "assets/favicon.ico"
+        url = f"{self.server}/{endpoint}/{file}"
+        result = self.session.get(url)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+
+    def test_genesets_config(self):
+        result = self.session.get(f"{self.URL_BASE}config")
+        config_data = result.json()
+        params = config_data["config"]["parameters"]
+        annotations_genesets = params["annotations_genesets"]
+        annotations_genesets_readonly = params["annotations_genesets_readonly"]
+        annotations_genesets_summary_methods = params["annotations_genesets_summary_methods"]
+        self.assertTrue(annotations_genesets)
+        self.assertEqual(annotations_genesets_readonly, self.GENESETS_READONLY)
+        self.assertEqual(annotations_genesets_summary_methods, ["mean"])
+
+    def test_get_genesets(self):
+        endpoint = "genesets"
+        url = f"{self.URL_BASE}{endpoint}"
+        result = self.session.get(url, headers={"Accept": "application/json"})
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/json")
+        result_data = result.json()
+        self.assertIsNotNone(result_data["genesets"])
+
+    def _setupClass(child_class, command_line):
+        child_class.ps, child_class.server = start_test_server(command_line)
+        child_class.URL_BASE = f"{child_class.server}/api/v0.2/"
+        child_class.session = requests.Session()
+        for i in range(90):
+            try:
+                result = child_class.session.get(f"{child_class.URL_BASE}schema")
+                child_class.schema = result.json()
+            except requests.exceptions.ConnectionError:
+                time.sleep(1)
+
+
+class EndPointsAnnotations(EndPoints):
+    def test_get_schema_existing_writable(self):
+        self._test_get_schema_writable("cluster-test")
+
+    def test_get_user_annotations_existing_obs_keys_fbs(self):
+        self._test_get_user_annotations_obs_keys_fbs(
+            "cluster-test",
+            {"unassigned", "one", "two", "three", "four", "five", "six", "seven"},
+        )
+
+    def test_put_user_annotations_obs_fbs(self):
+        endpoint = "annotations/obs"
+        query = "annotation-collection-name=test_annotations"
+        url = f"{self.URL_BASE}{endpoint}?{query}"
+        n_rows = self.data.get_shape()[0]
+        fbs = make_fbs({"cat_A": pd.Series(["label_A"] * n_rows, dtype="category")})
+        result = self.session.put(url, data=zlib.compress(fbs))
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/json")
+        self.assertEqual(result.json(), {"status": "OK"})
+        self._test_get_schema_writable("cat_A")
+        self._test_get_user_annotations_obs_keys_fbs("cat_A", {"label_A"})
+
+    def _test_get_user_annotations_obs_keys_fbs(self, annotation_name, columns):
+        endpoint = "annotations/obs"
+        query = f"annotation-name={annotation_name}"
+        url = f"{self.URL_BASE}{endpoint}?{query}"
+        header = {"Accept": "application/octet-stream"}
+        result = self.session.get(url, headers=header)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
+        df = decode_fbs.decode_matrix_FBS(result.content)
+        self.assertEqual(df["n_rows"], 2638)
+        self.assertEqual(df["n_cols"], 1)
+        self.assertListEqual(df["col_idx"], [annotation_name])
+        self.assertEqual(set(df["columns"][0]), columns)
+        self.assertIsNone(df["row_idx"])
+        self.assertEqual(len(df["columns"]), df["n_cols"])
+
+    def _test_get_schema_writable(self, cluster_name):
+        endpoint = "schema"
+        url = f"{self.URL_BASE}{endpoint}"
+        result = self.session.get(url)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/json")
+        result_data = result.json()
+        columns = result_data["schema"]["annotations"]["obs"]["columns"]
+        matching_columns = [c for c in columns if c["name"] == cluster_name]
+        self.assertEqual(len(matching_columns), 1)
+        self.assertTrue(matching_columns[0]["writable"])
+
+
+@parameterized_class(
+    [
+        {"data_locator": f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad"},
+        {"data_locator": f"{FIXTURES_ROOT}/pbmc3k_64.h5ad"},
+        {"data_locator": f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad"},
+        {"data_locator": f"{FIXTURES_ROOT}/pbmc3k-CSR-gz.h5ad"},
+    ]
+)
+class EndPointsAnndata(unittest.TestCase, EndPoints):
+    """Test Case for endpoints"""
+
+    ANNOTATIONS_ENABLED = False
+    GENESETS_READONLY = True
+
+    @classmethod
+    def setUpClass(cls):
+        if cls == EndPointsAnndata:
+            raise unittest.SkipTest("`parameterized_class` bug")
+
+        cls._setupClass(
+            cls,
+            [
+                cls.data_locator,
+                "--disable-annotations",
+                "--disable-gene-sets-save",
+            ],
+        )
+
+    @classmethod
+    def tearDownClass(cls):
+        stop_test_server(cls.ps)
+
+    @property
+    def annotations_enabled(self):
+        return False
+
+    def test_diff_exp(self):
+        endpoint = "diffexp/obs"
+        url = f"{self.URL_BASE}{endpoint}"
+        params = {
+            "mode": "topN",
+            "set1": {"filter": {"obs": {"annotation_value": [{"name": "louvain", "values": ["NK cells"]}]}}},
+            "set2": {"filter": {"obs": {"annotation_value": [{"name": "louvain", "values": ["CD8 T cells"]}]}}},
+            "count": 7,
+        }
+        result = self.session.post(url, json=params)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/json")
+        result_data = result.json()
+        self.assertEqual(len(result_data["positive"]), 7)
+        self.assertEqual(len(result_data["negative"]), 7)
+
+    def test_diff_exp_indices(self):
+        endpoint = "diffexp/obs"
+        url = f"{self.URL_BASE}{endpoint}"
+        params = {
+            "mode": "topN",
+            "count": 10,
+            "set1": {"filter": {"obs": {"index": [[0, 500]]}}},
+            "set2": {"filter": {"obs": {"index": [[500, 1000]]}}},
+        }
+        result = self.session.post(url, json=params)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/json")
+        result_data = result.json()
+        self.assertEqual(len(result_data["positive"]), 10)
+        self.assertEqual(len(result_data["negative"]), 10)
+
+    def test_get_summaryvar(self):
+        index_col_name = self.schema["schema"]["annotations"]["var"]["index"]
+        endpoint = "summarize/var"
+
+        # single column
+        filter = f"var:{index_col_name}=F5"
+        query = f"method=mean&{filter}"
+        query_hash = hashlib.sha1(query.encode()).hexdigest()
+        url = f"{self.URL_BASE}{endpoint}?{query}"
+        header = {"Accept": "application/octet-stream"}
+        result = self.session.get(url, headers=header)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
+        df = decode_fbs.decode_matrix_FBS(result.content)
+        self.assertEqual(df["n_rows"], 2638)
+        self.assertEqual(df["n_cols"], 1)
+        self.assertEqual(df["col_idx"], [query_hash])
+        self.assertAlmostEqual(df["columns"][0][0], -0.110451095)
+
+        # multi-column
+        col_names = ["F5", "BEB3", "SIK1"]
+        filter = "&".join([f"var:{index_col_name}={name}" for name in col_names])
+        query = f"method=mean&{filter}"
+        query_hash = hashlib.sha1(query.encode()).hexdigest()
+        url = f"{self.URL_BASE}{endpoint}?{query}"
+        header = {"Accept": "application/octet-stream"}
+        result = self.session.get(url, headers=header)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
+        df = decode_fbs.decode_matrix_FBS(result.content)
+        self.assertEqual(df["n_rows"], 2638)
+        self.assertEqual(df["n_cols"], 1)
+        self.assertEqual(df["col_idx"], [query_hash])
+        self.assertAlmostEqual(df["columns"][0][0], -0.16628358)
+
+    def test_post_summaryvar(self):
+        index_col_name = self.schema["schema"]["annotations"]["var"]["index"]
+        endpoint = "summarize/var"
+        headers = {"Content-Type": "application/x-www-form-urlencoded", "Accept": "application/octet-stream"}
+
+        # single column
+        filter = f"var:{index_col_name}=F5"
+        query = f"method=mean&{filter}"
+        query_hash = hashlib.sha1(query.encode()).hexdigest()
+        url = f"{self.URL_BASE}{endpoint}?key={query_hash}"
+        result = self.session.post(url, headers=headers, data=query)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
+        df = decode_fbs.decode_matrix_FBS(result.content)
+        self.assertEqual(df["n_rows"], 2638)
+        self.assertEqual(df["n_cols"], 1)
+        self.assertEqual(df["col_idx"], [query_hash])
+        self.assertAlmostEqual(df["columns"][0][0], -0.110451095)
+
+        # multi-column
+        col_names = ["F5", "BEB3", "SIK1"]
+        filter = "&".join([f"var:{index_col_name}={name}" for name in col_names])
+        query = f"method=mean&{filter}"
+        query_hash = hashlib.sha1(query.encode()).hexdigest()
+        url = f"{self.URL_BASE}{endpoint}?key={query_hash}"
+        result = self.session.post(url, headers=headers, data=query)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
+        df = decode_fbs.decode_matrix_FBS(result.content)
+        self.assertEqual(df["n_rows"], 2638)
+        self.assertEqual(df["n_cols"], 1)
+        self.assertEqual(df["col_idx"], [query_hash])
+        self.assertAlmostEqual(df["columns"][0][0], -0.16628358)
+
+
+class EndPointsAnndataAnnotations(unittest.TestCase, EndPointsAnnotations):
+    """Test Case for endpoints"""
+
+    ANNOTATIONS_ENABLED = True
+    GENESETS_READONLY = False
+
+    @classmethod
+    def setUpClass(cls):
+        cls.data, cls.tmp_dir, cls.annotations = data_with_tmp_annotations(
+            MatrixDataType.H5AD, annotations_fixture=True
+        )
+        cls._setupClass(cls, ["--annotations-file", cls.annotations.label_output_file, cls.data.get_location()])
+
+    @classmethod
+    def tearDownClass(cls):
+        shutil.rmtree(cls.tmp_dir)
+        stop_test_server(cls.ps)
+
+
+class EndPointsAnnDataGenesets(unittest.TestCase, EndPoints):
+    ANNOTATIONS_ENABLED = False
+    GENESETS_READONLY = False
+
+    @classmethod
+    def setUpClass(cls):
+        cls.tmp_dir = tempfile.mkdtemp()
+        genesets_file = path.join(cls.tmp_dir, "test_genesets.csv")
+        shutil.copyfile(f"{FIXTURES_ROOT}/pbmc3k-genesets.csv", genesets_file)
+        cls._setupClass(
+            cls,
+            [
+                f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad",
+                "--disable-annotations",
+                "--gene-sets-file",
+                genesets_file,
+            ],
+        )
+
+    @classmethod
+    def tearDownClass(cls):
+        shutil.rmtree(cls.tmp_dir)
+        stop_test_server(cls.ps)
+
+    def test_get_genesets_json(self):
+        endpoint = "genesets"
+        url = f"{self.URL_BASE}{endpoint}"
+        result = self.session.get(url, headers={"Accept": "application/json"})
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/json")
+        result_data = result.json()
+        self.assertIsNotNone(result_data["genesets"])
+        self.assertIsNotNone(result_data["tid"])
+
+        self.assertEqual(
+            result_data,
+            {
+                "genesets": [
+                    {
+                        "genes": [
+                            {"gene_description": " a gene_description", "gene_symbol": "F5"},
+                            {"gene_description": "", "gene_symbol": "SUMO3"},
+                            {"gene_description": "", "gene_symbol": "SRM"},
+                        ],
+                        "geneset_description": "a description",
+                        "geneset_name": "first gene set name",
+                    },
+                    {
+                        "genes": [
+                            {"gene_description": "", "gene_symbol": "RER1"},
+                            {"gene_description": "", "gene_symbol": "SIK1"},
+                        ],
+                        "geneset_description": "",
+                        "geneset_name": "second_gene_set",
+                    },
+                    {"genes": [], "geneset_description": "", "geneset_name": "third gene set"},
+                    {"genes": [], "geneset_description": "fourth description", "geneset_name": "fourth_gene_set"},
+                    {"genes": [], "geneset_description": "", "geneset_name": "fifth_dataset"},
+                    {
+                        "genes": [
+                            {"gene_description": "", "gene_symbol": "ACD"},
+                            {"gene_description": "", "gene_symbol": "AATF"},
+                            {"gene_description": "", "gene_symbol": "F5"},
+                            {"gene_description": "", "gene_symbol": "PIGU"},
+                        ],
+                        "geneset_description": "",
+                        "geneset_name": "summary test",
+                    },
+                    {"genes": [], "geneset_description": "", "geneset_name": "geneset_to_delete"},
+                    {"genes": [], "geneset_description": "", "geneset_name": "geneset_to_edit"},
+                    {
+                        "genes": [],
+                        "geneset_description": "",
+                        "geneset_name": "fill_this_geneset",
+                    },
+                    {
+                        "genes": [{"gene_description": "", "gene_symbol": "SIK1"}],
+                        "geneset_description": "",
+                        "geneset_name": "empty_this_geneset",
+                    },
+                    {
+                        "genes": [{"gene_description": "", "gene_symbol": "SIK1"}],
+                        "geneset_description": "",
+                        "geneset_name": "brush_this_gene",
+                    },
+                ],
+                "tid": 0,
+            },
+        )
+
+    def test_get_genesets_csv(self):
+        endpoint = "genesets"
+        url = f"{self.URL_BASE}{endpoint}"
+        result = self.session.get(url, headers={"Accept": "text/csv"})
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "text/csv")
+        self.assertEqual(
+            result.text,
+            """gene_set_name,gene_set_description,gene_symbol,gene_description\r
+first gene set name,a description,F5, a gene_description\r
+first gene set name,a description,SUMO3,\r
+first gene set name,a description,SRM,\r
+second_gene_set,,RER1,\r
+second_gene_set,,SIK1,\r
+third gene set,,,\r
+fourth_gene_set,fourth description,,\r
+fifth_dataset,,,\r
+summary test,,ACD,\r
+summary test,,AATF,\r
+summary test,,F5,\r
+summary test,,PIGU,\r
+geneset_to_delete,,,\r
+geneset_to_edit,,,\r
+fill_this_geneset,,,\r
+empty_this_geneset,,SIK1,\r
+brush_this_gene,,SIK1,\r
+""",
+        )
+
+    def test_put_genesets(self):
+        endpoint = "genesets"
+        url = f"{self.URL_BASE}{endpoint}"
+
+        # assume we start with TID 0
+        result = self.session.get(url, headers={"Accept": "application/json"})
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.json()["tid"], 0)
+
+        test1 = {"tid": 3, "genesets": []}
+        result = self.session.put(url, json=test1)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        result = self.session.get(url, headers={"Accept": "application/json"})
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.json(), test1)
+
+        # stale TID
+        result = self.session.put(url, json=test1)
+        self.assertEqual(result.status_code, HTTPStatus.NOT_FOUND)
+
+        test2 = {
+            "tid": 4,
+            "genesets": [
+                {"geneset_name": "foobar", "genes": []},
+                {"geneset_name": "contains a space", "genes": []},
+                {"geneset_name": "contains_weird_characters: #$%^&*()_+=-!@<>,./?';:\"[]{}|\\", "genes": []},
+            ],
+        }
+        test2_response = {
+            "tid": 4,
+            "genesets": [
+                {"geneset_name": "foobar", "geneset_description": "", "genes": []},
+                {"geneset_name": "contains a space", "geneset_description": "", "genes": []},
+                {
+                    "geneset_name": "contains_weird_characters: #$%^&*()_+=-!@<>,./?';:\"[]{}|\\",
+                    "geneset_description": "",
+                    "genes": [],
+                },
+            ],
+        }
+        result = self.session.put(url, json=test2)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        result = self.session.get(url, headers={"Accept": "application/json"})
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.json(), test2_response)
+
+        test3 = {
+            "tid": 5,
+            "genesets": [
+                {
+                    "geneset_name": "foobar",
+                    "geneset_description": "",
+                    "genes": [
+                        {
+                            "gene_symbol": "F5",
+                            "gene_description": "",
+                        }
+                    ],
+                }
+            ],
+        }
+        result = self.session.put(url, json=test3)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        result = self.session.get(url, headers={"Accept": "application/json"})
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.json(), test3)
+
+    def test_put_genesets_malformed(self):
+        """test malformed submissions that we expect the backend to catch/tolerate"""
+        endpoint = "genesets"
+        url = f"{self.URL_BASE}{endpoint}"
+
+        result = self.session.get(url, headers={"Accept": "application/json"})
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        original_data = result.json()
+        tid = original_data["tid"]
+
+        def test_case(test, expected_code, original_data):
+            """check for expected error AND that no change was made to the original state"""
+            result = self.session.put(url, json=test)
+            self.assertEqual(result.status_code, expected_code)
+            result = self.session.get(url, headers={"Accept": "application/json"})
+            self.assertEqual(result.status_code, HTTPStatus.OK)
+            self.assertEqual(result.json(), original_data)
+
+        # missing or malformed genesets
+        test_case(
+            {"tid": tid + 1},
+            HTTPStatus.BAD_REQUEST,
+            original_data,
+        )
+        test_case(
+            {"tid": tid + 1, "genesets": 99},
+            HTTPStatus.BAD_REQUEST,
+            original_data,
+        )
+
+        # illegal geneset_name
+        test_case(
+            {"tid": tid + 1, "genesets": [{"geneset_name": " foo", "genes": []}]},
+            HTTPStatus.BAD_REQUEST,
+            original_data,
+        )
+        test_case(
+            {"tid": tid + 1, "genesets": [{"geneset_name": "foo ", "genes": []}]},
+            HTTPStatus.BAD_REQUEST,
+            original_data,
+        )
+        test_case(
+            {"tid": tid + 1, "genesets": [{"geneset_name": "f  oo", "genes": []}]},
+            HTTPStatus.BAD_REQUEST,
+            original_data,
+        )
+        test_case(
+            {"tid": tid + 1, "genesets": [{"geneset_name": "f\too", "genes": []}]},
+            HTTPStatus.BAD_REQUEST,
+            original_data,
+        )
+        test_case(
+            {"tid": tid + 1, "genesets": [{"geneset_name": "f\roo", "genes": []}]},
+            HTTPStatus.BAD_REQUEST,
+            original_data,
+        )
+        test_case(
+            {"tid": tid + 1, "genesets": [{"geneset_name": "f\noo", "genes": []}]},
+            HTTPStatus.BAD_REQUEST,
+            original_data,
+        )
+        test_case(
+            {"tid": tid + 1, "genesets": [{"geneset_name": "f\voo", "genes": []}]},
+            HTTPStatus.BAD_REQUEST,
+            original_data,
+        )
+
+        # duplicate geneset_name
+        test_case(
+            {
+                "tid": tid + 1,
+                "genesets": [
+                    {"geneset_name": "foo", "genes": []},
+                    {"geneset_name": "foo", "genes": []},
+                ],
+            },
+            HTTPStatus.BAD_REQUEST,
+            original_data,
+        )
+
+        # missing geneset_name
+        test_case(
+            {"tid": tid + 1, "genesets": [{"genes": []}]},
+            HTTPStatus.BAD_REQUEST,
+            original_data,
+        )
+
+        # non-numeric TID
+        test_case(
+            {"tid": [], "genesets": [{"geneset_name": "foo", "genes": []}]},
+            HTTPStatus.BAD_REQUEST,
+            original_data,
+        )
+        test_case(
+            {"tid": None, "genesets": [{"geneset_name": "foo", "genes": []}]},
+            HTTPStatus.BAD_REQUEST,
+            original_data,
+        )
+        test_case(
+            {"tid": "not a number", "genesets": [{"geneset_name": "foo", "genes": []}]},
+            HTTPStatus.BAD_REQUEST,
+            original_data,
+        )
+
+        # duplicate gene_symbol
+        test_case(
+            {
+                "tid": "not a number",
+                "genesets": [{"geneset_name": "foo", "genes": [{"gene_symbol": "SIK1"}, {"gene_symbol": "SIK1"}]}],
+            },
+            HTTPStatus.BAD_REQUEST,
+            original_data,
+        )
+
+        # gene_symbol is not a string
+        test_case(
+            {
+                "tid": "not a number",
+                "genesets": [{"geneset_name": "foo", "genes": [{"gene_symbol": 99}]}],
+            },
+            HTTPStatus.BAD_REQUEST,
+            original_data,
+        )
+
+    def test_get_geneset_summary_edge_cases(self):
+        # attempt to summarize _all_ genesets, including edge cases with zero or one gene
+        result = self.session.get(f"{self.URL_BASE}genesets", headers={"Accept": "application/json"})
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        genesets = result.json()["genesets"]
+
+        endpoint = "summarize/var"
+        index_col_name = self.schema["schema"]["annotations"]["var"]["index"]
+        for gs in genesets:
+            genes = [g["gene_symbol"] for g in gs["genes"]]
+            filter = "&".join([f"var:{index_col_name}={gene}" for gene in genes])
+            query = f"method=mean&{filter}"
+            query_hash = hashlib.sha1(query.encode()).hexdigest()
+            url = f"{self.URL_BASE}{endpoint}?{query}"
+
+            result = self.session.get(url, headers={"Accept": "application/octet-stream"})
+            self.assertEqual(result.status_code, HTTPStatus.OK)
+            self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
+            df = decode_fbs.decode_matrix_FBS(result.content)
+            self.assertEqual(df["n_rows"], 2638)
+            self.assertEqual(df["n_cols"], 1)
+            self.assertEqual(df["col_idx"], [query_hash])

test/unit/common/test_corpora.py

@@ -0,0 +1,165 @@
+import json
+import shutil
+import tempfile
+import unittest
+from http import HTTPStatus
+
+import anndata
+import requests
+
+from server.common.corpora import (
+    corpora_get_versions_from_anndata,
+    corpora_is_version_supported,
+    corpora_get_props_from_anndata,
+)
+from test.unit import start_test_server, stop_test_server
+from test import PROJECT_ROOT
+
+VERSION = "v0.2"
+
+
+class CorporaAPITest(unittest.TestCase):
+    def test_corpora_get_versions_from_anndata(self):
+        adata = self._get_h5ad()
+
+        if "version" in adata.uns:
+            del adata.uns["version"]
+        self.assertIsNone(corpora_get_versions_from_anndata(adata))
+
+        # something bogus
+        adata.uns["version"] = 99
+        self.assertIsNone(corpora_get_versions_from_anndata(adata))
+
+        # something legit
+        adata.uns["version"] = {"corpora_schema_version": "0.0.0", "corpora_encoding_version": "9.9.9"}
+        self.assertEqual(corpora_get_versions_from_anndata(adata), ["0.0.0", "9.9.9"])
+
+    def test_corpora_is_version_supported(self):
+        self.assertTrue(corpora_is_version_supported("1.0.0", "0.1.0"))
+        self.assertFalse(corpora_is_version_supported("0.0.0", "0.1.0"))
+        self.assertFalse(corpora_is_version_supported("1.0.0", "0.0.0"))
+
+    def test_corpora_get_props_from_anndata(self):
+        adata = self._get_h5ad()
+
+        if "version" in adata.uns:
+            del adata.uns["version"]
+        self.assertIsNone(corpora_get_props_from_anndata(adata))
+
+        # something bogus
+        adata.uns["version"] = 99
+        self.assertIsNone(corpora_get_props_from_anndata(adata))
+
+        # unsupported version, but missing required values
+        adata.uns["version"] = {"corpora_schema_version": "99.0.0", "corpora_encoding_version": "32.1.0"}
+        with self.assertRaises(ValueError):
+            corpora_get_props_from_anndata(adata)
+
+        # legit version, but missing required values
+        adata.uns["version"] = {"corpora_schema_version": "1.0.0", "corpora_encoding_version": "0.1.0"}
+        with self.assertRaises(KeyError):
+            corpora_get_props_from_anndata(adata)
+
+        some_fields = {
+            "version": {"corpora_schema_version": "1.0.0", "corpora_encoding_version": "0.1.0"},
+            "title": "title",
+            "layer_descriptions": "layer_descriptions",
+            "organism": "organism",
+            "organism_ontology_term_id": "organism_ontology_term_id",
+            "project_name": "project_name",
+            "project_description": "project_description",
+            "contributors": json.dumps([{"contributors": "contributors"}]),
+            "project_links": json.dumps([{"link_name": "link_name", "link_url": "link_url", "link_type": "SUMMARY"}]),
+        }
+        for k in some_fields:
+            adata.uns[k] = some_fields[k]
+        some_fields["contributors"] = json.loads(some_fields["contributors"])
+        some_fields["project_links"] = json.loads(some_fields["project_links"])
+        self.assertEqual(corpora_get_props_from_anndata(adata), some_fields)
+
+    def test_corpora_get_props_from_anndata_v110(self):
+        adata = self._get_h5ad()
+
+        if "version" in adata.uns:
+            del adata.uns["version"]
+        self.assertIsNone(corpora_get_props_from_anndata(adata))
+
+        # legit version, but missing required values
+        adata.uns["version"] = {"corpora_schema_version": "1.1.0", "corpora_encoding_version": "0.1.0"}
+        with self.assertRaises(KeyError):
+            corpora_get_props_from_anndata(adata)
+
+        # Metadata following schema 1.1.0, which removes some fields relative to 1.1.0
+        some_110_fields = {
+            "version": {"corpora_schema_version": "1.0.0", "corpora_encoding_version": "0.1.0"},
+            "title": "title",
+            "layer_descriptions": "layer_descriptions",
+            "organism": "organism",
+            "organism_ontology_term_id": "organism_ontology_term_id",
+        }
+        for k in some_110_fields:
+            adata.uns[k] = some_110_fields[k]
+        self.assertEqual(corpora_get_props_from_anndata(adata), some_110_fields)
+
+    def _get_h5ad(self):
+        return anndata.read_h5ad(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad")
+
+
+class CorporaRESTAPITest(unittest.TestCase):
+    """ Confirm endpoints reflect Corpora-specific features """
+
+    @classmethod
+    def setCorporaFields(cls, path):
+        adata = anndata.read_h5ad(path)
+        corpora_props = {
+            "version": {"corpora_schema_version": "1.0.0", "corpora_encoding_version": "0.1.0"},
+            "title": "PBMC3K",
+            "contributors": json.dumps([{"name": "name"}]),
+            "layer_descriptions": {"X": "raw counts"},
+            "organism": "human",
+            "organism_ontology_term_id": "unknown",
+            "project_name": "test project",
+            "project_description": "test description",
+            "project_links": json.dumps(
+                [{"link_name": "test link", "link_type": "SUMMARY", "link_url": "https://a.u.r.l/"}]
+            ),
+            "default_embedding": "X_tsne",
+        }
+        adata.uns.update(corpora_props)
+        adata.write(path)
+
+    @classmethod
+    def setUpClass(cls):
+        cls.tmp_dir = tempfile.TemporaryDirectory()
+        src = f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad"
+        dst = f"{cls.tmp_dir.name}/pbmc3k.h5ad"
+        shutil.copyfile(src, dst)
+        cls.setCorporaFields(dst)
+        cls.ps, cls.server = start_test_server([dst])
+
+    @classmethod
+    def tearDownClass(cls):
+        stop_test_server(cls.ps)
+        cls.tmp_dir.cleanup()
+
+    def setUp(self):
+        self.session = requests.Session()
+        self.url_base = f"{self.server}/api/{VERSION}/"
+
+    def test_config(self):
+        endpoint = "config"
+        url = f"{self.url_base}{endpoint}"
+        result = self.session.get(url)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/json")
+
+        result_data = result.json()
+        self.assertIsInstance(result_data["config"]["corpora_props"], dict)
+        self.assertIsInstance(result_data["config"]["parameters"], dict)
+
+        corpora_props = result_data["config"]["corpora_props"]
+        parameters = result_data["config"]["parameters"]
+
+        self.assertEqual(corpora_props["version"]["corpora_schema_version"], "1.0.0")
+        self.assertEqual(corpora_props["organism"], "human")
+        self.assertEqual(parameters["default_embedding"], "tsne")

test/unit/common/test_nan_rest.py

@@ -0,0 +1,61 @@
+from http import HTTPStatus
+import unittest
+import math
+from test.unit import start_test_server, stop_test_server
+from test import FIXTURES_ROOT
+import test.decode_fbs as decode_fbs
+
+import requests
+
+VERSION = "v0.2"
+BAD_FILTER = {"filter": {"obs": {"annotation_value": [{"name": "xyz"}]}}}
+
+
+class WithNaNs(unittest.TestCase):
+    """Test Case for endpoints"""
+
+    @classmethod
+    def setUpClass(cls):
+        cls.ps, cls.server = start_test_server([f"{FIXTURES_ROOT}/nan.h5ad"])
+
+    @classmethod
+    def tearDownClass(cls):
+        stop_test_server(cls.ps)
+
+    def setUp(self):
+        self.session = requests.Session()
+        self.url_base = f"{self.server}/api/{VERSION}/"
+
+    def test_initialize(self):
+        endpoint = "schema"
+        url = f"{self.url_base}{endpoint}"
+        result = self.session.get(url)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+
+    def test_data(self):
+        endpoint = "data/var"
+        url = f"{self.url_base}{endpoint}"
+        filter = {"filter": {"var": {"index": [[0, 20]]}}}
+        result = self.session.put(url, json=filter)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
+        df = decode_fbs.decode_matrix_FBS(result.content)
+        self.assertTrue(math.isnan(df["columns"][3][3]))
+
+    def test_annotation_obs(self):
+        endpoint = "annotations/obs"
+        url = f"{self.url_base}{endpoint}"
+        result = self.session.get(url)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
+        df = decode_fbs.decode_matrix_FBS(result.content)
+        self.assertTrue(math.isnan(df["columns"][2][0]))
+
+    def test_annotation_var(self):
+        endpoint = "annotations/var"
+        url = f"{self.url_base}{endpoint}"
+        result = self.session.get(url)
+        self.assertEqual(result.status_code, HTTPStatus.OK)
+        self.assertEqual(result.headers["Content-Type"], "application/octet-stream")
+        df = decode_fbs.decode_matrix_FBS(result.content)
+        self.assertTrue(math.isnan(df["columns"][2][0]))

test/unit/common/test_rest.py

@@ -0,0 +1,79 @@
+import unittest
+from urllib.parse import parse_qs
+from werkzeug.datastructures import MultiDict
+from server.common.errors import FilterError
+from server.common.rest import _query_parameter_to_filter
+
+
+def _qsparse(qs):
+    """ emulate what Flask/Werkzeug do to our QS """
+    return MultiDict(parse_qs(qs))
+
+
+class FilterParseTests(unittest.TestCase):
+    """ Test cases for various filter parsing """
+
+    def test_queryparam_to_filter_parse(self):
+        # categories
+        self.assertEqual(
+            _query_parameter_to_filter(_qsparse("obs:foo=bar&var:baz=133&var:baz=A&obs:baz=foo")),
+            {
+                "obs": {"annotation_value": [{"name": "foo", "values": ["bar"]}, {"name": "baz", "values": ["foo"]}]},
+                "var": {"annotation_value": [{"name": "baz", "values": ["133", "A"]}]},
+            },
+        )
+
+        # ranges
+        self.assertEqual(
+            _query_parameter_to_filter(_qsparse("obs:A=1,99&obs:B=*,100&obs:C=0,*")),
+            {
+                "obs": {
+                    "annotation_value": [
+                        {"name": "A", "min": 1, "max": 99.0},
+                        {"name": "B", "max": 100.0},
+                        {"name": "C", "min": 0.0},
+                    ]
+                },
+            },
+        )
+
+        # combo
+        self.assertEqual(
+            _query_parameter_to_filter(_qsparse("var:B=YES&var:A=1,99&var:B=NO")),
+            {
+                "var": {
+                    "annotation_value": [
+                        {"name": "B", "values": ["YES", "NO"]},
+                        {"name": "A", "min": 1.0, "max": 99.0},
+                    ]
+                },
+            },
+        )
+
+    def test_queryparam_to_filter_escaping(self):
+        self.assertEqual(
+            _query_parameter_to_filter(_qsparse("obs:var=%2521%252C%253AOK%253D&obs:A%2521=YO")),
+            {"obs": {"annotation_value": [{"name": "var", "values": ["!,:OK="]}, {"name": "A!", "values": ["YO"]}]}},
+        )
+
+    def test_queryparam_to_filter_errors(self):
+
+        # should raise FilterError
+        filter_errors = [
+            "foo=bar",  # no axis
+            "X=&Y=3",  # no value
+            "X&Y=3",  # no value
+            "moo:foo=bar",  # bad axis
+            "obs:x=1,A",  # non-numeric range
+            "var:X=1,2&var:X=3,4",  # duplicate ranges
+            "var:Y=,",
+            "var:Y=2,",
+            "var:Y=,5",
+            "var:Y=*,",
+            "var:Y=,*",
+            "var:Y=*,*",
+        ]
+
+        for qs in filter_errors:
+            with self.assertRaises(FilterError):
+                _query_parameter_to_filter(_qsparse(qs))

test/unit/common/test_writable_annotation.py

@@ -0,0 +1,168 @@
+import json
+import shutil
+import unittest
+from os import path, listdir
+
+import numpy as np
+import pandas as pd
+
+import test.decode_fbs as decode_fbs
+from server.common.rest import annotations_put_fbs_helper, schema_get_helper
+from server.data_common.matrix_loader import MatrixDataType
+from test.unit import data_with_tmp_annotations, make_fbs
+
+
+class WritableAnnotationTest(unittest.TestCase):
+    def setUp(self):
+        self.data, self.tmp_dir, self.annotations = data_with_tmp_annotations(MatrixDataType.H5AD)
+        self.data.dataset_config.user_annotations = self.annotations
+
+    def tearDown(self):
+        shutil.rmtree(self.tmp_dir)
+
+    def annotation_put_fbs(self, fbs):
+        annotations_put_fbs_helper(self.data, fbs)
+        res = json.dumps({"status": "OK"})
+        return res
+
+    def test_error_checks(self):
+        # verify that the expected errors are generated
+        n_rows = self.data.get_shape()[0]
+        fbs_bad = make_fbs({"louvain": pd.Series(["undefined"] * n_rows, dtype="category")})
+
+        # ensure we catch attempt to overwrite non-writable data
+        with self.assertRaises(KeyError):
+            self.annotation_put_fbs(fbs_bad)
+
+    def test_write_to_file(self):
+        # verify the file is written as expected
+        n_rows = self.data.get_shape()[0]
+        fbs = make_fbs(
+            {
+                "cat_A": pd.Series(["label_A"] * n_rows, dtype="category"),
+                "cat_B": pd.Series(["label_B"] * n_rows, dtype="category"),
+            }
+        )
+        res = self.annotation_put_fbs(fbs)
+        self.assertEqual(res, json.dumps({"status": "OK"}))
+        self.assertTrue(path.exists(self.annotations.label_output_file))
+        df = pd.read_csv(self.annotations.label_output_file, index_col=0, header=0, comment="#")
+        self.assertEqual(df.shape, (n_rows, 2))
+        self.assertEqual(set(df.columns), {"cat_A", "cat_B"})
+        self.assertTrue(self.data.original_obs_index.equals(df.index))
+        self.assertTrue(np.all(df["cat_A"] == ["label_A"] * n_rows))
+        self.assertTrue(np.all(df["cat_B"] == ["label_B"] * n_rows))
+
+        # verify complete overwrite on second attempt, AND rotation occurs
+        fbs = make_fbs(
+            {
+                "cat_A": pd.Series(["label_A1"] * n_rows, dtype="category"),
+                "cat_C": pd.Series(["label_C"] * n_rows, dtype="category"),
+            }
+        )
+        res = self.annotation_put_fbs(fbs)
+        self.assertEqual(res, json.dumps({"status": "OK"}))
+        self.assertTrue(path.exists(self.annotations.label_output_file))
+        df = pd.read_csv(self.annotations.label_output_file, index_col=0, header=0, comment="#")
+        self.assertEqual(set(df.columns), {"cat_A", "cat_C"})
+        self.assertTrue(np.all(df["cat_A"] == ["label_A1"] * n_rows))
+        self.assertTrue(np.all(df["cat_C"] == ["label_C"] * n_rows))
+
+        # rotation
+        name, ext = path.splitext(self.annotations.label_output_file)
+        backup_dir = f"{name}-backups"
+        self.assertTrue(path.isdir(backup_dir))
+        found_files = listdir(backup_dir)
+        self.assertEqual(len(found_files), 1)
+
+    def test_file_rotation_to_max_9(self):
+        # verify we stop rotation at 9
+        n_rows = self.data.get_shape()[0]
+        fbs = make_fbs(
+            {
+                "cat_A": pd.Series(["label_A"] * n_rows, dtype="category"),
+                "cat_B": pd.Series(["label_B"] * n_rows, dtype="category"),
+            }
+        )
+        for i in range(0, 11):
+            res = self.annotation_put_fbs(fbs)
+            self.assertEqual(res, json.dumps({"status": "OK"}))
+
+        name, ext = path.splitext(self.annotations.label_output_file)
+        backup_dir = f"{name}-backups"
+        self.assertTrue(path.isdir(backup_dir))
+        found_files = listdir(backup_dir)
+        self.assertTrue(len(found_files) <= 9)
+
+    def test_put_get_roundtrip(self):
+        # verify that OBS PUTs (annotation_put_fbs) are accessible via
+        # GET (annotation_to_fbs_matrix)
+
+        n_rows = self.data.get_shape()[0]
+        fbs = make_fbs(
+            {
+                "cat_A": pd.Series(["label_A"] * n_rows, dtype="category"),
+                "cat_B": pd.Series(["label_B"] * n_rows, dtype="category"),
+            }
+        )
+
+        # put
+        res = self.annotation_put_fbs(fbs)
+        self.assertEqual(res, json.dumps({"status": "OK"}))
+
+        # get
+        labels = self.annotations.read_labels(None)
+        fbsAll = self.data.annotation_to_fbs_matrix("obs", None, labels)
+        schema = schema_get_helper(self.data)
+        annotations = decode_fbs.decode_matrix_FBS(fbsAll)
+        obs_index_col_name = schema["annotations"]["obs"]["index"]
+        self.assertEqual(annotations["n_rows"], n_rows)
+        self.assertEqual(annotations["n_cols"], 7)
+        self.assertIsNone(annotations["row_idx"])
+        self.assertEqual(
+            annotations["col_idx"],
+            [obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain", "cat_A", "cat_B"],
+        )
+        col_idx = annotations["col_idx"]
+        self.assertEqual(annotations["columns"][col_idx.index("cat_A")], ["label_A"] * n_rows)
+        self.assertEqual(annotations["columns"][col_idx.index("cat_B")], ["label_B"] * n_rows)
+
+        # verify the schema was updated
+        all_col_schema = {c["name"]: c for c in schema["annotations"]["obs"]["columns"]}
+        self.assertEqual(
+            all_col_schema["cat_A"],
+            {"name": "cat_A", "type": "categorical", "categories": ["label_A"], "writable": True},
+        )
+        self.assertEqual(
+            all_col_schema["cat_B"],
+            {"name": "cat_B", "type": "categorical", "categories": ["label_B"], "writable": True},
+        )
+
+    def test_put_float_data(self):
+        # verify that OBS PUTs (annotation_put_fbs) are accessible via
+        # GET (annotation_to_fbs_matrix)
+
+        n_rows = self.data.get_shape()[0]
+
+        # verifies that floating point with decimals fail.
+        fbs = make_fbs({"cat_F_FAIL": pd.Series([1.1] * n_rows, dtype=np.dtype("float"))})
+        with self.assertRaises(ValueError) as exception_context:
+            res = self.annotation_put_fbs(fbs)
+        self.assertEqual(str(exception_context.exception), "Columns may not have floating point types")
+
+        # verifies that floating point that can be converted to int passes
+        fbs = make_fbs({"cat_F_PASS": pd.Series([1.0] * n_rows, dtype="float")})
+        res = self.annotation_put_fbs(fbs)
+        self.assertEqual(res, json.dumps({"status": "OK"}))
+
+        # check read_labels
+        labels = self.annotations.read_labels(None)
+        fbsAll = self.data.annotation_to_fbs_matrix("obs", None, labels)
+        schema = schema_get_helper(self.data)
+        annotations = decode_fbs.decode_matrix_FBS(fbsAll)
+        self.assertEqual(annotations["n_rows"], n_rows)
+        all_col_schema = {c["name"]: c for c in schema["annotations"]["obs"]["columns"]}
+        self.assertEqual(
+            all_col_schema["cat_F_PASS"],
+            {"name": "cat_F_PASS", "type": "int32", "writable": True},
+        )

test/unit/common/utils/test_utils.py

@@ -0,0 +1,34 @@
+import os
+import shutil
+import unittest
+
+from server.common.utils.utils import import_plugins
+from test import PROJECT_ROOT, random_string
+
+
+class TestPlugins(unittest.TestCase):
+    """ Test plugin import functionality """
+
+    plugins_dir = f"{PROJECT_ROOT}/test/plugins"
+    test_plugin_path = f"{plugins_dir}/foo.py"
+    secret = random_string(8)
+
+    @classmethod
+    def setUpClass(cls) -> None:
+        if not os.path.isdir(cls.plugins_dir):
+            os.mkdir(cls.plugins_dir)
+        with open(cls.test_plugin_path, "w") as fh:
+            fh.write(f'SECRET = "{cls.secret}"\n')
+
+    @classmethod
+    def tearDownClass(cls) -> None:
+        if os.path.isdir(cls.plugins_dir):
+            shutil.rmtree(cls.plugins_dir)
+
+    def test_import_plugins(self):
+        self.assertTrue(os.path.isfile(self.test_plugin_path))
+        loaded_modules = import_plugins("test.plugins")
+        # test that import plugins found the file
+        self.assertEqual(["test.plugins.foo"], [ele.__name__ for ele in loaded_modules])
+        # test that the module was properly executed
+        self.assertEqual(self.secret, loaded_modules[0].SECRET)

test/unit/compute/test_diffexp_h5ad.py

@@ -0,0 +1,105 @@
+import unittest
+
+import numpy as np
+
+from server.common.compute import diffexp_generic
+from server.data_common.matrix_loader import MatrixDataLoader
+from test.unit import app_config
+from test import PROJECT_ROOT
+
+
+class DiffExpTest(unittest.TestCase):
+    """Tests the diffexp returns the expected results for one test case, using the h5ad
+    adaptor types and different algorithms."""
+
+    def load_dataset(self, path, extra_server_config={}, extra_dataset_config={}):
+        config = app_config(path, extra_server_config=extra_server_config, extra_dataset_config=extra_dataset_config)
+        loader = MatrixDataLoader(path)
+        adaptor = loader.open(config)
+        return adaptor
+
+    def get_mask(self, adaptor, start, stride):
+        """Simple function to return a mask or rows"""
+        rows = adaptor.get_shape()[0]
+        sel = list(range(start, rows, stride))
+        mask = np.zeros(rows, dtype=bool)
+        mask[sel] = True
+        return mask
+
+    def compare_diffexp_results(self, results, expects):
+        self.assertEqual(len(results), len(expects))
+        for result, expect in zip(results, expects):
+            self.assertEqual(result[0], expect[0])
+            self.assertTrue(np.isclose(result[1], expect[1], 1e-6, 1e-4))
+            self.assertTrue(np.isclose(result[2], expect[2], 1e-6, 1e-4))
+            self.assertTrue(np.isclose(result[3], expect[3], 1e-6, 1e-4))
+
+    def check_1_10_2_10(self, results):
+        """Checks the results for a specific set of rows selections"""
+
+        positive_expects = [
+            [1712, 0.24104056, 0.0051788902660723345, 1.0],
+            [1575, 0.2615018, 0.007830310753043345, 1.0],
+            [693, 0.23106655, 0.008715846769131548, 1.0],
+            [916, 0.2395215, 0.009080596532247588, 1.0],
+            [77, 0.22927025, 0.010070392939027756, 1.0],
+            [782, 0.20581803, 0.010161745218916036, 1.0],
+            [913, 0.23841085, 0.010782030711612685, 1.0],
+            [910, 0.21493295, 0.014596411069229197, 1.0],
+            [1727, 0.21911663, 0.015168372104237176, 1.0],
+            [1443, 0.19814226, 0.015337080567465522, 1.0],
+        ]
+        negative_expects = [
+            [956, -0.29662406, 0.0008649321884808977, 1.0],
+            [1124, -0.2607333, 0.0011717216548271284, 1.0],
+            [1809, -0.24854594, 0.0019304405196777848, 1.0],
+            [1754, -0.24683577, 0.005691734062127954, 1.0],
+            [948, -0.18708363, 0.006622111055981219, 1.0],
+            [1810, -0.2172082, 0.007055917428377063, 1.0],
+            [779, -0.21150622, 0.007202934422407284, 1.0],
+            [576, -0.19008157, 0.008272092578813124, 1.0],
+            [538, -0.21803819, 0.01062259019889307, 1.0],
+            [436, -0.2100364, 0.01127515110543434, 1.0],
+        ]
+
+        self.compare_diffexp_results(results["positive"], positive_expects)
+        self.compare_diffexp_results(results["negative"], negative_expects)
+
+    def get_X_col(self, adaptor, cols):
+        varmask = np.zeros(adaptor.get_shape()[1], dtype=bool)
+        varmask[cols] = True
+        return adaptor.get_X_array(None, varmask)
+
+    def test_anndata_default(self):
+        """Test an anndata adaptor with its default diffexp algorithm (diffexp_generic)"""
+        adaptor = self.load_dataset(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad")
+        maskA = self.get_mask(adaptor, 1, 10)
+        maskB = self.get_mask(adaptor, 2, 10)
+        results = adaptor.compute_diffexp_ttest(maskA, maskB, 10)
+        self.check_1_10_2_10(results)
+
+
+def test_h5ad_default(self):
+    """Test a h5ad adaptor with its default diffexp algorithm (diffexp_cxg)"""
+    adaptor = self.load_dataset(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad")
+    maskA = self.get_mask(adaptor, 1, 10)
+    maskB = self.get_mask(adaptor, 2, 10)
+
+    # run it through the adaptor
+    results = adaptor.compute_diffexp_ttest(maskA, maskB, 10)
+    self.check_1_10_2_10(results)
+
+    # run it directly
+
+    results = diffexp_generic.diffexp_ttest(adaptor, maskA, maskB, 10)
+    self.check_1_10_2_10(results)
+
+
+def test_h5ad_generic(self):
+    """Test a h5ad adaptor with the generic adaptor"""
+    adaptor = self.load_dataset(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad")
+    maskA = self.get_mask(adaptor, 1, 10)
+    maskB = self.get_mask(adaptor, 2, 10)
+    # run it directly
+    results = diffexp_generic.diffexp_ttest(adaptor, maskA, maskB, 10)
+    self.check_1_10_2_10(results)

test/unit/compute/test_est_dist.py

@@ -0,0 +1,120 @@
+import unittest
+import numpy as np
+from scipy import sparse
+from server.common.compute.estimate_distribution import estimate_approximate_distribution
+from server.common.constants import XApproximateDistribution
+from server.data_common.matrix_loader import MatrixDataLoader
+from test.unit import app_config
+from test import PROJECT_ROOT
+
+
+class EstDistTest(unittest.TestCase):
+    """Tests the diffexp returns the expected results for one test case, using the h5ad
+    adaptor types and different algorithms."""
+
+    def load_dataset(self, path, extra_server_config={}, extra_dataset_config={}):
+        config = app_config(path, extra_server_config=extra_server_config, extra_dataset_config=extra_dataset_config)
+        loader = MatrixDataLoader(path)
+        adaptor = loader.open(config)
+        return adaptor
+
+    def test_adaptestimate_approximate_distribution(self):
+        adaptor = self.load_dataset(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad")
+        self.assertEqual(adaptor.get_X_approximate_distribution(), XApproximateDistribution.NORMAL)
+
+    def test_estimate_approximate_distribution(self):
+        raw = np.random.exponential(scale=1000, size=(100, 40))
+
+        # empty
+        self.assertEqual(estimate_approximate_distribution(np.zeros((0,))), XApproximateDistribution.NORMAL)
+
+        # ndarray
+        self.assertEqual(estimate_approximate_distribution(raw), XApproximateDistribution.COUNT)
+        self.assertEqual(estimate_approximate_distribution(np.log1p(raw)), XApproximateDistribution.NORMAL)
+
+        # csr_matrix
+        self.assertEqual(estimate_approximate_distribution(sparse.csr_matrix(raw)), XApproximateDistribution.COUNT)
+        self.assertEqual(
+            estimate_approximate_distribution(sparse.csr_matrix(np.log1p(raw))), XApproximateDistribution.NORMAL
+        )
+
+        # csc_matrix
+        self.assertEqual(estimate_approximate_distribution(sparse.csc_matrix(raw)), XApproximateDistribution.COUNT)
+        self.assertEqual(
+            estimate_approximate_distribution(sparse.csc_matrix(np.log1p(raw))), XApproximateDistribution.NORMAL
+        )
+
+        # BIG (ie, trigger MT)
+        big = np.random.exponential(scale=100, size=(1_000_000, 100))
+        self.assertEqual(estimate_approximate_distribution(big), XApproximateDistribution.COUNT)
+        self.assertEqual(estimate_approximate_distribution(np.log1p(big)), XApproximateDistribution.NORMAL)
+
+    def test_unsupported_throws(self):
+        # dtypes and matrix formats we do not support
+        with self.assertRaises(TypeError):
+            estimate_approximate_distribution(np.array(["a", "b"]))
+        with self.assertRaises(TypeError):
+            estimate_approximate_distribution(sparse.coo_matrix(np.array([[0, 1, 2], [3, 0, 2]])))
+
+    def test_nonfinites(self):
+        def put(arr, ind, vals):
+            # like np.put, but creates and returns a modified copy of original array
+            a = arr.copy()
+            np.put(a, ind, vals)
+            return a
+
+        # non-finites
+        self.assertEqual(estimate_approximate_distribution(np.array([np.nan])), XApproximateDistribution.NORMAL)
+        self.assertEqual(estimate_approximate_distribution(np.array([np.PINF])), XApproximateDistribution.NORMAL)
+        self.assertEqual(estimate_approximate_distribution(np.array([np.NINF])), XApproximateDistribution.NORMAL)
+        self.assertEqual(
+            estimate_approximate_distribution(np.array([np.PINF, np.NINF, 0])), XApproximateDistribution.NORMAL
+        )
+        self.assertEqual(
+            estimate_approximate_distribution(np.array([np.nan, np.PINF, np.NINF])), XApproximateDistribution.NORMAL
+        )
+
+        raw = np.random.exponential(scale=1000, size=(50, 3))
+        logged = np.log1p(raw)
+
+        self.assertEqual(
+            estimate_approximate_distribution(put(raw, [1], [np.nan])),
+            XApproximateDistribution.COUNT,
+        )
+        self.assertEqual(
+            estimate_approximate_distribution(put(raw, [1], [np.PINF])),
+            XApproximateDistribution.COUNT,
+        )
+        self.assertEqual(
+            estimate_approximate_distribution(put(raw, [1], [np.NINF])),
+            XApproximateDistribution.COUNT,
+        )
+        self.assertEqual(
+            estimate_approximate_distribution(put(raw, [1, 3, 88], [np.nan, np.PINF, np.NINF])),
+            XApproximateDistribution.COUNT,
+        )
+        self.assertEqual(
+            estimate_approximate_distribution(put(raw, [0, 1], [np.nan, np.nan])),
+            XApproximateDistribution.COUNT,
+        )
+
+        self.assertEqual(
+            estimate_approximate_distribution(put(logged, [1], [np.nan])),
+            XApproximateDistribution.NORMAL,
+        )
+        self.assertEqual(
+            estimate_approximate_distribution(put(logged, [1], [np.PINF])),
+            XApproximateDistribution.NORMAL,
+        )
+        self.assertEqual(
+            estimate_approximate_distribution(put(logged, [1], [np.NINF])),
+            XApproximateDistribution.NORMAL,
+        )
+        self.assertEqual(
+            estimate_approximate_distribution(put(logged, [1, 3, 88], [np.nan, np.PINF, np.NINF])),
+            XApproximateDistribution.NORMAL,
+        )
+        self.assertEqual(
+            estimate_approximate_distribution(put(logged, [0, 1], [np.nan, np.nan])),
+            XApproximateDistribution.NORMAL,
+        )

test/unit/data_anndata/test_anndata_adaptor.py

@@ -0,0 +1,213 @@
+import json
+import sys
+import time
+import unittest
+
+import numpy as np
+import pandas as pd
+import pytest
+from parameterized import parameterized_class
+
+import test.decode_fbs as decode_fbs
+from server.common.utils.data_locator import DataLocator
+from server.common.errors import FilterError
+from server.data_anndata.anndata_adaptor import AnndataAdaptor
+from test import PROJECT_ROOT, FIXTURES_ROOT
+from test.unit import app_config
+from test.fixtures.fixtures import pbmc3k_colors
+
+"""
+Test the anndata adaptor using the pbmc3k data set.
+"""
+
+
+@parameterized_class(
+    ("data_locator", "backed", "X_approximate_distribution"),
+    [
+        (f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad", False, "auto"),
+        (f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad", False, "auto"),
+        (f"{FIXTURES_ROOT}/pbmc3k-CSR-gz.h5ad", False, "auto"),
+        (f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad", True, "auto"),
+        (f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad", True, "auto"),
+        (f"{FIXTURES_ROOT}/pbmc3k-CSR-gz.h5ad", True, "auto"),
+        (f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad", False, "normal"),
+        (f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad", False, "normal"),
+        (f"{FIXTURES_ROOT}/pbmc3k-CSR-gz.h5ad", False, "normal"),
+        (f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad", True, "normal"),
+        (f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad", True, "normal"),
+        (f"{FIXTURES_ROOT}/pbmc3k-CSR-gz.h5ad", True, "normal"),
+        (f"{FIXTURES_ROOT}/pbmc3k_64.h5ad", False, "auto"),  # 64 bit conversion tests
+    ],
+)
+class AdaptorTest(unittest.TestCase):
+    def setUp(self):
+        config = app_config(
+            self.data_locator,
+            self.backed,
+            extra_dataset_config=dict(X_approximate_distribution=self.X_approximate_distribution),
+        )
+        self.data = AnndataAdaptor(DataLocator(self.data_locator), config)
+
+    def test_init(self):
+        self.assertEqual(self.data.cell_count, 2638)
+        self.assertEqual(self.data.gene_count, 1838)
+        epsilon = 0.000_005
+        self.assertTrue(self.data.data.X[0, 0] - -0.171_469_51 < epsilon)
+
+    def test_mandatory_annotations(self):
+        obs_index_col_name = self.data.get_schema()["annotations"]["obs"]["index"]
+        self.assertIn(obs_index_col_name, self.data.data.obs)
+        self.assertEqual(list(self.data.data.obs.index), list(range(2638)))
+        var_index_col_name = self.data.get_schema()["annotations"]["var"]["index"]
+        self.assertIn(var_index_col_name, self.data.data.var)
+        self.assertEqual(list(self.data.data.var.index), list(range(1838)))
+
+    @pytest.mark.filterwarnings("ignore:Anndata data matrix")
+    def test_data_type(self):
+        # don't run the test on the more exotic data types, as they don't
+        # support the astype() interface (used by this test, but not underlying app)
+        if isinstance(self.data.data.X, np.ndarray):
+            self.data.data.X = self.data.data.X.astype("float64")
+            with self.assertWarns(UserWarning):
+                self.data._validate_data_types()
+
+    def test_filter_idx(self):
+        filter_ = {"filter": {"var": {"index": [1, 99, [200, 300]]}}}
+        fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
+        data = decode_fbs.decode_matrix_FBS(fbs)
+        self.assertEqual(data["n_rows"], 2638)
+        self.assertEqual(data["n_cols"], 102)
+
+    def test_filter_complex(self):
+        filter_ = {
+            "filter": {"var": {"annotation_value": [{"name": "n_cells", "min": 10}], "index": [1, 99, [200, 300]]}}
+        }
+        fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
+        data = decode_fbs.decode_matrix_FBS(fbs)
+        self.assertEqual(data["n_rows"], 2638)
+        self.assertEqual(data["n_cols"], 91)
+
+    def test_obs_and_var_names(self):
+        self.assertEqual(np.sum(self.data.data.var[self.data.get_schema()["annotations"]["var"]["index"]].isna()), 0)
+        self.assertEqual(np.sum(self.data.data.obs[self.data.get_schema()["annotations"]["obs"]["index"]].isna()), 0)
+
+    def test_get_colors(self):
+        self.assertEqual(self.data.get_colors(), pbmc3k_colors)
+
+    def test_get_schema(self):
+        with open(f"{FIXTURES_ROOT}/schema.json") as fh:
+            schema = json.load(fh)
+            self.assertDictEqual(self.data.get_schema(), schema)
+
+    def test_schema_produces_error(self):
+        self.data.data.obs["time"] = pd.Series(
+            list([time.time() for i in range(self.data.cell_count)]),
+            dtype="datetime64[ns]",
+        )
+        with pytest.raises(TypeError):
+            self.data._create_schema()
+
+    def test_layout(self):
+        fbs = self.data.layout_to_fbs_matrix(fields=None)
+        layout = decode_fbs.decode_matrix_FBS(fbs)
+        self.assertEqual(layout["n_cols"], 6)
+        self.assertEqual(layout["n_rows"], 2638)
+
+        X = layout["columns"][0]
+        self.assertTrue((X >= 0).all() and (X <= 1).all())
+        Y = layout["columns"][1]
+        self.assertTrue((Y >= 0).all() and (Y <= 1).all())
+
+    def test_layout_fields(self):
+        """X_pca, X_tsne, X_umap are available"""
+        fbs = self.data.layout_to_fbs_matrix(["pca"])
+        layout = decode_fbs.decode_matrix_FBS(fbs)
+        self.assertEqual(layout["n_cols"], 2)
+        self.assertEqual(layout["n_rows"], 2638)
+        self.assertCountEqual(layout["col_idx"], ["pca_0", "pca_1"])
+
+        fbs = self.data.layout_to_fbs_matrix(["tsne", "pca"])
+        layout = decode_fbs.decode_matrix_FBS(fbs)
+        self.assertEqual(layout["n_cols"], 4)
+        self.assertEqual(layout["n_rows"], 2638)
+        self.assertCountEqual(layout["col_idx"], ["tsne_0", "tsne_1", "pca_0", "pca_1"])
+
+    def test_annotations(self):
+        fbs = self.data.annotation_to_fbs_matrix("obs")
+        annotations = decode_fbs.decode_matrix_FBS(fbs)
+        self.assertEqual(annotations["n_rows"], 2638)
+        self.assertEqual(annotations["n_cols"], 5)
+        obs_index_col_name = self.data.get_schema()["annotations"]["obs"]["index"]
+        self.assertEqual(
+            annotations["col_idx"],
+            [obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain"],
+        )
+
+        fbs = self.data.annotation_to_fbs_matrix("var")
+        annotations = decode_fbs.decode_matrix_FBS(fbs)
+        self.assertEqual(annotations["n_rows"], 1838)
+        self.assertEqual(annotations["n_cols"], 2)
+        var_index_col_name = self.data.get_schema()["annotations"]["var"]["index"]
+        self.assertEqual(annotations["col_idx"], [var_index_col_name, "n_cells"])
+
+    def test_annotation_fields(self):
+        fbs = self.data.annotation_to_fbs_matrix("obs", ["n_genes", "n_counts"])
+        annotations = decode_fbs.decode_matrix_FBS(fbs)
+        self.assertEqual(annotations["n_rows"], 2638)
+        self.assertEqual(annotations["n_cols"], 2)
+
+        var_index_col_name = self.data.get_schema()["annotations"]["var"]["index"]
+        fbs = self.data.annotation_to_fbs_matrix("var", [var_index_col_name])
+        annotations = decode_fbs.decode_matrix_FBS(fbs)
+        self.assertEqual(annotations["n_rows"], 1838)
+        self.assertEqual(annotations["n_cols"], 1)
+
+    def test_diffexp_topN(self):
+        f1 = {"filter": {"obs": {"index": [[0, 500]]}}}
+        f2 = {"filter": {"obs": {"index": [[500, 1000]]}}}
+        result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"]))
+        self.assertEqual(len(result["positive"]), 10)
+        self.assertEqual(len(result["negative"]), 10)
+
+        result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"], 20))
+        self.assertEqual(len(result["positive"]), 20)
+        self.assertEqual(len(result["negative"]), 20)
+
+    def test_data_frame(self):
+        f1 = {"var": {"index": [[0, 10]]}}
+        fbs = self.data.data_frame_to_fbs_matrix(f1, "var")
+        data = decode_fbs.decode_matrix_FBS(fbs)
+        self.assertEqual(data["n_rows"], 2638)
+        self.assertEqual(data["n_cols"], 10)
+
+        with self.assertRaises(ValueError):
+            self.data.data_frame_to_fbs_matrix(None, "obs")
+
+    def test_filtered_data_frame(self):
+        filter_ = {"filter": {"var": {"annotation_value": [{"name": "n_cells", "min": 100}]}}}
+        fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
+        data = decode_fbs.decode_matrix_FBS(fbs)
+        self.assertEqual(data["n_rows"], 2638)
+        self.assertEqual(data["n_cols"], 1040)
+
+        filter_ = {"filter": {"obs": {"annotation_value": [{"name": "n_counts", "min": 3000}]}}}
+        with self.assertRaises(FilterError):
+            self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
+
+    def test_data_named_gene(self):
+        var_index_col_name = self.data.get_schema()["annotations"]["var"]["index"]
+        filter_ = {"filter": {"var": {"annotation_value": [{"name": var_index_col_name, "values": ["RER1"]}]}}}
+        fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
+        data = decode_fbs.decode_matrix_FBS(fbs)
+        self.assertEqual(data["n_rows"], 2638)
+        self.assertEqual(data["n_cols"], 1)
+        self.assertEqual(data["col_idx"], [4])
+
+        filter_ = {
+            "filter": {"var": {"annotation_value": [{"name": var_index_col_name, "values": ["SPEN", "TYMP", "PRMT2"]}]}}
+        }
+        fbs = self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
+        data = decode_fbs.decode_matrix_FBS(fbs)
+        self.assertEqual(data["n_rows"], 2638)
+        self.assertEqual(data["n_cols"], 3)
+        self.assertTrue((data["col_idx"] == [15, 1818, 1837]).all())

test/unit/data_anndata/test_anndata_adaptor_data_load.py

@@ -0,0 +1,86 @@
+import unittest
+import json
+
+from server.common.utils.data_locator import DataLocator
+from server.data_anndata.anndata_adaptor import AnndataAdaptor
+from server.common.config.app_config import AppConfig
+from test import PROJECT_ROOT
+
+
+class DataLoadAdaptorTest(unittest.TestCase):
+    """
+    Test file loading, including deferred loading/update.
+    """
+
+    def setUp(self):
+        self.data_file = DataLocator(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad")
+        config = AppConfig()
+        config.update_server_config(single_dataset__datapath=self.data_file.path)
+        config.update_server_config(app__flask_secret_key="secret")
+        config.complete_config()
+        self.data = AnndataAdaptor(self.data_file, config)
+
+    def test_delayed_load_data(self):
+        self.data._create_schema()
+        self.assertEqual(self.data.cell_count, 2638)
+        self.assertEqual(self.data.gene_count, 1838)
+        epsilon = 0.000_005
+        self.assertTrue(self.data.data.X[0, 0] - -0.171_469_51 < epsilon)
+
+    def test_diffexp_topN(self):
+        f1 = {"filter": {"obs": {"index": [[0, 500]]}}}
+        f2 = {"filter": {"obs": {"index": [[500, 1000]]}}}
+        result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"]))
+        self.assertEqual(len(result["positive"]), 10)
+        self.assertEqual(len(result["negative"]), 10)
+        result = json.loads(self.data.diffexp_topN(f1["filter"], f2["filter"], 20))
+        self.assertEqual(len(result["positive"]), 20)
+        self.assertEqual(len(result["negative"]), 20)
+
+
+class DataLocatorAdaptorTest(unittest.TestCase):
+    """
+    Test various types of data locators we expect to consume
+    """
+
+    def get_basic_config(self):
+        config = AppConfig()
+        config.update_server_config(
+            single_dataset__obs_names=None,
+            single_dataset__var_names=None,
+        )
+        config.update_server_config(app__flask_secret_key="secret")
+        config.update_dataset_config(
+            embeddings__names=["umap"],
+            presentation__max_categories=100,
+            diffexp__lfc_cutoff=0.01,
+        )
+        return config
+
+    def stdAsserts(self, data):
+        """ run these each time we load the data """
+        self.assertIsNotNone(data)
+        self.assertEqual(data.cell_count, 2638)
+        self.assertEqual(data.gene_count, 1838)
+
+    def test_posix_file(self):
+        locator = DataLocator("example-dataset/pbmc3k.h5ad")
+        config = self.get_basic_config()
+        config.update_server_config(single_dataset__datapath=locator.path)
+        config.complete_config()
+        data = AnndataAdaptor(locator, config)
+        self.stdAsserts(data)
+
+    def test_url_https(self):
+        url = "https://raw.githubusercontent.com/chanzuckerberg/cellxgene/main/example-dataset/pbmc3k.h5ad"
+        locator = DataLocator(url)
+        config = self.get_basic_config()
+        data = AnndataAdaptor(locator, config)
+        self.stdAsserts(data)
+
+    def test_url_http(self):
+        url = "http://raw.githubusercontent.com/chanzuckerberg/cellxgene/main/example-dataset/pbmc3k.h5ad"
+        locator = DataLocator(url)
+        config = self.get_basic_config()
+        data = AnndataAdaptor(locator, config)
+        self.stdAsserts(data)

test/unit/data_anndata/test_nan_anndata_adaptor.py

@@ -0,0 +1,65 @@
+import math
+import unittest
+import warnings
+
+import pytest
+
+import test.decode_fbs as decode_fbs
+from server.common.utils.data_locator import DataLocator
+from server.common.errors import FilterError
+from server.data_anndata.anndata_adaptor import AnndataAdaptor
+from test import FIXTURES_ROOT
+from test.unit import app_config
+
+
+class NaNTest(unittest.TestCase):
+    def setUp(self):
+        self.data_locator = DataLocator(f"{FIXTURES_ROOT}/nan.h5ad")
+        self.config = app_config(self.data_locator.path)
+
+        with warnings.catch_warnings():
+            warnings.simplefilter("ignore", category=UserWarning)
+            self.data = AnndataAdaptor(self.data_locator, self.config)
+            self.data._create_schema()
+
+    def test_load(self):
+        with self.assertLogs(level="WARN") as logger:
+            self.data = AnndataAdaptor(self.data_locator, self.config)
+            self.assertTrue(logger.output)
+
+    def test_init(self):
+        self.assertEqual(self.data.cell_count, 100)
+        self.assertEqual(self.data.gene_count, 100)
+        epsilon = 0.000_005
+        self.assertTrue(self.data.data.X[0, 0] - -0.171_469_51 < epsilon)
+
+    def test_dataframe(self):
+        data_frame_var = decode_fbs.decode_matrix_FBS(self.data.data_frame_to_fbs_matrix(None, "var"))
+        self.assertIsNotNone(data_frame_var)
+        self.assertEqual(data_frame_var["n_rows"], 100)
+        self.assertEqual(data_frame_var["n_cols"], 100)
+        self.assertTrue(math.isnan(data_frame_var["columns"][3][3]))
+
+        with pytest.raises(FilterError):
+            self.data.data_frame_to_fbs_matrix("an erroneous filter", "var")
+        with pytest.raises(FilterError):
+            filter_ = {"filter": {"obs": {"index": [1, 99, [200, 300]]}}}
+            self.data.data_frame_to_fbs_matrix(filter_["filter"], "var")
+
+    def test_dataframe_obs_not_implemented(self):
+        with self.assertRaises(ValueError) as cm:
+            decode_fbs.decode_matrix_FBS(self.data.data_frame_to_fbs_matrix(None, "obs"))
+        self.assertIsNotNone(cm.exception)
+
+    def test_annotation(self):
+        annotations = decode_fbs.decode_matrix_FBS(self.data.annotation_to_fbs_matrix("obs"))
+        obs_index_col_name = self.data.schema["annotations"]["obs"]["index"]
+        self.assertEqual(annotations["col_idx"], [obs_index_col_name, "n_genes", "percent_mito", "n_counts", "louvain"])
+        self.assertEqual(annotations["n_rows"], 100)
+        self.assertTrue(math.isnan(annotations["columns"][2][0]))
+
+        annotations = decode_fbs.decode_matrix_FBS(self.data.annotation_to_fbs_matrix("var"))
+        var_index_col_name = self.data.schema["annotations"]["var"]["index"]
+        self.assertEqual(annotations["col_idx"], [var_index_col_name, "n_cells", "var_with_nans"])
+        self.assertEqual(annotations["n_rows"], 100)
+        self.assertTrue(math.isnan(annotations["columns"][2][0]))

test/unit/fbs/test_matrix.py

@@ -0,0 +1,216 @@
+import unittest
+import pandas as pd
+import numpy as np
+from scipy import sparse
+from parameterized import parameterized_class
+import json
+
+from test import decode_fbs
+from server.common.fbs.matrix import encode_matrix_fbs, decode_matrix_fbs
+from server.common.utils.type_conversion_utils import get_dtypes_and_schemas_of_dataframe
+import server.common.fbs as fbs
+
+
+class FbsTests(unittest.TestCase):
+    """Test Case for Matrix FBS data encode/decode"""
+
+    def test_encode_boundary(self):
+        """test various boundary checks"""
+
+        # row indexing is unsupported
+        with self.assertRaises(ValueError):
+            encode_matrix_fbs(matrix=pd.DataFrame(), row_idx=[])
+
+        # matrix must be 2D
+        with self.assertRaises(ValueError):
+            encode_matrix_fbs(matrix=np.zeros((3, 2, 1)))
+        with self.assertRaises(ValueError):
+            encode_matrix_fbs(matrix=np.ones((10,)))
+
+    def fbs_checks(self, fbs, dims, expected_types, expected_column_idx):
+        d = decode_fbs.decode_matrix_FBS(fbs)
+        self.assertEqual(d["n_rows"], dims[0])
+        self.assertEqual(d["n_cols"], dims[1])
+        self.assertIsNone(d["row_idx"])
+        self.assertEqual(len(d["columns"]), dims[1])
+        for i in range(0, len(d["columns"])):
+            self.assertEqual(len(d["columns"][i]), dims[0])
+            self.assertIsInstance(d["columns"][i], expected_types[i][0])
+            if expected_types[i][1] is not None:
+                self.assertEqual(d["columns"][i].dtype, expected_types[i][1])
+        if expected_column_idx is not None:
+            self.assertSetEqual(set(expected_column_idx), set(d["col_idx"]))
+
+    def test_encode_DataFrame(self):
+        df = pd.DataFrame(
+            data={
+                "a": np.zeros((10,), dtype=np.float32),
+                "b": np.ones((10,), dtype=np.int64),
+                "c": np.array([i for i in range(0, 10)], dtype=np.uint16),
+                "d": pd.Series(["x", "y", "z", "x", "y", "z", "a", "x", "y", "z"], dtype="category"),
+            }
+        )
+        expected_types = ((np.ndarray, np.float32), (np.ndarray, np.int32), (np.ndarray, np.int32), (list, None))
+        fbs = encode_matrix_fbs(matrix=df, row_idx=None, col_idx=df.columns)
+        self.fbs_checks(fbs, (10, 4), expected_types, ["a", "b", "c", "d"])
+
+    def test_encode_ndarray(self):
+        arr = np.zeros((3, 2), dtype=np.float32)
+        expected_types = ((np.ndarray, np.float32), (np.ndarray, np.float32), (np.ndarray, np.float32))
+        fbs = encode_matrix_fbs(matrix=arr, row_idx=None, col_idx=None)
+        self.fbs_checks(fbs, (3, 2), expected_types, None)
+
+    def test_encode_sparse(self):
+        csc = sparse.csc_matrix(np.array([[0, 1, 2], [3, 0, 4]]))
+        expected_types = ((np.ndarray, np.int32), (np.ndarray, np.int32), (np.ndarray, np.int32))
+        fbs = encode_matrix_fbs(matrix=csc, row_idx=None, col_idx=None)
+        self.fbs_checks(fbs, (2, 3), expected_types, None)
+
+    def test_roundtrip(self):
+        dfSrc = pd.DataFrame(
+            data={
+                "a": np.zeros((10,), dtype=np.float32),
+                "b": np.ones((10,), dtype=np.int64),
+                "c": np.array([i for i in range(0, 10)], dtype=np.uint16),
+                "d": pd.Series(["x", "y", "z", "x", "y", "z", "a", "x", "y", "z"], dtype="category"),
+            }
+        )
+        dfDst = decode_matrix_fbs(encode_matrix_fbs(matrix=dfSrc, col_idx=dfSrc.columns))
+        self.assertEqual(dfSrc.shape, dfDst.shape)
+        self.assertEqual(set(dfSrc.columns), set(dfDst.columns))
+        for c in dfSrc.columns:
+            self.assertTrue(c in dfDst.columns)
+            if isinstance(dfSrc[c], pd.Series):
+                self.assertTrue(np.all(dfSrc[c] == dfDst[c]))
+            else:
+                self.assertEqual(dfSrc[c], dfDst[c])
+
+
+"""
+Test type consistency between FBS encoding and the underlying schema hint.
+
+Basic assertion:  the FBS type returned by encode_matrix_fbs() will be consistent
+with the schema hint returned by type_conversion_utils (which is in turn used
+to create the client schema).
+
+The following test cases are all dicts which contain the following keys:
+    - dataframe - the dataframe used as input for encode_matrix_fbs
+    - expected_fbs_types - upon success, dict of FBS column types expected (eg, Float32Array)
+    - expected_schema_hints - upon success, dict of schema hint
+All are keyed by column name.
+"""
+
+# simple tests that we convert all ints to int32
+int_dtypes = [np.dtype(d) for d in [np.int8, np.uint8, np.int16, np.uint16, np.int32, np.uint32, np.int64, np.uint64]]
+int_test_cases = [
+    {
+        "dataframe": pd.DataFrame({dtype.name: np.zeros((10,), dtype=dtype) for dtype in int_dtypes}),
+        "expected_fbs_types": dict(
+            [(dtype.name, fbs.NetEncoding.TypedArray.TypedArray.Int32Array) for dtype in int_dtypes]
+        ),
+        "expected_schema_hints": dict([(dtype.name, {"type": "int32"}) for dtype in int_dtypes]),
+    }
+]
+
+# simple tests that we convert all floats to float32
+float_dtypes = [np.dtype(d) for d in [np.float16, np.float32, np.float64]]
+float_test_cases = [
+    {
+        "dataframe": pd.DataFrame({dtype.name: np.zeros((10,), dtype=dtype) for dtype in float_dtypes}),
+        "expected_fbs_types": dict(
+            [(dtype.name, fbs.NetEncoding.TypedArray.TypedArray.Float32Array) for dtype in float_dtypes]
+        ),
+        "expected_schema_hints": dict([(dtype.name, {"type": "float32"}) for dtype in float_dtypes]),
+    }
+]
+
+# boolean - should be encoded as an uint32
+bool_dtypes = [np.dtype(d) for d in [np.bool_, bool]]
+bool_test_cases = [
+    {
+        "dataframe": pd.DataFrame({dtype.name: np.ones((10,), dtype=dtype) for dtype in bool_dtypes}),
+        "expected_fbs_types": dict(
+            [(dtype.name, fbs.NetEncoding.TypedArray.TypedArray.Uint32Array) for dtype in bool_dtypes]
+        ),
+        "expected_schema_hints": dict([(dtype.name, {"type": "boolean"}) for dtype in bool_dtypes]),
+    }
+]
+
+cat_test_cases = [
+    {
+        "dataframe": pd.DataFrame({"a": pd.Series(["a", "b", "c", "a", "b", "c"], dtype="category")}),
+        "expected_fbs_types": {"a": fbs.NetEncoding.TypedArray.TypedArray.JSONEncodedArray},
+        "expected_schema_hints": {"a": {"type": "categorical", "categories": ["a", "b", "c"]}},
+    },
+    {
+        "dataframe": pd.DataFrame(
+            {"a": pd.Series(["a", "b", "c", "a", "b", "c"], dtype="category").cat.remove_categories("b")}
+        ),
+        "expected_fbs_types": {"a": fbs.NetEncoding.TypedArray.TypedArray.JSONEncodedArray},
+        "expected_schema_hints": {"a": {"type": "categorical", "categories": ["a", "c"]}},
+    },
+    {
+        "dataframe": pd.DataFrame({"a": pd.Series(np.arange(0, 10, dtype=np.int64), dtype="category")}),
+        "expected_fbs_types": {"a": fbs.NetEncoding.TypedArray.TypedArray.Int32Array},
+        "expected_schema_hints": {"a": {"type": "categorical"}},
+    },
+    {
+        "dataframe": pd.DataFrame(
+            {"a": pd.Series(np.arange(0, 10, dtype=np.int64), dtype="category").cat.remove_categories(2)}
+        ),
+        "expected_fbs_types": {"a": fbs.NetEncoding.TypedArray.TypedArray.Float32Array},
+        "expected_schema_hints": {"a": {"type": "categorical"}},
+    },
+    {
+        "dataframe": pd.DataFrame({"a": pd.Series(np.arange(0, 10, dtype=np.float64), dtype="category")}),
+        "expected_fbs_types": {"a": fbs.NetEncoding.TypedArray.TypedArray.Float32Array},
+        "expected_schema_hints": {"a": {"type": "categorical"}},
+    },
+    {
+        "dataframe": pd.DataFrame(
+            {"a": pd.Series(np.arange(0, 10, dtype=np.float64), dtype="category").cat.remove_categories(2)}
+        ),
+        "expected_fbs_types": {"a": fbs.NetEncoding.TypedArray.TypedArray.Float32Array},
+        "expected_schema_hints": {"a": {"type": "categorical"}},
+    },
+]
+
+
+test_cases = [
+    *int_test_cases,
+    *float_test_cases,
+    *bool_test_cases,
+    *cat_test_cases,
+]
+
+
+@parameterized_class(test_cases)
+class TestTypeConversionConsistency(unittest.TestCase):
+    def test_type_conversion_consistency(self):
+        self.assertEqual(self.dataframe.shape[1], len(self.expected_fbs_types))
+        self.assertEqual(self.dataframe.shape[1], len(self.expected_schema_hints))
+
+        buf = encode_matrix_fbs(matrix=self.dataframe, col_idx=self.dataframe.columns)
+        encoding_dtypes, schema_hints = get_dtypes_and_schemas_of_dataframe(self.dataframe)
+
+        # check schema hints
+        # print(schema_hints)
+        # print(self.expected_schema_hints)
+        self.assertEqual(schema_hints, self.expected_schema_hints)
+
+        # inspect the FBS types
+        matrix = fbs.NetEncoding.Matrix.Matrix.GetRootAsMatrix(buf, 0)
+        columns_length = matrix.ColumnsLength()
+        self.assertEqual(columns_length, self.dataframe.shape[1])
+
+        self.assertEqual(matrix.ColIndexType(), fbs.NetEncoding.TypedArray.TypedArray.JSONEncodedArray)
+        col_labels_arr = fbs.NetEncoding.JSONEncodedArray.JSONEncodedArray()
+        col_labels_arr.Init(matrix.ColIndex().Bytes, matrix.ColIndex().Pos)
+        col_index_labels = json.loads(col_labels_arr.DataAsNumpy().tobytes().decode("utf-8"))
+        self.assertEqual(len(col_index_labels), self.dataframe.shape[1])
+
+        for col_idx in range(0, columns_length):
+            col_label = col_index_labels[col_idx]
+            col = matrix.Columns(col_idx)
+            col_type = col.UType()
+            self.assertEqual(self.expected_fbs_types[col_label], col_type)

test/unit/test_colors.py

@@ -0,0 +1,41 @@
+import unittest
+
+import anndata
+
+from server.common.colors import convert_color_to_hex_format, convert_anndata_category_colors_to_cxg_category_colors
+from server.common.errors import ColorFormatException
+from test import PROJECT_ROOT
+from test.fixtures.fixtures import pbmc3k_colors
+
+
+class ColorsTest(unittest.TestCase):
+    """ Test color helper functions """
+
+    def test_convert_color_to_hex_format(self):
+        self.assertEqual(convert_color_to_hex_format("wheat"), "#f5deb3")
+        self.assertEqual(convert_color_to_hex_format("WHEAT"), "#f5deb3")
+        self.assertEqual(convert_color_to_hex_format((245, 222, 179)), "#f5deb3")
+        self.assertEqual(convert_color_to_hex_format([245, 222, 179]), "#f5deb3")
+        self.assertEqual(convert_color_to_hex_format("#f5deb3"), "#f5deb3")
+        self.assertEqual(
+            convert_color_to_hex_format([0.9607843137254902, 0.8705882352941177, 0.7019607843137254]), "#f5deb3"
+        )
+        for bad_input in ["foo", "BAR", "#AABB", "#AABBCCDD", "#AABBGG", (1, 2), [1, 2], (1, 2, 3, 4), [1, 2, 3, 4]]:
+            with self.assertRaises(ColorFormatException):
+                convert_color_to_hex_format(bad_input)
+
+    def test_anndata_colors_to_cxg_colors(self):
+        # test standard behavior
+        adata = self._get_h5ad()
+        self.assertEqual(convert_anndata_category_colors_to_cxg_category_colors(adata), pbmc3k_colors)
+        # test that invalid color formats raise an exception
+        adata.uns["louvain_colors"][0] = "#NOTCOOL"
+        with self.assertRaises(ColorFormatException):
+            convert_anndata_category_colors_to_cxg_category_colors(adata)
+        # test that colors without a matching obs category are skipped
+        adata = self._get_h5ad()
+        del adata.obs["louvain"]
+        self.assertEqual(convert_anndata_category_colors_to_cxg_category_colors(adata), {})
+
+    def _get_h5ad(self):
+        return anndata.read_h5ad(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad")

test/unit/utils/test_jsonify_strict.py

@@ -0,0 +1,58 @@
+import unittest
+import numpy as np
+
+from server.common.utils.utils import (
+    jsonify_strict,
+)
+
+
+class TestJsonifyStrict(unittest.TestCase):
+    def test_jsonify_numpy_general_cases(self):
+        self.assertEqual(jsonify_strict({}), "{}")
+        self.assertEqual(jsonify_strict({"a": [], "b": "hello", "c": True}), '{"a": [], "b": "hello", "c": true}')
+
+    def test_jsonify_numpy_float_edges(self):
+        with self.assertRaises(ValueError):
+            jsonify_strict({"nan": [np.nan]})
+
+        with self.assertRaises(ValueError):
+            jsonify_strict({"pinf": [np.PINF]})
+
+        with self.assertRaises(ValueError):
+            jsonify_strict({"ninf": [np.NINF]})
+
+    def test_jsonify_numpy_ndarray(self):
+        values = {
+            "integer": [
+                np.int8(0),
+                np.int16(1),
+                np.int32(2),
+                np.int64(3),
+                np.uint8(4),
+                np.uint16(5),
+                np.uint32(6),
+                np.uint64(7),
+            ],
+            "floating": [
+                np.float16(100.0),
+                np.float32(101.0),
+                np.float64(102.0),
+            ],
+        }
+        # these just confirm our test assumptions
+        self.assertTrue(isinstance(values["floating"][0], np.float16))
+        self.assertTrue(isinstance(values["floating"][1], np.float32))
+        self.assertTrue(isinstance(values["floating"][2], np.float64))
+        self.assertTrue(isinstance(values["integer"][0], np.int8))
+        self.assertTrue(isinstance(values["integer"][1], np.int16))
+        self.assertTrue(isinstance(values["integer"][2], np.int32))
+        self.assertTrue(isinstance(values["integer"][3], np.int64))
+        self.assertTrue(isinstance(values["integer"][4], np.uint8))
+        self.assertTrue(isinstance(values["integer"][5], np.uint16))
+        self.assertTrue(isinstance(values["integer"][6], np.uint32))
+        self.assertTrue(isinstance(values["integer"][7], np.uint64))
+        # the actual test!
+        self.assertEqual(
+            jsonify_strict(values),
+            '{"floating": [100.0, 101.0, 102.0], "integer": [0, 1, 2, 3, 4, 5, 6, 7]}',
+        )

test/unit/utils/test_type_conversion_utils.py

@@ -0,0 +1,324 @@
+import unittest
+import logging
+from parameterized import parameterized_class
+
+import numpy as np
+import pandas as pd
+from pandas import Series, DataFrame
+from scipy import sparse
+
+from server.common.utils.type_conversion_utils import (
+    get_encoding_dtype_of_array,
+    get_schema_type_hint_of_array,
+    get_dtypes_and_schemas_of_dataframe,
+    get_dtype_and_schema_of_array,
+    get_schema_type_hint_from_dtype,
+)
+
+
+class TestTypeConversionUtils(unittest.TestCase):
+    def test__get_dtypes_and_schemas_of_dataframe__dtype_and_schema_returns_as_expected(self):
+        float_array = Series(data=[1, 2, 3], dtype=np.dtype(np.float64))
+        category_array = Series(data=["a", "b", "b"], dtype="category")
+        dataframe = DataFrame({"float_array": float_array, "category_array": category_array})
+
+        expected_data_types_dict = {"float_array": np.float32, "category_array": str}
+        expected_schema_type_hints_dict = {
+            "float_array": {"type": "float32"},
+            "category_array": {"type": "categorical", "categories": ["a", "b"]},
+        }
+
+        actual_dataframe_data_types, actual_dataframe_schema_type_hints = get_dtypes_and_schemas_of_dataframe(dataframe)
+
+        self.assertEqual(expected_data_types_dict, actual_dataframe_data_types)
+        self.assertEqual(expected_schema_type_hints_dict, actual_dataframe_schema_type_hints)
+
+    def test__get_schema_type_hint_from_dtype(self):
+        self.assertEqual(get_schema_type_hint_from_dtype(np.dtype(np.bool_)), {"type": "boolean"})
+
+        for dtype in [np.int8, np.int8, np.int16, np.uint16, np.int32]:
+            self.assertEqual(get_schema_type_hint_from_dtype(np.dtype(dtype)), {"type": "int32"})
+        for dtype in [np.uint32, np.int64, np.uint64]:
+            with self.assertRaises(TypeError):
+                get_schema_type_hint_from_dtype(np.dtype(dtype))
+
+        for dtype in [np.float16, np.float32, np.float64]:
+            self.assertEqual(get_schema_type_hint_from_dtype(np.dtype(dtype)), {"type": "float32"})
+
+        for dtype in [np.dtype(object), np.dtype(str)]:
+            self.assertEqual(get_schema_type_hint_from_dtype(dtype), {"type": "string"})
+
+
+# Credit: https://stackoverflow.com/questions/35871815/python-3-unit-testing-assert-logger-not-called/64774103#64774103
+class AssertNoLog:
+    def assertNoLogs(self, logger, level):
+        """functions as a context manager.  To be introduced in python 3.10"""
+
+        class AssertNoLogsContext(unittest.TestCase):
+            def __init__(self, logger, level):
+                self.logger = logger
+                self.level = level
+                self.context = self.assertLogs(logger, level)
+
+            def __enter__(self):
+                """enter self.assertLogs as context manager, and log something"""
+                self.initial_logmsg = "sole message"
+                self.cm = self.context.__enter__()
+                self.logger.log(self.level, self.initial_logmsg)
+                return self.cm
+
+            def __exit__(self, exc_type, exc_val, exc_tb):
+                """cleanup logs, and then check nothing extra was logged"""
+                # assertLogs.__exit__ should never fail because of initial msg
+                self.context.__exit__(exc_type, exc_val, exc_tb)
+                if len(self.cm.output) > 1:
+                    """override any exception passed to __exit__"""
+                    self.context._raiseFailure(
+                        "logs of level {} or higher triggered on {} : {}".format(
+                            logging.getLevelName(self.level), self.logger.name, self.cm.output[1:]
+                        )
+                    )
+
+        return AssertNoLogsContext(logger, level)
+
+
+"""
+See table of expected cases in type_conversion_utils.py.
+
+This probes all edge cases. Each case is a dict containing keys:
+    - data - the array to be introspected
+    - throws - if not None, the expected Error (eg, TypeError)
+    - expected_encoding_dtype - upon success
+    - expected_schema_hint - upon success
+    - logs - if not None, specify expected log output
+"""
+
+bool_OK_cases = [
+    {
+        "data": data,
+        "expected_encoding_dtype": np.uint8,
+        "expected_schema_hint": {"type": "boolean"},
+    }
+    for data in [
+        np.array([0, 1, 0, 1], dtype=np.bool_),
+        pd.Series(np.array([0, 1, 0, 1], dtype=np.bool_)),
+        # pd.Index with bools doesn't really make any sense...and becomes dtype=object
+    ]
+]
+
+int_OK_cases = [
+    {
+        "data": data,
+        "expected_encoding_dtype": np.int32,
+        "expected_schema_hint": {"type": "int32"},
+    }
+    for dtype in [np.int8, np.uint8, np.int16, np.uint16, np.int32, np.uint32, np.int64, np.uint64]
+    for data in [
+        np.arange(0, 1000, dtype=dtype),
+        pd.Series(np.arange(0, 1000, dtype=dtype)),
+        pd.Index(np.arange(0, 1000, dtype=dtype)),
+        sparse.csr_matrix((10, 100), dtype=dtype),
+    ]
+]
+
+float_OK_cases = [
+    {
+        "data": data,
+        "expected_encoding_dtype": np.float32,
+        "expected_schema_hint": {"type": "float32"},
+        "logs": None if data.dtype != np.float64 else {"level": logging.WARNING, "output": "may lose precision"},
+    }
+    for dtype in [np.float16, np.float32, np.float64]
+    for data in [
+        np.arange(-128, 1000, dtype=dtype),
+        pd.Series(np.arange(-128, 1000, dtype=dtype)),
+        pd.Index(np.arange(-129, 1000, dtype=dtype)),
+        np.array([-np.nan, np.NINF, -1, np.NZERO, 0, np.PZERO, 1, np.PINF, np.nan], dtype=dtype),
+        np.array([np.finfo(dtype).min, 0, np.finfo(dtype).max], dtype=dtype),
+        sparse.csr_matrix((10, 100), dtype=dtype),
+    ]
+]
+
+
+numeric_ERR_cases = [
+    {
+        "data": data,
+        "throws": TypeError,
+    }
+    for data in [
+        np.array([np.iinfo(np.int64).min, np.iinfo(np.int64).max], dtype=np.int64),
+        np.array([np.iinfo(np.uint64).min, np.iinfo(np.uint64).max], dtype=np.uint64),
+        np.array([np.iinfo(np.uint32).min, np.iinfo(np.uint32).max], dtype=np.uint32),
+    ]
+]
+
+
+string_OK_cases = [
+    {
+        "data": data,
+        "expected_encoding_dtype": np.dtype(str),
+        "expected_schema_hint": {"type": "string"},
+    }
+    for data in [
+        np.array(["a", "b", "c"]),
+        np.array(["a", "b", "c"], dtype="object"),
+        pd.Series(["a", "b", "c"]),
+        pd.Index(["a", "b", "c"]),
+        np.array(["a", [], {}, None, True, False, 383.2], dtype="object"),
+    ]
+]
+
+category_nonnumeric_OK_cases = [
+    {
+        "data": data,
+        "expected_encoding_dtype": np.dtype(str),
+        "expected_schema_hint": {"type": "categorical", "categories": data.dtype.categories.to_list()},
+    }
+    for data in [
+        pd.Series(["a", "b", "c"], dtype="category"),
+        pd.Series(["a", "b", "c", 0, 1, 2], dtype="category"),
+        pd.Series(["a", "b", "c"], dtype="category").cat.remove_categories(["b"]),
+        pd.Series(["a", "b", "c", 0, 1, 2], dtype="category").cat.remove_categories(["b", 0]),
+    ]
+]
+
+category_numeric_OK_cases = [
+    # numeric, no NA/NaN, int
+    *[
+        {
+            "data": data,
+            "expected_encoding_dtype": np.int32,
+            "expected_schema_hint": {"type": "categorical"},
+        }
+        for dtype in [np.int8, np.uint8, np.int16, np.uint16, np.int32, np.uint32, np.int64, np.uint64]
+        for data in [
+            pd.Series(np.array([0, 1, 2], dtype=dtype), dtype="category"),
+        ]
+    ],
+    # numeric, no NA/NaN, float
+    *[
+        {
+            "data": data,
+            "expected_encoding_dtype": np.float32,
+            "expected_schema_hint": {"type": "categorical"},
+            "logs": {"level": logging.WARNING, "output": "may lose precision"},
+        }
+        for dtype in [np.float16, np.float32, np.float64]
+        for data in [
+            pd.Series(np.array([0, 1, 2], dtype=dtype), dtype="category"),
+            pd.Series(np.array([0, 1, 2], dtype=dtype), dtype="category").cat.remove_categories([1]),
+            pd.Categorical(np.array([0, 1, 2], dtype=dtype)),
+        ]
+    ],
+    # numeric, has NA-induced cast to float32
+    *[
+        {
+            "data": data,
+            "expected_encoding_dtype": np.float32,
+            "expected_schema_hint": {"type": "categorical"},
+            "logs": {"level": logging.WARNING, "output": "may lose precision"},
+        }
+        for dtype in [
+            np.int8,
+            np.uint8,
+            np.int16,
+            np.uint16,
+            np.int32,
+            np.uint32,
+            np.int64,
+            np.uint64,
+            np.float16,
+            np.float32,
+            np.float64,
+        ]
+        for data in [
+            pd.Series(np.array([0, 1, 2], dtype=dtype), dtype="category").cat.remove_categories([1]),
+            pd.Categorical(np.array([0, 1, 2], dtype=dtype), categories=np.array([0, 1], dtype=dtype)),
+        ]
+    ],
+]
+
+category_ERR_cases = [
+    # catch expected categorical exceptions for Int64(etc) that have large values
+    {
+        "data": data,
+        "throws": TypeError,
+    }
+    for data in [
+        pd.Categorical(np.array([np.iinfo(np.int64).min, np.iinfo(np.int64).max], dtype=np.int64)),
+        pd.Categorical(np.array([np.iinfo(np.uint64).min, np.iinfo(np.uint64).max], dtype=np.uint64)),
+        pd.Categorical(np.array([np.iinfo(np.uint32).min, np.iinfo(np.uint32).max], dtype=np.uint32)),
+    ]
+]
+
+object_OK_cases = [
+    {
+        "data": data,
+        "expected_encoding_dtype": np.dtype(str),
+        "expected_schema_hint": {"type": "string"},
+    }
+    for data in [
+        np.array(["a", True, 1, [], {}], dtype="object"),
+        pd.Series(["a", True, 1, [], {}], dtype="object"),
+        pd.Index(["a", True, 1, [], {}], dtype="object"),
+    ]
+]
+
+err_cases = [
+    {"data": np.array, "throws": TypeError}
+    for data in [
+        np.ones((10,), dtype=np.complex64),
+        np.ones((10,), dtype=np.complex128),
+        np.array([b"foobar"], dtype=np.bytes_),
+        np.ones((10,), dtype=np.void),
+        np.arange("2005-02", "2005-03", dtype="datetime64[D]"),
+        np.arange("2005-02", "2005-03", dtype="datetime64[D]") - np.datetime64("2008-01-01"),
+        [],
+        {},
+    ]
+]
+
+test_cases = [
+    *bool_OK_cases,
+    *int_OK_cases,
+    *float_OK_cases,
+    *numeric_ERR_cases,
+    *string_OK_cases,
+    *category_nonnumeric_OK_cases,
+    *category_numeric_OK_cases,
+    *category_ERR_cases,
+    *object_OK_cases,
+    *err_cases,
+]
+
+
+@parameterized_class(test_cases)
+class TestTypeInference(unittest.TestCase, AssertNoLog):
+    def test_type_inference(self):
+        throws = getattr(self, "throws", None)
+        if throws:
+            with self.assertRaises(throws):
+                get_dtype_and_schema_of_array(self.data)
+            with self.assertRaises(throws):
+                get_encoding_dtype_of_array(self.data)
+            with self.assertRaises(throws):
+                get_schema_type_hint_of_array(self.data)
+
+        else:
+            logs = getattr(self, "logs", None)
+            if logs is not None:
+                with self.assertLogs(level=logs["level"]) as logger:
+                    encoding_dtype, schema_hint = get_dtype_and_schema_of_array(self.data)
+                    self.assertEqual(encoding_dtype, self.expected_encoding_dtype)
+                    self.assertEqual(schema_hint, self.expected_schema_hint)
+                    self.assertIn(logs["output"], logger.output[0])
+
+            else:
+                with self.assertNoLogs(logging.getLogger(), logging.WARNING):
+                    encoding_dtype, schema_hint = get_dtype_and_schema_of_array(self.data)
+                    self.assertEqual(encoding_dtype, self.expected_encoding_dtype)
+                    self.assertEqual(schema_hint, self.expected_schema_hint)
+
+            # also test the other public API
+            self.assertEqual(get_encoding_dtype_of_array(self.data), self.expected_encoding_dtype)
+            self.assertEqual(get_schema_type_hint_of_array(self.data), self.expected_schema_hint)