config refactor (#1854)

* split out config

* add tests for base and app config, refactor client config out of app config

* refactor default config retrieval

* create config test class and helper functions

* move default_config into server to fix import issue

server/test/unit/common/config/__init__.py

@@ -0,0 +1,246 @@
+import os
+import shutil
+import unittest
+import random
+from unittest import mock
+
+from server.test import FIXTURES_ROOT
+
+
+def mockenv(**envvars):
+    return mock.patch.dict(os.environ, envvars)
+
+
+class ConfigTests(unittest.TestCase):
+    tmp_fixtures_directory = os.path.join(FIXTURES_ROOT, "tmp_dir")
+
+    @classmethod
+    def tearDownClass(cls) -> None:
+        shutil.rmtree(cls.tmp_fixtures_directory)
+
+    @classmethod
+    def setUpClass(cls) -> None:
+        os.makedirs(cls.tmp_fixtures_directory)
+
+    def custom_server_config(
+        self,
+        verbose="false",
+        debug="false",
+        host="localhost",
+        port="null",
+        open_browser="false",
+        force_https="false",
+        flask_secret_key="null",
+        generate_cache_control_headers="false",
+        server_timing_headers="false",
+        csp_directives="null",
+        api_base_url="null",
+        web_base_url="null",
+        auth_type="session",
+        oauth_api_base_url="null",
+        client_id="null",
+        client_secret="null",
+        jwt_decode_options="null",
+        session_cookie="true",
+        cookie="null",
+        dataroot="null",
+        index="false",
+        allowed_matrix_types=[],
+        max_cached_datasets=5,
+        timelimit_s=5,
+        dataset_datapath="null",
+        obs_names="null",
+        var_names="null",
+        about="null",
+        title="null",
+        diffexp_max_workers=64,
+        cpu_multiplier=4,
+        target_workunit="16_000_000",
+        data_locater_region_name="us-east-1",
+        cxg_tile_cache_size=8589934592,
+        cxg_num_reader_threads=32,
+        anndata_backed="false",
+        column_request_max=32,
+        diffexp_cellcount_max="null",
+        config_file_name="server_config.yaml",
+    ):
+        configfile = os.path.join(self.tmp_fixtures_directory, config_file_name)
+        server_config_outline_path = os.path.join(FIXTURES_ROOT, "server_config_outline.py")
+        with open(server_config_outline_path, "r") as config_skeleton:
+            config = config_skeleton.read()
+            server_config = eval(config)
+        with open(configfile, "w") as server_config_file:
+            server_config_file.write(server_config)
+        return configfile
+
+    def custom_app_config(
+        self,
+        verbose="false",
+        debug="false",
+        host="localhost",
+        port="null",
+        open_browser="false",
+        force_https="false",
+        flask_secret_key="null",
+        generate_cache_control_headers="false",
+        server_timing_headers="false",
+        csp_directives="null",
+        api_base_url="null",
+        web_base_url="null",
+        auth_type="session",
+        oauth_api_base_url="null",
+        client_id="null",
+        client_secret="null",
+        jwt_decode_options="null",
+        session_cookie="true",
+        cookie="null",
+        dataroot="null",
+        index="false",
+        allowed_matrix_types=[],
+        max_cached_datasets=5,
+        timelimit_s=5,
+        dataset_datapath="null",
+        obs_names="null",
+        var_names="null",
+        about="null",
+        title="null",
+        diffexp_max_workers=64,
+        cpu_multiplier=4,
+        target_workunit="16_000_000",
+        data_locater_region_name="us-east-1",
+        cxg_tile_cache_size=8589934592,
+        cxg_num_reader_threads=32,
+        anndata_backed="false",
+        column_request_max=32,
+        diffexp_cellcount_max="null",
+        scripts=[],
+        inline_scripts=[],
+        about_legal_tos="null",
+        about_legal_privacy="null",
+        authentication_enable="true",
+        max_categories=1000,
+        custom_colors="true",
+        enable_users_annotations="true",
+        annotation_type="local_file_csv",
+        db_uri="null",
+        hosted_file_directory="null",
+        local_file_csv_directory="null",
+        local_file_csv_file="null",
+        ontology_enabled="false",
+        obo_location="null",
+        embedding_names=[],
+        enable_reembedding="false",
+        enable_difexp="true",
+        lfc_cutoff=0.01,
+        top_n=10,
+        config_file_name="app_config.yml",
+    ):
+        random_num = random.randrange(999999)
+        configfile = os.path.join(self.tmp_fixtures_directory, config_file_name)
+        server_config = self.custom_server_config(
+            verbose=verbose,
+            debug=debug,
+            host=host,
+            port=port,
+            open_browser=open_browser,
+            force_https=force_https,
+            flask_secret_key=flask_secret_key,
+            generate_cache_control_headers=generate_cache_control_headers,
+            server_timing_headers=server_timing_headers,
+            csp_directives=csp_directives,
+            api_base_url=api_base_url,
+            web_base_url=web_base_url,
+            auth_type=auth_type,
+            oauth_api_base_url=oauth_api_base_url,
+            client_id=client_id,
+            client_secret=client_secret,
+            jwt_decode_options=jwt_decode_options,
+            session_cookie=session_cookie,
+            cookie=cookie,
+            dataroot=dataroot,
+            index=index,
+            allowed_matrix_types=allowed_matrix_types,
+            max_cached_datasets=max_cached_datasets,
+            timelimit_s=timelimit_s,
+            dataset_datapath=dataset_datapath,
+            obs_names=obs_names,
+            var_names=var_names,
+            about=about,
+            title=title,
+            diffexp_max_workers=diffexp_max_workers,
+            cpu_multiplier=cpu_multiplier,
+            target_workunit=target_workunit,
+            data_locater_region_name=data_locater_region_name,
+            cxg_tile_cache_size=cxg_tile_cache_size,
+            cxg_num_reader_threads=cxg_num_reader_threads,
+            anndata_backed=anndata_backed,
+            column_request_max=column_request_max,
+            diffexp_cellcount_max=diffexp_cellcount_max,
+            config_file_name=f"temp_server_config_{random_num}.yml",
+        )
+        dataset_config = self.custom_dataset_config(
+            scripts=scripts,
+            inline_scripts=inline_scripts,
+            about_legal_tos=about_legal_tos,
+            about_legal_privacy=about_legal_privacy,
+            authentication_enable=authentication_enable,
+            max_categories=max_categories,
+            custom_colors=custom_colors,
+            enable_users_annotations=enable_users_annotations,
+            annotation_type=annotation_type,
+            db_uri=db_uri,
+            hosted_file_directory=hosted_file_directory,
+            local_file_csv_directory=local_file_csv_directory,
+            local_file_csv_file=local_file_csv_file,
+            ontology_enabled=ontology_enabled,
+            obo_location=obo_location,
+            embedding_names=embedding_names,
+            enable_reembedding=enable_reembedding,
+            enable_difexp=enable_difexp,
+            lfc_cutoff=lfc_cutoff,
+            top_n=top_n,
+            config_file_name=f"temp_dataset_config_{random_num}.yml",
+        )
+        with open(server_config) as server_config:
+            with open(dataset_config) as dataset_config:
+                with open(configfile, "w") as app_config_file:
+                    for line in server_config:
+                        app_config_file.write(line)
+                    for line in dataset_config:
+                        app_config_file.write(line)
+
+        return configfile
+
+    def custom_dataset_config(
+        self,
+        scripts=[],
+        inline_scripts=[],
+        about_legal_tos="null",
+        about_legal_privacy="null",
+        authentication_enable="true",
+        max_categories=1000,
+        custom_colors="true",
+        enable_users_annotations="true",
+        annotation_type="local_file_csv",
+        db_uri="null",
+        hosted_file_directory="null",
+        local_file_csv_directory="null",
+        local_file_csv_file="null",
+        ontology_enabled="false",
+        obo_location="null",
+        embedding_names=[],
+        enable_reembedding="false",
+        enable_difexp="true",
+        lfc_cutoff=0.01,
+        top_n=10,
+        config_file_name="dataset_config.yml",
+    ):
+        configfile = os.path.join(self.tmp_fixtures_directory, config_file_name)
+        dataset_config_outline_path = os.path.join(FIXTURES_ROOT, "dataset_config_outline.py")
+        with open(dataset_config_outline_path, "r") as config_skeleton:
+            config = config_skeleton.read()
+            dataset_config = eval(config)
+        with open(configfile, "w") as dataset_config_file:
+            dataset_config_file.write(dataset_config)
+
+        return configfile

server/test/unit/common/config/test_app_config.py

@@ -0,0 +1,140 @@
+import os
+import tempfile
+import unittest
+
+import yaml
+
+from server.default_config import default_config
+from server.common.config.app_config import AppConfig
+from server.test.unit.common.config import ConfigTests
+from server.common.errors import ConfigurationError
+from server.test import FIXTURES_ROOT
+
+
+class AppConfigTest(ConfigTests):
+    def setUp(self):
+        self.config_file_name = f"{unittest.TestCase.id(self).split('.')[-1]}.yml"
+        self.config = AppConfig()
+        self.config.update_server_config(multi_dataset__dataroot=FIXTURES_ROOT)
+        self.server_config = self.config.server_config
+        self.config.complete_config()
+
+        message_list = []
+
+        def noop(message):
+            message_list.append(message)
+
+        messagefn = noop
+        self.context = dict(messagefn=messagefn, messages=message_list)
+
+    def get_config(self, **kwargs):
+        file_name = self.custom_app_config(
+            dataroot=f"{FIXTURES_ROOT}", config_file_name=self.config_file_name, **kwargs
+        )
+        config = AppConfig()
+        config.update_from_config_file(file_name)
+        return config
+
+    def test_get_default_config_correctly_reads_default_config_file(self):
+        app_default_config = AppConfig().default_config
+
+        expected_config = yaml.load(default_config, Loader=yaml.Loader)
+
+        server_config = app_default_config['server']
+        dataset_config = app_default_config['dataset']
+
+        expected_server_config = expected_config['server']
+        expected_dataset_config = expected_config['dataset']
+
+        self.assertDictEqual(app_default_config, expected_config)
+        self.assertDictEqual(server_config, expected_server_config)
+        self.assertDictEqual(dataset_config, expected_dataset_config)
+
+    def test_get_dataset_config_returns_default_dataset_config_for_single_datasets(self):
+        datapath = f"{FIXTURES_ROOT}/1e4dfec4-c0b2-46ad-a04e-ff3ffb3c0a8f.h5ad"
+        file_name = self.custom_app_config(dataset_datapath=datapath, config_file_name=self.config_file_name)
+        config = AppConfig()
+        config.update_from_config_file(file_name)
+
+        self.assertEqual(config.get_dataset_config(""), config.default_dataset_config)
+
+    def test_update_server_config_updates_server_config_and_config_status(self):
+        config = self.get_config()
+        config.complete_config()
+        config.check_config()
+        config.update_server_config(multi_dataset__dataroot=FIXTURES_ROOT)
+        with self.assertRaises(ConfigurationError):
+            config.server_config.check_config()
+
+    def test_write_config_outputs_yaml_with_all_config_vars(self):
+        config = self.get_config()
+        config.write_config(f"{FIXTURES_ROOT}/tmp_dir/write_config.yml")
+        with open(f"{FIXTURES_ROOT}/tmp_dir/{self.config_file_name}", "r") as default_config:
+            default_config_yml = yaml.safe_load(default_config)
+
+        with open(f"{FIXTURES_ROOT}/tmp_dir/write_config.yml", "r") as output_config:
+            output_config_yml = yaml.safe_load(output_config)
+        self.maxDiff = None
+        self.assertEqual(default_config_yml, output_config_yml)
+
+    def test_update_app_config(self):
+        config = AppConfig()
+        config.update_server_config(app__verbose=True, multi_dataset__dataroot="datadir")
+        vars = config.server_config.changes_from_default()
+        self.assertCountEqual(vars, [("app__verbose", True, False), ("multi_dataset__dataroot", "datadir", None)])
+
+        config = AppConfig()
+        config.update_default_dataset_config(app__scripts=(), app__inline_scripts=())
+        vars = config.server_config.changes_from_default()
+        self.assertCountEqual(vars, [])
+
+        config = AppConfig()
+        config.update_default_dataset_config(app__scripts=[], app__inline_scripts=[])
+        vars = config.default_dataset_config.changes_from_default()
+        self.assertCountEqual(vars, [])
+
+        config = AppConfig()
+        config.update_default_dataset_config(app__scripts=("a", "b"), app__inline_scripts=["c", "d"])
+        vars = config.default_dataset_config.changes_from_default()
+        self.assertCountEqual(vars, [("app__scripts", ["a", "b"], []), ("app__inline_scripts", ["c", "d"], [])])
+
+    def test_configfile_no_dataset_section(self):
+        # test a config file without a dataset section
+
+        with tempfile.TemporaryDirectory() as tempdir:
+            configfile = os.path.join(tempdir, "config.yaml")
+            with open(configfile, "w") as fconfig:
+                config = """
+                server:
+                    multi_dataset:
+                        dataroot: test_dataroot
+
+                """
+                fconfig.write(config)
+
+            app_config = AppConfig()
+            app_config.update_from_config_file(configfile)
+            server_changes = app_config.server_config.changes_from_default()
+            dataset_changes = app_config.default_dataset_config.changes_from_default()
+            self.assertEqual(server_changes, [("multi_dataset__dataroot", "test_dataroot", None)])
+            self.assertEqual(dataset_changes, [])
+
+    def test_configfile_no_server_section(self):
+        # test a config file without a dataset section
+
+        with tempfile.TemporaryDirectory() as tempdir:
+            configfile = os.path.join(tempdir, "config.yaml")
+            with open(configfile, "w") as fconfig:
+                config = """
+                dataset:
+                    user_annotations:
+                        enable: false
+                """
+                fconfig.write(config)
+
+            app_config = AppConfig()
+            app_config.update_from_config_file(configfile)
+            server_changes = app_config.server_config.changes_from_default()
+            dataset_changes = app_config.default_dataset_config.changes_from_default()
+            self.assertEqual(server_changes, [])
+            self.assertEqual(dataset_changes, [("user_annotations__enable", False, True)])

server/test/unit/common/config/test_base_config.py

@@ -0,0 +1,63 @@
+import unittest
+
+from server.common.config.app_config import AppConfig
+from server.test import FIXTURES_ROOT
+from server.test.unit.common.config import ConfigTests
+from server.common.errors import ConfigurationError
+
+
+class BaseConfigTest(ConfigTests):
+    def setUp(self):
+        self.config_file_name = f"{unittest.TestCase.id(self).split('.')[-1]}.yml"
+        self.config = AppConfig()
+        self.config.update_server_config(multi_dataset__dataroot=FIXTURES_ROOT)
+        self.server_config = self.config.server_config
+        self.config.complete_config()
+
+        message_list = []
+
+        def noop(message):
+            message_list.append(message)
+
+        messagefn = noop
+        self.context = dict(messagefn=messagefn, messages=message_list)
+
+    def get_config(self, **kwargs):
+        file_name = self.custom_app_config(
+            dataroot=f"{FIXTURES_ROOT}", config_file_name=self.config_file_name, **kwargs
+        )
+        config = AppConfig()
+        config.update_from_config_file(file_name)
+        return config
+
+    def test_mapping_creation_returns_map_of_server_and_dataset_config(self):
+        config = AppConfig()
+        mapping = config.default_dataset_config.create_mapping(config.default_config)
+        self.assertIsNotNone(mapping["server__app__verbose"])
+        self.assertIsNotNone(mapping["dataset__presentation__max_categories"])
+        self.assertIsNotNone(mapping["dataset__user_annotations__ontology__obo_location"])
+        self.assertIsNotNone(mapping["server__multi_dataset__allowed_matrix_types"])
+
+    def test_changes_from_default_returns_list_of_nondefault_config_values(self):
+        config = self.get_config(verbose="true", lfc_cutoff=0.05)
+        server_changes = config.server_config.changes_from_default()
+        dataset_changes = config.default_dataset_config.changes_from_default()
+
+        self.assertEqual(
+            server_changes,
+            [
+                ("app__verbose", True, False),
+                ("multi_dataset__dataroot", FIXTURES_ROOT, None),
+                ("multi_dataset__matrix_cache__timelimit_s", 5, 30),
+                ("data_locator__s3__region_name", "us-east-1", True),
+            ],
+        )
+        self.assertEqual(dataset_changes, [("diffexp__lfc_cutoff", 0.05, 0.01)])
+
+    def test_check_config_throws_error_if_attr_has_not_been_checked(self):
+        config = self.get_config(verbose="true")
+        config.complete_config()
+        config.check_config()
+        config.update_server_config(app__verbose=False)
+        with self.assertRaises(ConfigurationError):
+            config.check_config()

server/test/unit/common/config/test_dataset_config.py

@@ -0,0 +1,260 @@
+import os
+import tempfile
+
+import requests
+import unittest
+from unittest.mock import patch
+
+from server.common.annotations.hosted_tiledb import AnnotationsHostedTileDB
+from server.common.annotations.local_file_csv import AnnotationsLocalFile
+from server.common.config.app_config import AppConfig
+from server.common.config.base_config import BaseConfig
+from server.test import test_server, PROJECT_ROOT, FIXTURES_ROOT
+
+from server.common.errors import ConfigurationError
+from server.test.unit.common.config import ConfigTests
+
+
+class TestDatasetConfig(ConfigTests):
+    def setUp(self):
+        self.config_file_name = f"{unittest.TestCase.id(self).split('.')[-1]}.yml"
+        self.config = AppConfig()
+        self.config.update_server_config(multi_dataset__dataroot=FIXTURES_ROOT)
+        self.dataset_config = self.config.default_dataset_config
+        self.config.complete_config()
+        message_list = []
+
+        def noop(message):
+            message_list.append(message)
+
+        messagefn = noop
+        self.context = dict(messagefn=messagefn, messages=message_list)
+
+    def get_config(self, **kwargs):
+        file_name = self.custom_app_config(
+            dataroot=f"{FIXTURES_ROOT}", config_file_name=self.config_file_name, **kwargs
+        )
+        config = AppConfig()
+        config.update_from_config_file(file_name)
+        return config
+
+    def test_init_datatset_config_sets_vars_from_default_config(self):
+        config = AppConfig()
+        self.assertEqual(config.default_dataset_config.presentation__max_categories, 1000)
+        self.assertEqual(config.default_dataset_config.user_annotations__type, "local_file_csv")
+        self.assertEqual(config.default_dataset_config.diffexp__lfc_cutoff, 0.01)
+        self.assertIsNone(config.default_dataset_config.user_annotations__ontology__obo_location)
+
+    @patch("server.common.config.dataset_config.BaseConfig.validate_correct_type_of_configuration_attribute")
+    def test_complete_config_checks_all_attr(self, mock_check_attrs):
+        mock_check_attrs.side_effect = BaseConfig.validate_correct_type_of_configuration_attribute()
+        self.dataset_config.complete_config(self.context)
+        self.assertEqual(mock_check_attrs.call_count, 21)
+
+    def test_app_sets_script_vars(self):
+        config = self.get_config(scripts=["path/to/script"])
+        config.default_dataset_config.handle_app()
+
+        self.assertEqual(config.default_dataset_config.app__scripts, [{"src": "path/to/script"}])
+
+        config = self.get_config(scripts=[{"src": "path/to/script", "more": "different/script/path"}])
+        config.default_dataset_config.handle_app()
+        self.assertEqual(
+            config.default_dataset_config.app__scripts, [{"src": "path/to/script", "more": "different/script/path"}]
+        )
+
+        config = self.get_config(scripts=["path/to/script", "different/script/path"])
+        config.default_dataset_config.handle_app()
+        # TODO @madison -- is this the desired functionality?
+        self.assertEqual(
+            config.default_dataset_config.app__scripts, [{"src": "path/to/script"}, {"src": "different/script/path"}]
+        )
+
+        config = self.get_config(scripts=[{"more": "different/script/path"}])
+        with self.assertRaises(ConfigurationError):
+            config.default_dataset_config.handle_app()
+
+    def test_handle_user_annotations_ensures_auth_is_enabled_with_valid_auth_type(self):
+        config = self.get_config(enable_users_annotations="true", authentication_enable="false")
+        config.server_config.complete_config(self.context)
+        with self.assertRaises(ConfigurationError):
+            config.default_dataset_config.handle_user_annotations(self.context)
+
+        config = self.get_config(enable_users_annotations="true", authentication_enable="true", auth_type="pretend")
+        with self.assertRaises(ConfigurationError):
+            config.server_config.complete_config(self.context)
+
+    def test_handle_user_annotations__adds_warning_message_if_annotation_vars_set_when_annotations_disabled(self):
+        config = self.get_config(
+            enable_users_annotations="false", authentication_enable="false", db_uri="shouldnt/be/set"
+        )
+        config.default_dataset_config.handle_user_annotations(self.context)
+
+        self.assertEqual(self.context["messages"], ["Warning: db_uri ignored as annotations are disabled."])
+
+    @patch("server.common.config.dataset_config.DbUtils")
+    def test_handle_user_annotations__instantiates_user_annotations_class_correctly(self, mock_db_utils):
+        mock_db_utils.return_value = "123"
+        config = self.get_config(
+            enable_users_annotations="true", authentication_enable="true", annotation_type="local_file_csv"
+        )
+        config.server_config.complete_config(self.context)
+        config.default_dataset_config.handle_user_annotations(self.context)
+        self.assertIsInstance(config.default_dataset_config.user_annotations, AnnotationsLocalFile)
+
+        config = self.get_config(
+            enable_users_annotations="true",
+            authentication_enable="true",
+            annotation_type="hosted_tiledb_array",
+            db_uri="gotta/set/this",
+            hosted_file_directory="and/this",
+        )
+        config.server_config.complete_config(self.context)
+        config.default_dataset_config.handle_user_annotations(self.context)
+        self.assertIsInstance(config.default_dataset_config.user_annotations, AnnotationsHostedTileDB)
+
+        config = self.get_config(
+            enable_users_annotations="true", authentication_enable="true", annotation_type="NOT_REAL"
+        )
+        config.server_config.complete_config(self.context)
+        with self.assertRaises(ConfigurationError):
+            config.default_dataset_config.handle_user_annotations(self.context)
+
+    def test_handle_local_file_csv_annotations__sets_dir_if_not_passed_in(self):
+        config = self.get_config(
+            enable_users_annotations="true", authentication_enable="true", annotation_type="local_file_csv"
+        )
+        config.server_config.complete_config(self.context)
+        config.default_dataset_config.handle_local_file_csv_annotations()
+        self.assertIsInstance(config.default_dataset_config.user_annotations, AnnotationsLocalFile)
+        cwd = os.getcwd()
+        self.assertEqual(config.default_dataset_config.user_annotations._get_output_dir(), cwd)
+
+    def test_handle_embeddings__checks_data_file_types(self):
+        file_name = self.custom_app_config(
+            embedding_names=["name1", "name2"],
+            enable_reembedding="true",
+            dataset_datapath=f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad",
+            anndata_backed="true",
+            config_file_name=self.config_file_name,
+        )
+        config = AppConfig()
+        config.update_from_config_file(file_name)
+        config.server_config.complete_config(self.context)
+        with self.assertRaises(ConfigurationError):
+            config.default_dataset_config.handle_embeddings()
+
+    def test_handle_diffexp__raises_warning_for_large_datasets(self):
+        config = self.get_config(lfc_cutoff=0.02, enable_difexp="true", top_n=15)
+        config.server_config.complete_config(self.context)
+        config.default_dataset_config.handle_diffexp(self.context)
+        self.assertEqual(len(self.context["messages"]), 0)
+
+    def test_multi_dataset(self):
+        config = AppConfig()
+        # test for illegal url_dataroots
+        for illegal in ("../b", "!$*", "\\n", "", "(bad)"):
+            config.update_server_config(
+                multi_dataset__dataroot={"tag": {"base_url": illegal, "dataroot": "{PROJECT_ROOT}/example-dataset"}}
+            )
+            with self.assertRaises(ConfigurationError):
+                config.complete_config()
+
+        # test for legal url_dataroots
+        for legal in ("d", "this.is-okay_", "a/b"):
+            config.update_server_config(
+                multi_dataset__dataroot={"tag": {"base_url": legal, "dataroot": "{PROJECT_ROOT}/example-dataset"}}
+            )
+            config.complete_config()
+
+        # test that multi dataroots work end to end
+        config.update_server_config(
+            multi_dataset__dataroot=dict(
+                s1=dict(dataroot=f"{PROJECT_ROOT}/example-dataset", base_url="set1/1/2"),
+                s2=dict(dataroot=f"{FIXTURES_ROOT}", base_url="set2"),
+                s3=dict(dataroot=f"{FIXTURES_ROOT}", base_url="set3"),
+            )
+        )
+
+        # Change this default to test if the dataroot overrides below work.
+        config.update_default_dataset_config(app__about_legal_tos="tos_default.html")
+
+        # specialize the configs for set1
+        config.add_dataroot_config(
+            "s1", user_annotations__enable=False, diffexp__enable=True, app__about_legal_tos="tos_set1.html"
+        )
+
+        # specialize the configs for set2
+        config.add_dataroot_config(
+            "s2", user_annotations__enable=True, diffexp__enable=False, app__about_legal_tos="tos_set2.html"
+        )
+
+        # no specializations for set3 (they get the default dataset config)
+        config.complete_config()
+
+        with test_server(app_config=config) as server:
+            session = requests.Session()
+
+            response = session.get(f"{server}/set1/1/2/pbmc3k.h5ad/api/v0.2/config")
+            data_config = response.json()
+            assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
+            assert data_config["config"]["parameters"]["annotations"] is False
+            assert data_config["config"]["parameters"]["disable-diffexp"] is False
+            assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_set1.html"
+
+            response = session.get(f"{server}/set2/pbmc3k.cxg/api/v0.2/config")
+            data_config = response.json()
+            assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
+            assert data_config["config"]["parameters"]["annotations"] is True
+            assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_set2.html"
+
+            response = session.get(f"{server}/set3/pbmc3k.cxg/api/v0.2/config")
+            data_config = response.json()
+            assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
+            assert data_config["config"]["parameters"]["annotations"] is True
+            assert data_config["config"]["parameters"]["disable-diffexp"] is False
+            assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_default.html"
+
+            response = session.get(f"{server}/health")
+            assert response.json()["status"] == "pass"
+
+    def test_configfile_with_specialization(self):
+        # test that per_dataset_config config load the default config, then the specialized config
+
+        with tempfile.TemporaryDirectory() as tempdir:
+            configfile = os.path.join(tempdir, "config.yaml")
+            with open(configfile, "w") as fconfig:
+                config = """
+                server:
+                    multi_dataset:
+                        dataroot:
+                            test:
+                                base_url: test
+                                dataroot: fake_dataroot
+
+                dataset:
+                    user_annotations:
+                        enable: false
+                        type: hosted_tiledb_array
+                        hosted_tiledb_array:
+                            db_uri: fake_db_uri
+                            hosted_file_directory: fake_dir
+
+                per_dataset_config:
+                    test:
+                        user_annotations:
+                            enable: true
+                """
+                fconfig.write(config)
+
+            app_config = AppConfig()
+            app_config.update_from_config_file(configfile)
+
+            test_config = app_config.dataroot_config["test"]
+
+            # test config from default
+            self.assertEqual(test_config.user_annotations__type, "hosted_tiledb_array")
+            self.assertEqual(test_config.user_annotations__hosted_tiledb_array__db_uri, "fake_db_uri")
+
+            # test config from specialization
+            self.assertTrue(test_config.user_annotations__enable)

server/test/unit/common/config/test_server_config.py

@@ -0,0 +1,335 @@
+import os
+import unittest
+from unittest import mock
+from unittest.mock import patch
+
+from server.common.config.base_config import BaseConfig
+from server.common.utils.utils import find_available_port
+from server.test import PROJECT_ROOT, FIXTURES_ROOT
+
+import requests
+
+from server.common.config.app_config import AppConfig
+from server.common.errors import ConfigurationError
+from server.test import test_server
+from server.test.unit.common.config import ConfigTests
+
+
+def mockenv(**envvars):
+    return mock.patch.dict(os.environ, envvars)
+
+
+class TestServerConfig(ConfigTests):
+    def setUp(self):
+        self.config_file_name = f"{unittest.TestCase.id(self).split('.')[-1]}.yml"
+        self.config = AppConfig()
+        self.config.update_server_config(multi_dataset__dataroot=FIXTURES_ROOT)
+        self.server_config = self.config.server_config
+        self.config.complete_config()
+
+        message_list = []
+
+        def noop(message):
+            message_list.append(message)
+
+        messagefn = noop
+        self.context = dict(messagefn=messagefn, messages=message_list)
+
+    def get_config(self, **kwargs):
+        file_name = self.custom_app_config(
+            dataroot=f"{FIXTURES_ROOT}", config_file_name=self.config_file_name, **kwargs
+        )
+        config = AppConfig()
+        config.update_from_config_file(file_name)
+        return config
+
+    def test_init_raises_error_if_default_config_is_invalid(self):
+        invalid_config = self.get_config(port="not_valid")
+        with self.assertRaises(ConfigurationError):
+            invalid_config.complete_config()
+
+    @patch("server.common.config.server_config.BaseConfig.validate_correct_type_of_configuration_attribute")
+    def test_complete_config_checks_all_attr(self, mock_check_attrs):
+        mock_check_attrs.side_effect = BaseConfig.validate_correct_type_of_configuration_attribute()
+        self.server_config.complete_config(self.context)
+        self.assertEqual(mock_check_attrs.call_count, 40)
+
+    def test_handle_app__throws_error_if_port_doesnt_exist(self):
+        config = self.get_config(port=99999999)
+        with self.assertRaises(ConfigurationError):
+            config.server_config.handle_app(self.context)
+
+    @patch("server.common.config.server_config.discover_s3_region_name")
+    def test_handle_data_locator_works_for_default_types(self, mock_discover_region_name):
+        mock_discover_region_name.return_value = None
+        # Default config
+        self.assertEqual(self.config.server_config.data_locator__s3__region_name, None)
+        # hard coded
+        config = self.get_config()
+        self.assertEqual(config.server_config.data_locator__s3__region_name, "us-east-1")
+        # incorrectly formatted
+        dataroot = {
+            "d1": {"base_url": "set1", "dataroot": "/path/to/set1_datasets/"},
+            "d2": {"base_url": "set2/subdir", "dataroot": "s3://shouldnt/work"},
+        }
+        file_name = self.custom_app_config(
+            dataroot=dataroot, config_file_name=self.config_file_name, data_locater_region_name="true"
+        )
+        config = AppConfig()
+        config.update_from_config_file(file_name)
+        with self.assertRaises(ConfigurationError):
+            config.server_config.handle_data_locator()
+
+    @patch("server.common.config.server_config.discover_s3_region_name")
+    def test_handle_data_locator_can_read_from_dataroot(self, mock_discover_region_name):
+        mock_discover_region_name.return_value = "us-west-2"
+        dataroot = {
+            "d1": {"base_url": "set1", "dataroot": "/path/to/set1_datasets/"},
+            "d2": {"base_url": "set2/subdir", "dataroot": "s3://hosted-cellxgene-dev"},
+        }
+        file_name = self.custom_app_config(
+            dataroot=dataroot, config_file_name=self.config_file_name, data_locater_region_name="true"
+        )
+        config = AppConfig()
+        config.update_from_config_file(file_name)
+        config.server_config.handle_data_locator()
+        self.assertEqual(config.server_config.data_locator__s3__region_name, "us-west-2")
+        mock_discover_region_name.assert_called_once_with("s3://hosted-cellxgene-dev")
+
+    def test_handle_app___can_use_envar_port(self):
+        config = self.get_config(port=24)
+        self.assertEqual(config.server_config.app__port, 24)
+
+        # Note if the port is set in the config file it will NOT be overwritten by a different envvar
+        os.environ["CXG_SERVER_PORT"] = "4008"
+        self.config = AppConfig()
+        self.config.server_config.handle_app(self.context)
+        self.assertEqual(self.config.server_config.app__port, 4008)
+
+    def test_handle_app__can_get_secret_key_from_envvar_or_config_file_with_envvar_given_preference(self):
+        config = self.get_config(flask_secret_key="KEY_FROM_FILE")
+        self.assertEqual(config.server_config.app__flask_secret_key, "KEY_FROM_FILE")
+
+        os.environ["CXG_SECRET_KEY"] = "KEY_FROM_ENV"
+        config.server_config.handle_app(self.context)
+        self.assertEqual(config.server_config.app__flask_secret_key, "KEY_FROM_ENV")
+
+    def test_handle_app__sets_web_base_url(self):
+        config = self.get_config(web_base_url="anything.com")
+        self.assertEqual(config.server_config.app__web_base_url, "anything.com")
+
+    def test_handle_auth__gets_client_secret_from_envvars_or_config_with_envvars_given_preference(self):
+        config = self.get_config(client_secret="KEY_FROM_FILE")
+        config.server_config.handle_authentication()
+        self.assertEqual(config.server_config.authentication__params_oauth__client_secret, "KEY_FROM_FILE")
+
+        os.environ["CXG_OAUTH_CLIENT_SECRET"] = "KEY_FROM_ENV"
+        config.server_config.handle_authentication()
+
+        self.assertEqual(config.server_config.authentication__params_oauth__client_secret, "KEY_FROM_ENV")
+
+    def test_handle_data_source__errors_when_passed_zero_or_two_dataroots(self):
+        file_name = self.custom_app_config(
+            dataroot=f"{FIXTURES_ROOT}",
+            config_file_name="two_data_roots.yml",
+            dataset_datapath=f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad",
+        )
+        config = AppConfig()
+        config.update_from_config_file(file_name)
+        with self.assertRaises(ConfigurationError):
+            config.server_config.handle_data_source()
+
+        file_name = self.custom_app_config(config_file_name="zero_roots.yml")
+        config = AppConfig()
+        config.update_from_config_file(file_name)
+        with self.assertRaises(ConfigurationError):
+            config.server_config.handle_data_source()
+
+    def test_get_api_base_url_works(self):
+
+        # test the api_base_url feature, and that it can contain a path
+        config = AppConfig()
+        backend_port = find_available_port("localhost", 10000)
+        config.update_server_config(
+            app__api_base_url=f"http://localhost:{backend_port}/additional/path",
+            multi_dataset__dataroot=f"{PROJECT_ROOT}/example-dataset",
+        )
+
+        config.complete_config()
+
+        with test_server(["-p", str(backend_port)], app_config=config) as server:
+            session = requests.Session()
+            self.assertEqual(server, f"http://localhost:{backend_port}")
+            response = session.get(f"{server}/additional/path/d/pbmc3k.h5ad/api/v0.2/config")
+            self.assertEqual(response.status_code, 200)
+            data_config = response.json()
+            self.assertEqual(data_config["config"]["displayNames"]["dataset"], "pbmc3k")
+
+            # test the health check at the correct url
+            response = session.get(f"{server}/additional/path/health")
+            assert response.json()["status"] == "pass"
+
+            # also check that the old URL still works.
+            # NOTE:  this old URL location will soon be deprecated, and when that happens
+            # this check can be removed.
+            response = session.get(f"{server}/health")
+            assert response.json()["status"] == "pass"
+
+    def test_get_web_base_url_works(self):
+        config = self.get_config(web_base_url="www.thisisawebsite.com")
+        web_base_url = config.server_config.get_web_base_url()
+        self.assertEqual(web_base_url, "www.thisisawebsite.com")
+
+        config = self.get_config(web_base_url="local", port=12)
+        web_base_url = config.server_config.get_web_base_url()
+        self.assertEqual(web_base_url, "http://localhost:12")
+
+        config = self.get_config(web_base_url="www.thisisawebsite.com/")
+        web_base_url = config.server_config.get_web_base_url()
+        self.assertEqual(web_base_url, "www.thisisawebsite.com")
+
+        config = self.get_config(api_base_url="www.api_base.com/")
+        web_base_url = config.server_config.get_web_base_url()
+        self.assertEqual(web_base_url, "www.api_base.com")
+
+    def test_config_for_single_dataset(self):
+        file_name = self.custom_app_config(
+            config_file_name="single_dataset.yml", dataset_datapath=f"{FIXTURES_ROOT}/pbmc3k.cxg"
+        )
+        config = AppConfig()
+        config.update_from_config_file(file_name)
+        config.server_config.handle_single_dataset(self.context)
+        self.assertIsNotNone(config.server_config.matrix_data_cache_manager)
+
+        file_name = self.custom_app_config(
+            config_file_name="single_dataset_with_about.yml",
+            about="www.cziscience.com",
+            dataset_datapath=f"{FIXTURES_ROOT}/pbmc3k.cxg",
+        )
+        config = AppConfig()
+        config.update_from_config_file(file_name)
+        with self.assertRaises(ConfigurationError):
+            config.server_config.handle_single_dataset(self.context)
+
+    def test_multi_dataset_raises_error_for_illegal_routes(self):
+        # test for illegal url_dataroots
+        for illegal in ("../b", "!$*", "\\n", "", "(bad)"):
+            self.config.update_server_config(
+                multi_dataset__dataroot={"tag": {"base_url": illegal, "dataroot": "{PROJECT_ROOT}/example-dataset"}}
+            )
+            with self.assertRaises(ConfigurationError):
+                self.config.complete_config()
+
+    def test_multidataset_works_for_legal_routes(self):
+        # test for legal url_dataroots
+        for legal in ("d", "this.is-okay_", "a/b"):
+            self.config.update_server_config(
+                multi_dataset__dataroot={"tag": {"base_url": legal, "dataroot": "{PROJECT_ROOT}/example-dataset"}}
+            )
+            self.config.complete_config()
+
+    def test_mulitdatasets_work_e2e(self):
+        # test that multi dataroots work end to end
+        self.config.update_server_config(
+            multi_dataset__dataroot=dict(
+                s1=dict(dataroot=f"{PROJECT_ROOT}/example-dataset", base_url="set1/1/2"),
+                s2=dict(dataroot=f"{FIXTURES_ROOT}", base_url="set2"),
+                s3=dict(dataroot=f"{FIXTURES_ROOT}", base_url="set3"),
+            )
+        )
+
+        # Change this default to test if the dataroot overrides below work.
+        self.config.update_default_dataset_config(app__about_legal_tos="tos_default.html")
+
+        # specialize the configs for set1
+        self.config.add_dataroot_config(
+            "s1", user_annotations__enable=False, diffexp__enable=True, app__about_legal_tos="tos_set1.html"
+        )
+
+        # specialize the configs for set2
+        self.config.add_dataroot_config(
+            "s2", user_annotations__enable=True, diffexp__enable=False, app__about_legal_tos="tos_set2.html"
+        )
+
+        # no specializations for set3 (they get the default dataset config)
+        self.config.complete_config()
+
+        with test_server(app_config=self.config) as server:
+            session = requests.Session()
+
+            response = session.get(f"{server}/set1/1/2/pbmc3k.h5ad/api/v0.2/config")
+            data_config = response.json()
+            assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
+            assert data_config["config"]["parameters"]["annotations"] is False
+            assert data_config["config"]["parameters"]["disable-diffexp"] is False
+            assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_set1.html"
+
+            response = session.get(f"{server}/set2/pbmc3k.cxg/api/v0.2/config")
+            data_config = response.json()
+            assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
+            assert data_config["config"]["parameters"]["annotations"] is True
+            assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_set2.html"
+
+            response = session.get(f"{server}/set3/pbmc3k.cxg/api/v0.2/config")
+            data_config = response.json()
+            assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
+            assert data_config["config"]["parameters"]["annotations"] is True
+            assert data_config["config"]["parameters"]["disable-diffexp"] is False
+            assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_default.html"
+
+            response = session.get(f"{server}/health")
+            assert response.json()["status"] == "pass"
+
+    @patch("server.common.config.server_config.diffexp_tiledb.set_config")
+    def test_handle_diffexp(self, mock_tiledb_config):
+        custom_config_file = self.custom_app_config(
+            dataroot=f"{FIXTURES_ROOT}",
+            cpu_multiplier=3,
+            diffexp_max_workers=1,
+            target_workunit=4,
+            config_file_name=self.config_file_name,
+        )
+        config = AppConfig()
+        config.update_from_config_file(custom_config_file)
+        config.server_config.handle_diffexp()
+        # called with the min of diffexp_max_workers and cpus*cpu_multiplier
+        mock_tiledb_config.assert_called_once_with(1, 4)
+
+    @patch("server.data_cxg.cxg_adaptor.CxgAdaptor.set_tiledb_context")
+    def test_handle_adaptor(self, mock_tiledb_context):
+        custom_config = self.custom_app_config(
+            dataroot=f"{FIXTURES_ROOT}", cxg_tile_cache_size=10, cxg_num_reader_threads=2
+        )
+        config = AppConfig()
+        config.update_from_config_file(custom_config)
+        config.server_config.handle_adaptor()
+        mock_tiledb_context.assert_called_once_with(
+            {"sm.tile_cache_size": 10, "sm.num_reader_threads": 2, "vfs.s3.region": "us-east-1"}
+        )
+
+    @mockenv(CXG_AWS_SECRET_NAME="TESTING", CXG_AWS_SECRET_REGION_NAME="TEST_REGION")
+    @patch("server.common.config.get_secret_key")
+    def test_get_config_vars_from_aws_secrets(self, mock_get_secret_key):
+        mock_get_secret_key.return_value = {
+            "flask_secret_key": "mock_flask_secret",
+            "oauth_client_secret": "mock_oauth_secret",
+            "db_uri": "mock_db_uri",
+        }
+
+        config = AppConfig()
+
+        with self.assertLogs(level="INFO") as logger:
+            from server.common.config import handle_config_from_secret
+
+            # should not throw error
+            # "AttributeError: 'XConfig' object has no attribute 'x'"
+            handle_config_from_secret(config)
+
+            # should log 3 lines (one for each var set from a secret)
+            self.assertEqual(len(logger.output), 3)
+            self.assertIn("INFO:root:set app__flask_secret_key from secret", logger.output[0])
+            self.assertIn("INFO:root:set authentication__params_oauth__client_secret from secret", logger.output[1])
+            self.assertIn("INFO:root:set user_annotations__hosted_tiledb_array__db_uri from secret", logger.output[2])
+            self.assertEqual(config.server_config.app__flask_secret_key, "mock_flask_secret")
+            self.assertEqual(config.server_config.authentication__params_oauth__client_secret, "mock_oauth_secret")
+            self.assertEqual(config.default_dataset_config.user_annotations__hosted_tiledb_array__db_uri, "mock_db_uri")

server/test/unit/common/test_api.py

@@ -8,8 +8,14 @@ import requests
 
 import server.test.unit.decode_fbs as decode_fbs
 from server.data_common.matrix_loader import MatrixDataType
-from server.test import (data_with_tmp_annotations, make_fbs, PROJECT_ROOT, FIXTURES_ROOT, start_test_server,
-                         stop_test_server)
+from server.test import (
+    data_with_tmp_annotations,
+    make_fbs,
+    PROJECT_ROOT,
+    FIXTURES_ROOT,
+    start_test_server,
+    stop_test_server,
+)
 from server.test.fixtures.fixtures import pbmc3k_colors
 
 BAD_FILTER = {"filter": {"obs": {"annotation_value": [{"name": "xyz"}]}}}
@@ -381,11 +387,14 @@ class EndPointsAnndata(unittest.TestCase, EndPoints):
 
     @classmethod
     def setUpClass(cls):
-        cls._setupClass(cls, [
-            f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad",
-            "--disable-annotations",
-            "--experimental-enable-reembedding",
-        ])
+        cls._setupClass(
+            cls,
+            [
+                f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad",
+                "--disable-annotations",
+                "--experimental-enable-reembedding",
+            ],
+        )
 
     @classmethod
     def tearDownClass(cls):
@@ -403,10 +412,7 @@ class EndPointsCxg(unittest.TestCase, EndPoints):
 
     @classmethod
     def setUpClass(cls):
-        cls._setupClass(cls, [
-            f"{FIXTURES_ROOT}/pbmc3k.cxg",
-            "--disable-annotations",
-        ])
+        cls._setupClass(cls, [f"{FIXTURES_ROOT}/pbmc3k.cxg", "--disable-annotations"])
 
     @classmethod
     def tearDownClass(cls):
@@ -423,7 +429,7 @@ class EndPointsAnndataAnnotations(unittest.TestCase, EndPointsAnnotations):
         cls.data, cls.tmp_dir, cls.annotations = data_with_tmp_annotations(
             MatrixDataType.H5AD, annotations_fixture=True
         )
-        cls._setupClass(cls, ["--annotations-file", cls.annotations.output_file, cls.data.get_location(), ])
+        cls._setupClass(cls, ["--annotations-file", cls.annotations.output_file, cls.data.get_location()])
 
     @classmethod
     def tearDownClass(cls):
@@ -439,11 +445,7 @@ class EndPointsCxgAnnotations(unittest.TestCase, EndPointsAnnotations):
     @classmethod
     def setUpClass(cls):
         cls.data, cls.tmp_dir, cls.annotations = data_with_tmp_annotations(MatrixDataType.CXG, annotations_fixture=True)
-        cls._setupClass(cls, [
-            "--annotations-file",
-            cls.annotations.output_file,
-            cls.data.get_location(),
-        ])
+        cls._setupClass(cls, ["--annotations-file", cls.annotations.output_file, cls.data.get_location()])
 
     @classmethod
     def tearDownClass(cls):

server/test/unit/common/test_app_config.py

@@ -1,249 +0,0 @@
-import os
-import unittest
-from unittest import mock
-from unittest.mock import patch
-import tempfile
-
-import requests
-
-from server.common.app_config import AppConfig
-from server.common.errors import ConfigurationError
-from server.common.utils.utils import find_available_port
-from server.test import PROJECT_ROOT, test_server, FIXTURES_ROOT
-
-
-# NOTE, there are more tests that should be written for AppConfig.
-# this is just a start.
-
-def mockenv(**envvars):
-    return mock.patch.dict(os.environ, envvars)
-
-
-class AppConfigTest(unittest.TestCase):
-    def test_update(self):
-        config = AppConfig()
-        config.update_server_config(app__verbose=True, multi_dataset__dataroot="datadir")
-        vars = config.server_config.changes_from_default()
-        self.assertCountEqual(vars, [("app__verbose", True, False), ("multi_dataset__dataroot", "datadir", None)])
-
-        config = AppConfig()
-        config.update_default_dataset_config(app__scripts=(), app__inline_scripts=())
-        vars = config.server_config.changes_from_default()
-        self.assertCountEqual(vars, [])
-
-        config = AppConfig()
-        config.update_default_dataset_config(app__scripts=[], app__inline_scripts=[])
-        vars = config.default_dataset_config.changes_from_default()
-        self.assertCountEqual(vars, [])
-
-        config = AppConfig()
-        config.update_default_dataset_config(app__scripts=("a", "b"), app__inline_scripts=["c", "d"])
-        vars = config.default_dataset_config.changes_from_default()
-        self.assertCountEqual(vars, [("app__scripts", ["a", "b"], []), ("app__inline_scripts", ["c", "d"], [])])
-
-    def test_multi_dataset(self):
-
-        config = AppConfig()
-        # test for illegal url_dataroots
-        for illegal in ("../b", "!$*", "\\n", "", "(bad)"):
-            config.update_server_config(
-                multi_dataset__dataroot={"tag": {"base_url": illegal, "dataroot": "{PROJECT_ROOT}/example-dataset"}}
-            )
-            with self.assertRaises(ConfigurationError):
-                config.complete_config()
-
-        # test for legal url_dataroots
-        for legal in ("d", "this.is-okay_", "a/b"):
-            config.update_server_config(
-                multi_dataset__dataroot={"tag": {"base_url": legal, "dataroot": "{PROJECT_ROOT}/example-dataset"}}
-            )
-            config.complete_config()
-
-        # test that multi dataroots work end to end
-        config.update_server_config(
-            multi_dataset__dataroot=dict(
-                s1=dict(dataroot=f"{PROJECT_ROOT}/example-dataset", base_url="set1/1/2"),
-                s2=dict(dataroot=f"{FIXTURES_ROOT}", base_url="set2"),
-                s3=dict(dataroot=f"{FIXTURES_ROOT}", base_url="set3"),
-            )
-        )
-
-        # Change this default to test if the dataroot overrides below work.
-        config.update_default_dataset_config(app__about_legal_tos="tos_default.html")
-
-        # specialize the configs for set1
-        config.add_dataroot_config(
-            "s1", user_annotations__enable=False, diffexp__enable=True, app__about_legal_tos="tos_set1.html"
-        )
-
-        # specialize the configs for set2
-        config.add_dataroot_config(
-            "s2", user_annotations__enable=True, diffexp__enable=False, app__about_legal_tos="tos_set2.html"
-        )
-
-        # no specializations for set3 (they get the default dataset config)
-        config.complete_config()
-
-        with test_server(app_config=config) as server:
-            session = requests.Session()
-
-            response = session.get(f"{server}/set1/1/2/pbmc3k.h5ad/api/v0.2/config")
-            data_config = response.json()
-            assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
-            assert data_config["config"]["parameters"]["annotations"] is False
-            assert data_config["config"]["parameters"]["disable-diffexp"] is False
-            assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_set1.html"
-
-            response = session.get(f"{server}/set2/pbmc3k.cxg/api/v0.2/config")
-            data_config = response.json()
-            assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
-            assert data_config["config"]["parameters"]["annotations"] is True
-            assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_set2.html"
-
-            response = session.get(f"{server}/set3/pbmc3k.cxg/api/v0.2/config")
-            data_config = response.json()
-            assert data_config["config"]["displayNames"]["dataset"] == "pbmc3k"
-            assert data_config["config"]["parameters"]["annotations"] is True
-            assert data_config["config"]["parameters"]["disable-diffexp"] is False
-            assert data_config["config"]["parameters"]["about_legal_tos"] == "tos_default.html"
-
-            response = session.get(f"{server}/health")
-            assert response.json()["status"] == "pass"
-
-    @mockenv(CXG_AWS_SECRET_NAME="TESTING", CXG_AWS_SECRET_REGION_NAME="TEST_REGION")
-    @patch('server.common.aws_secret_utils.get_secret_key')
-    def test_get_config_vars_from_aws_secrets(self, mock_get_secret_key):
-        mock_get_secret_key.return_value = {
-            "flask_secret_key": "mock_flask_secret",
-            "oauth_client_secret": "mock_oauth_secret",
-            "db_uri": "mock_db_uri"
-        }
-
-        config = AppConfig()
-
-        with self.assertLogs(level="INFO") as logger:
-            from server.common.aws_secret_utils import handle_config_from_secret
-            # should not throw error
-            # "AttributeError: 'XConfig' object has no attribute 'x'"
-            handle_config_from_secret(config)
-
-            # should log 3 lines (one for each var set from a secret)
-            self.assertEqual(len(logger.output), 3)
-            self.assertIn('INFO:root:set app__flask_secret_key from secret', logger.output[0])
-            self.assertIn('INFO:root:set authentication__params_oauth__client_secret from secret', logger.output[1])
-            self.assertIn('INFO:root:set user_annotations__hosted_tiledb_array__db_uri from secret', logger.output[2])
-            self.assertEqual(config.server_config.app__flask_secret_key, "mock_flask_secret")
-            self.assertEqual(config.server_config.authentication__params_oauth__client_secret, "mock_oauth_secret")
-            self.assertEqual(config.default_dataset_config.user_annotations__hosted_tiledb_array__db_uri, "mock_db_uri")
-
-    def test_api_base_url(self):
-
-        # test the api_base_url feature, and that it can contain a path
-        config = AppConfig()
-        backend_port = find_available_port("localhost", 10000)
-        config.update_server_config(
-            app__api_base_url=f"http://localhost:{backend_port}/additional/path",
-            multi_dataset__dataroot=f"{PROJECT_ROOT}/example-dataset"
-        )
-
-        config.complete_config()
-
-        with test_server(["-p", str(backend_port)], app_config=config) as server:
-            session = requests.Session()
-            self.assertEqual(server, f"http://localhost:{backend_port}")
-            response = session.get(f"{server}/additional/path/d/pbmc3k.h5ad/api/v0.2/config")
-            self.assertEqual(response.status_code, 200)
-            data_config = response.json()
-            self.assertEqual(data_config["config"]["displayNames"]["dataset"], "pbmc3k")
-
-            # test the health check at the correct url
-            response = session.get(f"{server}/additional/path/health")
-            assert response.json()["status"] == "pass"
-
-            # also check that the old URL still works.
-            # NOTE:  this old URL location will soon be deprecated, and when that happens
-            # this check can be removed.
-            response = session.get(f"{server}/health")
-            assert response.json()["status"] == "pass"
-
-    def test_configfile_with_specialization(self):
-        # test that per_dataset_config config load the default config, then the specialized config
-
-        with tempfile.TemporaryDirectory() as tempdir:
-            configfile = os.path.join(tempdir, "config.yaml")
-            with open(configfile, "w") as fconfig:
-                config = """
-                server:
-                    multi_dataset:
-                        dataroot:
-                            test:
-                                base_url: test
-                                dataroot: fake_dataroot
-
-                dataset:
-                    user_annotations:
-                        enable: false
-                        type: hosted_tiledb_array
-                        hosted_tiledb_array:
-                            db_uri: fake_db_uri
-                            hosted_file_directory: fake_dir
-
-                per_dataset_config:
-                    test:
-                        user_annotations:
-                            enable: true
-                """
-                fconfig.write(config)
-
-            app_config = AppConfig()
-            app_config.update_from_config_file(configfile)
-
-            test_config = app_config.dataroot_config["test"]
-
-            # test config from default
-            self.assertEqual(test_config.user_annotations__type, "hosted_tiledb_array")
-            self.assertEqual(test_config.user_annotations__hosted_tiledb_array__db_uri, "fake_db_uri")
-
-            # test config from specialization
-            self.assertTrue(test_config.user_annotations__enable)
-
-    def test_configfile_no_dataset_section(self):
-        # test a config file without a dataset section
-
-        with tempfile.TemporaryDirectory() as tempdir:
-            configfile = os.path.join(tempdir, "config.yaml")
-            with open(configfile, "w") as fconfig:
-                config = """
-                server:
-                    multi_dataset:
-                        dataroot: test_dataroot
-
-                """
-                fconfig.write(config)
-
-            app_config = AppConfig()
-            app_config.update_from_config_file(configfile)
-            server_changes = app_config.server_config.changes_from_default()
-            dataset_changes = app_config.default_dataset_config.changes_from_default()
-            self.assertEqual(server_changes, [("multi_dataset__dataroot", "test_dataroot", None)])
-            self.assertEqual(dataset_changes, [])
-
-    def test_configfile_no_server_section(self):
-        # test a config file without a dataset section
-
-        with tempfile.TemporaryDirectory() as tempdir:
-            configfile = os.path.join(tempdir, "config.yaml")
-            with open(configfile, "w") as fconfig:
-                config = """
-                dataset:
-                    user_annotations:
-                        enable: false
-                """
-                fconfig.write(config)
-
-            app_config = AppConfig()
-            app_config.update_from_config_file(configfile)
-            server_changes = app_config.server_config.changes_from_default()
-            dataset_changes = app_config.default_dataset_config.changes_from_default()
-            self.assertEqual(server_changes, [])
-            self.assertEqual(dataset_changes, [("user_annotations__enable", False, True)])

server/test/unit/common/test_corpora.py

@@ -87,24 +87,17 @@ class CorporaRESTAPITest(unittest.TestCase):
     def setCorporaFields(cls, path):
         adata = anndata.read_h5ad(path)
         corpora_props = {
-            "version": {
-                "corpora_schema_version": "1.0.0",
-                "corpora_encoding_version": "0.1.0"
-            },
+            "version": {"corpora_schema_version": "1.0.0", "corpora_encoding_version": "0.1.0"},
             "title": "PBMC3K",
-            "contributors": json.dumps([
-                {"name": "name"}
-            ]),
-            "layer_descriptions": {
-                "X": "raw counts"
-            },
+            "contributors": json.dumps([{"name": "name"}]),
+            "layer_descriptions": {"X": "raw counts"},
             "organism": "human",
             "organism_ontology_term_id": "unknown",
             "project_name": "test project",
             "project_description": "test description",
-            "project_links": json.dumps([
-                {"link_name": "test link", "link_type": "SUMMARY", "link_url": "https://a.u.r.l/"}
-            ]),
+            "project_links": json.dumps(
+                [{"link_name": "test link", "link_type": "SUMMARY", "link_url": "https://a.u.r.l/"}]
+            ),
             "default_embedding": "X_tsne",
         }
         adata.uns.update(corpora_props)

server/test/unit/common/test_writable_annotation.py

@@ -27,7 +27,7 @@ class auth(object):
 
 class WritableTileDBStoredAnnotationTest(unittest.TestCase):
     def setUp(self):
-        self.user_id = '1234'
+        self.user_id = "1234"
         self.data, self.tmp_dir, self.annotations = data_with_tmp_tiledb_annotations(MatrixDataType.H5AD)
         self.data.dataset_config.user_annotations = self.annotations
         self.db = self.annotations.db
@@ -38,7 +38,7 @@ class WritableTileDBStoredAnnotationTest(unittest.TestCase):
         }
         self.fbs = make_fbs(self.test_dict)
         self.df = pd.DataFrame(self.test_dict)
-        self.app = Flask('fake_app')
+        self.app = Flask("fake_app")
         self.app.__setattr__("auth", auth)
 
     def tearDown(self):
@@ -65,8 +65,7 @@ class WritableTileDBStoredAnnotationTest(unittest.TestCase):
             self.annotations.write_labels(self.df, self.data)
             dataset_id = self.db.query([CellxGeneDataset], [CellxGeneDataset.name == self.data.get_location()])[0].id
             annotation = self.db.query_for_most_recent(
-                Annotation,
-                [Annotation.user_id == self.user_id, Annotation.dataset_id == str(dataset_id)]
+                Annotation, [Annotation.user_id == self.user_id, Annotation.dataset_id == str(dataset_id)]
             )
             # retrieve tiledb array
             df = tiledb.open(annotation.tiledb_uri)
@@ -78,7 +77,7 @@ class WritableTileDBStoredAnnotationTest(unittest.TestCase):
 
     def test_write_labels_creates_a_dataset_if_it_doesnt_exist(self):
         with self.app.test_request_context():
-            new_name = 'new_dataset/location'
+            new_name = "new_dataset/location"
             self.data.get_location = MagicMock(return_value=new_name)
             num_datasets = len(self.db.query([CellxGeneDataset]))
             self.annotation_put_fbs(self.fbs)
@@ -130,15 +129,14 @@ class WritableTileDBStoredAnnotationTest(unittest.TestCase):
             with self.assertRaises(KeyError):
                 self.annotation_put_fbs(fbs_bad)
 
-    @patch('server.common.annotations.hosted_tiledb.current_app')
+    @patch("server.common.annotations.hosted_tiledb.current_app")
     def test_write_labels_stores_df_as_tiledb_array(self, mock_user_id):
-        mock_user_id.auth.get_user_id.return_value = '1234'
+        mock_user_id.auth.get_user_id.return_value = "1234"
         self.annotations.write_labels(self.df, self.data)
         # get uri
         dataset_id = self.db.query([CellxGeneDataset], [CellxGeneDataset.name == self.data.get_location()])[0].id
         annotation = self.db.query_for_most_recent(
-            Annotation,
-            [Annotation.user_id == '1234', Annotation.dataset_id == str(dataset_id)]
+            Annotation, [Annotation.user_id == "1234", Annotation.dataset_id == str(dataset_id)]
         )
 
         df = tiledb.open(annotation.tiledb_uri)

server/test/unit/common/utils/test_cxg_generation_utils.py

@@ -8,8 +8,12 @@ import numpy as np
 import tiledb
 from pandas import Series, DataFrame
 
-from server.common.utils.cxg_generation_utils import (convert_dictionary_to_cxg_group, convert_dataframe_to_cxg_array,
-                                                      convert_ndarray_to_cxg_dense_array, convert_matrix_to_cxg_array)
+from server.common.utils.cxg_generation_utils import (
+    convert_dictionary_to_cxg_group,
+    convert_dataframe_to_cxg_array,
+    convert_ndarray_to_cxg_dense_array,
+    convert_matrix_to_cxg_array,
+)
 
 PROJECT_ROOT = popen("git rev-parse --show-toplevel").read().strip()
 
@@ -28,8 +32,9 @@ class TestCxgGenerationUtils(unittest.TestCase):
         dictionary_name = "favorite_desserts"
         expected_array_directory = f"{self.testing_cxg_temp_directory}/{dictionary_name}"
 
-        convert_dictionary_to_cxg_group(self.testing_cxg_temp_directory, random_dictionary,
-                                        group_metadata_name=dictionary_name)
+        convert_dictionary_to_cxg_group(
+            self.testing_cxg_temp_directory, random_dictionary, group_metadata_name=dictionary_name
+        )
 
         array = tiledb.open(expected_array_directory)
         actual_stored_metadata = dict(array.meta.items())
@@ -44,13 +49,16 @@ class TestCxgGenerationUtils(unittest.TestCase):
         random_dataframe_name = f"random_dataframe_{uuid4()}"
         random_dataframe = DataFrame(data={"int_category": random_int_category, "bool_category": random_bool_category})
 
-        convert_dataframe_to_cxg_array(self.testing_cxg_temp_directory, random_dataframe_name, random_dataframe,
-                                       "int_category", tiledb.Ctx())
+        convert_dataframe_to_cxg_array(
+            self.testing_cxg_temp_directory, random_dataframe_name, random_dataframe, "int_category", tiledb.Ctx()
+        )
 
         expected_array_directory = f"{self.testing_cxg_temp_directory}/{random_dataframe_name}"
         expected_array_metadata = {
-            "cxg_schema": json.dumps({"int_category": {"type": "int32"}, "bool_category": {"type": "boolean"},
-                                      "index": "int_category"})}
+            "cxg_schema": json.dumps(
+                {"int_category": {"type": "int32"}, "bool_category": {"type": "boolean"}, "index": "int_category"}
+            )
+        }
 
         actual_stored_dataframe_array = tiledb.open(expected_array_directory)
         actual_stored_dataframe_metadata = dict(actual_stored_dataframe_array.meta.items())
@@ -95,7 +103,7 @@ class TestCxgGenerationUtils(unittest.TestCase):
 
         self.assertTrue(path.isdir(matrix_name))
         self.assertTrue(isinstance(actual_stored_array, tiledb.SparseArray))
-        self.assertTrue(actual_stored_array[:, :][''].size == 0)
+        self.assertTrue(actual_stored_array[:, :][""].size == 0)
 
     def test__convert_matrix_to_cxg_array__sparse_array_only_store_nonzeros(self):
         matrix = np.zeros([3, 3])
@@ -110,10 +118,10 @@ class TestCxgGenerationUtils(unittest.TestCase):
 
         self.assertTrue(path.isdir(matrix_name))
         self.assertTrue(isinstance(actual_stored_array, tiledb.SparseArray))
-        self.assertTrue(actual_stored_array[0, 0][''] == 1)
-        self.assertTrue(actual_stored_array[1, 1][''] == 1)
-        self.assertTrue(actual_stored_array[2, 2][''] == 2)
-        self.assertTrue(actual_stored_array[:, :][''].size == 3)
+        self.assertTrue(actual_stored_array[0, 0][""] == 1)
+        self.assertTrue(actual_stored_array[1, 1][""] == 1)
+        self.assertTrue(actual_stored_array[2, 2][""] == 2)
+        self.assertTrue(actual_stored_array[:, :][""].size == 3)
 
     def test__convert_matrix_to_cxg_array__sparse_array_with_column_encoding_empty_array(self):
         matrix_name = f"{self.testing_cxg_temp_directory}/awesome_column_shift_matrix_{uuid4()}"
@@ -122,14 +130,15 @@ class TestCxgGenerationUtils(unittest.TestCase):
         # a matrix of zeros which is sparse.
         column_shift = np.ones((3, 2))
 
-        convert_matrix_to_cxg_array(matrix_name, matrix, True, tiledb.Ctx(),
-                                    column_shift_for_sparse_encoding=column_shift)
+        convert_matrix_to_cxg_array(
+            matrix_name, matrix, True, tiledb.Ctx(), column_shift_for_sparse_encoding=column_shift
+        )
 
         actual_stored_array = tiledb.open(matrix_name)
 
         self.assertTrue(path.isdir(matrix_name))
         self.assertTrue(isinstance(actual_stored_array, tiledb.SparseArray))
-        self.assertTrue(actual_stored_array[:, :][''].size == 0)
+        self.assertTrue(actual_stored_array[:, :][""].size == 0)
 
     def test__convert_matrix_to_cxg_array__sparse_array_with_column_encoding_partial_array(self):
         matrix_name = f"{self.testing_cxg_temp_directory}/awesome_column_shift_matrix_{uuid4()}"
@@ -137,13 +146,14 @@ class TestCxgGenerationUtils(unittest.TestCase):
         # Only column shift the first column of ones.
         column_shift = np.array([[1, 0], [1, 0]])
 
-        convert_matrix_to_cxg_array(matrix_name, matrix, True, tiledb.Ctx(),
-                                    column_shift_for_sparse_encoding=column_shift)
+        convert_matrix_to_cxg_array(
+            matrix_name, matrix, True, tiledb.Ctx(), column_shift_for_sparse_encoding=column_shift
+        )
 
         actual_stored_array = tiledb.open(matrix_name)
 
         self.assertTrue(path.isdir(matrix_name))
         self.assertTrue(isinstance(actual_stored_array, tiledb.SparseArray))
-        self.assertTrue(actual_stored_array[0, 1][''] == 1)
-        self.assertTrue(actual_stored_array[1, 1][''] == 1)
-        self.assertTrue(actual_stored_array[:, :][''].size == 2)
+        self.assertTrue(actual_stored_array[0, 1][""] == 1)
+        self.assertTrue(actual_stored_array[1, 1][""] == 1)
+        self.assertTrue(actual_stored_array[:, :][""].size == 2)

server/test/unit/common/utils/test_matrix_utils.py

@@ -6,7 +6,6 @@ from server.common.utils.matrix_utils import is_matrix_sparse, get_column_shift_
 
 
 class TestMatrixUtils(unittest.TestCase):
-
     def test__is_matrix_sparse__zero_and_one_hundred_percent_threshold(self):
         matrix = np.array([1, 2, 3])
 

server/test/unit/common/utils/test_sanitization_utils.py

@@ -4,7 +4,6 @@ from server.common.utils.sanitization_utils import sanitize_values_in_list, sani
 
 
 class TestSanitizationUtils(unittest.TestCase):
-
     def test__sanitize_values_in_list__not_strings_raises_exception(self):
         keys_to_sanitize = [1, 2, 3]
 

server/test/unit/common/utils/test_type_conversion_utils.py

@@ -5,12 +5,17 @@ from unittest.mock import patch
 import numpy as np
 from pandas import Series, DataFrame
 
-from server.common.utils.type_conversion_utils import can_cast_to_float32, can_cast_to_int32, get_dtype_of_array, \
-    get_schema_type_hint_of_array, get_dtypes_and_schemas_of_dataframe, convert_pandas_series_to_numpy
+from server.common.utils.type_conversion_utils import (
+    can_cast_to_float32,
+    can_cast_to_int32,
+    get_dtype_of_array,
+    get_schema_type_hint_of_array,
+    get_dtypes_and_schemas_of_dataframe,
+    convert_pandas_series_to_numpy,
+)
 
 
 class TestTypeConversionUtils(unittest.TestCase):
-
     def test__can_cast_to_float32__string_is_false(self):
         array_to_convert = Series(data=["1", "2", "3"], dtype=str)
 
@@ -97,8 +102,9 @@ class TestTypeConversionUtils(unittest.TestCase):
         expected_dtypes = [np.float32, np.int32, np.uint8, np.unicode]
 
         for test_type_index in range(len(types)):
-            with self.subTest(f"Testing get_dtype_of_array with type {types[test_type_index].__name__}",
-                              i=test_type_index):
+            with self.subTest(
+                f"Testing get_dtype_of_array with type {types[test_type_index].__name__}", i=test_type_index
+            ):
                 array = Series(data=[], dtype=types[test_type_index])
                 self.assertEqual(get_dtype_of_array(array), expected_dtypes[test_type_index])
 
@@ -123,8 +129,9 @@ class TestTypeConversionUtils(unittest.TestCase):
         expected_dtypes = [np.float32, np.int32]
 
         for test_type_index in range(len(types)):
-            with self.subTest(f"Testing get_dtype_of_array with castable type {types[test_type_index].__name__}",
-                              i=test_type_index):
+            with self.subTest(
+                f"Testing get_dtype_of_array with castable type {types[test_type_index].__name__}", i=test_type_index
+            ):
                 array = Series(data=[], dtype=types[test_type_index])
                 self.assertEqual(get_dtype_of_array(array), expected_dtypes[test_type_index])
 
@@ -141,8 +148,9 @@ class TestTypeConversionUtils(unittest.TestCase):
         expected_schema_hints = [{"type": "float32"}, {"type": "int32"}, {"type": "boolean"}, {"type": "string"}]
 
         for test_type_index in range(len(types)):
-            with self.subTest(f"Testing get_schema_type_hint_of_array with type {types[test_type_index].__name__}",
-                              i=test_type_index):
+            with self.subTest(
+                f"Testing get_schema_type_hint_of_array with type {types[test_type_index].__name__}", i=test_type_index
+            ):
                 array = Series(data=[], dtype=types[test_type_index])
                 self.assertEqual(get_schema_type_hint_of_array(array), expected_schema_hints[test_type_index])
 
@@ -160,8 +168,9 @@ class TestTypeConversionUtils(unittest.TestCase):
 
         for test_type_index in range(len(types)):
             with self.subTest(
-                    f"Testing get_schema_type_hint_of_array with castable type {types[test_type_index].__name__}",
-                    i=test_type_index):
+                f"Testing get_schema_type_hint_of_array with castable type {types[test_type_index].__name__}",
+                i=test_type_index,
+            ):
                 array = Series(data=[], dtype=types[test_type_index])
                 self.assertEqual(get_schema_type_hint_of_array(array), expected_schema_hints[test_type_index])
 
@@ -171,8 +180,10 @@ class TestTypeConversionUtils(unittest.TestCase):
         dataframe = DataFrame({"float_array": float_array, "category_array": category_array})
 
         expected_data_types_dict = {"float_array": np.float32, "category_array": np.unicode}
-        expected_schema_type_hints_dict = {"float_array": {"type": "float32"},
-                                           "category_array": {"type": "categorical", "categories": ["a", "b"]}}
+        expected_schema_type_hints_dict = {
+            "float_array": {"type": "float32"},
+            "category_array": {"type": "categorical", "categories": ["a", "b"]},
+        }
 
         actual_dataframe_data_types, actual_dataframe_schema_type_hints = get_dtypes_and_schemas_of_dataframe(dataframe)
 
@@ -201,5 +212,6 @@ class TestTypeConversionUtils(unittest.TestCase):
         with self.assertLogs(level="ERROR") as logger:
             convert_pandas_series_to_numpy(int_series, np.int32)
 
-            self.assertIn("Cannot convert a pandas Series object to an integer dtype if it contains NaNs",
-                          logger.output[0])
+            self.assertIn(
+                "Cannot convert a pandas Series object to an integer dtype if it contains NaNs", logger.output[0]
+            )