mirror of
https://github.com/chanzuckerberg/cellxgene.git
synced 2026-10-04 09:38:11 +08:00
Split out the local backend (#2052)
This splits the backend into two parts: the local backend for desktop cellxgene and the AWS backend for hosted cellxgene. The local backend is in local_server while the hosted remains in server. The general idea is to copy everything from server to local_server, pull unneeded stuff out of local_server, and keep server as-is for this PR. Not touching server means all the infra and deployment code will continue working just as it did before so we can make those changes incrementally.
This commit is contained in:
Marcus Kinsella
@@ -0,0 +1,222 @@
|
||||
import os
|
||||
import shutil
|
||||
import unittest
|
||||
import random
|
||||
from unittest import mock
|
||||
import yaml
|
||||
|
||||
from local_server.test import FIXTURES_ROOT
|
||||
|
||||
|
||||
def mockenv(**envvars):
|
||||
return mock.patch.dict(os.environ, envvars)
|
||||
|
||||
|
||||
class ConfigTests(unittest.TestCase):
|
||||
tmp_fixtures_directory = os.path.join(FIXTURES_ROOT, "tmp_dir")
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls) -> None:
|
||||
shutil.rmtree(cls.tmp_fixtures_directory)
|
||||
|
||||
@classmethod
|
||||
def setUpClass(cls) -> None:
|
||||
os.makedirs(cls.tmp_fixtures_directory)
|
||||
|
||||
def custom_server_config(
|
||||
self,
|
||||
verbose="false",
|
||||
debug="false",
|
||||
host="localhost",
|
||||
port="null",
|
||||
open_browser="false",
|
||||
force_https="false",
|
||||
flask_secret_key="secret",
|
||||
auth_type="session",
|
||||
insecure_test_environment="false",
|
||||
index="false",
|
||||
allowed_matrix_types=[],
|
||||
max_cached_datasets=5,
|
||||
timelimit_s=5,
|
||||
dataset_datapath="null",
|
||||
obs_names="null",
|
||||
var_names="null",
|
||||
about="null",
|
||||
title="null",
|
||||
data_locater_region_name="us-east-1",
|
||||
anndata_backed="false",
|
||||
column_request_max=32,
|
||||
diffexp_cellcount_max="null",
|
||||
config_file_name="server_config.yaml",
|
||||
):
|
||||
configfile = os.path.join(self.tmp_fixtures_directory, config_file_name)
|
||||
server_config_outline_path = os.path.join(FIXTURES_ROOT, "server_config_outline.py")
|
||||
with open(server_config_outline_path, "r") as config_skeleton:
|
||||
config = config_skeleton.read()
|
||||
server_config = eval(config)
|
||||
with open(configfile, "w") as server_config_file:
|
||||
server_config_file.write(server_config)
|
||||
return configfile
|
||||
|
||||
def custom_app_config(
|
||||
self,
|
||||
verbose="false",
|
||||
debug="false",
|
||||
host="localhost",
|
||||
port="null",
|
||||
open_browser="false",
|
||||
force_https="false",
|
||||
flask_secret_key="secret",
|
||||
auth_type="session",
|
||||
index="false",
|
||||
allowed_matrix_types=[],
|
||||
max_cached_datasets=5,
|
||||
timelimit_s=5,
|
||||
dataset_datapath="null",
|
||||
obs_names="null",
|
||||
var_names="null",
|
||||
about="null",
|
||||
title="null",
|
||||
data_locater_region_name="us-east-1",
|
||||
anndata_backed="false",
|
||||
column_request_max=32,
|
||||
diffexp_cellcount_max="null",
|
||||
scripts=[],
|
||||
inline_scripts=[],
|
||||
authentication_enable="true",
|
||||
max_categories=1000,
|
||||
custom_colors="true",
|
||||
enable_users_annotations="true",
|
||||
annotation_type="local_file_csv",
|
||||
db_uri="null",
|
||||
hosted_file_directory="null",
|
||||
local_file_csv_directory="null",
|
||||
local_file_csv_file="null",
|
||||
ontology_enabled="false",
|
||||
obo_location="null",
|
||||
embedding_names=[],
|
||||
enable_reembedding="false",
|
||||
enable_difexp="true",
|
||||
lfc_cutoff=0.01,
|
||||
top_n=10,
|
||||
environment=None,
|
||||
aws_secrets_manager_region=None,
|
||||
aws_secrets_manager_secrets=[],
|
||||
config_file_name="app_config.yml",
|
||||
):
|
||||
random_num = random.randrange(999999)
|
||||
configfile = os.path.join(self.tmp_fixtures_directory, config_file_name)
|
||||
server_config = self.custom_server_config(
|
||||
verbose=verbose,
|
||||
debug=debug,
|
||||
host=host,
|
||||
port=port,
|
||||
open_browser=open_browser,
|
||||
force_https=force_https,
|
||||
flask_secret_key=flask_secret_key,
|
||||
auth_type=auth_type,
|
||||
index=index,
|
||||
allowed_matrix_types=allowed_matrix_types,
|
||||
max_cached_datasets=max_cached_datasets,
|
||||
timelimit_s=timelimit_s,
|
||||
dataset_datapath=dataset_datapath,
|
||||
obs_names=obs_names,
|
||||
var_names=var_names,
|
||||
about=about,
|
||||
title=title,
|
||||
data_locater_region_name=data_locater_region_name,
|
||||
anndata_backed=anndata_backed,
|
||||
column_request_max=column_request_max,
|
||||
diffexp_cellcount_max=diffexp_cellcount_max,
|
||||
config_file_name=f"temp_server_config_{random_num}.yml",
|
||||
)
|
||||
dataset_config = self.custom_dataset_config(
|
||||
scripts=scripts,
|
||||
inline_scripts=inline_scripts,
|
||||
authentication_enable=authentication_enable,
|
||||
max_categories=max_categories,
|
||||
custom_colors=custom_colors,
|
||||
enable_users_annotations=enable_users_annotations,
|
||||
annotation_type=annotation_type,
|
||||
db_uri=db_uri,
|
||||
hosted_file_directory=hosted_file_directory,
|
||||
local_file_csv_directory=local_file_csv_directory,
|
||||
local_file_csv_file=local_file_csv_file,
|
||||
ontology_enabled=ontology_enabled,
|
||||
obo_location=obo_location,
|
||||
embedding_names=embedding_names,
|
||||
enable_reembedding=enable_reembedding,
|
||||
enable_difexp=enable_difexp,
|
||||
lfc_cutoff=lfc_cutoff,
|
||||
top_n=top_n,
|
||||
config_file_name=f"temp_dataset_config_{random_num}.yml",
|
||||
)
|
||||
external_config = self.custom_external_config(
|
||||
environment=environment,
|
||||
aws_secrets_manager_region=aws_secrets_manager_region,
|
||||
aws_secrets_manager_secrets=aws_secrets_manager_secrets,
|
||||
config_file_name=f"temp_external_config_{random_num}.yml",
|
||||
)
|
||||
|
||||
with open(configfile, "w") as app_config_file:
|
||||
app_config_file.write(open(server_config).read())
|
||||
app_config_file.write(open(dataset_config).read())
|
||||
app_config_file.write(open(external_config).read())
|
||||
|
||||
return configfile
|
||||
|
||||
def custom_dataset_config(
|
||||
self,
|
||||
scripts=[],
|
||||
inline_scripts=[],
|
||||
authentication_enable="true",
|
||||
max_categories=1000,
|
||||
custom_colors="true",
|
||||
enable_users_annotations="true",
|
||||
annotation_type="local_file_csv",
|
||||
db_uri="null",
|
||||
hosted_file_directory="null",
|
||||
local_file_csv_directory="null",
|
||||
local_file_csv_file="null",
|
||||
ontology_enabled="false",
|
||||
obo_location="null",
|
||||
embedding_names=[],
|
||||
enable_reembedding="false",
|
||||
enable_difexp="true",
|
||||
lfc_cutoff=0.01,
|
||||
top_n=10,
|
||||
config_file_name="dataset_config.yml",
|
||||
):
|
||||
configfile = os.path.join(self.tmp_fixtures_directory, config_file_name)
|
||||
dataset_config_outline_path = os.path.join(FIXTURES_ROOT, "dataset_config_outline.py")
|
||||
with open(dataset_config_outline_path, "r") as config_skeleton:
|
||||
config = config_skeleton.read()
|
||||
dataset_config = eval(config)
|
||||
with open(configfile, "w") as dataset_config_file:
|
||||
dataset_config_file.write(dataset_config)
|
||||
|
||||
return configfile
|
||||
|
||||
def custom_external_config(
|
||||
self,
|
||||
environment=None,
|
||||
aws_secrets_manager_region=None,
|
||||
aws_secrets_manager_secrets=[],
|
||||
config_file_name="external_config.yaml",
|
||||
):
|
||||
# set to the default if environment is None
|
||||
if environment is None:
|
||||
environment = [
|
||||
dict(name="CXG_SECRET_KEY", path=["server", "app", "flask_secret_key"], required=False),
|
||||
]
|
||||
external_config = {
|
||||
"external": {
|
||||
"environment": environment,
|
||||
"aws_secrets_manager": {"region": aws_secrets_manager_region, "secrets": aws_secrets_manager_secrets},
|
||||
}
|
||||
}
|
||||
|
||||
configfile = os.path.join(self.tmp_fixtures_directory, config_file_name)
|
||||
with open(configfile, "w") as external_config_file:
|
||||
yaml.dump(external_config, external_config_file)
|
||||
return configfile
|
||||
@@ -0,0 +1,153 @@
|
||||
import os
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
import yaml
|
||||
|
||||
from local_server.default_config import default_config
|
||||
from local_server.common.config.app_config import AppConfig
|
||||
from local_server.test.unit.common.config import ConfigTests
|
||||
from local_server.common.errors import ConfigurationError
|
||||
from local_server.test import FIXTURES_ROOT, H5AD_FIXTURE
|
||||
|
||||
|
||||
class AppConfigTest(ConfigTests):
|
||||
def setUp(self):
|
||||
self.config_file_name = f"{unittest.TestCase.id(self).split('.')[-1]}.yml"
|
||||
self.config = AppConfig()
|
||||
self.config.update_server_config(app__flask_secret_key="secret")
|
||||
self.config.update_server_config(single_dataset__datapath=H5AD_FIXTURE)
|
||||
self.server_config = self.config.server_config
|
||||
self.config.complete_config()
|
||||
|
||||
message_list = []
|
||||
|
||||
def noop(message):
|
||||
message_list.append(message)
|
||||
|
||||
messagefn = noop
|
||||
self.context = dict(messagefn=messagefn, messages=message_list)
|
||||
|
||||
def get_config(self, **kwargs):
|
||||
file_name = self.custom_app_config(
|
||||
dataset_datapath=H5AD_FIXTURE, config_file_name=self.config_file_name, **kwargs
|
||||
)
|
||||
config = AppConfig()
|
||||
config.update_from_config_file(file_name)
|
||||
return config
|
||||
|
||||
def test_get_default_config_correctly_reads_default_config_file(self):
|
||||
app_default_config = AppConfig().default_config
|
||||
|
||||
expected_config = yaml.load(default_config, Loader=yaml.Loader)
|
||||
|
||||
server_config = app_default_config["server"]
|
||||
dataset_config = app_default_config["dataset"]
|
||||
|
||||
expected_server_config = expected_config["server"]
|
||||
expected_dataset_config = expected_config["dataset"]
|
||||
|
||||
self.assertDictEqual(app_default_config, expected_config)
|
||||
self.assertDictEqual(server_config, expected_server_config)
|
||||
self.assertDictEqual(dataset_config, expected_dataset_config)
|
||||
|
||||
def test_get_dataset_config_returns_dataset_config_for_single_datasets(self):
|
||||
datapath = f"{FIXTURES_ROOT}/1e4dfec4-c0b2-46ad-a04e-ff3ffb3c0a8f.h5ad"
|
||||
file_name = self.custom_app_config(dataset_datapath=datapath, config_file_name=self.config_file_name)
|
||||
config = AppConfig()
|
||||
config.update_from_config_file(file_name)
|
||||
|
||||
self.assertEqual(config.get_dataset_config(), config.dataset_config)
|
||||
|
||||
def test_update_server_config_updates_server_config_and_config_status(self):
|
||||
config = self.get_config()
|
||||
config.complete_config()
|
||||
config.check_config()
|
||||
config.update_server_config(single_dataset__datapath=H5AD_FIXTURE)
|
||||
with self.assertRaises(ConfigurationError):
|
||||
config.server_config.check_config()
|
||||
|
||||
def test_write_config_outputs_yaml_with_all_config_vars(self):
|
||||
config = self.get_config()
|
||||
config.write_config(f"{FIXTURES_ROOT}/tmp_dir/write_config.yml")
|
||||
with open(f"{FIXTURES_ROOT}/tmp_dir/{self.config_file_name}", "r") as default_config:
|
||||
default_config_yml = yaml.safe_load(default_config)
|
||||
|
||||
with open(f"{FIXTURES_ROOT}/tmp_dir/write_config.yml", "r") as output_config:
|
||||
output_config_yml = yaml.safe_load(output_config)
|
||||
self.maxDiff = None
|
||||
self.assertEqual(default_config_yml, output_config_yml)
|
||||
|
||||
def test_update_app_config(self):
|
||||
config = AppConfig()
|
||||
config.update_server_config(app__verbose=True, single_dataset__datapath="datapath")
|
||||
vars = config.server_config.changes_from_default()
|
||||
self.assertCountEqual(vars, [("app__verbose", True, False), ("single_dataset__datapath", "datapath", None)])
|
||||
|
||||
config = AppConfig()
|
||||
config.update_dataset_config(app__scripts=(), app__inline_scripts=())
|
||||
vars = config.server_config.changes_from_default()
|
||||
self.assertCountEqual(vars, [])
|
||||
|
||||
config = AppConfig()
|
||||
config.update_dataset_config(app__scripts=[], app__inline_scripts=[])
|
||||
vars = config.dataset_config.changes_from_default()
|
||||
self.assertCountEqual(vars, [])
|
||||
|
||||
config = AppConfig()
|
||||
config.update_dataset_config(app__scripts=("a", "b"), app__inline_scripts=["c", "d"])
|
||||
vars = config.dataset_config.changes_from_default()
|
||||
self.assertCountEqual(vars, [("app__scripts", ["a", "b"], []), ("app__inline_scripts", ["c", "d"], [])])
|
||||
|
||||
def test_configfile_no_server_section(self):
|
||||
# test a config file without a dataset section
|
||||
|
||||
with tempfile.TemporaryDirectory() as tempdir:
|
||||
configfile = os.path.join(tempdir, "config.yaml")
|
||||
with open(configfile, "w") as fconfig:
|
||||
config = """
|
||||
dataset:
|
||||
user_annotations:
|
||||
enable: false
|
||||
"""
|
||||
fconfig.write(config)
|
||||
|
||||
app_config = AppConfig()
|
||||
app_config.update_from_config_file(configfile)
|
||||
server_changes = app_config.server_config.changes_from_default()
|
||||
dataset_changes = app_config.dataset_config.changes_from_default()
|
||||
self.assertEqual(server_changes, [])
|
||||
self.assertEqual(dataset_changes, [("user_annotations__enable", False, True)])
|
||||
|
||||
def test_simple_update_single_config_from_path_and_value(self):
|
||||
"""Update a simple config parameter"""
|
||||
|
||||
config = AppConfig()
|
||||
config.server_config.single_dataset__datapath = "my/data/path"
|
||||
|
||||
# test simple value in server
|
||||
config.update_single_config_from_path_and_value(["server", "app", "flask_secret_key"], "mysecret")
|
||||
self.assertEqual(config.server_config.app__flask_secret_key, "mysecret")
|
||||
|
||||
# test simple value in default dataset
|
||||
config.update_single_config_from_path_and_value(
|
||||
["dataset", "user_annotations", "ontology", "obo_location"], "dummy_location",
|
||||
)
|
||||
self.assertEqual(config.dataset_config.user_annotations__ontology__obo_location, "dummy_location")
|
||||
|
||||
# error checking
|
||||
bad_paths = [
|
||||
(
|
||||
["dataset", "does", "not", "exist"],
|
||||
"unknown config parameter at path: '['dataset', 'does', 'not', 'exist']'",
|
||||
),
|
||||
(["does", "not", "exist"], "path must start with 'server', or 'dataset'"),
|
||||
([], "path must start with 'server', or 'dataset'"),
|
||||
([1, 2, 3], "path must be a list of strings, got '[1, 2, 3]'"),
|
||||
("string", "path must be a list of strings, got 'string'"),
|
||||
]
|
||||
for bad_path, error_message in bad_paths:
|
||||
with self.assertRaises(ConfigurationError) as config_error:
|
||||
config.update_single_config_from_path_and_value(bad_path, "value")
|
||||
|
||||
self.assertEqual(config_error.exception.message, error_message)
|
||||
@@ -0,0 +1,63 @@
|
||||
import unittest
|
||||
|
||||
from local_server.common.config.app_config import AppConfig
|
||||
from local_server.test import H5AD_FIXTURE
|
||||
from local_server.test.unit.common.config import ConfigTests
|
||||
from local_server.common.errors import ConfigurationError
|
||||
|
||||
|
||||
class BaseConfigTest(ConfigTests):
|
||||
def setUp(self):
|
||||
self.config_file_name = f"{unittest.TestCase.id(self).split('.')[-1]}.yml"
|
||||
self.config = AppConfig()
|
||||
self.config.update_server_config(app__flask_secret_key="secret")
|
||||
self.config.update_server_config(single_dataset__datapath=H5AD_FIXTURE)
|
||||
self.server_config = self.config.server_config
|
||||
self.config.complete_config()
|
||||
|
||||
message_list = []
|
||||
|
||||
def noop(message):
|
||||
message_list.append(message)
|
||||
|
||||
messagefn = noop
|
||||
self.context = dict(messagefn=messagefn, messages=message_list)
|
||||
|
||||
def get_config(self, **kwargs):
|
||||
file_name = self.custom_app_config(
|
||||
dataset_datapath=f"{H5AD_FIXTURE}", config_file_name=self.config_file_name, **kwargs
|
||||
)
|
||||
config = AppConfig()
|
||||
config.update_from_config_file(file_name)
|
||||
return config
|
||||
|
||||
def test_mapping_creation_returns_map_of_server_and_dataset_config(self):
|
||||
config = AppConfig()
|
||||
mapping = config.dataset_config.create_mapping(config.default_config)
|
||||
self.assertIsNotNone(mapping["server__app__verbose"])
|
||||
self.assertIsNotNone(mapping["dataset__presentation__max_categories"])
|
||||
self.assertIsNotNone(mapping["dataset__user_annotations__ontology__obo_location"])
|
||||
|
||||
def test_changes_from_default_returns_list_of_nondefault_config_values(self):
|
||||
config = self.get_config(verbose="true", lfc_cutoff=0.05)
|
||||
server_changes = config.server_config.changes_from_default()
|
||||
dataset_changes = config.dataset_config.changes_from_default()
|
||||
|
||||
self.assertEqual(
|
||||
server_changes,
|
||||
[
|
||||
("app__verbose", True, False),
|
||||
("app__flask_secret_key", "secret", None),
|
||||
("single_dataset__datapath", H5AD_FIXTURE, None),
|
||||
('data_locator__s3__region_name', 'us-east-1', True)
|
||||
],
|
||||
)
|
||||
self.assertEqual(dataset_changes, [("diffexp__lfc_cutoff", 0.05, 0.01)])
|
||||
|
||||
def test_check_config_throws_error_if_attr_has_not_been_checked(self):
|
||||
config = self.get_config(verbose="true")
|
||||
config.complete_config()
|
||||
config.check_config()
|
||||
config.update_server_config(app__verbose=False)
|
||||
with self.assertRaises(ConfigurationError):
|
||||
config.check_config()
|
||||
@@ -0,0 +1,157 @@
|
||||
import os
|
||||
import tempfile
|
||||
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from local_server.common.annotations.local_file_csv import AnnotationsLocalFile
|
||||
from local_server.common.config.app_config import AppConfig
|
||||
from local_server.common.config.base_config import BaseConfig
|
||||
from local_server.test import FIXTURES_ROOT, H5AD_FIXTURE
|
||||
|
||||
from local_server.common.errors import ConfigurationError
|
||||
from local_server.test.unit.common.config import ConfigTests
|
||||
|
||||
|
||||
class TestDatasetConfig(ConfigTests):
|
||||
def setUp(self):
|
||||
self.config_file_name = f"{unittest.TestCase.id(self).split('.')[-1]}.yml"
|
||||
self.config = AppConfig()
|
||||
self.config.update_server_config(app__flask_secret_key="secret")
|
||||
self.config.update_server_config(single_dataset__datapath=H5AD_FIXTURE)
|
||||
self.dataset_config = self.config.dataset_config
|
||||
self.config.complete_config()
|
||||
message_list = []
|
||||
|
||||
def noop(message):
|
||||
message_list.append(message)
|
||||
|
||||
messagefn = noop
|
||||
self.context = dict(messagefn=messagefn, messages=message_list)
|
||||
|
||||
def get_config(self, **kwargs):
|
||||
file_name = self.custom_app_config(dataset_datapath=H5AD_FIXTURE, **kwargs)
|
||||
config = AppConfig()
|
||||
config.update_from_config_file(file_name)
|
||||
return config
|
||||
|
||||
def test_init_datatset_config_sets_vars_from_config(self):
|
||||
config = AppConfig()
|
||||
self.assertEqual(config.dataset_config.presentation__max_categories, 1000)
|
||||
self.assertEqual(config.dataset_config.user_annotations__type, "local_file_csv")
|
||||
self.assertEqual(config.dataset_config.diffexp__lfc_cutoff, 0.01)
|
||||
self.assertIsNone(config.dataset_config.user_annotations__ontology__obo_location)
|
||||
|
||||
@patch("local_server.common.config.dataset_config.BaseConfig.validate_correct_type_of_configuration_attribute")
|
||||
def test_complete_config_checks_all_attr(self, mock_check_attrs):
|
||||
mock_check_attrs.side_effect = BaseConfig.validate_correct_type_of_configuration_attribute()
|
||||
self.dataset_config.complete_config(self.context)
|
||||
self.assertIsNotNone(self.config.server_config.data_adaptor)
|
||||
self.assertEqual(mock_check_attrs.call_count, 17)
|
||||
|
||||
def test_app_sets_script_vars(self):
|
||||
config = self.get_config(scripts=["path/to/script"])
|
||||
config.dataset_config.handle_app()
|
||||
|
||||
self.assertEqual(config.dataset_config.app__scripts, [{"src": "path/to/script"}])
|
||||
|
||||
config = self.get_config(scripts=[{"src": "path/to/script", "more": "different/script/path"}])
|
||||
config.dataset_config.handle_app()
|
||||
self.assertEqual(
|
||||
config.dataset_config.app__scripts, [{"src": "path/to/script", "more": "different/script/path"}]
|
||||
)
|
||||
|
||||
config = self.get_config(scripts=["path/to/script", "different/script/path"])
|
||||
config.dataset_config.handle_app()
|
||||
# TODO @madison -- is this the desired functionality?
|
||||
self.assertEqual(
|
||||
config.dataset_config.app__scripts, [{"src": "path/to/script"}, {"src": "different/script/path"}]
|
||||
)
|
||||
|
||||
config = self.get_config(scripts=[{"more": "different/script/path"}])
|
||||
with self.assertRaises(ConfigurationError):
|
||||
config.dataset_config.handle_app()
|
||||
|
||||
def test_handle_user_annotations_ensures_auth_is_enabled_with_valid_auth_type(self):
|
||||
config = self.get_config(enable_users_annotations="true", authentication_enable="false")
|
||||
config.server_config.complete_config(self.context)
|
||||
with self.assertRaises(ConfigurationError):
|
||||
config.dataset_config.handle_user_annotations(self.context)
|
||||
|
||||
config = self.get_config(enable_users_annotations="true", authentication_enable="true", auth_type="pretend")
|
||||
with self.assertRaises(ConfigurationError):
|
||||
config.server_config.complete_config(self.context)
|
||||
|
||||
def test_handle_user_annotations__instantiates_user_annotations_class_correctly(self):
|
||||
config = self.get_config(
|
||||
enable_users_annotations="true", authentication_enable="true", annotation_type="local_file_csv"
|
||||
)
|
||||
config.server_config.complete_config(self.context)
|
||||
config.dataset_config.handle_user_annotations(self.context)
|
||||
self.assertIsInstance(config.dataset_config.user_annotations, AnnotationsLocalFile)
|
||||
|
||||
config = self.get_config(
|
||||
enable_users_annotations="true", authentication_enable="true", annotation_type="NOT_REAL"
|
||||
)
|
||||
config.server_config.complete_config(self.context)
|
||||
with self.assertRaises(ConfigurationError):
|
||||
config.dataset_config.handle_user_annotations(self.context)
|
||||
|
||||
def test_handle_local_file_csv_annotations__sets_dir_if_not_passed_in(self):
|
||||
config = self.get_config(
|
||||
enable_users_annotations="true", authentication_enable="true", annotation_type="local_file_csv"
|
||||
)
|
||||
config.server_config.complete_config(self.context)
|
||||
config.dataset_config.handle_local_file_csv_annotations()
|
||||
self.assertIsInstance(config.dataset_config.user_annotations, AnnotationsLocalFile)
|
||||
cwd = os.getcwd()
|
||||
self.assertEqual(config.dataset_config.user_annotations._get_output_dir(), cwd)
|
||||
|
||||
def test_handle_embeddings__checks_data_file_types(self):
|
||||
file_name = self.custom_app_config(
|
||||
embedding_names=["name1", "name2"],
|
||||
enable_reembedding="true",
|
||||
dataset_datapath=f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad",
|
||||
anndata_backed="true",
|
||||
config_file_name=self.config_file_name,
|
||||
)
|
||||
config = AppConfig()
|
||||
config.update_from_config_file(file_name)
|
||||
config.server_config.complete_config(self.context)
|
||||
with self.assertRaises(ConfigurationError):
|
||||
config.dataset_config.handle_embeddings()
|
||||
|
||||
def test_handle_diffexp__raises_warning_for_large_datasets(self):
|
||||
config = self.get_config(lfc_cutoff=0.02, enable_difexp="true", top_n=15)
|
||||
config.server_config.complete_config(self.context)
|
||||
config.dataset_config.handle_diffexp(self.context)
|
||||
self.assertEqual(len(self.context["messages"]), 1)
|
||||
|
||||
def test_configfile_with_specialization(self):
|
||||
# test that per_dataset_config config load the default config, then the specialized config
|
||||
|
||||
with tempfile.TemporaryDirectory() as tempdir:
|
||||
configfile = os.path.join(tempdir, "config.yaml")
|
||||
with open(configfile, "w") as fconfig:
|
||||
config = """
|
||||
server:
|
||||
single_dataset:
|
||||
datapath: fake_datapath
|
||||
dataset:
|
||||
user_annotations:
|
||||
enable: false
|
||||
type: local_file_csv
|
||||
local_file_csv:
|
||||
file: fake_file
|
||||
directory: fake_dir
|
||||
"""
|
||||
fconfig.write(config)
|
||||
|
||||
app_config = AppConfig()
|
||||
app_config.update_from_config_file(configfile)
|
||||
|
||||
test_config = app_config.dataset_config
|
||||
|
||||
# test config from default
|
||||
self.assertEqual(test_config.user_annotations__type, "local_file_csv")
|
||||
self.assertEqual(test_config.user_annotations__local_file_csv__file, "fake_file")
|
||||
@@ -0,0 +1,215 @@
|
||||
import os
|
||||
from unittest.mock import patch
|
||||
|
||||
import requests
|
||||
|
||||
from local_server.common.errors import ConfigurationError
|
||||
from local_server.common.config.app_config import AppConfig
|
||||
from local_server.test import test_server, FIXTURES_ROOT
|
||||
from local_server.common.utils.type_conversion_utils import convert_string_to_value
|
||||
from local_server.test.unit.common.config import ConfigTests
|
||||
|
||||
|
||||
class TestExternalConfig(ConfigTests):
|
||||
def test_type_convert(self):
|
||||
# The values from environment variables and aws secrets are returned as strings.
|
||||
# These values need to be converted to the proper types.
|
||||
|
||||
self.assertEqual(convert_string_to_value("1"), int(1))
|
||||
self.assertEqual(convert_string_to_value("1.1"), float(1.1))
|
||||
self.assertEqual(convert_string_to_value("string"), "string")
|
||||
self.assertEqual(convert_string_to_value("true"), True)
|
||||
self.assertEqual(convert_string_to_value("True"), True)
|
||||
self.assertEqual(convert_string_to_value("false"), False)
|
||||
self.assertEqual(convert_string_to_value("False"), False)
|
||||
self.assertEqual(convert_string_to_value("null"), None)
|
||||
self.assertEqual(convert_string_to_value("None"), None)
|
||||
self.assertEqual(convert_string_to_value("{'a':10, 'b':'string'}"), dict(a=int(10), b="string"))
|
||||
|
||||
def test_environment_variable(self):
|
||||
configfile = self.custom_external_config(
|
||||
environment=[
|
||||
dict(name="DATAPATH", path=["server", "single_dataset", "datapath"], required=True),
|
||||
dict(name="DIFFEXP", path=["dataset", "diffexp", "enable"], required=True),
|
||||
],
|
||||
config_file_name="environment_external_config.yaml",
|
||||
)
|
||||
|
||||
env = os.environ
|
||||
env["DATAPATH"] = f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad"
|
||||
env["DIFFEXP"] = "False"
|
||||
with test_server(command_line_args=["-c", configfile], env=env) as server:
|
||||
session = requests.Session()
|
||||
response = session.get(f"{server}/api/v0.2/config")
|
||||
data_config = response.json()
|
||||
self.assertEqual(data_config["config"]["displayNames"]["dataset"], "pbmc3k-CSC-gz")
|
||||
self.assertTrue(data_config["config"]["parameters"]["disable-diffexp"])
|
||||
|
||||
env["DATAPATH"] = f"{FIXTURES_ROOT}/a95c59b4-7f5d-4b80-ad53-a694834ca18b.h5ad"
|
||||
env["DIFFEXP"] = "True"
|
||||
with test_server(command_line_args=["-c", configfile], env=env) as server:
|
||||
session = requests.Session()
|
||||
response = session.get(f"{server}/api/v0.2/config")
|
||||
data_config = response.json()
|
||||
self.assertEqual(data_config["config"]["displayNames"]["dataset"], "a95c59b4-7f5d-4b80-ad53-a694834ca18b")
|
||||
self.assertFalse(data_config["config"]["parameters"]["disable-diffexp"])
|
||||
|
||||
def test_environment_variable_errors(self):
|
||||
|
||||
# no name
|
||||
app_config = AppConfig()
|
||||
app_config.external_config.environment = [dict(required=True, path=["this", "is", "a", "path"])]
|
||||
with self.assertRaises(ConfigurationError) as config_error:
|
||||
app_config.complete_config()
|
||||
self.assertEqual(config_error.exception.message, "environment: 'name' is missing")
|
||||
|
||||
# required has wrong type
|
||||
app_config = AppConfig()
|
||||
app_config.external_config.environment = [
|
||||
dict(name="myenvar", required="optional", path=["this", "is", "a", "path"])
|
||||
]
|
||||
with self.assertRaises(ConfigurationError) as config_error:
|
||||
app_config.complete_config()
|
||||
self.assertEqual(config_error.exception.message, "environment: 'required' must be a bool")
|
||||
|
||||
# no path
|
||||
app_config = AppConfig()
|
||||
app_config.external_config.environment = [dict(name="myenvar", required=True)]
|
||||
with self.assertRaises(ConfigurationError) as config_error:
|
||||
app_config.complete_config()
|
||||
self.assertEqual(config_error.exception.message, "environment: 'path' is missing")
|
||||
|
||||
# required environment variable is not set
|
||||
app_config = AppConfig()
|
||||
app_config.external_config.environment = [
|
||||
dict(name="THIS_ENV_IS_NOT_SET", required=True, path=["this", "is", "a", "path"])
|
||||
]
|
||||
with self.assertRaises(ConfigurationError) as config_error:
|
||||
app_config.complete_config()
|
||||
self.assertEqual(config_error.exception.message, "required environment variable 'THIS_ENV_IS_NOT_SET' not set")
|
||||
|
||||
@patch("local_server.common.config.external_config.get_secret_key")
|
||||
def test_aws_secrets_manager(self, mock_get_secret_key):
|
||||
mock_get_secret_key.return_value = {
|
||||
"flask_secret_key": "mock_flask_secret_key",
|
||||
}
|
||||
configfile = self.custom_external_config(
|
||||
aws_secrets_manager_region="us-west-2",
|
||||
aws_secrets_manager_secrets=[
|
||||
dict(
|
||||
name="my_secret",
|
||||
values=[
|
||||
dict(key="flask_secret_key", path=["server", "app", "flask_secret_key"], required=True),
|
||||
],
|
||||
)
|
||||
],
|
||||
config_file_name="secret_external_config.yaml",
|
||||
)
|
||||
|
||||
app_config = AppConfig()
|
||||
app_config.update_from_config_file(configfile)
|
||||
app_config.server_config.single_dataset__datapath = f"{FIXTURES_ROOT}/pbmc3k-CSC-gz.h5ad"
|
||||
|
||||
app_config.complete_config()
|
||||
|
||||
self.assertEqual(app_config.server_config.app__flask_secret_key, "mock_flask_secret_key")
|
||||
|
||||
@patch("local_server.common.config.external_config.get_secret_key")
|
||||
def test_aws_secrets_manager_error(self, mock_get_secret_key):
|
||||
mock_get_secret_key.return_value = {
|
||||
"db_uri": "mock_db_uri",
|
||||
}
|
||||
|
||||
# no region
|
||||
app_config = AppConfig()
|
||||
app_config.external_config.aws_secrets_manager__region = None
|
||||
app_config.external_config.aws_secrets_manager__secrets = [
|
||||
dict(name="secret1", values=[dict(key="key1", required=True, path=["this", "is", "my", "path"])])
|
||||
]
|
||||
with self.assertRaises(ConfigurationError) as config_error:
|
||||
app_config.complete_config()
|
||||
self.assertEqual(
|
||||
config_error.exception.message,
|
||||
"Invalid type for attribute: aws_secrets_manager__region, expected type str, got NoneType",
|
||||
)
|
||||
|
||||
# missing secret name
|
||||
app_config = AppConfig()
|
||||
app_config.external_config.aws_secrets_manager__region = "us-west-2"
|
||||
app_config.external_config.aws_secrets_manager__secrets = [
|
||||
dict(values=[dict(key="db_uri", required=True, path=["this", "is", "my", "path"])])
|
||||
]
|
||||
with self.assertRaises(ConfigurationError) as config_error:
|
||||
app_config.complete_config()
|
||||
self.assertEqual(config_error.exception.message, "aws_secrets_manager: 'name' is missing")
|
||||
|
||||
# secret name wrong type
|
||||
app_config = AppConfig()
|
||||
app_config.external_config.aws_secrets_manager__region = "us-west-2"
|
||||
app_config.external_config.aws_secrets_manager__secrets = [
|
||||
dict(name=1, values=[dict(key="db_uri", required=True, path=["this", "is", "my", "path"])])
|
||||
]
|
||||
with self.assertRaises(ConfigurationError) as config_error:
|
||||
app_config.complete_config()
|
||||
self.assertEqual(config_error.exception.message, "aws_secrets_manager: 'name' must be a string")
|
||||
|
||||
# missing values name
|
||||
app_config = AppConfig()
|
||||
app_config.external_config.aws_secrets_manager__region = "us-west-2"
|
||||
app_config.external_config.aws_secrets_manager__secrets = [dict(name="mysecret")]
|
||||
with self.assertRaises(ConfigurationError) as config_error:
|
||||
app_config.complete_config()
|
||||
self.assertEqual(config_error.exception.message, "aws_secrets_manager: 'values' is missing")
|
||||
|
||||
# values wrong type
|
||||
app_config = AppConfig()
|
||||
app_config.external_config.aws_secrets_manager__region = "us-west-2"
|
||||
app_config.external_config.aws_secrets_manager__secrets = [
|
||||
dict(name="mysecret", values=dict(key="db_uri", required=True, path=["this", "is", "my", "path"]))
|
||||
]
|
||||
with self.assertRaises(ConfigurationError) as config_error:
|
||||
app_config.complete_config()
|
||||
self.assertEqual(config_error.exception.message, "aws_secrets_manager: 'values' must be a list")
|
||||
|
||||
# entry missing key
|
||||
app_config = AppConfig()
|
||||
app_config.external_config.aws_secrets_manager__region = "us-west-2"
|
||||
app_config.external_config.aws_secrets_manager__secrets = [
|
||||
dict(name="mysecret", values=[dict(required=True, path=["this", "is", "my", "path"])])
|
||||
]
|
||||
with self.assertRaises(ConfigurationError) as config_error:
|
||||
app_config.complete_config()
|
||||
self.assertEqual(config_error.exception.message, "missing 'key' in secret values: mysecret")
|
||||
|
||||
# entry required is wrong type
|
||||
app_config = AppConfig()
|
||||
app_config.external_config.aws_secrets_manager__region = "us-west-2"
|
||||
app_config.external_config.aws_secrets_manager__secrets = [
|
||||
dict(name="mysecret", values=[dict(key="db_uri", required="optional", path=["this", "is", "my", "path"])])
|
||||
]
|
||||
with self.assertRaises(ConfigurationError) as config_error:
|
||||
app_config.complete_config()
|
||||
self.assertEqual(config_error.exception.message, "wrong type for 'required' in secret values: mysecret")
|
||||
|
||||
# entry missing path
|
||||
app_config = AppConfig()
|
||||
app_config.external_config.aws_secrets_manager__region = "us-west-2"
|
||||
app_config.external_config.aws_secrets_manager__secrets = [
|
||||
dict(name="mysecret", values=[dict(key="db_uri", required=True)])
|
||||
]
|
||||
with self.assertRaises(ConfigurationError) as config_error:
|
||||
app_config.complete_config()
|
||||
self.assertEqual(config_error.exception.message, "missing 'path' in secret values: mysecret")
|
||||
|
||||
# secret missing required key
|
||||
app_config = AppConfig()
|
||||
app_config.external_config.aws_secrets_manager__region = "us-west-2"
|
||||
app_config.external_config.aws_secrets_manager__secrets = [
|
||||
dict(
|
||||
name="mysecret",
|
||||
values=[dict(key="KEY_DOES_NOT_EXIST", required=True, path=["this", "is", "a", "path"])],
|
||||
)
|
||||
]
|
||||
with self.assertRaises(ConfigurationError) as config_error:
|
||||
app_config.complete_config()
|
||||
self.assertEqual(config_error.exception.message, "required secret 'mysecret:KEY_DOES_NOT_EXIST' not set")
|
||||
@@ -0,0 +1,122 @@
|
||||
import os
|
||||
import unittest
|
||||
from unittest import mock
|
||||
from unittest.mock import patch
|
||||
|
||||
from local_server.common.config.base_config import BaseConfig
|
||||
from local_server.test import H5AD_FIXTURE
|
||||
|
||||
from local_server.common.config.app_config import AppConfig
|
||||
from local_server.common.errors import ConfigurationError
|
||||
from local_server.test.unit.common.config import ConfigTests
|
||||
|
||||
|
||||
def mockenv(**envvars):
|
||||
return mock.patch.dict(os.environ, envvars)
|
||||
|
||||
|
||||
class TestServerConfig(ConfigTests):
|
||||
def setUp(self):
|
||||
self.config_file_name = f"{unittest.TestCase.id(self).split('.')[-1]}.yml"
|
||||
self.config = AppConfig()
|
||||
self.config.update_server_config(app__flask_secret_key="secret")
|
||||
self.config.update_server_config(single_dataset__datapath=H5AD_FIXTURE)
|
||||
self.server_config = self.config.server_config
|
||||
self.config.complete_config()
|
||||
|
||||
message_list = []
|
||||
|
||||
def noop(message):
|
||||
message_list.append(message)
|
||||
|
||||
messagefn = noop
|
||||
self.context = dict(messagefn=messagefn, messages=message_list)
|
||||
|
||||
def get_config(self, **kwargs):
|
||||
file_name = self.custom_app_config(
|
||||
dataset_datapath=f"{H5AD_FIXTURE}", config_file_name=self.config_file_name, **kwargs
|
||||
)
|
||||
config = AppConfig()
|
||||
config.update_from_config_file(file_name)
|
||||
return config
|
||||
|
||||
def test_init_raises_error_if_default_config_is_invalid(self):
|
||||
invalid_config = self.get_config(port="not_valid")
|
||||
with self.assertRaises(ConfigurationError):
|
||||
invalid_config.complete_config()
|
||||
|
||||
@patch("local_server.common.config.server_config.BaseConfig.validate_correct_type_of_configuration_attribute")
|
||||
def test_complete_config_checks_all_attr(self, mock_check_attrs):
|
||||
mock_check_attrs.side_effect = BaseConfig.validate_correct_type_of_configuration_attribute()
|
||||
self.server_config.complete_config(self.context)
|
||||
self.assertEqual(mock_check_attrs.call_count, 20)
|
||||
|
||||
def test_handle_app__throws_error_if_port_doesnt_exist(self):
|
||||
config = self.get_config(port=99999999)
|
||||
with self.assertRaises(ConfigurationError):
|
||||
config.server_config.handle_app(self.context)
|
||||
|
||||
@patch("local_server.common.config.server_config.discover_s3_region_name")
|
||||
def test_handle_data_locator_works_for_default_types(self, mock_discover_region_name):
|
||||
mock_discover_region_name.return_value = None
|
||||
# Default config
|
||||
self.assertEqual(self.config.server_config.data_locator__s3__region_name, None)
|
||||
# hard coded
|
||||
config = self.get_config()
|
||||
self.assertEqual(config.server_config.data_locator__s3__region_name, "us-east-1")
|
||||
# incorrectly formatted
|
||||
datapath = "s3://shouldnt/work"
|
||||
file_name = self.custom_app_config(
|
||||
dataset_datapath=datapath, config_file_name=self.config_file_name, data_locater_region_name="true"
|
||||
)
|
||||
config = AppConfig()
|
||||
config.update_from_config_file(file_name)
|
||||
with self.assertRaises(ConfigurationError):
|
||||
config.server_config.handle_data_locator()
|
||||
|
||||
def test_handle_app___can_use_envar_port(self):
|
||||
config = self.get_config(port=24)
|
||||
self.assertEqual(config.server_config.app__port, 24)
|
||||
|
||||
# Note if the port is set in the config file it will NOT be overwritten by a different envvar
|
||||
os.environ["CXG_SERVER_PORT"] = "4008"
|
||||
self.config = AppConfig()
|
||||
self.config.update_server_config(app__flask_secret_key="secret")
|
||||
self.config.server_config.handle_app(self.context)
|
||||
self.assertEqual(self.config.server_config.app__port, 4008)
|
||||
del os.environ["CXG_SERVER_PORT"]
|
||||
|
||||
def test_handle_app__can_get_secret_key_from_envvar_or_config_file_with_envvar_given_preference(self):
|
||||
config = self.get_config(flask_secret_key="KEY_FROM_FILE")
|
||||
self.assertEqual(config.server_config.app__flask_secret_key, "KEY_FROM_FILE")
|
||||
|
||||
os.environ["CXG_SECRET_KEY"] = "KEY_FROM_ENV"
|
||||
config.external_config.handle_environment(self.context)
|
||||
self.assertEqual(config.server_config.app__flask_secret_key, "KEY_FROM_ENV")
|
||||
|
||||
def test_config_for_single_dataset(self):
|
||||
file_name = self.custom_app_config(
|
||||
config_file_name="single_dataset.yml", dataset_datapath=f"{H5AD_FIXTURE}"
|
||||
)
|
||||
config = AppConfig()
|
||||
config.update_from_config_file(file_name)
|
||||
config.server_config.handle_single_dataset(self.context)
|
||||
|
||||
file_name = self.custom_app_config(
|
||||
config_file_name="single_dataset_with_about.yml",
|
||||
about="www.cziscience.com",
|
||||
dataset_datapath=f"{H5AD_FIXTURE}",
|
||||
)
|
||||
config = AppConfig()
|
||||
config.update_from_config_file(file_name)
|
||||
with self.assertRaises(ConfigurationError):
|
||||
config.server_config.handle_single_dataset(self.context)
|
||||
|
||||
def test_test_auth_only_in_insecure(self):
|
||||
|
||||
config = self.get_config(auth_type="test")
|
||||
with self.assertRaises(ConfigurationError):
|
||||
config.complete_config()
|
||||
|
||||
config.update_server_config(authentication__insecure_test_environment=True)
|
||||
config.complete_config()
|
||||
Reference in New Issue
Block a user