diff --git a/server/common/aws_secret_utils.py b/server/common/aws_secret_utils.py new file mode 100644 index 00000000..a88d9e7d --- /dev/null +++ b/server/common/aws_secret_utils.py @@ -0,0 +1,67 @@ +import logging +import os +import sys + +import boto3 +from flask import json + +from server.common.data_locator import discover_s3_region_name + + +def handle_config_from_secret(app_config): + """Update configuration from the secret manager""" + secret_name = os.getenv("CXG_AWS_SECRET_NAME") + if not secret_name: + return + + # need to find the secret manager region. + # 1. from CXG_AWS_SECRET_REGION_NAME + # 2. discover from dataroot location (if on s3) + # 3. discover from config file location (if on s3) + secret_region_name = os.getenv("CXG_AWS_SECRET_REGION_NAME") + if secret_region_name is None: + secret_region_name = discover_s3_region_name(app_config.multi_dataset__dataroot) + if not secret_region_name: + from server.eb.app import config_file + secret_region_name = discover_s3_region_name(config_file) + if not secret_region_name: + logging.error("Could not determine the AWS Secret Manager region") + sys.exit(1) + + secrets = get_secret_key(secret_region_name, secret_name) + + if not secrets: + return + + keyattrs = ( + ("flask_secret_key", "app__flask_secret_key"), + ("oauth_client_secret", "authentication__params_oauth__client_secret"), + ) + + for key, attr in keyattrs: + cur_val = getattr(app_config.server_config, attr) + if cur_val: + continue + + # replace the attr with the secret if it is not set + val = secrets.get(key) + if val: + logging.error(f"set {attr} from secret") + app_config.update_server_config(**{attr : val}) + + +def get_secret_key(region_name, secret_name): + session = boto3.session.Session() + client = session.client(service_name="secretsmanager", region_name=region_name) + + try: + get_secret_value_response = client.get_secret_value(SecretId=secret_name) + if "SecretString" in get_secret_value_response: + var = get_secret_value_response["SecretString"] + secret = json.loads(var) + return secret + except Exception: + logging.critical("Caught exception during get_secret_key", exc_info=True) + sys.exit(1) + + return None diff --git a/server/eb/README.md b/server/eb/README.md index 5fa635df..2e7827c9 100644 --- a/server/eb/README.md +++ b/server/eb/README.md @@ -203,7 +203,7 @@ $ EB_INSTANCE=m5.large $ CXG_DATAROOT= $ CXG_CONFIG_FILE= -# Potentially also set envvars for the sercret key. +# Potentially also set envvars for the secret key. $ eb create $EB_ENV --instance-type $EB_INSTANCE \ --envvars CXG_DATAROOT=$CXG_DATAROOT,CXG_CONFIG_FILE=$CXG_CONFIG_FILE diff --git a/server/eb/app.py b/server/eb/app.py index 933277ec..1e047010 100644 --- a/server/eb/app.py +++ b/server/eb/app.py @@ -7,8 +7,8 @@ import base64 from flask import json import logging from flask_talisman import Talisman -import boto3 +from server.common.aws_secret_utils import handle_config_from_secret if os.path.isdir("/opt/python/log"): # This is the standard location where Amazon EC2 instances store the application logs. @@ -31,64 +31,6 @@ except Exception: sys.exit(1) -def get_secret_key(region_name, secret_name): - session = boto3.session.Session() - client = session.client(service_name="secretsmanager", region_name=region_name) - - try: - get_secret_value_response = client.get_secret_value(SecretId=secret_name) - if "SecretString" in get_secret_value_response: - var = get_secret_value_response["SecretString"] - secret = json.loads(var) - return secret - except Exception: - logging.critical("Caught exception during get_secret_key", exc_info=True) - sys.exit(1) - - return None - - -def handle_config_from_secret(app_config): - """Update configuration from the secret manager""" - secret_name = os.getenv("CXG_AWS_SECRET_NAME") - if not secret_name: - return - - # need to find the secret manager region. - # 1. from CXG_AWS_SECRET_REGION_NAME - # 2. discover from dataroot location (if on s3) - # 3. discover from config file location (if on s3) - secret_region_name = os.getenv("CXG_AWS_SECRET_REGION_NAME") - if secret_region_name is None: - secret_region_name = discover_s3_region_name(app_config.multi_dataset__dataroot) - if not secret_region_name: - secret_region_name = discover_s3_region_name(config_file) - if not secret_region_name: - logging.error("Could not determine the AWS Secret Manager region") - sys.exit(1) - - secrets = get_secret_key(secret_region_name, secret_name) - if not secrets: - return - - keyattrs = ( - ("flask_secret_key", "app__flask_secret_key"), - ("oauth_client_secret", "authentication__params_oauth__client_secret"), - ("db_uri", "user_annotations__hosted_tiledb_array__db_uri"), - ) - - for key, attr in keyattrs: - curval = getattr(app_config.server_config, attr) - if curval: - continue - - # replace the attr with the secret if it is not set - val = secrets.get(key) - if val: - logging.error(f"set {attr} from secret") - app_config.update_server_config(**{attr : val}) - - class WSGIServer(Server): def __init__(self, app_config): super().__init__(app_config) diff --git a/server/test/unit/common/test_app_config.py b/server/test/unit/common/test_app_config.py index d1cc9496..ed21e3cc 100644 --- a/server/test/unit/common/test_app_config.py +++ b/server/test/unit/common/test_app_config.py @@ -1,12 +1,20 @@ +import os import unittest +from unittest import mock +from unittest.mock import patch + from server.common.app_config import AppConfig from server.common.errors import ConfigurationError from server.test import PROJECT_ROOT, test_server, FIXTURES_ROOT import requests + # NOTE, there are more tests that should be written for AppConfig. # this is just a start. +def mockenv(**envvars): + return mock.patch.dict(os.environ, envvars) + class AppConfigTest(unittest.TestCase): def test_update(self): @@ -98,3 +106,25 @@ class AppConfigTest(unittest.TestCase): r = session.get(f"{server}/health") assert r.json()["status"] == "pass" + + @mockenv(CXG_AWS_SECRET_NAME="TESTING", CXG_AWS_SECRET_REGION_NAME="TEST_REGION") + @patch('server.common.aws_secret_utils.get_secret_key') + def test_get_config_vars_from_aws_secrets(self, mock_get_secret_key): + mock_get_secret_key.return_value = { + "flask_secret_key": "mock_flask_secret", + "oauth_client_secret": "mock_oauth_secret" + } + + config = AppConfig() + + with self.assertLogs(level="ERROR") as logger: + + from server.common.aws_secret_utils import handle_config_from_secret + # should not throw error + # "AttributeError: 'ServerConfig' object has no attribute 'user_annotations__hosted_tiledb_array__db_uri'" + handle_config_from_secret(config) + + # should throw 2 errors (one for each var set from a secret) + self.assertEqual(len(logger.output), 2) + self.assertIn('ERROR:root:set app__flask_secret_key from secret', logger.output[0]) + self.assertIn('ERROR:root:set authentication__params_oauth__client_secret from secret', logger.output[1])