fork/cellxgene
1
0
Fork 0
mirror of https://github.com/chanzuckerberg/cellxgene.git synced 2026-09-24 09:18:11 +08:00
Code Issues Packages Projects Releases Wiki Activity

remove db_uri secret (#1751)

Browse Source 
* remove db_uri secret

* add test to catch bug in future
This commit is contained in:
Madison Dunitz
2020-08-14 12:38:46 -05:00
committed by GitHub
parent 018f653ec6
commit 6a82030558
4 changed files with 99 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/common/aws_secret_utils.py
+67
View File
@@ -0,0 +1,67 @@
import logging
import os
import sys
import boto3
from flask import json
from server.common.data_locator import discover_s3_region_name
def handle_config_from_secret(app_config):
"""Update configuration from the secret manager"""
secret_name = os.getenv("CXG_AWS_SECRET_NAME")
if not secret_name:
return
# need to find the secret manager region.
# 1. from CXG_AWS_SECRET_REGION_NAME
# 2. discover from dataroot location (if on s3)
# 3. discover from config file location (if on s3)
secret_region_name = os.getenv("CXG_AWS_SECRET_REGION_NAME")
if secret_region_name is None:
secret_region_name = discover_s3_region_name(app_config.multi_dataset__dataroot)
if not secret_region_name:
from server.eb.app import config_file
secret_region_name = discover_s3_region_name(config_file)
if not secret_region_name:
logging.error("Could not determine the AWS Secret Manager region")
sys.exit(1)
secrets = get_secret_key(secret_region_name, secret_name)
if not secrets:
return
keyattrs = (
("flask_secret_key", "app__flask_secret_key"),
("oauth_client_secret", "authentication__params_oauth__client_secret"),
)
for key, attr in keyattrs:
cur_val = getattr(app_config.server_config, attr)
if cur_val:
continue
# replace the attr with the secret if it is not set
val = secrets.get(key)
if val:
logging.error(f"set {attr} from secret")
app_config.update_server_config(**{attr : val})
def get_secret_key(region_name, secret_name):
session = boto3.session.Session()
client = session.client(service_name="secretsmanager", region_name=region_name)
try:
get_secret_value_response = client.get_secret_value(SecretId=secret_name)
if "SecretString" in get_secret_value_response:
var = get_secret_value_response["SecretString"]
secret = json.loads(var)
return secret
except Exception:
logging.critical("Caught exception during get_secret_key", exc_info=True)
sys.exit(1)
return None
server/eb/README.md
+1 -1
View File
@@ -203,7 +203,7 @@ $ EB_INSTANCE=m5.large
$ CXG_DATAROOT=<location to your S3 bucket>
$ CXG_CONFIG_FILE=<location to your config file>
# Potentially also set envvars for the sercret key.
# Potentially also set envvars for the secret key.
$ eb create $EB_ENV --instance-type $EB_INSTANCE \
--envvars CXG_DATAROOT=$CXG_DATAROOT,CXG_CONFIG_FILE=$CXG_CONFIG_FILE
server/eb/app.py
+1 -59
View File
@@ -7,8 +7,8 @@ import base64
from flask import json
import logging
from flask_talisman import Talisman
import boto3
from server.common.aws_secret_utils import handle_config_from_secret
if os.path.isdir("/opt/python/log"):
# This is the standard location where Amazon EC2 instances store the application logs.
@@ -31,64 +31,6 @@ except Exception:
sys.exit(1)
def get_secret_key(region_name, secret_name):
session = boto3.session.Session()
client = session.client(service_name="secretsmanager", region_name=region_name)
try:
get_secret_value_response = client.get_secret_value(SecretId=secret_name)
if "SecretString" in get_secret_value_response:
var = get_secret_value_response["SecretString"]
secret = json.loads(var)
return secret
except Exception:
logging.critical("Caught exception during get_secret_key", exc_info=True)
sys.exit(1)
return None
def handle_config_from_secret(app_config):
"""Update configuration from the secret manager"""
secret_name = os.getenv("CXG_AWS_SECRET_NAME")
if not secret_name:
return
# need to find the secret manager region.
# 1. from CXG_AWS_SECRET_REGION_NAME
# 2. discover from dataroot location (if on s3)
# 3. discover from config file location (if on s3)
secret_region_name = os.getenv("CXG_AWS_SECRET_REGION_NAME")
if secret_region_name is None:
secret_region_name = discover_s3_region_name(app_config.multi_dataset__dataroot)
if not secret_region_name:
secret_region_name = discover_s3_region_name(config_file)
if not secret_region_name:
logging.error("Could not determine the AWS Secret Manager region")
sys.exit(1)
secrets = get_secret_key(secret_region_name, secret_name)
if not secrets:
return
keyattrs = (
("flask_secret_key", "app__flask_secret_key"),
("oauth_client_secret", "authentication__params_oauth__client_secret"),
("db_uri", "user_annotations__hosted_tiledb_array__db_uri"),
)
for key, attr in keyattrs:
curval = getattr(app_config.server_config, attr)
if curval:
continue
# replace the attr with the secret if it is not set
val = secrets.get(key)
if val:
logging.error(f"set {attr} from secret")
app_config.update_server_config(**{attr : val})
class WSGIServer(Server):
def __init__(self, app_config):
super().__init__(app_config)
server/test/unit/common/test_app_config.py
+30
View File
@@ -1,12 +1,20 @@
import os
import unittest
from unittest import mock
from unittest.mock import patch
from server.common.app_config import AppConfig
from server.common.errors import ConfigurationError
from server.test import PROJECT_ROOT, test_server, FIXTURES_ROOT
import requests
# NOTE, there are more tests that should be written for AppConfig.
# this is just a start.
def mockenv(**envvars):
return mock.patch.dict(os.environ, envvars)
class AppConfigTest(unittest.TestCase):
def test_update(self):
@@ -98,3 +106,25 @@ class AppConfigTest(unittest.TestCase):
r = session.get(f"{server}/health")
assert r.json()["status"] == "pass"
@mockenv(CXG_AWS_SECRET_NAME="TESTING", CXG_AWS_SECRET_REGION_NAME="TEST_REGION")
@patch('server.common.aws_secret_utils.get_secret_key')
def test_get_config_vars_from_aws_secrets(self, mock_get_secret_key):
mock_get_secret_key.return_value = {
"flask_secret_key": "mock_flask_secret",
"oauth_client_secret": "mock_oauth_secret"
}
config = AppConfig()
with self.assertLogs(level="ERROR") as logger:
from server.common.aws_secret_utils import handle_config_from_secret
# should not throw error
# "AttributeError: 'ServerConfig' object has no attribute 'user_annotations__hosted_tiledb_array__db_uri'"
handle_config_from_secret(config)
# should throw 2 errors (one for each var set from a secret)
self.assertEqual(len(logger.output), 2)
self.assertIn('ERROR:root:set app__flask_secret_key from secret', logger.output[0])
self.assertIn('ERROR:root:set authentication__params_oauth__client_secret from secret', logger.output[1])