Enhance the AppConfig with external config sources. (#1904)

* Enhance the AppConfig with external config sources.

The external config sources are currently environment variables
and AWS secrets manager.

The config file can be augmented with a section describing how
environmen variables and secrets can update config parameters.

benefits:
 - it will enable the config to draw from more than one secret.  This is useful
   for shared secrets between cellxgene and data portal, as well as auth0 secrets.
 - it will make it very straightforward to check the config before a deployment.

 Part of #1859

server/common/aws_secret_utils.py

@@ -18,6 +18,6 @@ def get_secret_key(region_name, secret_name):
             return secret
     except Exception as e:
         logging.critical(f"Caught exception during get_secret_key, {e}", exc_info=True)
-        raise SecretKeyRetrievalError
+        raise SecretKeyRetrievalError(str(e))
 
     return None

server/common/config/app_config.py

@@ -4,6 +4,7 @@ from flatten_dict import unflatten
 from server.default_config import get_default_config
 from server.common.config.dataset_config import DatasetConfig
 from server.common.config.server_config import ServerConfig
+from server.common.config.external_config import ExternalConfig
 from server.common.errors import ConfigurationError
 
 
@@ -44,6 +45,9 @@ class AppConfig(object):
         #   dataroot config
         self.dataroot_config = {}
 
+        #  external config
+        self.external_config = ExternalConfig(self, self.default_config["external"])
+
         # Set to true when config_completed is called
         self.is_completed = False
 
@@ -61,6 +65,7 @@ class AppConfig(object):
         self.default_dataset_config.check_config()
         for dataset_config in self.dataroot_config.values():
             dataset_config.check_config()
+        self.external_config.check_config()
 
     def update_server_config(self, **kw):
         self.server_config.update(**kw)
@@ -73,6 +78,51 @@ class AppConfig(object):
             value.update(**kw)
         self.is_complete = False
 
+    def update_single_config_from_path_and_value(self, path, value):
+        """Update a single config parameter with the value.
+        Path is a list of string, that gives a path to the config parameter to be updated.
+        For example, path may be ["server","app","port"].
+        """
+        self.is_complete = False
+        if not isinstance(path, list):
+            raise ConfigurationError(f"path must be a list of strings, got '{str(path)}'")
+        for part in path:
+            if not isinstance(part, str):
+                raise ConfigurationError(f"path must be a list of strings, got '{str(path)}'")
+
+        if len(path) < 1 or path[0] not in ("server", "dataset", "per_dataset_config"):
+            raise ConfigurationError("path must start with 'server', 'dataset', or 'per_dataset_config'")
+
+        if path[0] == "server":
+            attr = "__".join(path[1:])
+            try:
+                self.update_server_config(**{attr: value})
+            except ConfigurationError:
+                raise ConfigurationError(f"unknown config parameter at path: '{str(path)}'")
+        elif path[0] == "dataset":
+            attr = "__".join(path[1:])
+            try:
+                self.update_default_dataset_config(**{attr: value})
+            except ConfigurationError:
+                raise ConfigurationError(f"unknown config parameter at path: '{str(path)}'")
+
+        elif path[0] == "per_dataset_config":
+            if len(path) < 2:
+                raise ConfigurationError(f"missing dataroot when using per_dataset_config: got '{path}'")
+            dataroot = path[1]
+            if dataroot not in self.dataroot_config:
+                dataroots = str(list(self.dataroot_config.keys()))
+                raise ConfigurationError(
+                    f"unknown dataroot when using per_dataset_config: got '{path}',"
+                    f" dataroots specified in config are {dataroots}"
+                )
+
+            attr = "__".join(path[2:])
+            try:
+                self.dataroot_config[dataroot].update(**{attr: value})
+            except ConfigurationError:
+                raise ConfigurationError(f"unknown config parameter at path: '{str(path)}'")
+
     def update_from_config_file(self, config_file):
         try:
             with open(config_file) as yml_file:
@@ -94,12 +144,16 @@ class AppConfig(object):
             # then apply the per dataset configuration
             self.dataroot_config[key].update_from_config(dataroot_config, f"per_dataset_config__{key}")
 
+        if config.get("external"):
+            self.external_config.update_from_config(config["external"], "external")
+
         self.is_complete = False
 
-    def write_config(self, config_file):
-        """output the config to a yaml file"""
+    def config_to_dict(self):
+        """return the configuration as an unflattened dict"""
         server = self.server_config.create_mapping(self.server_config.default_config)
         dataset = self.default_dataset_config.create_mapping(self.default_dataset_config.default_config)
+        external = self.external_config.create_mapping(self.external_config.default_config)
         config = dict(server={}, dataset={})
         for attrname in server.keys():
             config["server__" + attrname] = getattr(self.server_config, attrname)
@@ -111,15 +165,23 @@ class AppConfig(object):
             dataset = dataroot_config.create_mapping(dataroot_config.default_config)
             for attrname in dataset.keys():
                 config[f"per_dataset_config__{dataroot_tag}__" + attrname] = getattr(dataroot_config, attrname)
+        for attrname in external.keys():
+            config["external__" + attrname] = getattr(self.external_config, attrname)
 
         config = unflatten(config, splitter=lambda key: key.split("__"))
+        return config
+
+    def write_config(self, config_file):
+        """output the config to a yaml file"""
+        config = self.config_to_dict()
         yaml.dump(config, open(config_file, "w"))
 
     def changes_from_default(self):
         """Return all the attribute that are different from the default"""
         diff_server = self.server_config.changes_from_default()
         diff_dataset = self.default_dataset_config.changes_from_default()
-        diff = dict(server=diff_server, dataset=diff_dataset)
+        diff_external = self.external.changes_from_default()
+        diff = dict(server=diff_server, dataset=diff_dataset, external=diff_external)
         return diff
 
     def add_dataroot_config(self, dataroot_tag, **kw):
@@ -154,6 +216,8 @@ class AppConfig(object):
         # messages we can give correct context for attributes with bad value.
         context = dict(messagefn=messagefn)
 
+        # complete config for external_config first, since this may update values in the other sections
+        self.external_config.complete_config(context)
         self.server_config.complete_config(context)
         self.default_dataset_config.complete_config(context)
         for dataroot_config in self.dataroot_config.values():

server/common/config/base_config.py

@@ -80,8 +80,27 @@ class BaseConfig(object):
                 raise ConfigurationError(f"The attr '{key}' has not been checked")
 
     def update(self, **kw):
+        """Update the attributes defined in kw with their new values."""
         for key, value in kw.items():
             if not hasattr(self, key):
+
+                # check if the key is setting into a dictval entry.
+                found_dictval = False
+                for dictval in self.dictval_cases:
+                    dictvalname = "__".join(dictval)
+                    if dictvalname + "__" in key:
+                        dictkey = key[len(dictvalname) + 2 :]
+                        curdictval = getattr(self, dictvalname)
+                        if curdictval is None:
+                            setattr(self, dictvalname, dict(dictkey=value))
+                        else:
+                            curdictval[dictkey] = value
+
+                        found_dictval = True
+                        break
+
+                if found_dictval:
+                    continue
                 raise ConfigurationError(f"unknown config parameter {key}.")
             try:
                 if type(value) == tuple:

server/common/config/dataset_config.py

@@ -30,22 +30,18 @@ class DatasetConfig(BaseConfig):
             self.user_annotations__type = default_config["user_annotations"]["type"]
             self.user_annotations__local_file_csv__directory = default_config["user_annotations"]["local_file_csv"][
                 "directory"
-            ]  # noqa E501
+            ]
             self.user_annotations__local_file_csv__file = default_config["user_annotations"]["local_file_csv"]["file"]
             self.user_annotations__ontology__enable = default_config["user_annotations"]["ontology"]["enable"]
             self.user_annotations__ontology__obo_location = default_config["user_annotations"]["ontology"][
                 "obo_location"
-            ]  # noqa E501
+            ]
             self.user_annotations__hosted_tiledb_array__db_uri = default_config["user_annotations"][
                 "hosted_tiledb_array"
-            ][
-                "db_uri"
-            ]  # noqa E501
+            ]["db_uri"]
             self.user_annotations__hosted_tiledb_array__hosted_file_directory = default_config["user_annotations"][
                 "hosted_tiledb_array"
-            ][
-                "hosted_file_directory"
-            ]  # noqa E501
+            ]["hosted_file_directory"]
 
             self.embeddings__names = default_config["embeddings"]["names"]
             self.embeddings__enable_reembedding = default_config["embeddings"]["enable_reembedding"]
@@ -98,20 +94,20 @@ class DatasetConfig(BaseConfig):
         self.validate_correct_type_of_configuration_attribute("user_annotations__type", str)
         self.validate_correct_type_of_configuration_attribute(
             "user_annotations__local_file_csv__directory", (type(None), str)
-        )  # noqa E501
+        )
         self.validate_correct_type_of_configuration_attribute(
             "user_annotations__local_file_csv__file", (type(None), str)
-        )  # noqa E501
+        )
         self.validate_correct_type_of_configuration_attribute("user_annotations__ontology__enable", bool)
         self.validate_correct_type_of_configuration_attribute(
             "user_annotations__ontology__obo_location", (type(None), str)
-        )  # noqa E501
+        )
         self.validate_correct_type_of_configuration_attribute(
             "user_annotations__hosted_tiledb_array__db_uri", (type(None), str)
-        )  # noqa E501
+        )
         self.validate_correct_type_of_configuration_attribute(
             "user_annotations__hosted_tiledb_array__hosted_file_directory", (type(None), str)
-        )  # noqa E501
+        )
         if self.user_annotations__enable:
             server_config = self.app_config.server_config
             if not self.app__authentication_enable:
@@ -166,7 +162,7 @@ class DatasetConfig(BaseConfig):
         self.validate_correct_type_of_configuration_attribute("user_annotations__hosted_tiledb_array__db_uri", str)
         self.validate_correct_type_of_configuration_attribute(
             "user_annotations__hosted_tiledb_array__hosted_file_directory", str
-        )  # noqa E501
+        )
         self.user_annotations = AnnotationsHostedTileDB(
             directory_path=self.user_annotations__hosted_tiledb_array__hosted_file_directory,
             db=DbUtils(self.user_annotations__hosted_tiledb_array__db_uri),

server/common/config/external_config.py

@@ -0,0 +1,96 @@
+import os
+
+from server.common.config.base_config import BaseConfig
+from server.common.errors import ConfigurationError
+from server.common.config import get_secret_key
+from server.common.errors import SecretKeyRetrievalError
+from server.common.utils.type_conversion_utils import convert_string_to_value
+
+
+class ExternalConfig(BaseConfig):
+    """Manages the config attribute associated with external configuration sources, such as
+    environment variables or the AWS Secrets Manager."""
+
+    def __init__(self, app_config, default_config):
+        super().__init__(app_config, default_config)
+        try:
+            self.environment = default_config["environment"]
+            self.aws_secrets_manager__region = default_config["aws_secrets_manager"]["region"]
+            self.aws_secrets_manager__secrets = default_config["aws_secrets_manager"]["secrets"]
+
+        except KeyError as e:
+            raise ConfigurationError(f"Unexpected config: {str(e)}")
+
+    def complete_config(self, context):
+        self.handle_environment(context)
+        self.handle_aws_secrets_manager(context)
+
+    def handle_environment(self, context):
+        """For each environment variable defined, get the value (if it is set),
+        and set the specified config parameter"""
+        self.validate_correct_type_of_configuration_attribute("environment", list)
+        for envdict in self.environment:
+            name = envdict.get("name")
+            if name is None:
+                raise ConfigurationError("environment: 'name' is missing")
+            required = envdict.get("required", False)
+            if type(required) != bool:
+                raise ConfigurationError("environment: 'required' must be a bool")
+            path = envdict.get("path")
+            if path is None:
+                raise ConfigurationError("environment: 'path' is missing")
+
+            value = os.environ.get(name)
+            if value is None:
+                if required:
+                    raise ConfigurationError(f"required environment variable '{name}' not set")
+            else:
+                value = convert_string_to_value(value)
+                self.app_config.update_single_config_from_path_and_value(path, value)
+
+    def handle_aws_secrets_manager(self, context):
+        """For each aws secret defined, get the key/values, and set the specified config parameter"""
+        self.validate_correct_type_of_configuration_attribute("aws_secrets_manager__region", (type(None), str))
+        self.validate_correct_type_of_configuration_attribute("aws_secrets_manager__secrets", list)
+
+        if not self.aws_secrets_manager__secrets:
+            return
+
+        self.validate_correct_type_of_configuration_attribute("aws_secrets_manager__region", str)
+
+        for secret in self.aws_secrets_manager__secrets:
+            secret_name = secret.get("name")
+            if secret_name is None:
+                raise ConfigurationError("aws_secrets_manager: 'name' is missing")
+            if not isinstance(secret_name, str):
+                raise ConfigurationError("aws_secrets_manager: 'name' must be a string")
+
+            try:
+                secret_dict = get_secret_key(self.aws_secrets_manager__region, secret_name)
+            except SecretKeyRetrievalError as e:
+                raise ConfigurationError(f"Unable to retrieve secret {secret_name}: {str(e)}")
+
+            values = secret.get("values")
+            if values is None:
+                raise ConfigurationError("aws_secrets_manager: 'values' is missing")
+            if not isinstance(values, list):
+                raise ConfigurationError("aws_secrets_manager: 'values' must be a list")
+
+            for value in values:
+                key = value.get("key")
+                if key is None:
+                    raise ConfigurationError(f"missing 'key' in secret values: {secret_name}")
+                path = value.get("path")
+                if path is None:
+                    raise ConfigurationError(f"missing 'path' in secret values: {secret_name}")
+                required = value.get("required", False)
+                if type(required) != bool:
+                    raise ConfigurationError(f"wrong type for 'required' in secret values: {secret_name}")
+
+                secret_value = secret_dict.get(key)
+                if secret_value is None:
+                    if required:
+                        raise ConfigurationError(f"required secret '{secret_name}:{key}' not set")
+                else:
+                    secret_value = convert_string_to_value(secret_value)
+                    self.app_config.update_single_config_from_path_and_value(path, secret_value)

server/common/config/server_config.py

@@ -44,17 +44,17 @@ class ServerConfig(BaseConfig):
             self.authentication__type = default_config["authentication"]["type"]
             self.authentication__params_oauth__oauth_api_base_url = default_config["authentication"]["params_oauth"][
                 "oauth_api_base_url"
-            ]  # noqa E501
+            ]
             self.authentication__params_oauth__client_id = default_config["authentication"]["params_oauth"]["client_id"]
             self.authentication__params_oauth__client_secret = default_config["authentication"]["params_oauth"][
                 "client_secret"
-            ]  # noqa E501
+            ]
             self.authentication__params_oauth__jwt_decode_options = default_config["authentication"]["params_oauth"][
                 "jwt_decode_options"
-            ]  # noqa E501
+            ]
             self.authentication__params_oauth__session_cookie = default_config["authentication"]["params_oauth"][
                 "session_cookie"
-            ]  # noqa E501
+            ]
             self.authentication__params_oauth__cookie = default_config["authentication"]["params_oauth"]["cookie"]
 
             self.multi_dataset__dataroot = default_config["multi_dataset"]["dataroot"]
@@ -62,10 +62,10 @@ class ServerConfig(BaseConfig):
             self.multi_dataset__allowed_matrix_types = default_config["multi_dataset"]["allowed_matrix_types"]
             self.multi_dataset__matrix_cache__max_datasets = default_config["multi_dataset"]["matrix_cache"][
                 "max_datasets"
-            ]  # noqa E501
+            ]
             self.multi_dataset__matrix_cache__timelimit_s = default_config["multi_dataset"]["matrix_cache"][
                 "timelimit_s"
-            ]  # noqa E501
+            ]
 
             self.single_dataset__datapath = default_config["single_dataset"]["datapath"]
             self.single_dataset__obs_names = default_config["single_dataset"]["obs_names"]
@@ -151,11 +151,6 @@ class ServerConfig(BaseConfig):
         if not self.app__verbose:
             sys.tracebacklimit = 0
 
-        # secret key:
-        #   first, from CXG_SECRET_KEY environment variable
-        #   second, from config file
-        self.app__flask_secret_key = os.environ.get("CXG_SECRET_KEY", self.app__flask_secret_key)
-
         # CSP Directives are a dict of string: list(string) or string: string
         if self.app__csp_directives is not None:
             for k, v in self.app__csp_directives.items():
@@ -178,25 +173,20 @@ class ServerConfig(BaseConfig):
         ptypes = str if self.authentication__type == "oauth" else (type(None), str)
         self.validate_correct_type_of_configuration_attribute(
             "authentication__params_oauth__oauth_api_base_url", ptypes
-        )  # noqa E501
+        )
         self.validate_correct_type_of_configuration_attribute("authentication__params_oauth__client_id", ptypes)
         self.validate_correct_type_of_configuration_attribute("authentication__params_oauth__client_secret", ptypes)
         self.validate_correct_type_of_configuration_attribute(
             "authentication__params_oauth__jwt_decode_options", (type(None), dict)
-        )  # noqa E501
+        )
         self.validate_correct_type_of_configuration_attribute("authentication__params_oauth__session_cookie", bool)
 
         if self.authentication__params_oauth__session_cookie:
             self.validate_correct_type_of_configuration_attribute(
                 "authentication__params_oauth__cookie", (type(None), dict)
-            )  # noqa E501
+            )
         else:
             self.validate_correct_type_of_configuration_attribute("authentication__params_oauth__cookie", dict)
-        #   secret key: first, from CXG_OAUTH_CLIENT_SECRET environment variable
-        #   second, from config file
-        self.authentication__params_oauth__client_secret = os.environ.get(
-            "CXG_OAUTH_CLIENT_SECRET", self.authentication__params_oauth__client_secret
-        )
 
         self.auth = AuthTypeFactory.create(self.authentication__type, self)
         if self.auth is None:
@@ -286,7 +276,7 @@ class ServerConfig(BaseConfig):
         self.validate_correct_type_of_configuration_attribute("multi_dataset__matrix_cache__max_datasets", int)
         self.validate_correct_type_of_configuration_attribute(
             "multi_dataset__matrix_cache__timelimit_s", (type(None), int, float)
-        )  # noqa E501
+        )
 
         if self.multi_dataset__dataroot is None:
             return

server/common/utils/type_conversion_utils.py

@@ -151,3 +151,17 @@ def convert_pandas_series_to_numpy(series_to_convert: pd.Series, dtype):
         logging.error("Cannot convert a pandas Series object to an integer dtype if it contains NaNs.")
 
     return series_to_convert.to_numpy(dtype)
+
+
+def convert_string_to_value(value: str):
+    """convert a string to value with the most appropriate type"""
+    if value.lower() == "true":
+        return True
+    if value.lower() == "false":
+        return False
+    if value == "null":
+        return None
+    try:
+        return eval(value)
+    except:  # noqa E722
+        return value