Provide a hook into the AWS Secret Manager for the flask secret key (#1398)

Also, the secret manager required a region name, so there was some
refactoring around how regions are handled.

Fixes #1239

server/common/app_config.py

@@ -6,8 +6,6 @@ import sys
 from urllib.parse import urlparse
 import yaml
 import copy
-import boto3
-import botocore
 
 from server.common.default_config import get_default_config
 from server.common.errors import ConfigurationError, DatasetAccessError, OntologyLoadFailure
@@ -17,6 +15,7 @@ import warnings
 from server.common.annotations import AnnotationsLocalFile
 from server.common.utils import custom_format_warning
 import server.compute.diffexp_cxg as diffexp_tiledb
+from server.common.data_locator import discover_s3_region_name
 
 DEFAULT_SERVER_PORT = int(os.environ.get("CXG_SERVER_PORT", "5005"))
 # anything bigger than this will generate a special message
@@ -266,19 +265,13 @@ class AppConfig(object):
         self.__check_attr("data_locator__s3__region_name", (type(None), bool, str))
         if self.data_locator__s3__region_name is True:
             path = self.single_dataset__datapath or self.multi_dataset__dataroot
-            if path and path.startswith("s3:"):
-                bucket = urlparse(path).netloc
-                client = boto3.client("s3")
-                try:
-                    res = client.head_bucket(Bucket=bucket)
-                except botocore.exceptions.ClientError:
-                    raise ConfigurationError(f"Unable to determine region from {path}")
-
-                region = res.get("ResponseMetadata", {}).get("HTTPHeaders", {}).get("x-amz-bucket-region")
-                if region:
-                    self.data_locator__s3__region_name = region
-                else:
-                    raise ConfigurationError(f"Unable to determine region from {path}")
+            if path.startswith("s3://"):
+                region_name = discover_s3_region_name(path)
+                if region_name is None:
+                    raise ConfigurationError(f"Unable to discover s3 region name from {path}")
+            else:
+                region_name = None
+            self.data_locator__s3__region_name = region_name
 
     def handle_presentation(self, context):
         self.__check_attr("presentation__max_categories", int)

server/common/data_locator.py

@@ -2,6 +2,9 @@ import os
 import tempfile
 import fsspec
 from datetime import datetime
+import boto3
+import botocore
+from urllib.parse import urlparse
 
 
 class DataLocator:
@@ -25,7 +28,7 @@ class DataLocator:
 
     """
 
-    def __init__(self, uri_or_path, app_config=None):
+    def __init__(self, uri_or_path, region_name=None):
         if isinstance(uri_or_path, DataLocator):
             locator = uri_or_path
             self.uri_or_path = locator.uri_or_path
@@ -39,9 +42,9 @@ class DataLocator:
             self.cname = self.path if self.protocol == "file" else self.uri_or_path
 
         # fsspec.filesystem will throw RuntimeError if the protocol is unsupported
-        if self.protocol == "s3" and app_config and app_config.data_locator__s3__region_name:
+        if self.protocol == "s3" and region_name:
             self.fs = fsspec.filesystem(
-                self.protocol, config_kwargs={"region_name": app_config.data_locator__s3__region_name}
+                self.protocol, config_kwargs={"region_name": region_name}
             )
         else:
             self.fs = fsspec.filesystem(self.protocol)
@@ -125,3 +128,25 @@ class LocalFilePath:
     def __exit__(self, *args):
         if self.delete:
             os.unlink(self.tmp_path)
+
+
+def discover_s3_region_name(uri):
+    """If this is an s3 protocol, discover and return the (aws) region name.
+    If a return name could not be discovered, or if the uri is not an s3 protocol, return None."""
+
+    protocol, _ = DataLocator._get_protocol_and_path(uri)
+    if protocol == "s3":
+        bucket = urlparse(uri).netloc
+        client = boto3.client("s3")
+        try:
+            res = client.head_bucket(Bucket=bucket)
+        except botocore.exceptions.ClientError:
+            return None
+
+        region = res.get("ResponseMetadata", {}).get("HTTPHeaders", {}).get("x-amz-bucket-region")
+        if region:
+            return region
+        else:
+            return None
+
+    return None

server/common/health.py

@@ -10,7 +10,7 @@ def _is_accessible(path, config):
         return True
 
     try:
-        dl = DataLocator(path, config)
+        dl = DataLocator(path, region_name=config.data_locator__s3__region_name)
         return dl.exists()
     except RuntimeError:
         return False