From b255e32548f3b3ea469e34a901ad9e3308ff38a2 Mon Sep 17 00:00:00 2001 From: bmccandless Date: Mon, 4 May 2020 09:52:03 -0700 Subject: [PATCH] Add a more consistent way to customize the eb deployment (#1446) * Add a more consistent way to customize the eb deployment Plus a few minor bugfixes --- server/common/app_config.py | 2 +- server/eb/Makefile | 22 +++++++------ server/eb/README.md | 63 ++++++++++++++++++++++--------------- server/eb/app.py | 3 ++ 4 files changed, 54 insertions(+), 36 deletions(-) diff --git a/server/common/app_config.py b/server/common/app_config.py index db13ea35..aec58aac 100644 --- a/server/common/app_config.py +++ b/server/common/app_config.py @@ -132,7 +132,7 @@ class AppConfig(object): del dc["adaptor"]["cxg_adaptor"]["tiledb_ctx"] # special case for csp_directives whose value is a dict. - val = config.get("server", {}).get("csp_directives", {}) + val = config.get("server", {}).get("csp_directives") if val is not None: mapping["server__csp_directives"] = (("server", "csp_directives"), val) del dc["server"]["csp_directives"] diff --git a/server/eb/Makefile b/server/eb/Makefile index 2efd2347..87b21add 100644 --- a/server/eb/Makefile +++ b/server/eb/Makefile @@ -20,15 +20,19 @@ build: clean cp app.py artifact.dir/application.py; \ cp ../requirements.txt artifact.dir; \ cp -r .ebextensions artifact.dir; \ - if [ -f config.yaml ] ; then \ - cp config.yaml artifact.dir; \ - fi; \ - if [ -d static ] ; then \ - mkdir -p artifact.dir/server/common/web/static/deploy; \ - cp -r static/* artifact.dir/server/common/web/static/deploy; \ - fi; \ - if [ -d scripts ] ; then \ - cp -r scripts/* artifact.dir/server/common/web/templates; \ + if [ -d customize ] ; then \ + if [ -f customize/config.yaml ] ; then \ + cp customize/config.yaml artifact.dir; \ + fi ; \ + if [ -d customize/deploy ] ; then \ + cp -r customize/deploy artifact.dir/server/common/web/static; \ + fi; \ + if [ -d customize/inline_scripts ] ; then \ + cp -r customize/inline_scripts/* artifact.dir/server/common/web/templates; \ + fi; \ + if [ -d customize/ebextensions ] ; then \ + cp -r customize/ebextensions/* artifact.dir/.ebextensions; \ + fi; \ fi; \ (cd artifact.dir; \ cp -r server/common/web/static static; \ diff --git a/server/eb/README.md b/server/eb/README.md index a2781f60..6256927b 100644 --- a/server/eb/README.md +++ b/server/eb/README.md @@ -47,8 +47,9 @@ There are many more options to these commands that may be important or necessary The config file may then be customized before the app is deployed. - If your config file is named "config.yaml" and exists in this directory, then it will be bundled with the - application zip file and installed along side the app on the EB servers. + If your config file is named "config.yaml" and exists in `customize/config.yaml`, + then it will be bundled with the application zip file and installed along + side the app on the EB servers. However, a potentially more flexible approach is to place your config file in a location accessible to the EB servers, such as along side the matrix files in S3. For example: s3://my-bucket/my-datasets/config.yaml. @@ -58,17 +59,26 @@ There are many more options to these commands that may be important or necessary is the location where the matrix files are located. This environment variable will override the dataroot in the config file (if specified). - - Note: Certain features, such as diffexp and user annotations, are automatically disabled by the EB app, + - Note: Certain features, such as user annotations, are automatically disabled by the EB app, and cannot be enabled using configuration. They may be enabled manually by modifying app.py, however this is not supported or recommended at this time. -4. Prepare static endpoints +4. Customization - The cellxgene server can server additional static webpages that may be associated with the deployment. - These include the about_legal_tos (terms of service), and bout_legal_privacy, for example. - To use this feature, do the following: +The deployment can be customized in several ways, by adding files to a directory called +`customize` which is placed in this directory. + +config file: - * In this directory, create a sub directory called "static". +This was described in the previous section. + +static files: + +The cellxgene server can serve additional static webpages that will be associated with the app. +These include the about_legal_tos (terms of service), and about_legal_privacy, for example. +To use this feature, do the following: + + * In this directory, create a sub directory called "customize/deploy/". * Copy the files you want to serve into this directory * modify your configuration file to set the location to these file: /static/deploy/ @@ -77,45 +87,46 @@ There are many more options to these commands that may be important or necessary ``` $ mkdir static - $ cp /tos.html static/tos.html - $ cp /privacy.html static/privacy.html + $ cp /tos.html customize/deploy/tos.html + $ cp /privacy.html customize/deploy/privacy.html # edit config.yaml $ grep "/static/deploy" config.yaml about_legal_tos: /static/deploy/tos.html about_legal_privacy: /static/deploy/privacy.html ``` - - The next step will place these files into the deployment. -5. Add additional script files to include in HTML page +Inline javascript scripts: -Additional scripts can be added using the server/scripts or server/inline_scripts config parameters. +Additional scripts can be added using the server/inline_scripts config parameters. To include these scripts in the deployment, use the following steps: - * In this directory, create a sub directory called "scripts". + * In this directory, create a sub directory called "customize/inline_scripts". * Copy the script files into this directory - * modify your configuration file to set the location to these file: - - for server/scripts, the path is: /templates/ - - for server/inline_scripts, the path is: + * modify your configuration file to set the location to these file (leaving off customize/inline_scripts) For example, to add an inline script called "myscript.js": ``` $ mkdir scripts - $ cp /myscript.js scripts/myscript.js + $ cp /myscript.js customize/inline_scripts/myscript.js # edit the config.yaml $ grep inline_scripts config.yaml inline_scripts : [ myscript.js ] ``` - -6. Create the artifact.zip file for the application + +ebextensions: + +Any additional config files intended for the `.ebextensions` directory of the artifact can be added +to the `customize/ebextensions` directory. Any file found here will be copied over. + +5. Create the artifact.zip file for the application ``` $ make build ``` -7. Flask secret key +6. Flask secret key The application requires as secret key to be provided to flask, the web framework used by cellxgene. There are three ways to provide the secret key: @@ -130,7 +141,7 @@ To include these scripts in the deployment, use the following steps: then the AWS Secret Manager region name can be specified in an environment variable: CXG_AWS_SECRET_REGION_NAME. -8. Create an environment +7. Create an environment ``` # name of the environment @@ -149,18 +160,18 @@ To include these scripts in the deployment, use the following steps: --envvars CXG_DATAROOT=$CXG_DATAROOT,CXG_CONFIG_FILE=$CXG_CONFIG_FILE ``` -9. Give the elastic beanstalk environment access to the S3 bucket. +8. Give the elastic beanstalk environment access to the S3 bucket. This link may provide some useful information: https://aws.amazon.com/premiumsupport/knowledge-center/elastic-beanstalk-s3-bucket-instance/ -10. Deploy the application +9. Deploy the application ``` $ eb deploy $EB_ENV ``` -11. Open the application in a browser +10. Open the application in a browser ``` $ eb open $EB_ENV diff --git a/server/eb/app.py b/server/eb/app.py index be9f20fa..3ee64aef 100644 --- a/server/eb/app.py +++ b/server/eb/app.py @@ -157,6 +157,9 @@ try: secret_region_name = discover_s3_region_name(app_config.multi_dataset__dataroot) if not secret_region_name: logging.error(f"Expected to discover the s3 region name from {app_config.multi_dataset__dataroot}") + secret_region_name = discover_s3_region_name(config_file) + if not secret_region_name: + logging.error(f"Expected to discover the s3 region name from {app_config.multi_dataset__dataroot}") flask_secret_key = get_flask_secret_key(secret_region_name, secret_name) app_config.update(server__flask_secret_key=flask_secret_key)