fork/cellxgene
1
0
Fork 0
mirror of https://github.com/chanzuckerberg/cellxgene.git synced 2026-10-09 19:20:57 +08:00
Code Issues Packages Projects Releases Wiki Activity

Update readme for eb server. (#1928)

Browse Source 
* Update readme for eb server.

Update the README with new way of handling secrets.
Update portions that were out of date.
Add a section for Authentication and a placeholder for User Annotations.

Also remove an obsolete function that processes the AWS secrets.

  #1522

Co-authored-by: Madison Dunitz <madison.dunitz@chanzuckerberg.com>
This commit is contained in:
bmccandless
2020-10-16 14:02:05 -07:00
committed by GitHub
co-authored by Madison Dunitz
parent c9f9549118
commit 6a741956e1
5 changed files with 123 additions and 155 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/common/config/__init__.py
+1 -63
View File
@@ -1,66 +1,4 @@
import logging from server.common.aws_secret_utils import get_secret_key # noqa F504
import os
import sys
from server.common.aws_secret_utils import get_secret_key
from server.common.data_locator import discover_s3_region_name
DEFAULT_SERVER_PORT = 5005 DEFAULT_SERVER_PORT = 5005
BIG_FILE_SIZE_THRESHOLD = 100 * 2 ** 20 # 100MB BIG_FILE_SIZE_THRESHOLD = 100 * 2 ** 20 # 100MB
def handle_config_from_secret(app_config):
"""Update configuration from the secret manager"""
secret_name = os.getenv("CXG_AWS_SECRET_NAME")
if not secret_name:
return
# need to find the secret manager region.
# 1. from CXG_AWS_SECRET_REGION_NAME
# 2. discover from dataroot location (if on s3)
# 3. discover from config file location (if on s3)
secret_region_name = os.getenv("CXG_AWS_SECRET_REGION_NAME")
if secret_region_name is None:
secret_region_name = discover_s3_region_name(app_config.multi_dataset__dataroot)
if not secret_region_name:
from server.eb.app import config_file
secret_region_name = discover_s3_region_name(config_file)
if not secret_region_name:
logging.error("Could not determine the AWS Secret Manager region")
sys.exit(1)
secrets = get_secret_key(secret_region_name, secret_name)
if not secrets:
return
server_attrs = (
("flask_secret_key", "app__flask_secret_key"),
("oauth_client_secret", "authentication__params_oauth__client_secret"),
)
default_dataset_attrs = (("db_uri", "user_annotations__hosted_tiledb_array__db_uri"),)
# update server configuration attributes
for key, attr in server_attrs:
cur_val = getattr(app_config.server_config, attr)
if cur_val:
continue
# replace the attr with the secret if it is not set
val = secrets.get(key)
if val:
logging.info(f"set {attr} from secret")
app_config.update_server_config(**{attr: val})
# update default dataset configuration attributes
for key, attr in default_dataset_attrs:
cur_val = getattr(app_config.default_dataset_config, attr)
if cur_val:
continue
# replace the attr with the secret if it is not set
val = secrets.get(key)
if val:
logging.info(f"set {attr} from secret")
app_config.update_default_dataset_config(**{attr: val})
server/eb/README.md
+118 -47
View File
@@ -1,12 +1,12 @@
# AWS Elastic Beanstalk # AWS Elastic Beanstalk
This directory contains script to aid in creating and deploying cellxgene on This directory contains scripts to aid in creating and deploying cellxgene on
an AWS Elastic Beanstalk instance. AWS Elastic Beanstalk.
This will result in a variant of cellxgene, running on AWS EC2 instances, serving data from S3. This will result in a variant of cellxgene, running on AWS EC2 instances, serving data from S3.
All datasets must be in the new CXG (tiledb) format - see the converter script cxgtool.py All datasets must be in the CXG (tiledb) format (see `cellxene convert --help`),
in server/converters - and located in a single S3 prefix, which is accessible to the instance. and located under a single S3 prefix, which is accessible to the instance.
In the current incarnation, no access control or authentication support is available In the current incarnation, no access control is available
(outside of anything you configure yourself), so this is most appropriate for public datasets. (outside of anything you configure yourself), so this is most appropriate for public datasets.
This is early development work, and will change significantly in the near future. This is early development work, and will change significantly in the near future.
@@ -17,10 +17,10 @@ We would love feedback on it, but please assume it will change.
1. Some familiarity with AWS EB, S3, and IAM are needed. 1. Some familiarity with AWS EB, S3, and IAM are needed.
2. Install the awsebcli. 2. Install the awsebcli.
Instruction are here: Instruction are here:
https://docs.aws.amazon.com/elasticbeanstalk/latest/dg/eb-cli3-install.html https://docs.aws.amazon.com/elasticbeanstalk/latest/dg/eb-cli3-install.html
3. In the top level directory, run ```make build-client``` to create the client static assets. 3. In the top level directory, run `make build-client` to create the client static assets.
## Steps ## Steps
@@ -31,20 +31,21 @@ There are many more options to these commands that may be important or necessary
The following choices are known to work. The following choices are known to work.
* S3 Bucket. - S3 Bucket.
* POSIX filesystem (such as Lustre) - POSIX filesystem (such as Lustre)
* Lustre filesystem backed by S3 - Lustre filesystem backed by S3
S3 is convenient and the relatively inexpensive option. S3 is convenient and the relatively inexpensive option.
Lustre is higher performance, but more expensive, and slightly more complex to setup and manage. Lustre is higher performance, but more expensive, and slightly more complex to setup and manage.
AWS supports a feature to back the Lustre filesystem with S3, which give an easy to manage and high AWS supports a feature to back the Lustre filesystem with S3, which gives an easy to manage, high
performance option. performance option.
Once the storage is in place, the next step is to copy your matrix files to that location. Once the storage is in place, the next step is to copy your data files to that location.
Currently cellxgene supports a flat file organization. Each matrix file is located from Currently cellxgene supports a flat file organization. Each matrix file is located under
the same s3 prefix or filesystem directory. This location is specified in the configuration as the dataroot. the same s3 prefix or filesystem directory. This location is specified in the configuration
as the dataroot.
### 2. Create an elastic beanstalk application. For example: ### 2. Create an elastic beanstalk application. For example:
``` ```
EB_APP=cellxgene-app EB_APP=cellxgene-app
@@ -54,9 +55,9 @@ eb init -p python-3.6 $EB_APP
### 3. Configuring cellxgene ### 3. Configuring cellxgene
All the cellxgene configuration options can be set from a configuration file. All the cellxgene configuration options can be set from a configuration file.
This file can be generated like this: A yaml config file containing all of the default configuration options can be generated like this:
```cellxgene launch --dump-default-config > myconfig.yaml``` `cellxgene launch --dump-default-config > myconfig.yaml`
The config file may then be customized before the app is deployed. The config file may then be customized before the app is deployed.
@@ -66,18 +67,14 @@ First, if your config file is named "config.yaml" and exists in `customize/confi
then it will be bundled with the application zip file and installed along then it will be bundled with the application zip file and installed along
side the app on the EB servers. side the app on the EB servers.
Second, a potentially more flexible approach is to place your config file in a location accessible to the EB Second, a potentially more flexible approach is to place your config file in a location accessible
servers, such as in S3. For example: s3://my-bucket/my-datasets/config.yaml. to the EB servers, such as in S3. For example: s3://my-bucket/my-datasets/config.yaml.
Set the CXG_CONFIG_FILE environment variable to specify this location. Set the CXG_CONFIG_FILE environment variable to specify this location.
Another option is to set the CXG_DATAROOT environment variable. The dataroot Another option is to set the CXG_DATAROOT environment variable. The dataroot
is the location where the matrix files are located. is the location where the matrix files are located.
This environment variable will override the dataroot in the config file (if specified). This environment variable will override the dataroot in the config file (if specified).
- Note: Certain features, such as user annotations, are automatically disabled by the EB app,
and cannot be enabled using configuration. They may be enabled manually by modifying app.py, however
this is not supported or recommended at this time.
### 4. Customization ### 4. Customization
The deployment can be customized in several ways, by adding files to a directory called The deployment can be customized in several ways, by adding files to a directory called
@@ -93,15 +90,15 @@ The cellxgene server can serve additional static webpages that will be associate
These include the about_legal_tos (terms of service), and about_legal_privacy, for example. These include the about_legal_tos (terms of service), and about_legal_privacy, for example.
To use this feature, do the following: To use this feature, do the following:
* In this directory, create a sub directory called "customize/deploy/". - In this directory, create a sub directory called "customize/deploy/".
* Copy the files you want to serve into this directory - Copy the files you want to serve into this directory
* modify your configuration file to set the location to these file: /static/cellxgene/deploy/<filename> - modify your configuration file to set the location to these file: /static/cellxgene/deploy/<filename>
Example: you want to include an "about_legal_tos" and "about_legal_privacy" page to cellxgene. Example: you want to include an "about_legal_tos" and "about_legal_privacy" page to cellxgene.
Assume files called "tos.html" and "privacy.html" exist. Assume files called "tos.html" and "privacy.html" exist.
``` ```
$ mkdir static $ mkdir -p customize/deploy
$ cp <source_dir>/tos.html customize/deploy/tos.html $ cp <source_dir>/tos.html customize/deploy/tos.html
$ cp <source_dir>/privacy.html customize/deploy/privacy.html $ cp <source_dir>/privacy.html customize/deploy/privacy.html
@@ -116,14 +113,14 @@ about_legal_privacy: /static/cellxgene/deploy/privacy.html
Additional scripts can be added using the server/inline_scripts config parameters. Additional scripts can be added using the server/inline_scripts config parameters.
To include these scripts in the deployment, use the following steps: To include these scripts in the deployment, use the following steps:
* In this directory, create a sub directory called "customize/inline_scripts". - In this directory, create a sub directory called "customize/inline_scripts".
* Copy the script files into this directory - Copy the script files into this directory
* modify your configuration file to set the location to these file (leaving off customize/inline_scripts) - Modify your configuration file to set the location to these file (leaving off customize/inline_scripts)
For example, to add an inline script called "myscript.js": For example, to add an inline script called "myscript.js":
``` ```
$ mkdir scripts $ mkdir -p customize/inline_scripts
$ cp <source_dir>/myscript.js customize/inline_scripts/myscript.js $ cp <source_dir>/myscript.js customize/inline_scripts/myscript.js
# edit the config.yaml # edit the config.yaml
$ grep inline_scripts config.yaml $ grep inline_scripts config.yaml
@@ -135,14 +132,14 @@ $ grep inline_scripts config.yaml
Optionally, you can add plugins to the server python code. To include a plugin in the deployment use the following steps: Optionally, you can add plugins to the server python code. To include a plugin in the deployment use the following steps:
``` ```
$ mkdir plugins $ mkdir -p customize/plugins
$ cp <source_dir>/<my_plugin>.py customize/plugins/<my_plugin>.py $ cp <source_dir>/<my_plugin>.py customize/plugins/<my_plugin>.py
``` ```
#### ebextensions #### ebextensions
Any additional config files intended for the `.ebextensions` directory of the artifact can be added Any additional config files intended for the `.ebextensions` directory of the artifact can be added
to the `customize/ebextensions` directory. Any file found here will be copied over. to the `customize/ebextensions` directory. Any file found here will be copied over.
#### requirements.txt #### requirements.txt
@@ -152,6 +149,7 @@ This is useful to ensure that the dependencies do not change from one deployment
Therefore the custom/requirements.txt must all have exact versions specified (e.g. anndata==0.7.1). Therefore the custom/requirements.txt must all have exact versions specified (e.g. anndata==0.7.1).
This file can be generated the first time using a process like this: This file can be generated the first time using a process like this:
``` ```
# assume you are running in this directory # assume you are running in this directory
$ virtualenv temp $ virtualenv temp
@@ -168,6 +166,20 @@ If a future cellxgene version updates its requirements by modifying a module ver
or adding a new dependency, then the `make build` process will detect any or adding a new dependency, then the `make build` process will detect any
incompatibilities and raise an error. incompatibilities and raise an error.
#### File structure for customizations
The following diagram shows the file structure for the customization directory.
```
customization
+-- config.yaml
+-- deploy/
+-- inline_scripts/
+-- plugins/
+-- ebextensions/
+-- requirements.txt
```
### 5. Create the artifact.zip file for the application ### 5. Create the artifact.zip file for the application
``` ```
@@ -176,19 +188,13 @@ $ make build
### 6. Flask secret key ### 6. Flask secret key
The application requires as secret key to be provided to flask, the web framework used by cellxgene. The application requires a secret key to be provided to flask, the web framework used by cellxgene.
There are three ways to provide the secret key: There are three ways to provide the secret key:
- In the configuration file: update the server/flask_secret_key attribute. - In the configuration file, update the server/flask_secret_key attribute.
- In the configuration file, update the external/aws_secrets_manager section to set the
secret name and key that defines the flask secret key.
- An environment variable: `CXG_SECRET_KEY` - An environment variable: `CXG_SECRET_KEY`
- Managed by the AWS Secret Manager
If using the AWS Secret Manager, then the secret name is passed as an environment variable: CXG_AWS_SECRET_NAME.
The secret must contain a key with the name "flask_secret_key".
The region name for the AWS Secret Manager must be specified (e.g. us-east-1).
The most straightforward way is to specified it with the CXG_AWS_SECRET_REGION_NAME environment variable.
If this environment variable is not defined, then the app attempts to determine the region from the
dataroot (if in s3), or the config file location (if in s3).
### 7. Create an environment ### 7. Create an environment
@@ -203,7 +209,8 @@ $ EB_INSTANCE=m5.large
$ CXG_DATAROOT=<location to your S3 bucket> $ CXG_DATAROOT=<location to your S3 bucket>
$ CXG_CONFIG_FILE=<location to your config file> $ CXG_CONFIG_FILE=<location to your config file>
# Potentially also set envvars for the secret key. # Potentially also set an environment variable for the flask secret key,
# and other environemet variable described in the configuration file.
$ eb create $EB_ENV --instance-type $EB_INSTANCE \ $ eb create $EB_ENV --instance-type $EB_INSTANCE \
--envvars CXG_DATAROOT=$CXG_DATAROOT,CXG_CONFIG_FILE=$CXG_CONFIG_FILE --envvars CXG_DATAROOT=$CXG_DATAROOT,CXG_CONFIG_FILE=$CXG_CONFIG_FILE
@@ -227,3 +234,67 @@ $ eb deploy $EB_ENV
``` ```
$ eb open $EB_ENV $ eb open $EB_ENV
``` ```
## Advanced Features
### Authentication
Authentication can be configured in the configuration file. Authentication is required
for User Annotations (see below). User Annotations is a feature where annotations can be
created by the user
, and
then associated with the user's id.
When the user revisits the site, their annotations will be available.
There are three main authentication modes: null, session, or oauth.
In the configuration file specify the authentication mode by setting
`server / authentication / type`.
#### null
Authentication is disabled: user annotations cannot be enabled.
#### session
The user is associated with their client browser session. This approach is
simple to setup, but not recommended for hosted cellxgene, since the user will not have access to
their annotations when running from a different browser, or if their cookies get cleared.
#### oauth
A user logs into cellxgene using an identity provider (like Google), or logs in using
an email/password. This is the best option, but requires making use of an oauth service and
additional configuration of the cellxgene server.
To see what this looks like, please look at https://cellxgene.cziscience.com/,
and view one of the cellxgene datasets.
For this server, Auth0 (auth0.com) is used for authentication, but there are other options.
There are good sources of documentation online that describe how to use one of these
services.
The `params_oauth` section in the configuration file describes characteristics of the
authentication service, like "client_id" and "client_secret".
For security, the client_secret needs to be protected. One option is to
store it in the AWS Secrets Manager.
### User Annotations
User annotations can be configured in the configuration file both generally and for a specific data route. The annotations feature is only available when Authorization is enabled.
To enable Annotations, it is necessary to create a relational database and add the database uri (typically `postgresql://[user[:password]@][netloc][:port][/dbname]`) to the secrets manager under `DB_URI`.
The hosted version of cellxgene runs on AWS's [Aurora PostgreSQL](https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/Aurora.AuroraPostgreSQL.html) but any sqlalchemy compatible relational database should work.
Once the database is set up apply the cellxgene schema to your database by running the following inside the cellxgene repo
`PROJECT_ROOT=$(git rev-parse --show-toplevel)`
`python3`
Inside the python console
`from sqlalchemy import create_engine`
`from server.db.cellxgene_orm import Base`
`uri = "[DB_URI]”`
`engine = create_engine(uri)`
Base.metadata.create_all(engine)`
To check the schema was properly applied (or just to check what is in the database at any point)
ssh into your database. For a postgres database this entails running:
`psql [DB_URI]`
You'll also need to update your IAM policies to allow the instance to write to the s3 bucket.
server/eb/app.py
+3 -17
View File
@@ -9,8 +9,6 @@ from flask import json
import logging import logging
from flask_talisman import Talisman from flask_talisman import Talisman
from flask_cors import CORS from flask_cors import CORS
from server.common.config import handle_config_from_secret
from server.common.errors import SecretKeyRetrievalError
if os.path.isdir("/opt/python/log"): if os.path.isdir("/opt/python/log"):
@@ -165,26 +163,14 @@ try:
logging.info("Configuration from CXG_DATAROOT") logging.info("Configuration from CXG_DATAROOT")
app_config.update_server_config(multi_dataset__dataroot=dataroot) app_config.update_server_config(multi_dataset__dataroot=dataroot)
# update from secret manager # overwrite configuration for the eb app
try:
handle_config_from_secret(app_config)
except SecretKeyRetrievalError:
sys.exit(1)
# features are unsupported in the current hosted server
app_config.update_default_dataset_config(embeddings__enable_reembedding=False,) app_config.update_default_dataset_config(embeddings__enable_reembedding=False,)
app_config.update_server_config(multi_dataset__allowed_matrix_types=["cxg"],) app_config.update_server_config(multi_dataset__allowed_matrix_types=["cxg"],)
# complete config
app_config.complete_config(logging.info) app_config.complete_config(logging.info)
if not app_config.server_config.app__flask_secret_key:
logging.critical(
"flask_secret_key is not provided. Either set in config file, CXG_SECRET_KEY environment variable, "
"or in AWS Secret Manager"
)
sys.exit(1)
server = WSGIServer(app_config) server = WSGIServer(app_config)
debug = False debug = False
application = server.app application = server.app
server/eb/check_config.py
+1 -1
View File
@@ -19,8 +19,8 @@ def main():
args = parser.parse_args() args = parser.parse_args()
app_config = AppConfig() app_config = AppConfig()
app_config.update_from_config_file(args.config_file)
try: try:
app_config.update_from_config_file(args.config_file)
app_config.complete_config() app_config.complete_config()
except Exception as e: except Exception as e:
print(f"Error: {str(e)}") print(f"Error: {str(e)}")
server/test/unit/common/config/test_server_config.py
-27
View File
@@ -310,30 +310,3 @@ class TestServerConfig(ConfigTests):
mock_tiledb_context.assert_called_once_with( mock_tiledb_context.assert_called_once_with(
{"sm.tile_cache_size": 10, "sm.num_reader_threads": 2, "vfs.s3.region": "us-east-1"} {"sm.tile_cache_size": 10, "sm.num_reader_threads": 2, "vfs.s3.region": "us-east-1"}
) )
@mockenv(CXG_AWS_SECRET_NAME="TESTING", CXG_AWS_SECRET_REGION_NAME="TEST_REGION")
@patch("server.common.config.get_secret_key")
def test_get_config_vars_from_aws_secrets(self, mock_get_secret_key):
mock_get_secret_key.return_value = {
"flask_secret_key": "mock_flask_secret",
"oauth_client_secret": "mock_oauth_secret",
"db_uri": "mock_db_uri",
}
config = AppConfig()
with self.assertLogs(level="INFO") as logger:
from server.common.config import handle_config_from_secret
# should not throw error
# "AttributeError: 'XConfig' object has no attribute 'x'"
handle_config_from_secret(config)
# should log 3 lines (one for each var set from a secret)
self.assertEqual(len(logger.output), 3)
self.assertIn("INFO:root:set app__flask_secret_key from secret", logger.output[0])
self.assertIn("INFO:root:set authentication__params_oauth__client_secret from secret", logger.output[1])
self.assertIn("INFO:root:set user_annotations__hosted_tiledb_array__db_uri from secret", logger.output[2])
self.assertEqual(config.server_config.app__flask_secret_key, "mock_flask_secret")
self.assertEqual(config.server_config.authentication__params_oauth__client_secret, "mock_oauth_secret")
self.assertEqual(config.default_dataset_config.user_annotations__hosted_tiledb_array__db_uri, "mock_db_uri")