diff --git a/Makefile b/Makefile index e4af098c..70e89531 100644 --- a/Makefile +++ b/Makefile @@ -34,25 +34,13 @@ build-client: build-cli: build-client git ls-files server/ | cpio -pdm $(BUILDDIR) cp -r client/build/ $(CLIENTBUILD) - mkdir -p $(SERVERBUILD)/common/web/static/img - mkdir -p $(SERVERBUILD)/common/web/templates/ - cp $(CLIENTBUILD)/index.html $(SERVERBUILD)/common/web/templates/ - cp -r $(CLIENTBUILD)/static $(SERVERBUILD)/common/web/ - cp $(CLIENTBUILD)/favicon.png $(SERVERBUILD)/common/web/static/img - cp $(CLIENTBUILD)/service-worker.js $(SERVERBUILD)/common/web/static/js/ + $(call copy_client_assets,$(CLIENTBUILD),$(SERVERBUILD)) cp MANIFEST.in README.md setup.cfg setup.py $(BUILDDIR) # If you are actively developing in the server folder use this, dirties the source tree .PHONY: build-for-server-dev build-for-server-dev: clean-server build-client - mkdir -p server/common/web/static/img - mkdir -p server/common/web/static/js - mkdir -p server/common/web/templates/ - cp client/build/index.html server/common/web/templates/ - cp -r client/build/static server/common/web/ - cp client/build/favicon.png server/common/web/static/img - cp client/build/service-worker.js server/common/web/static/js/ - + $(call copy_client_assets,client/build,server) # TESTING .PHONY: test diff --git a/common.mk b/common.mk index e2bca695..ded8c17d 100644 --- a/common.mk +++ b/common.mk @@ -17,3 +17,17 @@ export JEST_ENV := $(call get_or_else_dev_env_default,JEST_ENV) export CXG_SERVER_PORT export CXG_CLIENT_PORT export JEST_ENV + +# copy the client assests to a location known to the server +# $(1) is the source of the client assets +# $(2) is the destination +define copy_client_assets + mkdir -p $(2)/common/web/static/img + mkdir -p $(2)/common/web/static/js + mkdir -p $(2)/common/web/templates/ + cp $(1)/index.html $(2)/common/web/templates/ + cp -r $(1)/static $(2)/common/web/ + cp $(1)/favicon.png $(2)/common/web/static/img + cp $(1)/service-worker.js $(2)/common/web/static/js/ +endef + diff --git a/server/common/app_config.py b/server/common/app_config.py index a2b1b632..11ae6ee6 100644 --- a/server/common/app_config.py +++ b/server/common/app_config.py @@ -28,7 +28,7 @@ class AppConfig(object): self.title = "" self.about = None self.scripts = [] - self.layout = None + self.layout = [] self.max_category_items = 100 self.diffexp_lfc_cutoff = 0.01 self.disable_diffexp = False diff --git a/server/eb/.ebextensions/enable_mod_deflate.config b/server/eb/.ebextensions/enable_mod_deflate.config new file mode 100644 index 00000000..3456869b --- /dev/null +++ b/server/eb/.ebextensions/enable_mod_deflate.config @@ -0,0 +1,30 @@ +files: + "/etc/httpd/conf.d/enable_mod_deflate.conf": + mode: "000644" + owner: root + group: root + content: | + + + AddOutputFilterByType DEFLATE text/plain + AddOutputFilterByType DEFLATE text/html + AddOutputFilterByType DEFLATE application/xhtml+xml + AddOutputFilterByType DEFLATE text/xml + AddOutputFilterByType DEFLATE application/xml + AddOutputFilterByType DEFLATE application/xml+rss + AddOutputFilterByType DEFLATE application/x-javascript + AddOutputFilterByType DEFLATE text/javascript + AddOutputFilterByType DEFLATE text/css + AddOutputFilterByType DEFLATE application/octet-stream + + DeflateCompressionLevel 9 + + BrowserMatch ^Mozilla/4 gzip-only-text/html + BrowserMatch ^Mozilla/4\.0[678] no-gzip + BrowserMatch \bMSI[E] !no-gzip !gzip-only-text/html + + + Header append Vary User-Agent env=!dont-vary + + + diff --git a/server/eb/.ebextensions/wsgi_app_group_global.config b/server/eb/.ebextensions/wsgi_app_group_global.config new file mode 100644 index 00000000..0befb288 --- /dev/null +++ b/server/eb/.ebextensions/wsgi_app_group_global.config @@ -0,0 +1,10 @@ +# Configue WSGI so that it will work with numpy, scanpy, etc, which all use the +# Python SWIG, and therefore will deadlock on start. For more information, see +# https://modwsgi.readthedocs.io/en/develop/user-guides/application-issues.html#python-simplified-gil-state-api +files: + "/etc/httpd/conf.d/wsgi_custom.conf": + mode: "000644" + owner: root + group: root + content: | + WSGIApplicationGroup %{GLOBAL} diff --git a/server/eb/Makefile b/server/eb/Makefile new file mode 100644 index 00000000..8cb17c0d --- /dev/null +++ b/server/eb/Makefile @@ -0,0 +1,26 @@ +include ../../common.mk + +.PHONY: clean +clean: + rm -f artifact.zip + rm -rf artifact.dir + + +# Build the ElasticBeanstalk configuration and deployment bundle, +# such that deployment can be done with a simple `eb deploy`. +# Presumes that a top-level `make build-client` has been done to +# create the client static assets. + +.PHONY: build +build: clean + mkdir artifact.dir; \ + (cd ../.. ; \ + git ls-files server/ | cpio -pdm server/eb/artifact.dir ; ); \ + $(call copy_client_assets,../../client/build,artifact.dir/server) ; \ + cp app.py artifact.dir/application.py; \ + cp ../requirements.txt artifact.dir; \ + cp -r .ebextensions artifact.dir; \ + (cd artifact.dir; zip -r ../artifact.zip . --exclude server/test/\* server/eb/\* ); \ + if ! grep artifact.zip .elasticbeanstalk/config.yml ; then \ + mkdir -p .elasticbeanstalk ; cat config_deploy.yaml >> .elasticbeanstalk/config.yml ; fi + diff --git a/server/eb/README.md b/server/eb/README.md new file mode 100644 index 00000000..5c05e6e4 --- /dev/null +++ b/server/eb/README.md @@ -0,0 +1,74 @@ +# AWS Elastic Beanstalk + +This directory contains script to aid in creating and deploying cellxgene on +an AWS Elastic Beanstalk instance. + +This will result in a variant of cellxgene, running on AWS EC2 instances, serving data from S3. +All datasets must be in the new CXG (tiledb) format - see the converter script cxgtool.py +in server/converters - and located in a single S3 prefix, which is accessible to the instance. +In the current incarnation, no access control or authentication support is available +(outside of anything you configure yourself), so this is most appropriate for public datasets. + +This is early development work, and will change significantly in the near future. +We would love feedback on it, but please assume it will change. + +## Prerequisites + +1. Some familiarity with AWS EB, S3, and IAM are needed. + +2. Install the awsebcli. +Instruction are here: +https://docs.aws.amazon.com/elasticbeanstalk/latest/dg/eb-cli3-install.html + +3. In the top level directory, run ```make build-client``` to create the client static assets. + +## Steps + +These steps are meant to serve as an example. +There are many more options to these commands that may be important or necessary for your environment. + +1. Create an S3 bucket + + Upload your matrix files to this bucket + +2. Create an elastic beanstalk application. For example: + + ``` + EB_APP=cellxgene-app + eb init -p python-3.6 $EB_APP + ``` + +3. Create the artifact.zip file for the application + + ``` + make build + ``` + +4. Create an environment + + ``` + # name of the environment + EB_ENV=cellxgene-env + # type of ec2 instance to run the cellxgene server. + EB_INSTANCE=m5.large + CXG_DATAROOT= + + eb create $EB_ENV --instance-type $EB_INSTANCE --envvars CXG_DATAROOT=$CXG_DATAROOT + ``` + +5. Give the elastic beanstalk environment access to the S3 bucket. + + This link may provide some useful information: + https://aws.amazon.com/premiumsupport/knowledge-center/elastic-beanstalk-s3-bucket-instance/ + +6. Deploy the application + + ``` + eb deploy $EB_ENV + ``` + +7. Open the application in a browser + + ``` + eb open + ``` diff --git a/server/eb/app.py b/server/eb/app.py new file mode 100644 index 00000000..7bca6b49 --- /dev/null +++ b/server/eb/app.py @@ -0,0 +1,69 @@ +"""cellxgene AWS elastic beanstalk application""" + +import sys +import os +import logging + +if os.path.isdir("/opt/python/log"): + # This is the standard location where Amazon EC2 instances store the application logs. + logging.basicConfig( + filename="/opt/python/log/app.log", + level=logging.DEBUG, + format="%(asctime)s.%(msecs)03d %(levelname)s %(module)s - %(funcName)s: %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", + ) + +# echo the logs to stdout. Useful for local testing +logging.getLogger().addHandler(logging.StreamHandler(sys.stdout)) + +SERVERDIR = os.path.dirname(os.path.realpath(__file__)) +sys.path.append(SERVERDIR) + +try: + from server.common.app_config import AppConfig + from server.app.app import Server + from server.data_common.matrix_loader import MatrixDataCacheManager +except Exception: + logging.exception("Exception importing server modules") + sys.exit(1) + +try: + dataroot = os.getenv("CXG_DATAROOT") + if dataroot is None: + logging.error("CXG_DATAROOT environment variable must be set") + sys.exit(1) + + app_config = AppConfig( + datapath=None, + dataroot=dataroot, + title="", + about=None, + scripts=[], + layout=[], + max_category_items=100, + diffexp_lfc_cutoff=0.01, + obs_names=None, + var_names=None, + anndata_backed=False, + disable_diffexp=False) + + matrix_data_cache_manager = MatrixDataCacheManager() + annotations = None + + server = Server(matrix_data_cache_manager, annotations, app_config) + + debug = False + application = server.app + +except Exception: + logging.exception("Caught exception during initialization") + sys.exit(1) + +logging.info(f"starting server with CXG_DATAROOT={dataroot}") + +if __name__ == "__main__": + try: + application.run(debug=debug, threaded=not debug, use_debugger=False) + except Exception: + logging.exception("Caught exception during server run") + sys.exit(1) diff --git a/server/eb/config_deploy.yaml b/server/eb/config_deploy.yaml new file mode 100644 index 00000000..8c6e4cb0 --- /dev/null +++ b/server/eb/config_deploy.yaml @@ -0,0 +1,2 @@ +deploy: + artifact: artifact.zip