hosted cellxgene (#38) (#1200)

* early, non-working eb config

* hosted cellxgene

In this PR, contains scripts and instructions for deploying cellxgene
for AWS elastic beanstalk.  It supports the multi-dataset option.

The Makefile in the server/eb directory creates an artifact.zip
file, which can be deploy at AWS EB.

The server/eb directory contains:
  app.py - flask app to run the server
  Makefile - which creates an artifact.zip file which can be deployed.
  README.md - instructions for setting up and deploying the eb app.

* hosted cellxgene (#38)

In this PR, contains scripts and instructions for deploying cellxgene
for AWS elastic beanstalk.  It supports the multi-dataset option.

The Makefile in the server/eb directory creates an artifact.zip
file, which can be deploy at AWS EB.

The server/eb directory contains:
  app.py - flask app to run the server
  Makefile - which creates an artifact.zip file which can be deployed.
  README.md - instructions for setting up and deploying the eb app.

* Update how artifact.zip is created

prune the server/test and server/eb directories

* Remove debugging print statements

* fixes from review comments

* fix lint

Co-authored-by: bkmartinjr <bruce@chanzuckerberg.com>

Makefile

@@ -34,25 +34,13 @@ build-client:
 build-cli: build-client
 	git ls-files server/ | cpio -pdm $(BUILDDIR)
 	cp -r client/build/  $(CLIENTBUILD)
-	mkdir -p $(SERVERBUILD)/common/web/static/img
-	mkdir -p $(SERVERBUILD)/common/web/templates/
-	cp $(CLIENTBUILD)/index.html $(SERVERBUILD)/common/web/templates/
-	cp -r $(CLIENTBUILD)/static $(SERVERBUILD)/common/web/
-	cp $(CLIENTBUILD)/favicon.png $(SERVERBUILD)/common/web/static/img
-	cp $(CLIENTBUILD)/service-worker.js $(SERVERBUILD)/common/web/static/js/
+	$(call copy_client_assets,$(CLIENTBUILD),$(SERVERBUILD))
 	cp MANIFEST.in README.md setup.cfg setup.py $(BUILDDIR)
 
 # If you are actively developing in the server folder use this, dirties the source tree
 .PHONY: build-for-server-dev
 build-for-server-dev: clean-server build-client
-	mkdir -p server/common/web/static/img
-	mkdir -p server/common/web/static/js
-	mkdir -p server/common/web/templates/
-	cp client/build/index.html server/common/web/templates/
-	cp -r client/build/static server/common/web/
-	cp client/build/favicon.png server/common/web/static/img
-	cp client/build/service-worker.js server/common/web/static/js/
-
+	$(call copy_client_assets,client/build,server)
 
 # TESTING
 .PHONY: test

common.mk

@@ -17,3 +17,17 @@ export JEST_ENV := $(call get_or_else_dev_env_default,JEST_ENV)
 export CXG_SERVER_PORT
 export CXG_CLIENT_PORT
 export JEST_ENV
+
+# copy the client assests to a location known to the server
+# $(1) is the source of the client assets
+# $(2) is the destination
+define copy_client_assets
+	mkdir -p $(2)/common/web/static/img
+	mkdir -p $(2)/common/web/static/js
+	mkdir -p $(2)/common/web/templates/
+	cp $(1)/index.html $(2)/common/web/templates/
+	cp -r $(1)/static $(2)/common/web/
+	cp $(1)/favicon.png $(2)/common/web/static/img
+	cp $(1)/service-worker.js $(2)/common/web/static/js/
+endef
+

server/common/app_config.py

@@ -28,7 +28,7 @@ class AppConfig(object):
         self.title = ""
         self.about = None
         self.scripts = []
-        self.layout = None
+        self.layout = []
         self.max_category_items = 100
         self.diffexp_lfc_cutoff = 0.01
         self.disable_diffexp = False

server/eb/.ebextensions/enable_mod_deflate.config

@@ -0,0 +1,30 @@
+files:
+  "/etc/httpd/conf.d/enable_mod_deflate.conf":
+    mode: "000644"
+    owner: root
+    group: root
+    content: |
+      <IfModule mod_deflate.c>
+
+        AddOutputFilterByType DEFLATE text/plain
+        AddOutputFilterByType DEFLATE text/html
+        AddOutputFilterByType DEFLATE application/xhtml+xml
+        AddOutputFilterByType DEFLATE text/xml
+        AddOutputFilterByType DEFLATE application/xml
+        AddOutputFilterByType DEFLATE application/xml+rss
+        AddOutputFilterByType DEFLATE application/x-javascript
+        AddOutputFilterByType DEFLATE text/javascript
+        AddOutputFilterByType DEFLATE text/css
+        AddOutputFilterByType DEFLATE application/octet-stream
+
+        DeflateCompressionLevel 9
+
+        BrowserMatch ^Mozilla/4 gzip-only-text/html
+        BrowserMatch ^Mozilla/4\.0[678] no-gzip
+        BrowserMatch \bMSI[E] !no-gzip !gzip-only-text/html
+
+        <IfModule mod_headers.c>
+          Header append Vary User-Agent env=!dont-vary
+        </IfModule>
+
+      </IfModule>

server/eb/.ebextensions/wsgi_app_group_global.config

@@ -0,0 +1,10 @@
+# Configue WSGI so that it will work with numpy, scanpy, etc, which all use the
+# Python SWIG, and therefore will deadlock on start.  For more information, see
+# https://modwsgi.readthedocs.io/en/develop/user-guides/application-issues.html#python-simplified-gil-state-api
+files:
+  "/etc/httpd/conf.d/wsgi_custom.conf":
+    mode: "000644"
+    owner: root
+    group: root
+    content: |
+      WSGIApplicationGroup %{GLOBAL}

server/eb/Makefile

@@ -0,0 +1,26 @@
+include ../../common.mk
+
+.PHONY: clean
+clean:
+	rm -f artifact.zip
+	rm -rf artifact.dir
+
+
+# Build the ElasticBeanstalk configuration and deployment bundle,
+# such that deployment can be done with a simple `eb deploy`.
+# Presumes that a top-level `make build-client` has been done to 
+# create the client static assets.
+
+.PHONY: build
+build: clean
+	mkdir artifact.dir; \
+	(cd ../.. ; \
+	    git ls-files server/ | cpio -pdm server/eb/artifact.dir ; ); \
+	$(call copy_client_assets,../../client/build,artifact.dir/server) ; \
+	cp app.py artifact.dir/application.py; \
+	cp ../requirements.txt artifact.dir; \
+	cp -r .ebextensions artifact.dir; \
+	(cd artifact.dir; zip -r ../artifact.zip . --exclude server/test/\* server/eb/\* ); \
+	if ! grep artifact.zip .elasticbeanstalk/config.yml ; then \
+	    mkdir -p .elasticbeanstalk ; cat config_deploy.yaml >> .elasticbeanstalk/config.yml ; fi
+

server/eb/README.md

@@ -0,0 +1,74 @@
+# AWS Elastic Beanstalk 
+
+This directory contains script to aid in creating and deploying cellxgene on
+an AWS Elastic Beanstalk instance.
+
+This will result in a variant of cellxgene, running on AWS EC2 instances, serving data from S3. 
+All datasets must be in the new CXG (tiledb) format - see the converter script cxgtool.py 
+in server/converters - and located in a single S3 prefix, which is accessible to the instance. 
+In the current incarnation, no access control or authentication support is available 
+(outside of anything you configure yourself), so this is most appropriate for public datasets.
+
+This is early development work, and will change significantly in the near future. 
+We would love feedback on it, but please assume it will change.
+
+## Prerequisites
+
+1. Some familiarity with AWS EB, S3, and IAM are needed.
+
+2. Install the awsebcli.
+Instruction are here:  
+https://docs.aws.amazon.com/elasticbeanstalk/latest/dg/eb-cli3-install.html
+
+3. In the top level directory, run ```make build-client``` to create the client static assets. 
+
+## Steps
+
+These steps are meant to serve as an example.  
+There are many more options to these commands that may be important or necessary for your environment.
+
+1. Create an S3 bucket
+
+   Upload your matrix files to this bucket
+   
+2. Create an elastic beanstalk application.  For example:
+
+    ```
+    EB_APP=cellxgene-app
+    eb init -p python-3.6 $EB_APP
+    ```
+
+3. Create the artifact.zip file for the application
+
+   ```
+   make build
+   ```
+   
+4. Create an environment
+
+    ```
+    # name of the environment
+    EB_ENV=cellxgene-env
+    # type of ec2 instance to run the cellxgene server.     
+    EB_INSTANCE=m5.large 
+    CXG_DATAROOT=<location to your S3 bucket>
+   
+    eb create $EB_ENV --instance-type $EB_INSTANCE --envvars CXG_DATAROOT=$CXG_DATAROOT
+    ```
+
+5. Give the elastic beanstalk environment access to the S3 bucket.
+
+    This link may provide some useful information:
+    https://aws.amazon.com/premiumsupport/knowledge-center/elastic-beanstalk-s3-bucket-instance/
+    
+6. Deploy the application
+
+    ```
+    eb deploy $EB_ENV 
+    ```
+   
+7. Open the application in a browser
+
+   ```
+   eb open
+   ```

server/eb/app.py

@@ -0,0 +1,69 @@
+"""cellxgene AWS elastic beanstalk application"""
+
+import sys
+import os
+import logging
+
+if os.path.isdir("/opt/python/log"):
+    # This is the standard location where Amazon EC2 instances store the application logs.
+    logging.basicConfig(
+        filename="/opt/python/log/app.log",
+        level=logging.DEBUG,
+        format="%(asctime)s.%(msecs)03d %(levelname)s %(module)s - %(funcName)s: %(message)s",
+        datefmt="%Y-%m-%d %H:%M:%S",
+    )
+
+# echo the logs to stdout.  Useful for local testing
+logging.getLogger().addHandler(logging.StreamHandler(sys.stdout))
+
+SERVERDIR = os.path.dirname(os.path.realpath(__file__))
+sys.path.append(SERVERDIR)
+
+try:
+    from server.common.app_config import AppConfig
+    from server.app.app import Server
+    from server.data_common.matrix_loader import MatrixDataCacheManager
+except Exception:
+    logging.exception("Exception importing server modules")
+    sys.exit(1)
+
+try:
+    dataroot = os.getenv("CXG_DATAROOT")
+    if dataroot is None:
+        logging.error("CXG_DATAROOT environment variable must be set")
+        sys.exit(1)
+
+    app_config = AppConfig(
+        datapath=None,
+        dataroot=dataroot,
+        title="",
+        about=None,
+        scripts=[],
+        layout=[],
+        max_category_items=100,
+        diffexp_lfc_cutoff=0.01,
+        obs_names=None,
+        var_names=None,
+        anndata_backed=False,
+        disable_diffexp=False)
+
+    matrix_data_cache_manager = MatrixDataCacheManager()
+    annotations = None
+
+    server = Server(matrix_data_cache_manager, annotations, app_config)
+
+    debug = False
+    application = server.app
+
+except Exception:
+    logging.exception("Caught exception during initialization")
+    sys.exit(1)
+
+logging.info(f"starting server with CXG_DATAROOT={dataroot}")
+
+if __name__ == "__main__":
+    try:
+        application.run(debug=debug, threaded=not debug, use_debugger=False)
+    except Exception:
+        logging.exception("Caught exception during server run")
+        sys.exit(1)

server/eb/config_deploy.yaml

@@ -0,0 +1,2 @@
+deploy:
+  artifact: artifact.zip