Introduce a config file to cellxgene (#1264)

* Introduce a config file to cellxgene

The config file format is in yaml.  The default config is located
in server/common/default_config.py.  A user may create a yaml file
that contains a subset of these fields.  It can be used during cellxgene
launch, or for hosted cellxgene.

The code has also been refactored.  Much of the logic to check arguments
has moved from launch to app config.

It is now possible to set the tiledb context parameters using the config
file.  Other feature will soon be handled in a similar way.

server/eb/Makefile

@@ -20,6 +20,9 @@ build: clean
 	cp app.py artifact.dir/application.py; \
 	cp ../requirements.txt artifact.dir; \
 	cp -r .ebextensions artifact.dir; \
+	if [ -f config.yaml ] ; then \
+	  cp config.yaml artifact.dir;  \
+	fi; \
 	(cd artifact.dir; \
 	  cp -r server/common/web/static static; \
 	  zip -r ../artifact.zip . --exclude server/test/\* server/eb/\* ; ); \

server/eb/README.md

@@ -40,6 +40,10 @@ There are many more options to these commands that may be important or necessary
 
 3. Create the artifact.zip file for the application
 
+    If you have additional configuration for the application,
+    place that application in server/eb/config.yaml.  This will
+    be included in the eb deployment.
+    
    ```
    make build
    ```

server/eb/app.py

@@ -24,49 +24,53 @@ try:
     from server.app.app import Server
     from server.data_common.matrix_loader import MatrixDataCacheManager
 except Exception:
-    logging.exception("Exception importing server modules")
+    logging.critical("Exception importing server modules", exc_info=True)
     sys.exit(1)
 
 try:
     dataroot = os.getenv("CXG_DATAROOT")
-    if dataroot is None:
-        logging.error("CXG_DATAROOT environment variable must be set")
-        sys.exit(1)
+    app_config = AppConfig()
 
-    app_config = AppConfig(
-        datapath=None,
-        dataroot=dataroot,
-        title="",
-        about=None,
-        scripts=[],
-        layout=[],
-        max_category_items=100,
-        diffexp_lfc_cutoff=0.01,
-        obs_names=None,
-        var_names=None,
-        anndata_backed=False,
-        disable_diffexp=True,
-        multi_dataset_index=None,
-        multi_dataset_allowed_matrix_type=["cxg"],
+    config_file = "config.yaml"
+    if os.path.exists(config_file):
+        logging.info(f"Configuration from {config_file}")
+        app_config.update_from_config_file(config_file)
+
+    if dataroot:
+        logging.info(f"Configuration from CXG_DATAROOT")
+        app_config.update(
+            multi_dataset__dataroot=dataroot,
+        )
+
+    # features are unsupported in the current hosted server
+    app_config.update(
+        diffexp__enable=False,
+        user_annotations__enable=False,
+        embeddings__enable_reembedding=False,
+        multi_dataset__allowed_matrix_types=["cxg"],
     )
 
     matrix_data_cache_manager = MatrixDataCacheManager()
-    annotations = None
+    app_config.complete_config(matrix_data_cache_manager, logging.info)
+    user_annotations = app_config.user_annotations
 
-    server = Server(matrix_data_cache_manager, annotations, app_config)
+    server = Server(matrix_data_cache_manager, user_annotations, app_config)
 
     debug = False
     application = server.app
 
 except Exception:
-    logging.exception("Caught exception during initialization")
+    logging.critical("Caught exception during initialization", exc_info=True)
     sys.exit(1)
 
-logging.info(f"starting server with CXG_DATAROOT={dataroot}")
+if app_config.multi_dataset__dataroot:
+    logging.info(f"starting server with multi_dataset__dataroot={app_config.multi_dataset__dataroot}")
+elif app_config.single_dataset__datapath:
+    logging.info(f"starting server with single_dataset__datapath={app_config.single_dataset__datapath}")
 
 if __name__ == "__main__":
     try:
         application.run(debug=debug, threaded=not debug, use_debugger=False)
     except Exception:
-        logging.exception("Caught exception during server run")
+        logging.critical("Caught exception during initialization", exc_info=True)
         sys.exit(1)