Add server plugin system (#1447)

* Add server plugin system

Plugins are optional modules loaded at runtime. Specification:
* Plugins are loaded from the server.plugins module (directory
  server/plugins)
* The import_plugins method is run as part of the initialization of the
  server module in __init__.py

* Add plugins to the EB build process

* Remove bit of dead code

* Respond to feedback from @bmccandless

server/__init__.py

@@ -1 +1,6 @@
+from server.common.utils import import_plugins
+
 __version__ = "0.15.0"
+
+
+import_plugins("server.plugins")

server/common/utils.py

@@ -1,12 +1,16 @@
 import contextlib
 import errno
-import socket
+import importlib.util
-from urllib.parse import urlsplit, urljoin
+import logging
 import os
+import pkgutil
+import socket
+import warnings
+
 from flask import json
+from urllib.parse import urlsplit, urljoin
 import numpy as np
 import pandas as pd
-import warnings
 
 
 def find_available_port(host, port=5005):
@@ -142,3 +146,24 @@ def series_to_schema(array):
         else:
             raise TypeError(f"Annotations of type {dtype} are unsupported.")
         return schema
+
+
+def import_plugins(plugin_module):
+    """
+    Load optional plugin modules from server.common.plugins
+
+    If you would like to customize cellxgene, you can add submodules to server.common.plugins before running the app.
+    This code will import each, loading the code in each. If no plugins are defined, initializing the app continues as
+    normal.
+    """
+    loaded_modules = []
+    try:
+        pkg = importlib.import_module(plugin_module)
+        for loader, name, is_pkg in pkgutil.walk_packages(pkg.__path__):
+            full_name = f"{plugin_module}.{name}"
+            module = importlib.import_module(full_name)
+            logging.info(f"Imported plugin {full_name}")
+            loaded_modules.append(module)
+    except ModuleNotFoundError:
+        logging.debug(f"No plugins found in module: {plugin_module}")
+    return loaded_modules

server/eb/Makefile

@@ -41,6 +41,9 @@ build: clean
 	      cp -r customize/ebextensions/* artifact.dir/.ebextensions;  \
 	  fi; \
 	fi; \
+	if [ -d customize/plugins ] ; then \
+	  cp -r customize/plugins artifact.dir/server/;  \
+	fi; \
 	(cd artifact.dir; \
 	  cp -r server/common/web/static static; \
 	  zip -r ../artifact.zip . --exclude server/test/\* server/eb/\* ; ); \

server/eb/README.md

@@ -27,7 +27,7 @@ https://docs.aws.amazon.com/elasticbeanstalk/latest/dg/eb-cli3-install.html
 These steps are meant to serve as an example.
 There are many more options to these commands that may be important or necessary for your environment.
 
-1. Make your matrix files available to the EB servers.
+### 1. Make your matrix files available to the EB servers.
 
 The following choices are known to work.
 
@@ -44,14 +44,14 @@ There are many more options to these commands that may be important or necessary
 Currently cellxgene supports a flat file organization.   Each matrix file is located from
 the same s3 prefix or filesystem directory.  This location is specified in the configuration as the dataroot.
 
-2. Create an elastic beanstalk application.  For example:
+### 2. Create an elastic beanstalk application.  For example:
 
 ```
 EB_APP=cellxgene-app
 eb init -p python-3.6 $EB_APP
 ```
 
-3. Configuring cellxgene
+### 3. Configuring cellxgene
 
 All the cellxgene configuration options can be set from a configuration file.
 This file can be generated like this:
@@ -78,16 +78,16 @@ There are many more options to these commands that may be important or necessary
 and cannot be enabled using configuration.  They may be enabled manually by modifying app.py, however
 this is not supported or recommended at this time.
 
-4. Customization
+### 4. Customization
 
 The deployment can be customized in several ways, by adding files to a directory called
 `customize` which is placed in this directory.
 
-config file: 
+#### config file
 
 This was described in the previous section.
 
-static files: 
+#### static files
 
 The cellxgene server can serve additional static webpages that will be associated with the app.
 These include the about_legal_tos (terms of service), and about_legal_privacy, for example.
@@ -111,7 +111,7 @@ To use this feature, do the following:
 about_legal_privacy: /static/deploy/privacy.html
 ```
 
-Inline javascript scripts:
+#### Inline javascript scripts
 
 Additional scripts can be added using the server/inline_scripts config parameters.
 To include these scripts in the deployment, use the following steps:
@@ -130,12 +130,21 @@ To include these scripts in the deployment, use the following steps:
   inline_scripts : [ myscript.js ]
 ```
 
-ebextensions:
+#### Plugins
+
+Optionally, you can add plugins to the server python code. To include a plugin in the deployment use the following steps:
+
+```
+$ mkdir plugins
+$ cp <source_dir>/<my_plugin>.py customize/plugins/<my_plugin>.py
+```
+
+#### ebextensions
 
 Any additional config files intended for the `.ebextensions` directory of the artifact can be added
 to the `customize/ebextensions` directory.  Any file found here will be copied over.
 
-requirements.txt:
+#### requirements.txt
 
 A custom requirements.txt can be supplied in customize/requirements.txt.
 This file must fully specify the versions of all the python modules used by the server in the deployment.
@@ -159,19 +168,19 @@ If a future cellxgene version updates its requirements by modifying a module ver
 or adding a new dependency, then the `make build` process will detect any
 incompatibilities and raise an error.
 
-5. Create the artifact.zip file for the application
+### 5. Create the artifact.zip file for the application
 
 ```
 $ make build
 ```
 
-6. Flask secret key
+### 6. Flask secret key
 
 The application requires as secret key to be provided to flask, the web framework used by cellxgene.
 There are three ways to provide the secret key:
 
 - In the configuration file: update the server/flask_secret_key attribute.
-   - An environment variable:  CXG_SECRET_KEY
+- An environment variable: `CXG_SECRET_KEY`
 - Managed by the AWS Secret Manager
 
 If using the AWS Secret Manager, then the secret name is passed as an environment variable:  CXG_AWS_SECRET_NAME.
@@ -181,8 +190,7 @@ incompatibilities and raise an error.
 If this environment variable is not defined, then the app attempts to determine the region from the
 dataroot (if in s3), or the config file location (if in s3).
 
-   
+### 7. Create an environment
-7. Create an environment
 
 ```
 # name of the environment
@@ -201,21 +209,20 @@ incompatibilities and raise an error.
    --envvars CXG_DATAROOT=$CXG_DATAROOT,CXG_CONFIG_FILE=$CXG_CONFIG_FILE
 ```
 
-8. Give the elastic beanstalk environment access to the dataroot. 
+### 8. Give the elastic beanstalk environment access to the dataroot.
 
 If using S3, this link may provide some useful information:
 https://aws.amazon.com/premiumsupport/knowledge-center/elastic-beanstalk-s3-bucket-instance/
-    
 If using Lustre, then this link may provide a place to start:
 https://aws.amazon.com/fsx/lustre/
 
-9. Deploy the application
+### 9. Deploy the application
 
 ```
 $ eb deploy $EB_ENV
 ```
 
-10. Open the application in a browser
+### 10. Open the application in a browser
 
 ```
 $ eb open $EB_ENV

server/test/__init__.py

@@ -1,4 +1,6 @@
+import random
 import shutil
+import string
 import tempfile
 from os import path, popen
 
@@ -74,3 +76,7 @@ def app_config(data_locator, backed=False):
     config.update(**args)
     config.complete_config()
     return config
+
+
+def random_string(n):
+    return "".join(random.choice(string.ascii_letters) for _ in range(n))

server/test/test_cxg_adaptor.py

@@ -7,7 +7,6 @@ from server.test.test_datasets.fixtures import pbmc3k_colors
 
 
 class TestCxgAdaptor(unittest.TestCase):
-
     def test_get_colors(self):
         data = self.get_data("pbmc3k.cxg")
         self.assertDictEqual(data.get_colors(), pbmc3k_colors)

server/test/test_cxgtool.py

@@ -1,6 +1,4 @@
-import random
 import shutil
-import string
 import unittest
 
 import anndata
@@ -8,7 +6,7 @@ import anndata
 from server.common.data_locator import DataLocator
 from server.converters.cxgtool import write_cxg
 from server.data_cxg.cxg_adaptor import CxgAdaptor
-from server.test import PROJECT_ROOT, app_config
+from server.test import PROJECT_ROOT, app_config, random_string
 from server.test.test_datasets.fixtures import pbmc3k_colors
 
 
@@ -31,8 +29,8 @@ class TestCxgAdaptor(unittest.TestCase):
         self.assertEqual(data.get_colors(), {})
 
     def convert_pbmc3k(self, **kwargs):
-        random_string = "".join(random.choice(string.ascii_letters) for _ in range(8))
+        rand_str = random_string(8)
-        data_locator = f"/tmp/test_{random_string}.cxg"
+        data_locator = f"/tmp/test_{rand_str}.cxg"
         self.fixtures.append(data_locator)
         source_h5ad = anndata.read_h5ad(f"{PROJECT_ROOT}/example-dataset/pbmc3k.h5ad")
         write_cxg(adata=source_h5ad, container=data_locator, title="pbmc3k", **kwargs)

server/test/test_plugins.py

@@ -0,0 +1,34 @@
+import os
+import shutil
+import unittest
+
+from server.common.utils import import_plugins
+from server.test import PROJECT_ROOT, random_string
+
+
+class TestPlugins(unittest.TestCase):
+    """ Test plugin import functionality """
+
+    plugins_dir = f"{PROJECT_ROOT}/server/test/plugins"
+    test_plugin_path = f"{plugins_dir}/foo.py"
+    secret = random_string(8)
+
+    @classmethod
+    def setUpClass(cls) -> None:
+        if not os.path.isdir(cls.plugins_dir):
+            os.mkdir(cls.plugins_dir)
+        with open(cls.test_plugin_path, "w") as fh:
+            fh.write(f'SECRET = "{cls.secret}"\n')
+
+    @classmethod
+    def tearDownClass(cls) -> None:
+        if os.path.isdir(cls.plugins_dir):
+            shutil.rmtree(cls.plugins_dir)
+
+    def test_import_plugins(self):
+        self.assertTrue(os.path.isfile(self.test_plugin_path))
+        loaded_modules = import_plugins("server.test.plugins")
+        # test that import plugins found the file
+        self.assertEqual(["server.test.plugins.foo"], [ele.__name__ for ele in loaded_modules])
+        # test that the module was properly executed
+        self.assertEqual(self.secret, loaded_modules[0].SECRET)