Add a process for specifying exact requirements for an EB deployment (#1451)

Add a process for keeping specifying exact requirements for an EB deployment

server/eb/Makefile

@@ -17,6 +17,7 @@ build: clean
 	(cd ../.. ; \
 	    git ls-files server/ | cpio -pdm server/eb/artifact.dir ; ); \
 	$(call copy_client_assets,../../client/build,artifact.dir/server) ; \
+	set -e ; \
 	cp app.py artifact.dir/application.py; \
 	cp ../requirements.txt artifact.dir; \
 	cp -r .ebextensions artifact.dir; \
@@ -24,6 +25,12 @@ build: clean
 	  if [ -f customize/config.yaml ] ; then \
 	     cp customize/config.yaml artifact.dir;  \
 	  fi ; \
+	  if [ -f customize/requirements.txt ] ; then \
+	     pip install requirements-parser ; \
+	     pip install packaging ; \
+	     python check_requirements.py ../requirements.txt customize/requirements.txt; \
+	     cp customize/requirements.txt artifact.dir;  \
+	  fi ; \
 	  if [ -d customize/deploy ] ; then \
 		  cp -r customize/deploy artifact.dir/server/common/web/static;  \
 	  fi; \

server/eb/README.md

@@ -135,6 +135,30 @@ ebextensions:
 Any additional config files intended for the `.ebextensions` directory of the artifact can be added
 to the `customize/ebextensions` directory.  Any file found here will be copied over.
 
+requirements.txt:
+
+A custom requirements.txt can be supplied in customize/requirements.txt.   
+This file must fully specify the versions of all the python modules used by the server in the deployment.
+This is useful to ensure that the dependencies do not change from one deployment to the next.
+Therefore the custom/requirements.txt must all have exact versions specified (e.g. anndata==0.7.1).
+
+This file can be generated the first time using a process like this:
+```
+   #  assume you are running in this directory
+   $  virtualenv temp
+   $  source temp/bin/activate
+   $  pip install -r ../requirements.txt 
+   $  mkdir -p customize
+   $  pip freeze > customize/requirements.txt
+   $  deactivate
+   $  rm -rf temp/
+```
+ 
+Keep the customize/requirememts.txt file, and reuse it for each deployment. 
+If a future cellxgene version updates its requirements by modifying a module version 
+or adding a new dependency, then the `make build` process will detect any 
+incompatibilities and raise an error. 
+
 5. Create the artifact.zip file for the application
 
    ```

server/eb/check_requirements.py

@@ -0,0 +1,104 @@
+"""This is a simple script to ensure the custom requirements.txt do not violate
+the server requirements.txt. A hosted cellxgene deployment may specify the exact
+version requirements on all the modules, and may add additional modules.
+This script is meant to aid in making that list of custom requirements easier to maintain.
+If cellxgene adds a new dependency, or changes the version requirements of an existing
+dependency, then this script can check if the custom requirements are still valid"""
+
+import sys
+import requirements
+from packaging.version import Version
+import pkg_resources
+
+
+def check(expected, custom):
+    """checks that the custom requirements meet all the requirements of the expected requirements.
+    The custom set of requirements may contain additional entries than expected.
+    The requirements in custom must all be exact (==).
+    An expected requirement must be present in custom, and must match all the specs
+    for that requirement.
+
+    expected : name of the expected requirement.txt file
+    custom : name of the custom requirements.txt file
+    """
+    edict = parse_requirements(expected)
+    cdict = parse_requirements(custom)
+
+    okay = True
+
+    # cdict must only have exact requirements (==)
+    for cname, cspecs in cdict.items():
+        if len(cspecs) != 1 or cspecs[0][0] != "==":
+            print(
+                f"Error, spec must be an exact requirement {custom}: {cname} {str(cspecs)}"
+            )
+            okay = False
+
+    for ename, especs in edict.items():
+        if ename not in cdict:
+            print(
+                f"Error, missing requirement from {custom}: {ename} {str(especs)}"
+            )
+            okay = False
+            continue
+
+        cver = Version(cdict[ename][0][1])
+        for espec in especs:
+            rokay = check_version(cver, espec[0], Version(espec[1]))
+            if not rokay:
+                print(
+                    f"Error, failed requirement from {custom}: {ename} {espec}, {cver}"
+                )
+                okay = False
+
+    if okay:
+        print("requirements check successful")
+        sys.exit(0)
+    else:
+        sys.exit(1)
+
+
+def parse_requirements(fname):
+    """Read a requirements file and return a dict of modules name / specification"""
+    try:
+        with open(fname, "r") as fd:
+            try:
+                # pylint: disable=no-member
+                rdict = {req.name: req.specs for req in requirements.parse(fd)}
+            except pkg_resources.RequirementParseError:
+                print(f"Unable to parse the requirements file: {fname}")
+                sys.exit(1)
+    except Exception as e:
+        print(f"Unable to open file {fname}: {str(e)}")
+        sys.exit(1)
+
+    return rdict
+
+
+# pylint: disable=too-many-return-statements
+def check_version(cver, optype, ever):
+    """
+    Simple version check.
+    Note: There is more complexity to comparing version (PEP440).
+    However the use cases in cellxgene are limited, and do not require a general solution.
+    """
+
+    if optype == "==":
+        return cver == ever
+    if optype == "!=":
+        return cver != ever
+    if optype == ">=":
+        return cver >= ever
+    if optype == ">":
+        return cver > ever
+    if optype == "<=":
+        return cver <= ever
+    if optype == "<":
+        return cver < ever
+
+    print(f"Error, optype not handled: {optype}")
+    return False
+
+
+if __name__ == "__main__":
+    check(sys.argv[1], sys.argv[2])