diff --git a/server/eb/Makefile b/server/eb/Makefile index 87b21add..ae15f80b 100644 --- a/server/eb/Makefile +++ b/server/eb/Makefile @@ -17,6 +17,7 @@ build: clean (cd ../.. ; \ git ls-files server/ | cpio -pdm server/eb/artifact.dir ; ); \ $(call copy_client_assets,../../client/build,artifact.dir/server) ; \ + set -e ; \ cp app.py artifact.dir/application.py; \ cp ../requirements.txt artifact.dir; \ cp -r .ebextensions artifact.dir; \ @@ -24,6 +25,12 @@ build: clean if [ -f customize/config.yaml ] ; then \ cp customize/config.yaml artifact.dir; \ fi ; \ + if [ -f customize/requirements.txt ] ; then \ + pip install requirements-parser ; \ + pip install packaging ; \ + python check_requirements.py ../requirements.txt customize/requirements.txt; \ + cp customize/requirements.txt artifact.dir; \ + fi ; \ if [ -d customize/deploy ] ; then \ cp -r customize/deploy artifact.dir/server/common/web/static; \ fi; \ diff --git a/server/eb/README.md b/server/eb/README.md index ce2c67d8..0a8d45d7 100644 --- a/server/eb/README.md +++ b/server/eb/README.md @@ -135,6 +135,30 @@ ebextensions: Any additional config files intended for the `.ebextensions` directory of the artifact can be added to the `customize/ebextensions` directory. Any file found here will be copied over. +requirements.txt: + +A custom requirements.txt can be supplied in customize/requirements.txt. +This file must fully specify the versions of all the python modules used by the server in the deployment. +This is useful to ensure that the dependencies do not change from one deployment to the next. +Therefore the custom/requirements.txt must all have exact versions specified (e.g. anndata==0.7.1). + +This file can be generated the first time using a process like this: +``` + # assume you are running in this directory + $ virtualenv temp + $ source temp/bin/activate + $ pip install -r ../requirements.txt + $ mkdir -p customize + $ pip freeze > customize/requirements.txt + $ deactivate + $ rm -rf temp/ +``` + +Keep the customize/requirememts.txt file, and reuse it for each deployment. +If a future cellxgene version updates its requirements by modifying a module version +or adding a new dependency, then the `make build` process will detect any +incompatibilities and raise an error. + 5. Create the artifact.zip file for the application ``` diff --git a/server/eb/check_requirements.py b/server/eb/check_requirements.py new file mode 100644 index 00000000..67d7ab8b --- /dev/null +++ b/server/eb/check_requirements.py @@ -0,0 +1,104 @@ +"""This is a simple script to ensure the custom requirements.txt do not violate +the server requirements.txt. A hosted cellxgene deployment may specify the exact +version requirements on all the modules, and may add additional modules. +This script is meant to aid in making that list of custom requirements easier to maintain. +If cellxgene adds a new dependency, or changes the version requirements of an existing +dependency, then this script can check if the custom requirements are still valid""" + +import sys +import requirements +from packaging.version import Version +import pkg_resources + + +def check(expected, custom): + """checks that the custom requirements meet all the requirements of the expected requirements. + The custom set of requirements may contain additional entries than expected. + The requirements in custom must all be exact (==). + An expected requirement must be present in custom, and must match all the specs + for that requirement. + + expected : name of the expected requirement.txt file + custom : name of the custom requirements.txt file + """ + edict = parse_requirements(expected) + cdict = parse_requirements(custom) + + okay = True + + # cdict must only have exact requirements (==) + for cname, cspecs in cdict.items(): + if len(cspecs) != 1 or cspecs[0][0] != "==": + print( + f"Error, spec must be an exact requirement {custom}: {cname} {str(cspecs)}" + ) + okay = False + + for ename, especs in edict.items(): + if ename not in cdict: + print( + f"Error, missing requirement from {custom}: {ename} {str(especs)}" + ) + okay = False + continue + + cver = Version(cdict[ename][0][1]) + for espec in especs: + rokay = check_version(cver, espec[0], Version(espec[1])) + if not rokay: + print( + f"Error, failed requirement from {custom}: {ename} {espec}, {cver}" + ) + okay = False + + if okay: + print("requirements check successful") + sys.exit(0) + else: + sys.exit(1) + + +def parse_requirements(fname): + """Read a requirements file and return a dict of modules name / specification""" + try: + with open(fname, "r") as fd: + try: + # pylint: disable=no-member + rdict = {req.name: req.specs for req in requirements.parse(fd)} + except pkg_resources.RequirementParseError: + print(f"Unable to parse the requirements file: {fname}") + sys.exit(1) + except Exception as e: + print(f"Unable to open file {fname}: {str(e)}") + sys.exit(1) + + return rdict + + +# pylint: disable=too-many-return-statements +def check_version(cver, optype, ever): + """ + Simple version check. + Note: There is more complexity to comparing version (PEP440). + However the use cases in cellxgene are limited, and do not require a general solution. + """ + + if optype == "==": + return cver == ever + if optype == "!=": + return cver != ever + if optype == ">=": + return cver >= ever + if optype == ">": + return cver > ever + if optype == "<=": + return cver <= ever + if optype == "<": + return cver < ever + + print(f"Error, optype not handled: {optype}") + return False + + +if __name__ == "__main__": + check(sys.argv[1], sys.argv[2])