From f737cc4ee45a25a17d26245d2f36e6b0dd9edbbb Mon Sep 17 00:00:00 2001 From: Charlotte Weaver Date: Mon, 4 Feb 2019 14:15:35 -0800 Subject: [PATCH] Build improvements (#577) --- .gitignore | 1 + .travis.yml | 4 +- bin/build-client | 15 ----- bin/clean | 18 ------ dev_docs/release_process.md | 34 +++++----- makefile | 122 ++++++++++++++++++++++++++++++++++++ 6 files changed, 144 insertions(+), 50 deletions(-) delete mode 100755 bin/build-client delete mode 100755 bin/clean create mode 100644 makefile diff --git a/.gitignore b/.gitignore index 32204627..98ed330f 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ coverage # production build/ +build-dev/ dist/ *.egg-info diff --git a/.travis.yml b/.travis.yml index a01d4b8d..ceb2399f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,8 +8,8 @@ cache: install: - set -eo pipefail - pip install flake8 - - ./bin/build-client - - pip install -e . + - make build + - make install - pip install -r server/requirements-dev.txt - docker build . script: diff --git a/bin/build-client b/bin/build-client deleted file mode 100755 index 33a900b9..00000000 --- a/bin/build-client +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" -CELLXGENE_DIR=$(dirname $DIR) - -cd $CELLXGENE_DIR - -npm install --prefix client/ client -npm run --prefix client build -rm -rf server/app/web/static -mkdir -p server/app/web/static/img -cp client/build/index.html server/app/web/templates/ -cp -r client/build/static server/app/web/ - -cp client/build/favicon.png server/app/web/static/img -cp client/build/service-worker.js server/app/web/static/js/ diff --git a/bin/clean b/bin/clean deleted file mode 100755 index 3ba2a395..00000000 --- a/bin/clean +++ /dev/null @@ -1,18 +0,0 @@ -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" -CELLXGENE_DIR=$(dirname $DIR) - -echo "Uninstalling cellxgene" -yes | pip uninstall cellxgene -echo "removing node_modules" -rm -rf $CELLXGENE_DIR/client/node_modules -echo "removing client_build" -rm -rf $CELLXGENE_DIR/client/build -echo "removing dist" -rm -rf $CELLXGENE_DIR/dist -echo "removing egg-info" -rm -rf $CELLXGENE_DIR/cellxgene.egg-info -echo "removing static files" -rm -f $CELLXGENE_DIR/server/app/web/templates/index.html -rm -rf $CELLXGENE_DIR/server/app/web/static -echo "cellxgene cleanup complete" - diff --git a/dev_docs/release_process.md b/dev_docs/release_process.md index 3668e958..ac770cef 100644 --- a/dev_docs/release_process.md +++ b/dev_docs/release_process.md @@ -20,18 +20,14 @@ Follow these steps to create a release. 1. Preparation: - python3.6 environment, and a cellxgene clone - - install required tools: `pip install -r requirements-dev.txt` - Define the release version number, using [semantic versioning](https://semver.org/), and specifying all three digits (eg, 0.3.0) - Write the release title and release notes and add to [release notes document](https://docs.google.com/document/d/1KnHwkYfhyWO5H8BDcMu7y3ogjvq5Yi4OwpmZ8DB6w0Y/edit) 2. Create a release branch, eg, `release-version` 3. In the release branch: - - Run `bumpversion --config-file .bumpversion.cfg [major | minor | patch]`, - where you choose major/minor/patch depending on which part of the version + - Run `make release-stage-1 PART=[major | minor | patch]` where you choose major/minor/patch depending on which part of the version is being bumped (eg, 0.2.9->0.3 is minor). - - Clean up existing environment using `bin/clean` - - Build the JS asserts using `bin/build-client` 4. Commit and push the new branch 5. Create a PR for the release. - [optional] As needed, conduct PR review. @@ -44,18 +40,26 @@ Follow these steps to create a release. - Type title `Release {version num}` - [optional] Check pre-release if this release is not ready for production - Publish Release -8. Publish to pypi by performing the following steps (assumes you have `setuptools` - and `twine` installed, that you have registered for pypi, and that you have - write access to the cellxgene pypi package): - - Build the distribution by calling `python setup.py sdist` - inside the top-level directory - - [optional] Upload the package to test pypi - `twine upload --repository-url https://test.pypi.org/legacy/ dist/*` - - [optional] Test the test installation in a fresh virtual environment using - `pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple cellxgene` - - Upload the package to real pypi using `twine upload dist/*` +8. Publish to pypi by performing the following steps (assumes you that you have registered for pypi, +and that you have write access to the cellxgene pypi package): + - Build the distribution and upload to test pypi `make release-stage-2` + - [optional] Test the test installation in a fresh virtual environment using `make install-release-test` + - Upload the package to real pypi using `make release-stage-final` - [optional] Test the installation in a fresh virtual environment using `pip install cellxgene` + - **Troubleshooting**: + - Fails to upload to test.pypi: pypi doesn't allow you to reupload a release with the same version number, + if you accidentally burned a release number you want to use on prod, you have a couple options. + 1) OPTION 1: Create distribution `make pydist`; test release locally `pip install dist/`; + then upload to prod `make release-stage-final`. + 2) OPTION 2: (DANGER) release directly to prod: `make release-burned`. + 3) OPTION 3: If the release was burned on prod as well run from Step 3 again with option + PART=patch until you get to an unburned version. + - The release doesn't install or fails your tests when you install it: Delete it from pypi - Go to pypi.org, sign in, + go to the cellxgene package, click manage, then in the options drop down, click delete and + follow the instructions. You will not be able to use that release number again. If it is a minor bug + and not a major regression, you can just release a patch. + The optional steps are for testing purposes, and are recommended for publishing any major releases, and any releases that significantly diff --git a/makefile b/makefile new file mode 100644 index 00000000..46cb6540 --- /dev/null +++ b/makefile @@ -0,0 +1,122 @@ +BUILDDIR := build +CLIENTBUILD := $(BUILDDIR)/client +SERVERBUILD := $(BUILDDIR)/server +CLEANFILES := $(BUILDDIR)/ client/build dist cellxgene.egg-info + +PART ?= patch + +# BUILDING PACKAGE + +build : clean build-server + @echo "done" + +build-server : build-client + mkdir -p $(SERVERBUILD) + cp -r server/* $(SERVERBUILD) + cp -r client/build/ $(CLIENTBUILD) + mkdir -p $(SERVERBUILD)/app/web/static/img + cp $(CLIENTBUILD)/index.html $(SERVERBUILD)/app/web/templates/ + cp -r $(CLIENTBUILD)/static $(SERVERBUILD)/app/web/ + cp $(CLIENTBUILD)/favicon.png $(SERVERBUILD)/app/web/static/img + cp $(CLIENTBUILD)/service-worker.js $(SERVERBUILD)/app/web/static/js/ + cp MANIFEST.in README.md setup.cfg setup.py $(BUILDDIR) + +build-client : + npm install --prefix client/ client + npm run --prefix client build + +# If you are actively developing in the server folder use this, dirties the source tree +build-for-server-dev : clean-server build-client + mkdir -p server/app/web/static/img + cp client/build/index.html server/app/web/templates/ + cp -r client/build/static server/app/web/ + cp client/build/favicon.png server/app/web/static/img + cp client/build/service-worker.js server/app/web/static/js/ + +clean : clean-lite clean-server + rm -rf client/node_modules + +# cleaning node_modules is the longest one, so we avoid that if possible +clean-lite : + rm -rf $(CLEANFILES) + +clean-server : + rm -f server/app/web/templates/index.html + rm -rf server/app/web/static + +.PHONY : build build-server build-client build-for-server-dev clean clean-lite clean-server + +# CREATING DISTRIBUTION RELEASE + +pydist : build + cd $(BUILDDIR); python setup.py sdist -d ../dist + @echo "done" + +.PHONY : pydist + +# RELEASE HELPERS + +# create new version to commit to master +release-stage-1 : dev-env bump clean-lite gen-package-lock + @echo "Version bumped part:$(PART) and client built. Ready to commit and push" + +# build dist and release to dev pypi +release-stage-2 : dev-env pydist twine + @echo "Dist built and uploaded to test.pypi.org" + @echo "Test the install `make install-release-test` and then upload to Pypi prod" + @echo "`make twine-prod`" + +release-stage-final: twine-prod + @echo "Release uploaded to pypi.org" + +# DANGER: releases directly to prod +# use this if you accidently burned a test release version number, +release-burned : dev-env pydist twine-prod + @echo "Dist built and uploaded to pypi.org" + @echo "Test the install `make install-release`" + +dev-env : + pip install -r server/requirements-dev.txt + +# give PART=[major, minor, part] as param to make bump +bump : + bumpversion --config-file .bumpversion.cfg $(PART) + +twine : + twine upload --repository-url https://test.pypi.org/legacy/ dist/* + +twine-prod : + twine upload dist/* + +# quicker than re-building client +gen-package-lock : + npm install --prefix client/ client + +.PHONY : release-stage-1 release-stage-2 release-stage-final release-burned dev-env bump twine twine-prod gen-package-lock + +# INSTALL + +# setup.py sucks when you have your library in a separate folder, adding these in to help setup envs + +# install from build directory +install : uninstall + cd $(BUILDDIR); pip install -e . + +# install from source tree for development +install-dev : uninstall + pip install -e . + +# install from test.pypi to test your release +install-release-test : uninstall + pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple cellxgene + @echo "Installed cellxgene from test.pypi.org, now run and smoke test" + +# install from pypi to test your release +install-release : uninstall + pip install cellxgene + @echo "Installed cellxgene from pypi.org" + +uninstall : + yes | pip uninstall cellxgene || true + +.PHONY : install install-dev install-release-test install-release uninstall