fork/cellxgene
1
0
Fork 0
mirror of https://github.com/chanzuckerberg/cellxgene.git synced 2026-10-07 04:38:11 +08:00
Code Issues Packages Projects Releases Wiki Activity

Refactor developer convenience scripts (#1377)

Browse Source 
* Cleanup the backend-dev convenience method

* Add the frontend_dev convenience method

frontend_dev is a soup-to-nuts convenience method for setting up the FE
development environment with node running a the client code on port 3000
with the a separate cellxgene package serving the API over port 5005 in
the background.

The script can be run from Finder.

* Update the developer scripts documentation

* Remove the 'test' make target in the client Makefile

Rationale:
* Given how long the smoke tests take to run, it is unlikely that
  developers will want to run all tests together.
* It is unlikely that developers will have set up the backend server
  properly for the tests to pass.
* Available commands should be safe-ish and not lend themselves to
  confusing errors.
* You can still group tests by concatenating them in a make command, as
  in `make unit-test smoke-test`.
* This target isn't used in any of our CI pipelines -- KISS.

* Some version of python3...

* Minor typos in docs

* Respond to feedback from @bkmartinjr

* Make adjustments so that DATASET path is predictable

* Simplify environment defaults a bit
This commit is contained in:
Matt Weiden
2020-04-15 10:32:19 -07:00
committed by GitHub
parent e5dfd6e8fa
commit 7d4d360e52
7 changed files with 146 additions and 113 deletions
Download Patch File Download Diff File Expand all files Collapse all files
client/Makefile
-27
View File
@@ -4,7 +4,6 @@ ifeq ($(shell uname),Darwin) # is Windows_NT on XP, 2000, 7, Vista, 10...
IS_DARWIN := "true" IS_DARWIN := "true"
endif endif
DATASET := $(if $(DATASET),$(DATASET),../example-dataset/pbmc3k.h5ad)
ANNOTATIONS := $(if $(ANNOTATIONS),$(ANNOTATIONS),../server/test/test_datasets/pbmc3k-annotations.csv) ANNOTATIONS := $(if $(ANNOTATIONS),$(ANNOTATIONS),../server/test/test_datasets/pbmc3k-annotations.csv)
ANNOTATIONS_FILENAME := $(shell basename $(ANNOTATIONS)) ANNOTATIONS_FILENAME := $(shell basename $(ANNOTATIONS))
@@ -38,36 +37,10 @@ lint-diff:
# Development convenience methods # Development convenience methods
.PHONY: server-requirements
server-requirements:
virtualenv -p python3 ../venv
source ../venv/bin/activate && \
pip install -r ../server/requirements-dev.txt && \
yes | pip uninstall cellxgene || true && \
pip install -e ..
.PHONY: start-frontend .PHONY: start-frontend
start-frontend: start-frontend:
node server/development.js node server/development.js
.PHONY: start-server
start-server:
cellxgene launch -p $(CXG_SERVER_PORT) $(CXG_OPTIONS) $(DATASET)
.PHONY: backend-dev
backend-dev: server-requirements
source ../venv/bin/activate && CXG_OPTIONS='--debug' $(MAKE) start-server
.PHONY: backend-dev-anno-ontology
backend-dev-anno-ontology: server-requirements
CXG_OPTIONS='--experimental-annotations-ontology --debug' \
$(MAKE) backend-dev
.PHONY: test
test:
node node_modules/jest/bin/jest.js
.PHONY: e2e .PHONY: e2e
e2e: e2e:
node node_modules/jest/bin/jest.js \ node node_modules/jest/bin/jest.js \
common.mk
+6 -4
View File
@@ -7,16 +7,18 @@ SHELL := env PATH='$(PATH)' /bin/bash
# - If a variable is defined, return its value # - If a variable is defined, return its value
# - Else return the default value from environment.dev # - Else return the default value from environment.dev
define get_or_else_dev_env_default define get_or_else_dev_env_default
$(if $($(1)),$($(1)),$(shell sed -n 's/$(1)=\(.*\)/\1/p' < $(PROJECT_ROOT)/environment.default)) $(if $($(1)),$($(1)),$(shell VAR=$$(sed -n 's/$(1)=\(.*\)/\1/p' $(PROJECT_ROOT)/environment.default); eval "echo \"$$VAR\""))
endef endef
export CXG_SERVER_PORT := $(call get_or_else_dev_env_default,CXG_SERVER_PORT) export CXG_SERVER_PORT := $(call get_or_else_dev_env_default,CXG_SERVER_PORT)
export CXG_CLIENT_PORT := $(call get_or_else_dev_env_default,CXG_CLIENT_PORT) export CXG_CLIENT_PORT := $(call get_or_else_dev_env_default,CXG_CLIENT_PORT)
export JEST_ENV := $(call get_or_else_dev_env_default,JEST_ENV) export JEST_ENV := $(call get_or_else_dev_env_default,JEST_ENV)
export DATASET := $(call get_or_else_dev_env_default,DATASET)
export CXG_OPTIONS := $(call get_or_else_dev_env_default,CXG_OPTIONS)
export CXG_SERVER_PORT .PHONY: start-server
export CXG_CLIENT_PORT start-server:
export JEST_ENV cellxgene launch -p $(CXG_SERVER_PORT) $(CXG_OPTIONS) $(DATASET)
# copy the client assests to a location known to the server # copy the client assests to a location known to the server
# $(1) is the source of the client assets # $(1) is the source of the client assets
dev_docs/developer_scripts.md
+74 -82
View File
@@ -1,122 +1,114 @@
# Developer convenience methods # Developer convenience scripts
## Makefile This document describes scripts for accelerating cellxgene development.
Documentation for the `Makefile` targets in the project root directory. Paths are relative to the root project directory. If you need to know what
this is, run `PROJECT_ROOT=$(git rev-parse --show-toplevel); echo
$PROJECT_ROOT`.
### Build commands ## Project-level scripts
builds source code ### Build
``` **Usage:** from the `$PROJECT_ROOT` directory run:
build - builds the whole app (client/ source and server/ source packaged together) * `make build` builds whole app client and server
build-client - runs the client webpack build * `make build-client` runs webpack build
build-for-server-dev - builds client and copies output directly into the server source tree (only for server devlopment) * `make build-for-server-dev` builds client and copies output directly into
``` source tree (only for server devlopment)
### Clean commands ### Clean
deletes generated files Deletes generated files.
``` **Usage:** from the `$PROJECT_ROOT` directory run:
clean - cleans everything including node modules (means build with take a while * `make clean` cleans everything including node modules (means build with take
clean-lite - cleans built directories a while
clean-server - cleans source tree * `make clean-lite` cleans built directories
``` * `make clean-server` cleans source tree
### Dist commands ### Distribution
creates distribution for python module to upload to pypi Creates distribution for python module to upload to pypi.
``` **Usage:** from the `$PROJECT_ROOT` directory run:
pydist - builds code and then builds sdist * `make pydist` builds code and then builds sdist
```
### Release commands ### Release
see release_process.md See `release_process.md`.
### Env commands ### Development environment
Installs requirements files Installs requirements files.
``` **Usage:** from the `$PROJECT_ROOT` directory run:
dev-env - installs requirements and requirments-dev (for building code) * `make dev-env` installs requirements and requirments-dev (for building code)
```
### install commands ### Installing cellxgene packages
Installs cellxgene from different locations **Usage:** from the `$PROJECT_ROOT` directory:
* `install-dev` - installs from local source tree
* `install-release-test` - installs from test pypi
* `install-release` - installs from pypi
* `install-dist` - installs from local dist folder
* `uninstall` - uninstalls cellxgene
``` ## Client-level scripts
install-dev - installs from local source tree
install-release-test - installs from test pypi
install-release - installs from pypi
install-dist - installs from local dist folder
uninstall - uninstalls cellxgene
```
## Client Makefile ### Running the client
The following phony `make` targets in `client/Makefile` are convenience methods for getting you up and developing.
### Runner scripts
#### start-frontend #### start-frontend
**About** Serve the current client javascript independently from the `server` code. **About** Serve the current client javascript independently from the `server` code.
**Requires** **Requires**
* The server to be running. Best way to do this is with `make backend-dev` * The server to be running. Best way to do this is with [backend_dev](#backend_dev).
* `make ci` to install the necessary node modules * `make ci` to install the necessary node modules
**Usage** `make start-frontend` **Usage:** from the `$PROJECT_ROOT/client` directory run `make start-frontend`
#### backend-dev #### backend_dev
**About** This script enables FE developers to run the REST API necessary to back the development server for the front end. It is intended to ensure that the FE developer gets the current version of the backend with a single command and no knowledge of python necessary. It creates and activates a virtual environment and installs cellxgene from the current branch. **About** This script enables FE developers to run the REST API necessary to
back the development server for the front end. It is intended to ensure that
the FE developer gets the current version of the backend with a single command
and no knowledge of python necessary. It creates and activates a virtual
environment and installs cellxgene from the current branch.
**Requires** Python3.6 - `virtual-env`, `pip` **Requires** `Python3.6+`, `virtual-env`, `pip`
**Usage** `make backend-dev`. Optionally, you can then launch the node development server to serve the current state of **Usage:** from the `$PROJECT_ROOT` directory run `./scripts/backend_dev`
the FE with `make start-frontend`. You can also select a specific dataset using `DATASET=<dataset path> make backend-dev`.
You can also use `CXG_OPTIONS` to pass options to the `cellxgene launch` command, as in
`CXG_OPTIONS='--experimental-annotations --experimental-annotations-file annotations.csv' make backend-dev`.
**Tips** Developers will probably want to run this in parallel with the node dev server. You can either do this by running each in a separate termanial window or by running `backend-dev` in the background (add an `&` at the end of the command to run in the background: `DATASET=<dataset> make backend-dev &`). **Options:**
* In parallel, you can then launch the node development server to serve the
current state of the FE with [`start-frontend`](#start-frontend), usually in
a different terminal tab.
* You can also select a specific dataset using `DATASET=<dataset path> ./scripts/backend_dev`.
* You can also use `CXG_OPTIONS` to pass options to the `cellxgene launch`
command, as in `CXG_OPTIONS='--disable-annotations' ./scripts/backend_dev`.
**Breakdown** **Breakdown**
| command | purpose | | command | purpose |
| ---------------------------------------- | --------------------------------------------------------- | | ---------------------------------------- | ---------------------------------------------------------- |
| python3.6 -m venv cellxgene | creates cellxgene virtual environment | | python3.6 -m venv cellxgene | creates cellxgene virtual environment |
| source cellxgene/bin/activate | activates virtual environment | | source cellxgene/bin/activate | activates virtual environment |
| yes \| pip uninstall cellxgene \|\| true | uninstalls cellxgene (if installed) | | yes \| pip uninstall cellxgene \|\| true | uninstalls cellxgene (if installed) |
| pip install -e . | installs current local version of cellxgene | | pip install -e . | installs current local version of cellxgene |
| cellxgene launch | launches cellxgene (must supply dataset as last parameter) | | cellxgene launch | launches cellxgene (must supply dataset as last parameter) |
### Test scripts ### Client test scripts
#### test Methods used to test the client javascript code
**About** Run test locally. In order for this command to succeed you will need to give it a specific unit test to run. It won't pass if you run all tests as may be expected. This is because the unit tests and end to end (e2e) tests require different testing environments. **Usage:** from the `$PROJECT_ROOT/client` directory run:
* `make unit-test` Runs all unit tests. It excludes any tests in the e2e
**Usage** `make test` folder. This is used by travis to run unit tests.
* `make smoke-test` Starts backend development server and runs end to end
#### unit-test tests. This is what travis runs. It depends on the `e2e` and the
`backend-dev` targets. One starts the server, the other runs the tests. If
**About** Runs all unit tests. It excludes any tests in the e2e folder. This is used by travis to run unit tests. developing a front-end feature and just checking if tests pass, this is
probabaly the one you want to run.
**Usage** `make unit-test` * `make e2e` Runs backend tests without starting the server. You will need to
start the rest api separately with the pbmc3k.h5ad file. Note you can use
#### smoke-test the `JEST_ENV` environment variable to change how JEST runs in the browser.
**About** Starts backend development server and runs end to end tests. This is what travis runs. It depends on the `e2e` and the `backend-dev` targets. One starts the server, the other runs the tests. If developing a front-end feature and just checking if tests pass, this is probabaly the one you want to run.
**Usage** `make smoke-test`
#### e2e
**About** Runs backend tests without starting the server. You will need to start the rest api separately with the pbmc3k.h5ad file. Note you can use the `JEST_ENV` environment variable to change how JEST runs in the browser.
**Usage** `make e2e`
environment.default
+2
View File
@@ -2,3 +2,5 @@
CXG_SERVER_PORT=5005 CXG_SERVER_PORT=5005
CXG_CLIENT_PORT=3000 CXG_CLIENT_PORT=3000
JEST_ENV=dev JEST_ENV=dev
DATASET="$(git rev-parse --show-toplevel)/example-dataset/pbmc3k.h5ad"
CXG_OPTIONS=--debug
scripts/backend_dev
Executable
+10
View File
@@ -0,0 +1,10 @@
#!/usr/bin/env bash
set -e
PROJECT_ROOT=$(git rev-parse --show-toplevel)
${PROJECT_ROOT}/scripts/dev_setup
source ${PROJECT_ROOT}/venv/bin/activate
make --file=${PROJECT_ROOT}/common.mk start-server
scripts/dev_setup
Executable
+12
View File
@@ -0,0 +1,12 @@
#!/usr/bin/env bash
set -e
PROJECT_ROOT=$(git rev-parse --show-toplevel)
cd ${PROJECT_ROOT}
virtualenv -p python3 venv
source venv/bin/activate
make uninstall dev-env-server build-for-server-dev install-dev
scripts/frontend_dev
Executable
+42
View File
@@ -0,0 +1,42 @@
#!/usr/bin/osascript
set DATASET to system attribute "DATASET"
set CXG_OPTIONS to system attribute "CXG_OPTIONS"
on gotoProjectRootDir()
tell application "Finder"
set current_path to container of (path to me) as alias
end tell
tell application "Terminal"
do script "cd " & (POSIX path of current_path) & ".." in front window
end tell
end gotoProjectRootDir
on newTab()
tell application "System Events" to keystroke "t" using command down
delay 0.05
end newTab
tell application "Terminal"
activate
reopen
my newTab()
my gotoProjectRootDir()
if DATASET is not equal to "" then
do script "export DATASET=" & DATASET in front window
end if
if CXG_OPTIONS is not equal to "" then
do script "export CXG_OPTIONS=" & CXG_OPTIONS in front window
end if
do script "./scripts/backend_dev" in front window
my newTab()
my gotoProjectRootDir()
do script "sleep 1
# This tab is for the client hot-reloading node server. It will wait until cellxgene is built before continuing. Please hang in there while the other tab completes.
while [ `ps aux | grep dev_setup | grep -v grep | awk '{ print $2 }'` ]; do sleep 1; done
cd client/
make start-frontend" in window 1
end tell