fork/cellxgene
1
0
Fork 0
mirror of https://github.com/chanzuckerberg/cellxgene.git synced 2026-09-15 12:47:56 +08:00
Code Issues Packages Projects Releases Wiki Activity

Refactor developer convenience scripts (#1377)

Browse Source 
* Cleanup the backend-dev convenience method

* Add the frontend_dev convenience method

frontend_dev is a soup-to-nuts convenience method for setting up the FE
development environment with node running a the client code on port 3000
with the a separate cellxgene package serving the API over port 5005 in
the background.

The script can be run from Finder.

* Update the developer scripts documentation

* Remove the 'test' make target in the client Makefile

Rationale:
* Given how long the smoke tests take to run, it is unlikely that
  developers will want to run all tests together.
* It is unlikely that developers will have set up the backend server
  properly for the tests to pass.
* Available commands should be safe-ish and not lend themselves to
  confusing errors.
* You can still group tests by concatenating them in a make command, as
  in `make unit-test smoke-test`.
* This target isn't used in any of our CI pipelines -- KISS.

* Some version of python3...

* Minor typos in docs

* Respond to feedback from @bkmartinjr

* Make adjustments so that DATASET path is predictable

* Simplify environment defaults a bit
This commit is contained in:
Matt Weiden
2020-04-15 10:32:19 -07:00
committed by GitHub
parent e5dfd6e8fa
commit 7d4d360e52
7 changed files with 146 additions and 113 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
client/Makefile
View File

@@ -4,7 +4,6 @@ ifeq ($(shell uname),Darwin) # is Windows_NT on XP, 2000, 7, Vista, 10...
IS_DARWIN := "true"
endif
DATASET := $(if $(DATASET),$(DATASET),../example-dataset/pbmc3k.h5ad)
ANNOTATIONS := $(if $(ANNOTATIONS),$(ANNOTATIONS),../server/test/test_datasets/pbmc3k-annotations.csv)
ANNOTATIONS_FILENAME := $(shell basename $(ANNOTATIONS))
@@ -38,36 +37,10 @@ lint-diff:
# Development convenience methods
.PHONY: server-requirements
server-requirements:
virtualenv -p python3 ../venv
source ../venv/bin/activate && \
pip install -r ../server/requirements-dev.txt && \
yes | pip uninstall cellxgene || true && \
pip install -e ..
.PHONY: start-frontend
start-frontend:
node server/development.js
.PHONY: start-server
start-server:
cellxgene launch -p $(CXG_SERVER_PORT) $(CXG_OPTIONS) $(DATASET)
.PHONY: backend-dev
backend-dev: server-requirements
source ../venv/bin/activate && CXG_OPTIONS='--debug' $(MAKE) start-server
.PHONY: backend-dev-anno-ontology
backend-dev-anno-ontology: server-requirements
CXG_OPTIONS='--experimental-annotations-ontology --debug' \
$(MAKE) backend-dev
.PHONY: test
test:
node node_modules/jest/bin/jest.js
.PHONY: e2e
e2e:
node node_modules/jest/bin/jest.js \

10
common.mk
View File

@@ -7,16 +7,18 @@ SHELL := env PATH='$(PATH)' /bin/bash
# - If a variable is defined, return its value
# - Else return the default value from environment.dev
define get_or_else_dev_env_default
$(if $($(1)),$($(1)),$(shell sed -n 's/$(1)=\(.*\)/\1/p' < $(PROJECT_ROOT)/environment.default))
$(if $($(1)),$($(1)),$(shell VAR=$$(sed -n 's/$(1)=\(.*\)/\1/p' $(PROJECT_ROOT)/environment.default); eval "echo \"$$VAR\""))
endef
export CXG_SERVER_PORT := $(call get_or_else_dev_env_default,CXG_SERVER_PORT)
export CXG_CLIENT_PORT := $(call get_or_else_dev_env_default,CXG_CLIENT_PORT)
export JEST_ENV := $(call get_or_else_dev_env_default,JEST_ENV)
export DATASET := $(call get_or_else_dev_env_default,DATASET)
export CXG_OPTIONS := $(call get_or_else_dev_env_default,CXG_OPTIONS)
export CXG_SERVER_PORT
export CXG_CLIENT_PORT
export JEST_ENV
.PHONY: start-server
start-server:
cellxgene launch -p $(CXG_SERVER_PORT) $(CXG_OPTIONS) $(DATASET)
# copy the client assests to a location known to the server
# $(1) is the source of the client assets

156
dev_docs/developer_scripts.md
View File

@@ -1,122 +1,114 @@
# Developer convenience methods
# Developer convenience scripts
## Makefile
This document describes scripts for accelerating cellxgene development.
Documentation for the `Makefile` targets in the project root directory.
Paths are relative to the root project directory. If you need to know what
this is, run `PROJECT_ROOT=$(git rev-parse --show-toplevel); echo
$PROJECT_ROOT`.
### Build commands
## Project-level scripts
builds source code
### Build
```
build - builds the whole app (client/ source and server/ source packaged together)
build-client - runs the client webpack build
build-for-server-dev - builds client and copies output directly into the server source tree (only for server devlopment)
```
**Usage:** from the `$PROJECT_ROOT` directory run:
* `make build` builds whole app client and server
* `make build-client` runs webpack build
* `make build-for-server-dev` builds client and copies output directly into
source tree (only for server devlopment)
### Clean commands
### Clean
deletes generated files
Deletes generated files.
```
clean - cleans everything including node modules (means build with take a while
clean-lite - cleans built directories
clean-server - cleans source tree
```
**Usage:** from the `$PROJECT_ROOT` directory run:
* `make clean` cleans everything including node modules (means build with take
a while
* `make clean-lite` cleans built directories
* `make clean-server` cleans source tree
### Dist commands
### Distribution
creates distribution for python module to upload to pypi
Creates distribution for python module to upload to pypi.
```
pydist - builds code and then builds sdist
```
**Usage:** from the `$PROJECT_ROOT` directory run:
* `make pydist` builds code and then builds sdist
### Release commands
### Release
see release_process.md
See `release_process.md`.
### Env commands
### Development environment
Installs requirements files
Installs requirements files.
```
dev-env - installs requirements and requirments-dev (for building code)
```
**Usage:** from the `$PROJECT_ROOT` directory run:
* `make dev-env` installs requirements and requirments-dev (for building code)
### install commands
### Installing cellxgene packages
Installs cellxgene from different locations
**Usage:** from the `$PROJECT_ROOT` directory:
* `install-dev` - installs from local source tree
* `install-release-test` - installs from test pypi
* `install-release` - installs from pypi
* `install-dist` - installs from local dist folder
* `uninstall` - uninstalls cellxgene
```
install-dev - installs from local source tree
install-release-test - installs from test pypi
install-release - installs from pypi
install-dist - installs from local dist folder
uninstall - uninstalls cellxgene
```
## Client-level scripts
## Client Makefile
The following phony `make` targets in `client/Makefile` are convenience methods for getting you up and developing.
### Runner scripts
### Running the client
#### start-frontend
**About** Serve the current client javascript independently from the `server` code.
**Requires**
* The server to be running. Best way to do this is with `make backend-dev`
* The server to be running. Best way to do this is with [backend_dev](#backend_dev).
* `make ci` to install the necessary node modules
**Usage** `make start-frontend`
**Usage:** from the `$PROJECT_ROOT/client` directory run `make start-frontend`
#### backend-dev
#### backend_dev
**About** This script enables FE developers to run the REST API necessary to back the development server for the front end. It is intended to ensure that the FE developer gets the current version of the backend with a single command and no knowledge of python necessary. It creates and activates a virtual environment and installs cellxgene from the current branch.
**About** This script enables FE developers to run the REST API necessary to
back the development server for the front end. It is intended to ensure that
the FE developer gets the current version of the backend with a single command
and no knowledge of python necessary. It creates and activates a virtual
environment and installs cellxgene from the current branch.
**Requires** Python3.6 - `virtual-env`, `pip`
**Requires** `Python3.6+`, `virtual-env`, `pip`
**Usage** `make backend-dev`. Optionally, you can then launch the node development server to serve the current state of
the FE with `make start-frontend`. You can also select a specific dataset using `DATASET=<dataset path> make backend-dev`.
You can also use `CXG_OPTIONS` to pass options to the `cellxgene launch` command, as in
`CXG_OPTIONS='--experimental-annotations --experimental-annotations-file annotations.csv' make backend-dev`.
**Usage:** from the `$PROJECT_ROOT` directory run `./scripts/backend_dev`
**Tips** Developers will probably want to run this in parallel with the node dev server. You can either do this by running each in a separate termanial window or by running `backend-dev` in the background (add an `&` at the end of the command to run in the background: `DATASET=<dataset> make backend-dev &`).
**Options:**
* In parallel, you can then launch the node development server to serve the
current state of the FE with [`start-frontend`](#start-frontend), usually in
a different terminal tab.
* You can also select a specific dataset using `DATASET=<dataset path> ./scripts/backend_dev`.
* You can also use `CXG_OPTIONS` to pass options to the `cellxgene launch`
command, as in `CXG_OPTIONS='--disable-annotations' ./scripts/backend_dev`.
**Breakdown**
| command | purpose |
| ---------------------------------------- | --------------------------------------------------------- |
| python3.6 -m venv cellxgene | creates cellxgene virtual environment |
| source cellxgene/bin/activate | activates virtual environment |
| yes \| pip uninstall cellxgene \|\| true | uninstalls cellxgene (if installed) |
| pip install -e . | installs current local version of cellxgene |
| command | purpose |
| ---------------------------------------- | ---------------------------------------------------------- |
| python3.6 -m venv cellxgene | creates cellxgene virtual environment |
| source cellxgene/bin/activate | activates virtual environment |
| yes \| pip uninstall cellxgene \|\| true | uninstalls cellxgene (if installed) |
| pip install -e . | installs current local version of cellxgene |
| cellxgene launch | launches cellxgene (must supply dataset as last parameter) |
### Test scripts
### Client test scripts
#### test
Methods used to test the client javascript code
**About** Run test locally. In order for this command to succeed you will need to give it a specific unit test to run. It won't pass if you run all tests as may be expected. This is because the unit tests and end to end (e2e) tests require different testing environments.
**Usage** `make test`
#### unit-test
**About** Runs all unit tests. It excludes any tests in the e2e folder. This is used by travis to run unit tests.
**Usage** `make unit-test`
#### smoke-test
**About** Starts backend development server and runs end to end tests. This is what travis runs. It depends on the `e2e` and the `backend-dev` targets. One starts the server, the other runs the tests. If developing a front-end feature and just checking if tests pass, this is probabaly the one you want to run.
**Usage** `make smoke-test`
#### e2e
**About** Runs backend tests without starting the server. You will need to start the rest api separately with the pbmc3k.h5ad file. Note you can use the `JEST_ENV` environment variable to change how JEST runs in the browser.
**Usage** `make e2e`
**Usage:** from the `$PROJECT_ROOT/client` directory run:
* `make unit-test` Runs all unit tests. It excludes any tests in the e2e
folder. This is used by travis to run unit tests.
* `make smoke-test` Starts backend development server and runs end to end
tests. This is what travis runs. It depends on the `e2e` and the
`backend-dev` targets. One starts the server, the other runs the tests. If
developing a front-end feature and just checking if tests pass, this is
probabaly the one you want to run.
* `make e2e` Runs backend tests without starting the server. You will need to
start the rest api separately with the pbmc3k.h5ad file. Note you can use
the `JEST_ENV` environment variable to change how JEST runs in the browser.

2
environment.default
View File

@@ -2,3 +2,5 @@
CXG_SERVER_PORT=5005
CXG_CLIENT_PORT=3000
JEST_ENV=dev
DATASET="$(git rev-parse --show-toplevel)/example-dataset/pbmc3k.h5ad"
CXG_OPTIONS=--debug

10
scripts/backend_dev Executable file
View File

@@ -0,0 +1,10 @@
#!/usr/bin/env bash
set -e
PROJECT_ROOT=$(git rev-parse --show-toplevel)
${PROJECT_ROOT}/scripts/dev_setup
source ${PROJECT_ROOT}/venv/bin/activate
make --file=${PROJECT_ROOT}/common.mk start-server

12
scripts/dev_setup Executable file
View File

@@ -0,0 +1,12 @@
#!/usr/bin/env bash
set -e
PROJECT_ROOT=$(git rev-parse --show-toplevel)
cd ${PROJECT_ROOT}
virtualenv -p python3 venv
source venv/bin/activate
make uninstall dev-env-server build-for-server-dev install-dev

42
scripts/frontend_dev Executable file
View File

@@ -0,0 +1,42 @@
#!/usr/bin/osascript
set DATASET to system attribute "DATASET"
set CXG_OPTIONS to system attribute "CXG_OPTIONS"
on gotoProjectRootDir()
tell application "Finder"
set current_path to container of (path to me) as alias
end tell
tell application "Terminal"
do script "cd " & (POSIX path of current_path) & ".." in front window
end tell
end gotoProjectRootDir
on newTab()
tell application "System Events" to keystroke "t" using command down
delay 0.05
end newTab
tell application "Terminal"
activate
reopen
my newTab()
my gotoProjectRootDir()
if DATASET is not equal to "" then
do script "export DATASET=" & DATASET in front window
end if
if CXG_OPTIONS is not equal to "" then
do script "export CXG_OPTIONS=" & CXG_OPTIONS in front window
end if
do script "./scripts/backend_dev" in front window
my newTab()
my gotoProjectRootDir()
do script "sleep 1
# This tab is for the client hot-reloading node server. It will wait until cellxgene is built before continuing. Please hang in there while the other tab completes.
while [ `ps aux | grep dev_setup | grep -v grep | awk '{ print $2 }'` ]; do sleep 1; done
cd client/
make start-frontend" in window 1
end tell