Simplify CI checks on git push, add nightly checks on cron (#1158)

* Switch push tests to Github Actions

* Add nightly tests

* Add anndata check

* Add test status badges

* Add anndata 0.6.22.post1 workaround

* Rename Nightly tests to Compatibility Tests

* Fix to anndata fix

* Turn on cron schedule

* Fix badge mardown in README.md

* Fix cron schedule

.github/workflows/compatibility_tests.yml

@@ -0,0 +1,101 @@
+name: Compatability Tests
+
+on:
+  schedule:
+    - cron: '0 8 7 * 2'
+
+env:
+  JEST_ENV: prod
+
+jobs:
+  docker-build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v1
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Build docker image
+        run: docker build .
+
+  cellxgene-master-with-python-and-anndata-versions:
+    name: python versions x anndata versions
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [3.6, 3.7, 3.8]
+        anndata-version: [0.6.22.post1, 0.7.1]
+        test-suite: [smoke-test, smoke-test-annotations]
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v1
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        sudo apt-get install libhdf5-serial-dev
+        make pydist install-dist dev-env
+        pip install anndata==${{ matrix.anndata-version }}
+        # workaround for anndata 0.6.22.post1 bug
+        [[ "0.6.22.post1" = "${{ matrix.anndata-version }}" ]] && pip install h5py==2.9.0 || true
+    - name: Tests
+      run: make unit-test ${{ matrix.test-suite }}
+
+  cellxgene-release-with-anndata-master:
+    name: cellxgene release with anndata master
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        test-suite: [smoke-test, smoke-test-annotations]
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python 3.7
+        uses: actions/setup-python@v1
+        with:
+          python-version: 3.7
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          path: cellxgene
+      - name: Checkout tools repo
+        uses: actions/checkout@v2
+        with:
+          repository: theislab/anndata
+          path: anndata
+      - name: Install dependencies
+        run: |
+          pip install --upgrade cellxgene
+          cd cellxgene && make dev-env
+          cd ../anndata && pip install -e .
+      - name: Tests
+        run: cd cellxgene && make unit-test ${{ matrix.test-suite }}
+
+  cellxgene-master-with-anndata-master:
+    name: cellxgene master with anndata master
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        test-suite: [smoke-test, smoke-test-annotations]
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python 3.7
+        uses: actions/setup-python@v1
+        with:
+          python-version: 3.7
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          path: cellxgene
+      - name: Checkout tools repo
+        uses: actions/checkout@v2
+        with:
+          repository: theislab/anndata
+          path: anndata
+      - name: Install dependencies
+        run: |
+          cd cellxgene && make pydist install-dist dev-env
+          cd ../anndata && pip install -e .
+      - name: Tests
+        run: cd cellxgene && make unit-test ${{ matrix.test-suite }}

.github/workflows/push_tests.yml

@@ -0,0 +1,101 @@
+name: Push Tests
+
+on: [push]
+
+env:
+  JEST_ENV: prod
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python 3.7
+      uses: actions/setup-python@v1
+      with:
+        python-version: 3.7
+    - name: Lint with flake8
+      run: |
+        pip install flake8
+        make lint
+
+  unit-test:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python 3.7
+      uses: actions/setup-python@v1
+      with:
+        python-version: 3.7
+    - name: Python cache
+      uses: actions/cache@v1
+      with:
+        path: ~/.cache/pip
+        key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}
+        restore-keys: |
+          ${{ runner.os }}-pip-
+    - name: Node cache
+      uses: actions/cache@v1
+      with:
+        path: ~/.npm
+        key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
+        restore-keys: |
+          ${{ runner.os }}-node-
+    - name: Install dependencies
+      run: make pydist install-dist dev-env-server
+    - name: Unit tests
+      run: make unit-test
+
+  smoke-tests:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python 3.7
+      uses: actions/setup-python@v1
+      with:
+        python-version: 3.7
+    - name: Python cache
+      uses: actions/cache@v1
+      with:
+        path: ~/.cache/pip
+        key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}
+        restore-keys: |
+          ${{ runner.os }}-pip-
+    - name: Node cache
+      uses: actions/cache@v1
+      with:
+        path: ~/.npm
+        key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
+        restore-keys: |
+          ${{ runner.os }}-node-
+    - name: Install dependencies
+      run: make pydist install-dist
+    - name: Smoke tests (without annotations feature)
+      run: cd client && make smoke-test
+
+  smoke-tests-annotations:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python 3.7
+      uses: actions/setup-python@v1
+      with:
+        python-version: 3.7
+    - name: Python cache
+      uses: actions/cache@v1
+      with:
+        path: ~/.cache/pip
+        key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt') }}
+        restore-keys: |
+          ${{ runner.os }}-pip-
+    - name: Node cache
+      uses: actions/cache@v1
+      with:
+        path: ~/.npm
+        key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
+        restore-keys: |
+          ${{ runner.os }}-node-
+    - name: Install dependencies
+      run: make pydist install-dist
+    - name: Smoke tests (with annotations feature)
+      run: cd client && make smoke-test-annotations

.travis.yml

@@ -1,31 +0,0 @@
-language: python
-dist: xenial
-sudo: required
-before_install:
-  - nvm install node
-cache:
-  - pip
-  - npm
-install:
-  - set -eo pipefail
-  - pip install flake8
-  - make pydist install-dist dev-env
-
-jobs:
-  include:
-    - name: "Branch Tests 3.7"
-      python: "3.7"
-      script: make build-client lint unit-test
-    - name: "Branch Tests 3.6"
-      python: "3.6"
-      script: make build-client lint unit-test
-    - name: "Docker Build"
-      install: skip
-      python: "3.6"
-      script: docker build .
-    - name: "Smoke Tests (with Annotations)"
-      python: "3.6"
-      script: cd client && make smoke-test
-    - name: "Smoke Tests (without Annotations)"
-      python: "3.6"
-      script: cd client && make smoke-test-annotations

Makefile

@@ -68,6 +68,10 @@ unit-test-%:
 smoke-test:
 	cd client && $(MAKE) smoke-test
 
+.PHONY: smoke-test-annotations
+smoke-test-annotations:
+	cd client && $(MAKE) smoke-test-annotations
+
 # FORMATTING CODE
 
 .PHOHY: fmt
@@ -123,8 +127,14 @@ release-directly-to-prod: dev-env pydist twine-prod
 	@echo "    make install-release"
 
 .PHONY: dev-env
-dev-env:
+dev-env: dev-env-client dev-env-server
+
+.PHONY: dev-env-client
+dev-env-client:
 	cd client && $(MAKE) ci
+
+.PHONY: dev-env-server
+dev-env-server:
 	pip install -r server/requirements-dev.txt
 
 .PHONY: gui-env

README.md

@@ -3,6 +3,8 @@
 _an interactive explorer for single-cell transcriptomics data_
 
 [![DOI](https://zenodo.org/badge/105615409.svg)](https://zenodo.org/badge/latestdoi/105615409) [![PyPI](https://img.shields.io/pypi/v/cellxgene)](https://pypi.org/project/cellxgene/) [![PyPI - Downloads](https://img.shields.io/pypi/dm/cellxgene)](https://pypistats.org/packages/cellxgene) [![GitHub last commit](https://img.shields.io/github/last-commit/chanzuckerberg/cellxgene)](https://github.com/chanzuckerberg/cellxgene/pulse)
+![Push Tests](https://github.com/chanzuckerberg/cellxgene/workflows/Push%20Tests/badge.svg)
+![Compatability Tests](https://github.com/chanzuckerberg/cellxgene/workflows/Compatability%20Tests/badge.svg)
 
 cellxgene (pronounced "cell-by-gene") is an interactive data explorer for single-cell transcriptomics datasets, such as those coming from the [Human Cell Atlas](https://humancellatlas.org). Leveraging modern web development techniques to enable fast visualizations of at least 1 million cells, we hope to enable biologists and computational researchers to explore their data.
 

scripts/await_port

@@ -16,7 +16,7 @@ import time
 parser = argparse.ArgumentParser(description='Wait for a port either to be bound to or to be free.')
 parser.add_argument('port', type=int, help='port to wait for')
 parser.add_argument('--sleep-increment', type=int, default=2.0, help='sleep duration in between checks of the port')
-parser.add_argument('--timeout', type=int, default=20.0, help='How long should we wait before timing out?')
+parser.add_argument('--timeout', type=int, default=60.0, help='How long should we wait before timing out?')
 parser.add_argument('--await-free', default=False, action='store_true',
                     help='Wait for the port to be active or free?')