diff --git a/.github/workflows/compatibility_tests.yml b/.github/workflows/compatibility_tests.yml index 72edc046..0f388595 100644 --- a/.github/workflows/compatibility_tests.yml +++ b/.github/workflows/compatibility_tests.yml @@ -7,10 +7,6 @@ on: branches: - main - # For debug - uncomment below to run on all PRs - # pull_request: - # branches: "*" - env: JEST_ENV: prod @@ -32,14 +28,13 @@ jobs: strategy: fail-fast: false matrix: - # note: The `macos-latest` is latest Catalina version, and not Big Sur. So we explicitly ask for Big Sur (`macos-11`) os: [ubuntu-latest, macos-latest, macos-13] - python-version: ["3.10", "3.11"] + python-version: ["3.10", "3.11", "3.12"] cellxgene_build: [main, latest] # add anndata pinned version test for subset of matrix configurations, # in order to reduce matrix cross-product explosion include: - - python-version: 3.11 + - python-version: 3.12 cellxgene_build: latest # TODO: dynamically use the literal version in requirements.txt, # to avoid having to update this in manually in the future @@ -100,7 +95,7 @@ jobs: # keep same pip pkg versions as in the cxg release sed -i'' -e 's/-r requirements.txt//' server/requirements-dev.txt pip install -r server/requirements-dev.txt - pip install --force-reinstall numpy==2.0.1 numba>=0.60.0 pandas + pip install --force-reinstall numpy==2.0.1 numba>=0.60.0 pandas flatbuffers==2.0.7 - name: Install anndata version per matrix variable run: pip install anndata${{ matrix.anndata_version }} - name: Install node diff --git a/.github/workflows/push_tests.yml b/.github/workflows/push_tests.yml index 5de6038b..20a77be5 100644 --- a/.github/workflows/push_tests.yml +++ b/.github/workflows/push_tests.yml @@ -17,10 +17,10 @@ jobs: - uses: actions/checkout@v4 - run: | git fetch --depth=1 origin +${{github.base_ref}} - - name: Set up Python 3.11 + - name: Set up Python 3.12 uses: actions/setup-python@v5 with: - python-version: 3.11 + python-version: 3.12 - name: Node cache uses: actions/cache@v4 with: @@ -46,10 +46,10 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - name: Set up Python 3.11 (pyenv) # pyenv needed for mlflow in cli annotate tests + - name: Set up Python 3.12 (pyenv) # pyenv needed for mlflow in cli annotate tests uses: gabrielfalcao/pyenv-action@v9 with: - default: 3.11 + default: 3.12 command: pip install -U pip # upgrade pip after installing python - run: pip install virtualenv # virtualenv needed for mlflow in cli annotate tests - name: Python cache @@ -79,10 +79,10 @@ jobs: timeout-minutes: 20 steps: - uses: actions/checkout@v4 - - name: Set up Python 3.11 + - name: Set up Python 3.12 uses: actions/setup-python@v5 with: - python-version: 3.11 + python-version: 3.12 - name: Python cache uses: actions/cache@v4 with: @@ -98,7 +98,9 @@ jobs: restore-keys: | ${{ runner.os }}-node- - name: Install dependencies - run: make pydist install-dist + run: | + pip install setuptools + make pydist install-dist - name: Smoke tests (without annotations feature) run: | cd client && make smoke-test diff --git a/README.md b/README.md index 8d6f480d..3686a0d6 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ Whether you need to visualize one thousand cells or one million, CELLxGENE Annot ### Quick start -To install CELLxGENE Annotate you need Python 3.6+. We recommend [installing Annotate into a conda or virtual environment.](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/desktop/install.md) +To install CELLxGENE Annotate you need Python 3.10+. We recommend [installing Annotate into a conda or virtual environment.](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/desktop/install.md) Install the package. @@ -66,22 +66,21 @@ For any errors, [report bugs on Github](https://github.com/chanzuckerberg/cellxg ### Contributing -We warmly welcome contributions from the community! Please see our [contributing guide](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/contribute.md) and don't hesitate to open an issue or send a pull request to improve CELLxGENE Annotate. Please see the [dev_docs](https://github.com/chanzuckerberg/cellxgene/tree/main/dev_docs) for pull request suggestions, unit test details, local documentation preview, and other development specifics. +We warmly welcome contributions from the community! Please see our [contributing guide](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/contribute.md) and don't hesitate to open an issue or send a pull request to improve CELLxGENE Annotate. Please see the [dev_docs](https://github.com/chanzuckerberg/cellxgene/tree/main/dev_docs) for pull request suggestions, unit test details, local documentation preview, and other development specifics. This project adheres to the Contributor Covenant [code of conduct](https://github.com/chanzuckerberg/.github/blob/master/CODE_OF_CONDUCT.md). By participating, you are expected to uphold this code. Please report unacceptable behavior to opensource@chanzuckerberg.com. ### Reuse -This project was started with the sole goal of empowering the scientific community to explore and understand their data. -As such, we encourage other scientific tool builders in academia or industry to adopt the patterns, tools, and code from +This project was started with the sole goal of empowering the scientific community to explore and understand their data. +As such, we encourage other scientific tool builders in academia or industry to adopt the patterns, tools, and code from this project. All code is freely available for reuse under the [MIT license](https://opensource.org/licenses/MIT). +Before extending CELLxGENE Annotate, we encourage you to reach out to us with ideas or questions. It might be possible that an +extension could be directly contributed, which would make it available for a wider audience, or that it's on our +[roadmap](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/roadmap.md) and under active development. -Before extending CELLxGENE Annotate, we encourage you to reach out to us with ideas or questions. It might be possible that an -extension could be directly contributed, which would make it available for a wider audience, or that it's on our -[roadmap](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/roadmap.md) and under active development. - -See the [CELLxGENE extensions](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/community-extensions.md) section of our documentation for examples of community use and CELLxGENE extensions. +See the [CELLxGENE extensions](https://github.com/chanzuckerberg/cellxgene-documentation/blob/main/community-extensions.md) section of our documentation for examples of community use and CELLxGENE extensions. ### Security diff --git a/client/.nvmrc b/client/.nvmrc index 3b3a53bf..39d00c05 100644 --- a/client/.nvmrc +++ b/client/.nvmrc @@ -1 +1 @@ -16.20.0 \ No newline at end of file +18.17.0 \ No newline at end of file diff --git a/client/jest-puppeteer.config.js b/client/jest-puppeteer.config.js index a3391a6f..c6eb18e4 100644 --- a/client/jest-puppeteer.config.js +++ b/client/jest-puppeteer.config.js @@ -14,6 +14,7 @@ const DEFAULT_LAUNCH_CONFIG = { headless: !isHeadful, args: ["--ignore-certificate-errors", "--ignore-ssl-errors"], ignoreHTTPSErrors: true, + timeout: 90000, defaultViewport: { width: 1280, height: 960, diff --git a/client/package.json b/client/package.json index 56472787..ebea28f6 100644 --- a/client/package.json +++ b/client/package.json @@ -18,8 +18,8 @@ }, "engineStrict": true, "engines": { - "npm": ">=3.0.0", - "node": "^16.0.0" + "npm": ">=9.6.7", + "node": "^18.17.0" }, "eslintConfig": { "extends": "./configuration/eslint/eslint.js" diff --git a/dev_docs/developer_guidelines.md b/dev_docs/developer_guidelines.md index 33e918d8..449251a5 100644 --- a/dev_docs/developer_guidelines.md +++ b/dev_docs/developer_guidelines.md @@ -3,7 +3,7 @@ ## Requirements - npm -- Python 3.6+ +- Python 3.10+ - Chrome [See dev section of README](../README.md) @@ -148,6 +148,6 @@ If you would like to run the smoke tests against a hot-reloaded version of the c ### Tips -- You can also install/launch the server side code from npm scrips (requires python3.6 with virtualenv) with the `scripts/backend_dev` script. +- You can also install/launch the server side code from npm scrips (requires python3.10 with virtualenv) with the `scripts/backend_dev` script. -- Check out [e2e Tests](e2e_tests.md) for more details \ No newline at end of file +- Check out [e2e Tests](e2e_tests.md) for more details diff --git a/dev_docs/developer_scripts.md b/dev_docs/developer_scripts.md index 864a2e58..3e271abb 100644 --- a/dev_docs/developer_scripts.md +++ b/dev_docs/developer_scripts.md @@ -11,9 +11,10 @@ $PROJECT_ROOT`. ### Build **Usage:** from the `$PROJECT_ROOT` directory run: -* `make build` builds whole app client and server -* `make build-client` runs webpack build -* `make build-for-server-dev` builds client and copies output directly into + +- `make build` builds whole app client and server +- `make build-client` runs webpack build +- `make build-for-server-dev` builds client and copies output directly into source tree (only for server devlopment) ### Clean @@ -21,17 +22,19 @@ $PROJECT_ROOT`. Deletes generated files. **Usage:** from the `$PROJECT_ROOT` directory run: -* `make clean` cleans everything including node modules (means build with take + +- `make clean` cleans everything including node modules (means build with take a while -* `make clean-lite` cleans built directories -* `make clean-server` cleans source tree +- `make clean-lite` cleans built directories +- `make clean-server` cleans source tree ### Distribution Creates distribution for python module to upload to pypi. **Usage:** from the `$PROJECT_ROOT` directory run: -* `make pydist` builds code and then builds sdist + +- `make pydist` builds code and then builds sdist ### Release @@ -42,16 +45,18 @@ See `release_process.md`. Installs requirements files. **Usage:** from the `$PROJECT_ROOT` directory run: -* `make dev-env` installs requirements and requirments-dev (for building code) + +- `make dev-env` installs requirements and requirments-dev (for building code) ### Installing cellxgene packages **Usage:** from the `$PROJECT_ROOT` directory: -* `install-dev` - installs from local source tree -* `install-release-test` - installs from test pypi -* `install-release` - installs from pypi -* `install-dist` - installs from local dist folder -* `uninstall` - uninstalls cellxgene + +- `install-dev` - installs from local source tree +- `install-release-test` - installs from test pypi +- `install-release` - installs from pypi +- `install-dist` - installs from local dist folder +- `uninstall` - uninstalls cellxgene ## Client-level scripts @@ -62,8 +67,9 @@ Installs requirements files. **About** Serve the current client javascript independently from the `server` code. **Requires** -* The server to be running. Best way to do this is with [backend_dev](#backend_dev). -* `make ci` to install the necessary node modules + +- The server to be running. Best way to do this is with [backend_dev](#backend_dev). +- `make ci` to install the necessary node modules **Usage:** from the `$PROJECT_ROOT/client` directory run `make start-frontend` @@ -75,23 +81,24 @@ the FE developer gets the current version of the backend with a single command and no knowledge of python necessary. It creates and activates a virtual environment and installs cellxgene from the current branch. -**Requires** `Python3.6+`, `virtual-env`, `pip` +**Requires** `Python3.10+`, `virtual-env`, `pip` **Usage:** from the `$PROJECT_ROOT` directory run `./scripts/backend_dev` **Options:** -* In parallel, you can then launch the node development server to serve the + +- In parallel, you can then launch the node development server to serve the current state of the FE with [`start-frontend`](#start-frontend), usually in a different terminal tab. -* You can also select a specific dataset using `DATASET= ./scripts/backend_dev`. -* You can also use `CXG_OPTIONS` to pass options to the `cellxgene launch` +- You can also select a specific dataset using `DATASET= ./scripts/backend_dev`. +- You can also use `CXG_OPTIONS` to pass options to the `cellxgene launch` command, as in `CXG_OPTIONS='--disable-annotations' ./scripts/backend_dev`. **Breakdown** | command | purpose | | ---------------------------------------- | ---------------------------------------------------------- | -| python3.6 -m venv cellxgene | creates cellxgene virtual environment | +| python3.12 -m venv cellxgene | creates cellxgene virtual environment | | source cellxgene/bin/activate | activates virtual environment | | yes \| pip uninstall cellxgene \|\| true | uninstalls cellxgene (if installed) | | pip install -e . | installs current local version of cellxgene | @@ -102,14 +109,15 @@ environment and installs cellxgene from the current branch. Methods used to test the client javascript code **Usage:** from the `$PROJECT_ROOT/client` directory run: -* `make unit-test` Runs all unit tests. It excludes any tests in the e2e + +- `make unit-test` Runs all unit tests. It excludes any tests in the e2e folder. This is used by travis to run unit tests. -* `make smoke-test` Starts backend development server and runs end to end +- `make smoke-test` Starts backend development server and runs end to end tests. This is what travis runs. It depends on the `e2e` and the `backend-dev` targets. One starts the server, the other runs the tests. If developing a front-end feature and just checking if tests pass, this is probabaly the one you want to run. -* `npm run e2e` Runs backend tests without starting the server. You will need to +- `npm run e2e` Runs backend tests without starting the server. You will need to start the rest api separately with the pbmc3k.h5ad file. Note you can use the `JEST_ENV` environment variable to change how JEST runs in the browser. The test runs against `localhost:3000` by default. You can use the diff --git a/server/requirements.txt b/server/requirements.txt index ddbdb3f3..2af0df3e 100644 --- a/server/requirements.txt +++ b/server/requirements.txt @@ -7,7 +7,7 @@ Flask-Cors>=3.0.9 Flask-RESTful>=0.3.6 flask-server-timing>=0.1.2 flask-talisman>=0.7.0 -flatbuffers>=1.11.0,<2.0.0 # cellxgene is not compatible with 2.0.0. Requires migration +flatbuffers==2.0.7 flatten-dict>=0.2.0 fsspec>0.8.0 gunicorn>=20.0.4 @@ -19,4 +19,5 @@ pandas>=2.2.2 PyYAML>=5.4 # CVE-2020-14343 requests>=2.22.0 s3fs==0.4.2 -scipy>=1.4 \ No newline at end of file +scipy>=1.4 +setuptools diff --git a/setup.py b/setup.py index 53b2c1b4..38c187c9 100644 --- a/setup.py +++ b/setup.py @@ -24,7 +24,7 @@ setup( long_description=long_description, long_description_content_type="text/markdown", install_requires=requirements, - python_requires=">=3.6", + python_requires=">=3.10", include_package_data=True, zip_safe=False, classifiers=[ @@ -37,8 +37,9 @@ setup( "Operating System :: MacOS :: MacOS X", "Programming Language :: JavaScript", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.6", - "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3 :: Only", "Topic :: Scientific/Engineering :: Bio-Informatics", ], diff --git a/test/unit/utils/test_type_conversion_utils.py b/test/unit/utils/test_type_conversion_utils.py index 9625f57a..a8a72670 100644 --- a/test/unit/utils/test_type_conversion_utils.py +++ b/test/unit/utils/test_type_conversion_utils.py @@ -134,7 +134,7 @@ float_OK_cases = [ np.arange(-128, 1000, dtype=dtype), pd.Series(np.arange(-128, 1000, dtype=dtype)), pd.Index(np.arange(-129, 1000, dtype=dtype)), - np.array([-np.nan, np.inf, -1, 0.0, 0, 0.0, 1, np.inf, np.nan], dtype=dtype), + np.array([-np.nan, -np.inf, -1, -0.0, 0, 0.0, 1, np.inf, np.nan], dtype=dtype), np.array([np.finfo(dtype).min, 0, np.finfo(dtype).max], dtype=dtype), sparse.csr_matrix((10, 100), dtype=dtype), ]